diff --git a/app/main.js b/app/main.js index 51faddd..0370d12 100644 --- a/app/main.js +++ b/app/main.js @@ -89,49 +89,63 @@ const getRowCount = rtc => (queryStr, cb) => { * invoked via electron remote * * arguments: - * pathname -- path to CSV file we are opening + * targetPath -- filename or sqlite URL we are opening * srcfile (optional) -- path we are opening from */ -const initMainAsync = async (options, pathname, srcfile) => { +const initMainAsync = async (options, targetPath, srcfile) => { let rtOptions = {} if (options['show-queries']) { rtOptions.showQueries = true } - const rtc = await reltabSqlite.getContext(rtOptions) + let rtc + let ti - // check if pathname exists - if (!fs.existsSync(pathname)) { - let found = false - let srcdir = null - let srcDirTarget = null - log.warn('initMain: pathname not found: ', pathname) - const basename = path.basename(pathname) - if (srcfile) { - srcdir = path.dirname(srcfile) - srcDirTarget = path.join(srcdir, basename) - if (fs.existsSync(srcDirTarget)) { - log.warn('initMain: using ' + srcDirTarget + ' instead') - pathname = srcDirTarget - found = true + const sqlUrlPrefix = 'sqlite://' + if (targetPath.startsWith(sqlUrlPrefix)) { + const urlPath = targetPath.slice(sqlUrlPrefix.length) + const tableSepIndex = urlPath.lastIndexOf('/') + const tableName = urlPath.slice(tableSepIndex + 1) + const dbFileName = urlPath.slice(0, tableSepIndex) + rtc = await reltabSqlite.getContext(dbFileName, rtOptions) + ti = await rtc.getTableInfo(tableName) + } else { + rtc = await reltabSqlite.getContext(':memory:', rtOptions) + let pathname = targetPath + // check if pathname exists + if (!fs.existsSync(pathname)) { + let found = false + let srcdir = null + let srcDirTarget = null + log.warn('initMain: pathname not found: ', pathname) + const basename = path.basename(pathname) + if (srcfile) { + srcdir = path.dirname(srcfile) + srcDirTarget = path.join(srcdir, basename) + if (fs.existsSync(srcDirTarget)) { + log.warn('initMain: using ' + srcDirTarget + ' instead') + pathname = srcDirTarget + found = true + } } - } - if (!found) { - let msg = '"' + pathname + '": file not found.' - if (srcdir) { - msg += '\n(Also tried "' + srcDirTarget + '")' + if (!found) { + let msg = '"' + pathname + '": file not found.' + if (srcdir) { + msg += '\n(Also tried "' + srcDirTarget + '")' + } + throw new Error(msg) } - throw new Error(msg) } - } - // could also call: csvimport.importSqlite(pathname) - const md = await csvimport.fastImport(pathname) - rtc.addImportedTable(md) + // could also call: csvimport.importSqlite(pathname) + const md = await csvimport.fastImport(pathname) + ti = csvimport.mkTableInfo(md) + } + rtc.registerTable(ti) // Now let's place a function in global so it can be run via remote: global.runQuery = runQuery(rtc) global.getRowCount = getRowCount(rtc) - const mdStr = JSON.stringify(md, null, 2) - return mdStr + const tiStr = JSON.stringify(ti, null, 2) + return tiStr } const mkInitMain = (options) => (pathname, srcfile, cb) => { @@ -154,8 +168,8 @@ const optionDefinitions = [ name: 'srcfile', type: String, defaultOption: true, - typeLabel: '[underline]{file}.csv or [underline]{file}.tad', - description: 'CSV file(.csv with header row) or Tad(.tad) file to view' + typeLabel: '[underline]{file}.csv or [underline]{file}.tad or sqlite://[underline]{file}/[underline]{table}', + description: 'CSV file(.csv with header row), Tad(.tad) file to view' }, { name: 'executed-from', @@ -201,7 +215,8 @@ const usageInfo = [ header: 'Synopsis', content: [ '$ tad [[italic]{options}] [underline]{file}.csv', - '$ tad [[italic]{options}] [underline]{file}.tad' + '$ tad [[italic]{options}] [underline]{file}.tad', + '$ tad [[italic]{options}] sqlite://[underline]{file}/[underline]{table}' ] }, { @@ -236,7 +251,9 @@ const errorDialog = (title: string, msg: string, fatal = false) => { const getTargetPath = (options, filePath) => { let targetPath = null const srcDir = options['executed-from'] - if (srcDir && filePath && !(filePath.startsWith('/'))) { + if (srcDir && filePath && + !(filePath.startsWith('/')) && + !(filePath.startsWith('sqlite://'))) { // relative path -- prepend executed-from targetPath = path.join(srcDir, filePath) } else { diff --git a/src/csvimport.js b/src/csvimport.js index ae5f51a..f371b48 100644 --- a/src/csvimport.js +++ b/src/csvimport.js @@ -3,7 +3,8 @@ * Import CSV files into sqlite */ -import type {ColumnType, FileMetadata} from './reltab' +import type {ColumnType, ColumnMetaMap, TableInfo} from './reltab' +import {Schema} from './reltab' // eslint-disable-line import csv from 'fast-csv' import * as _ from 'lodash' import * as path from 'path' @@ -12,7 +13,6 @@ import through from 'through' import * as fs from 'fs' import db from 'sqlite' import Gauge from 'gauge' - /* * regex to match a float or int: * allows commas and leading $ @@ -20,6 +20,47 @@ import Gauge from 'gauge' const intRE = /[-+]?[$]?[0-9,]+/ const realRE = /[-+]?[$]?[0-9,]*\.?[0-9]+([eE][-+]?[0-9]+)?/ +/* + * FileMetaData is an array of unique column IDs, column display names and + * ColumnType for each column in a CSV file. + * The possible null for ColumnType deals with an empty file (no rows) + * + */ +export type FileMetadata = { + columnIds: Array, + columnNames: Array, + columnTypes: Array, + rowCount: number, + tableName: string, + csvOptions: Object +} + +function assertDefined (x: ?A): A { + if (x == null) { + throw new Error('unexpected null value') + } + return x +} + +export const mkTableInfo = (md: FileMetadata): TableInfo => { + const extendCMap = (cmm: ColumnMetaMap, + cnm: string, idx: number): ColumnMetaMap => { + const cType = md.columnTypes[idx] + if (cType == null) { + console.error('mkTableInfo: No column type for "' + cnm + '", index: ' + idx) + } + const cmd = { + displayName: md.columnNames[idx], + type: assertDefined(cType) + } + cmm[cnm] = cmd + return cmm + } + const cmMap = md.columnIds.reduce(extendCMap, {}) + const schema = new Schema(md.columnIds, cmMap) + return { tableName: md.tableName, schema } +} + /** * Given the current guess (or null) for a column type and cell value string cs * make a conservative guess at column type. diff --git a/src/reltab-sqlite.js b/src/reltab-sqlite.js index 76db0be..2d48049 100644 --- a/src/reltab-sqlite.js +++ b/src/reltab-sqlite.js @@ -1,9 +1,15 @@ /* @flow */ import sqlite from 'sqlite' -import * as reltab from './reltab' -import { TableRep, Schema, FilterExp, QueryExp } from './reltab' // eslint-disable-line -import type { FileMetadata, TableInfoMap, ValExp, Row, AggColSpec, SubExp, ColumnMetaMap, ColumnMapInfo, ColumnExtendVal, Connection } from './reltab' // eslint-disable-line +import { TableRep, QueryExp, Schema } from './reltab' +import type { TableInfoMap, TableInfo, ValExp, Row, AggColSpec, SubExp, ColumnMetaMap, ColumnMapInfo, ColumnExtendVal, Connection } from './reltab' // eslint-disable-line + +function assertDefined (x: ?A): A { + if (x == null) { + throw new Error('unexpected null value') + } + return x +} class SqliteContext { db: any @@ -16,8 +22,8 @@ class SqliteContext { this.showQueries = (options && options.showQueries) } - addImportedTable (md: FileMetadata) { - this.tableMap[md.tableName] = reltab.mkTableInfo(md) + registerTable (ti: TableInfo) { + this.tableMap[ti.tableName] = ti } evalQuery (query: QueryExp, offset: number = -1, limit: number = -1): Promise { @@ -68,10 +74,37 @@ class SqliteContext { return ret }) } + + // use table_info pragma to construct a TableInfo: + getTableInfo (tableName: string): Promise { + const tiQuery = `PRAGMA table_info(${tableName})` + const qp = this.db.all(tiQuery) + return qp.then(rows => { + console.log('getTableInfo: ', rows) + const extendCMap = (cmm: ColumnMetaMap, + row: any, idx: number): ColumnMetaMap => { + const cnm = row.name + const cType = row.type.toLocaleLowerCase() + if (cType == null) { + console.error('mkTableInfo: No column type for "' + cnm + '", index: ' + idx) + } + const cmd = { + displayName: cnm, + type: assertDefined(cType) + } + cmm[cnm] = cmd + return cmm + } + const cmMap = rows.reduce(extendCMap, {}) + const columnIds = rows.map(r => r.name) + const schema = new Schema(columnIds, cmMap) + return { tableName, schema } + }) + } } -const init = async (options: Object = {}): Connection => { - await sqlite.open(':memory:') +const init = async (dbfile, options: Object = {}): Connection => { + await sqlite.open(dbfile) const ctx = new SqliteContext(sqlite, options) return ctx } @@ -79,9 +112,9 @@ const init = async (options: Object = {}): Connection => { // get (singleton) connection to sqlite: let ctxPromise: ?Promise = null -export const getContext = (options: Object = {}): Promise => { +export const getContext = (dbfile: string, options: Object = {}): Promise => { if (!ctxPromise) { - ctxPromise = init(options) + ctxPromise = init(dbfile, options) } return ctxPromise } diff --git a/src/reltab.js b/src/reltab.js index 00c7287..1bcdbda 100644 --- a/src/reltab.js +++ b/src/reltab.js @@ -1029,54 +1029,10 @@ export class Schema { return outSchema } } -/* - * FileMetaData is an array of unique column IDs, column display names and - * ColumnType for each column in a CSV file. - * The possible null for ColumnType deals with an empty file (no rows) - * - * TODO: This began life in csvimport, but moved here because TablInfoMap did, - * which we need for QueryExp.getSchema(). - * This distinct data structure in reltab should perhaps just go away; we could just - * use Schema everywhere. - */ -export type FileMetadata = { - columnIds: Array, - columnNames: Array, - columnTypes: Array, - rowCount: number, - tableName: string, - csvOptions: Object -} -export type TableInfo = { tableName: string, schema: Schema, md: FileMetadata } +export type TableInfo = { tableName: string, schema: Schema } export type TableInfoMap = { [tableName: string]: TableInfo } -function assertDefined (x: ?A): A { - if (x == null) { - throw new Error('unexpected null value') - } - return x -} - -export const mkTableInfo = (md: FileMetadata): TableInfo => { - const extendCMap = (cmm: ColumnMetaMap, - cnm: string, idx: number): ColumnMetaMap => { - const cType = md.columnTypes[idx] - if (cType == null) { - console.error('mkTableInfo: No column type for "' + cnm + '", index: ' + idx) - } - const cmd = { - displayName: md.columnNames[idx], - type: assertDefined(cType) - } - cmm[cnm] = cmd - return cmm - } - const cmMap = md.columnIds.reduce(extendCMap, {}) - const schema = new Schema(md.columnIds, cmMap) - return { tableName: md.tableName, schema, md } -} - export class TableRep { schema: Schema rowData: Array diff --git a/src/renderMain.js b/src/renderMain.js index 6f9b30c..a279066 100644 --- a/src/renderMain.js +++ b/src/renderMain.js @@ -31,15 +31,15 @@ const remoteErrorDialog = remote.getGlobal('errorDialog') const ipcRenderer = require('electron').ipcRenderer -const initMainProcess = (targetPath, srcFile): Promise => { +const initMainProcess = (targetPath, srcFile): Promise => { return new Promise((resolve, reject) => { - remoteInitMain(targetPath, srcFile, (err, mdStr) => { + remoteInitMain(targetPath, srcFile, (err, tiStr) => { if (err) { console.error('initMain error: ', err) reject(err) } else { - const md = JSON.parse(mdStr) - resolve(md) + const ti = JSON.parse(tiStr) + resolve(ti) } }) }) @@ -72,8 +72,8 @@ const init = () => { // and kick off main process initialization: initMainProcess(targetPath, srcFile) - .then(md => { - const tableName = md.tableName + .then(ti => { + const tableName = ti.tableName const baseQuery = reltab.tableQuery(tableName) const rtc = reltabElectron.init() @@ -81,7 +81,7 @@ const init = () => { // module local to keep alive: var pivotRequester: ?PivotRequester = null // eslint-disable-line - actions.initAppState(rtc, md.tableName, baseQuery, viewParams, updater) + actions.initAppState(rtc, ti.tableName, baseQuery, viewParams, updater) .then(() => { pivotRequester = new PivotRequester(stateRef) // eslint-disable-line @@ -96,12 +96,12 @@ const init = () => { { requestId, contents: serState }) }) ipcRenderer.on('set-show-hidden-cols', (event, val) => { - console.log('got set-show-hidden-cols: ', val) actions.setShowHiddenCols(val, updater) }) }) }) .catch(err => { + console.error('renderMain: caught error during initialization: ', err.message, err.stack) remoteErrorDialog('Error initializing Tad', err.message, true) }) } diff --git a/test/reltabSqliteTests.js b/test/reltabSqliteTests.js index b87ad0a..8a1c723 100644 --- a/test/reltabSqliteTests.js +++ b/test/reltabSqliteTests.js @@ -21,9 +21,10 @@ const sqliteTestSetup = (htest) => { htest('sqlite test setup', async (t) => { try { const showQueries = global.showQueries - const rtc = await reltabSqlite.getContext({showQueries}) + const rtc = await reltabSqlite.getContext(':memory:', {showQueries}) const md = await csvimport.importSqlite(testPath) - rtc.addImportedTable(md) + const ti = csvimport.mkTableInfo(md) + rtc.registerTable(ti) sharedRtc = rtc console.log('set rtc: ', sharedRtc) t.ok(true, 'setup and import complete') diff --git a/test/runQuery.js b/test/runQuery.js index d9a2795..673433c 100644 --- a/test/runQuery.js +++ b/test/runQuery.js @@ -7,10 +7,12 @@ import 'console.table' import db from 'sqlite' import * as csvimport from '../src/csvimport' -// const testPath = 'csv/bart-comp-all.csv' -const testPath = '/Users/antony/data/uber-raw-data-apr14.csv' +const testPath = 'csv/bart-comp-all.csv' +// const testPath = '/Users/antony/data/uber-raw-data-apr14.csv' -// const tq = 'select * from \'bart-comp-all\' limit 10' +// const tq = 'select * from \'bart_comp_all\' limit 10' + +const tq = 'PRAGMA table_info(bart_comp_all)' /* const tq = ` @@ -61,35 +63,35 @@ SELECT "Name", "Title", "Base", "OT", "Other", "MDV", "ER", "EE", "DC", "Misc", FROM 'bart-comp-all' WHERE "Title"='Department Manager Gov''t & Comm Rel'` */ - +/* const tq = ` SELECT * - FROM 'uber-raw-data-apr14' + FROM 'uber_raw_data_apr14' LIMIT 10` +*/ + +const main = async () => { + try { + const hrProcStart = process.hrtime() + let hrQueryStart = 0 + await db.open(':memory:') + await csvimport.importSqlite(testPath) + // await db.open('/Users/antony/data/testdb.sqlite') + const [es, ens] = process.hrtime(hrProcStart) + console.info('runQuery: import completed in %ds %dms', es, ens / 1e6) + // console.log('table import complete: ', md.tableName) + console.log('running query:\n', tq) + hrQueryStart = process.hrtime() -const main = () => { - const hrProcStart = process.hrtime() - let hrQueryStart = 0 - db.open(':memory:') - .then(() => csvimport.importSqlite(testPath)) - .then(md => { - const [es, ens] = process.hrtime(hrProcStart) - console.info('runQuery: import completed in %ds %dms', es, ens / 1e6) - console.log('table import complete: ', md.tableName) - console.log('running query:\n', tq) - hrQueryStart = process.hrtime() - return db.all(tq) - }) - .then(rows => { - const [es, ens] = process.hrtime(hrQueryStart) - console.log('read rows from sqlite table.') - console.table(rows) - console.info('runQuery: evaluated query in %ds %dms', es, ens / 1e6) - }) - .then(() => db.close()) - .catch(err => { - console.error('caught exception in promise chain: ', err, err.stack) - }) + const rows = await db.all(tq) + const [qes, qens] = process.hrtime(hrQueryStart) + console.log('read rows from sqlite table.') + console.table(rows) + console.info('runQuery: evaluated query in %ds %dms', qes, qens / 1e6) + await db.close() + } catch (err) { + console.error('caught exception running query: ', err, err.stack) + } } main()