Add support for direct viewing of sqlite files via sqlite URL scheme

app/main.js

@@ -89,49 +89,63 @@ const getRowCount = rtc => (queryStr, cb) => {
  * invoked via electron remote
  *
  * arguments:
- * pathname -- path to CSV file we are opening
+ * targetPath -- filename or sqlite URL we are opening
  * srcfile (optional) -- path we are opening from
  */
-const initMainAsync = async (options, pathname, srcfile) => {
+const initMainAsync = async (options, targetPath, srcfile) => {
   let rtOptions = {}
   if (options['show-queries']) {
     rtOptions.showQueries = true
   }
-  const rtc = await reltabSqlite.getContext(rtOptions)
+  let rtc
+  let ti
 
-  // check if pathname exists
-  if (!fs.existsSync(pathname)) {
-    let found = false
-    let srcdir = null
-    let srcDirTarget = null
-    log.warn('initMain: pathname not found: ', pathname)
-    const basename = path.basename(pathname)
-    if (srcfile) {
-      srcdir = path.dirname(srcfile)
-      srcDirTarget = path.join(srcdir, basename)
-      if (fs.existsSync(srcDirTarget)) {
-        log.warn('initMain: using ' + srcDirTarget + ' instead')
-        pathname = srcDirTarget
-        found = true
+  const sqlUrlPrefix = 'sqlite://'
+  if (targetPath.startsWith(sqlUrlPrefix)) {
+    const urlPath = targetPath.slice(sqlUrlPrefix.length)
+    const tableSepIndex = urlPath.lastIndexOf('/')
+    const tableName = urlPath.slice(tableSepIndex + 1)
+    const dbFileName = urlPath.slice(0, tableSepIndex)
+    rtc = await reltabSqlite.getContext(dbFileName, rtOptions)
+    ti = await rtc.getTableInfo(tableName)
+  } else {
+    rtc = await reltabSqlite.getContext(':memory:', rtOptions)
+    let pathname = targetPath
+    // check if pathname exists
+    if (!fs.existsSync(pathname)) {
+      let found = false
+      let srcdir = null
+      let srcDirTarget = null
+      log.warn('initMain: pathname not found: ', pathname)
+      const basename = path.basename(pathname)
+      if (srcfile) {
+        srcdir = path.dirname(srcfile)
+        srcDirTarget = path.join(srcdir, basename)
+        if (fs.existsSync(srcDirTarget)) {
+          log.warn('initMain: using ' + srcDirTarget + ' instead')
+          pathname = srcDirTarget
+          found = true
+        }
       }
-    }
-    if (!found) {
-      let msg = '"' + pathname + '": file not found.'
-      if (srcdir) {
-        msg += '\n(Also tried "' + srcDirTarget + '")'
+      if (!found) {
+        let msg = '"' + pathname + '": file not found.'
+        if (srcdir) {
+          msg += '\n(Also tried "' + srcDirTarget + '")'
+        }
+        throw new Error(msg)
       }
-      throw new Error(msg)
     }
-  }
 
-  // could also call: csvimport.importSqlite(pathname)
-  const md = await csvimport.fastImport(pathname)
-  rtc.addImportedTable(md)
+    // could also call: csvimport.importSqlite(pathname)
+    const md = await csvimport.fastImport(pathname)
+    ti = csvimport.mkTableInfo(md)
+  }
+  rtc.registerTable(ti)
   // Now let's place a function in global so it can be run via remote:
   global.runQuery = runQuery(rtc)
   global.getRowCount = getRowCount(rtc)
-  const mdStr = JSON.stringify(md, null, 2)
-  return mdStr
+  const tiStr = JSON.stringify(ti, null, 2)
+  return tiStr
 }
 
 const mkInitMain = (options) => (pathname, srcfile, cb) => {
@@ -154,8 +168,8 @@ const optionDefinitions = [
     name: 'srcfile',
     type: String,
     defaultOption: true,
-    typeLabel: '[underline]{file}.csv or [underline]{file}.tad',
-    description: 'CSV file(.csv with header row) or Tad(.tad) file to view'
+    typeLabel: '[underline]{file}.csv or [underline]{file}.tad or sqlite://[underline]{file}/[underline]{table}',
+    description: 'CSV file(.csv with header row), Tad(.tad) file to view'
   },
   {
     name: 'executed-from',
@@ -201,7 +215,8 @@ const usageInfo = [
     header: 'Synopsis',
     content: [
       '$ tad [[italic]{options}] [underline]{file}.csv',
-      '$ tad [[italic]{options}] [underline]{file}.tad'
+      '$ tad [[italic]{options}] [underline]{file}.tad',
+      '$ tad [[italic]{options}] sqlite://[underline]{file}/[underline]{table}'
     ]
   },
   {
@@ -236,7 +251,9 @@ const errorDialog = (title: string, msg: string, fatal = false) => {
 const getTargetPath = (options, filePath) => {
   let targetPath = null
   const srcDir = options['executed-from']
-  if (srcDir && filePath && !(filePath.startsWith('/'))) {
+  if (srcDir && filePath &&
+    !(filePath.startsWith('/')) &&
+    !(filePath.startsWith('sqlite://'))) {
     // relative path -- prepend executed-from
     targetPath = path.join(srcDir, filePath)
   } else {

src/csvimport.js

@@ -3,7 +3,8 @@
  * Import CSV files into sqlite
  */
 
-import type {ColumnType, FileMetadata} from './reltab'
+import type {ColumnType, ColumnMetaMap, TableInfo} from './reltab'
+import {Schema} from './reltab' // eslint-disable-line
 import csv from 'fast-csv'
 import * as _ from 'lodash'
 import * as path from 'path'
@@ -12,14 +13,54 @@ import through from 'through'
 import * as fs from 'fs'
 import db from 'sqlite'
 import Gauge from 'gauge'
-
 /*
  * regex to match a float or int:
  * allows commas and leading $
  */
 const intRE = /[-+]?[$]?[0-9,]+/
 const realRE = /[-+]?[$]?[0-9,]*\.?[0-9]+([eE][-+]?[0-9]+)?/
 
+/*
+ * FileMetaData is an array of unique column IDs, column display names and
+ * ColumnType for each column in a CSV file.
+ * The possible null for ColumnType deals with an empty file (no rows)
+ *
+ */
+export type FileMetadata = {
+  columnIds: Array<string>,
+  columnNames: Array<string>,
+  columnTypes: Array<?ColumnType>,
+  rowCount: number,
+  tableName: string,
+  csvOptions: Object
+}
+
+function assertDefined<A> (x: ?A): A {
+  if (x == null) {
+    throw new Error('unexpected null value')
+  }
+  return x
+}
+
+export const mkTableInfo = (md: FileMetadata): TableInfo => {
+  const extendCMap = (cmm: ColumnMetaMap,
+        cnm: string, idx: number): ColumnMetaMap => {
+    const cType = md.columnTypes[idx]
+    if (cType == null) {
+      console.error('mkTableInfo: No column type for "' + cnm + '", index: ' + idx)
+    }
+    const cmd = {
+      displayName: md.columnNames[idx],
+      type: assertDefined(cType)
+    }
+    cmm[cnm] = cmd
+    return cmm
+  }
+  const cmMap = md.columnIds.reduce(extendCMap, {})
+  const schema = new Schema(md.columnIds, cmMap)
+  return { tableName: md.tableName, schema }
+}
+
 /**
  * Given the current guess (or null) for a column type and cell value string cs
  * make a conservative guess at column type.

src/reltab-sqlite.js

@@ -1,9 +1,15 @@
 /* @flow */
 
 import sqlite from 'sqlite'
-import * as reltab from './reltab'
-import {  TableRep, Schema, FilterExp, QueryExp } from './reltab'  // eslint-disable-line
-import type { FileMetadata, TableInfoMap, ValExp, Row, AggColSpec, SubExp, ColumnMetaMap, ColumnMapInfo, ColumnExtendVal, Connection } from './reltab' // eslint-disable-line
+import { TableRep, QueryExp, Schema } from './reltab'
+import type { TableInfoMap, TableInfo, ValExp, Row, AggColSpec, SubExp, ColumnMetaMap, ColumnMapInfo, ColumnExtendVal, Connection } from './reltab' // eslint-disable-line
+
+function assertDefined<A> (x: ?A): A {
+  if (x == null) {
+    throw new Error('unexpected null value')
+  }
+  return x
+}
 
 class SqliteContext {
   db: any
@@ -16,8 +22,8 @@ class SqliteContext {
     this.showQueries = (options && options.showQueries)
   }
 
-  addImportedTable (md: FileMetadata) {
-    this.tableMap[md.tableName] = reltab.mkTableInfo(md)
+  registerTable (ti: TableInfo) {
+    this.tableMap[ti.tableName] = ti
   }
 
   evalQuery (query: QueryExp, offset: number = -1, limit: number = -1): Promise<TableRep> {
@@ -68,20 +74,47 @@ class SqliteContext {
       return ret
     })
   }
+
+  // use table_info pragma to construct a TableInfo:
+  getTableInfo (tableName: string): Promise<TableInfo> {
+    const tiQuery = `PRAGMA table_info(${tableName})`
+    const qp = this.db.all(tiQuery)
+    return qp.then(rows => {
+      console.log('getTableInfo: ', rows)
+      const extendCMap = (cmm: ColumnMetaMap,
+            row: any, idx: number): ColumnMetaMap => {
+        const cnm = row.name
+        const cType = row.type.toLocaleLowerCase()
+        if (cType == null) {
+          console.error('mkTableInfo: No column type for "' + cnm + '", index: ' + idx)
+        }
+        const cmd = {
+          displayName: cnm,
+          type: assertDefined(cType)
+        }
+        cmm[cnm] = cmd
+        return cmm
+      }
+      const cmMap = rows.reduce(extendCMap, {})
+      const columnIds = rows.map(r => r.name)
+      const schema = new Schema(columnIds, cmMap)
+      return { tableName, schema }
+    })
+  }
 }
 
-const init = async (options: Object = {}): Connection => {
-  await sqlite.open(':memory:')
+const init = async (dbfile, options: Object = {}): Connection => {
+  await sqlite.open(dbfile)
   const ctx = new SqliteContext(sqlite, options)
   return ctx
 }
 
 // get (singleton) connection to sqlite:
 let ctxPromise: ?Promise<Connection> = null
 
-export const getContext = (options: Object = {}): Promise<Connection> => {
+export const getContext = (dbfile: string, options: Object = {}): Promise<Connection> => {
   if (!ctxPromise) {
-    ctxPromise = init(options)
+    ctxPromise = init(dbfile, options)
   }
   return ctxPromise
 }

src/reltab.js

@@ -1029,54 +1029,10 @@ export class Schema {
     return outSchema
   }
 }
-/*
- * FileMetaData is an array of unique column IDs, column display names and
- * ColumnType for each column in a CSV file.
- * The possible null for ColumnType deals with an empty file (no rows)
- *
- * TODO: This began life in csvimport, but moved here because TablInfoMap did,
- * which we need for QueryExp.getSchema().
- * This distinct data structure in reltab should perhaps just go away; we could just
- * use Schema everywhere.
- */
-export type FileMetadata = {
-  columnIds: Array<string>,
-  columnNames: Array<string>,
-  columnTypes: Array<?ColumnType>,
-  rowCount: number,
-  tableName: string,
-  csvOptions: Object
-}
 
-export type TableInfo = { tableName: string, schema: Schema, md: FileMetadata }
+export type TableInfo = { tableName: string, schema: Schema }
 export type TableInfoMap = { [tableName: string]: TableInfo }
 
-function assertDefined<A> (x: ?A): A {
-  if (x == null) {
-    throw new Error('unexpected null value')
-  }
-  return x
-}
-
-export const mkTableInfo = (md: FileMetadata): TableInfo => {
-  const extendCMap = (cmm: ColumnMetaMap,
-        cnm: string, idx: number): ColumnMetaMap => {
-    const cType = md.columnTypes[idx]
-    if (cType == null) {
-      console.error('mkTableInfo: No column type for "' + cnm + '", index: ' + idx)
-    }
-    const cmd = {
-      displayName: md.columnNames[idx],
-      type: assertDefined(cType)
-    }
-    cmm[cnm] = cmd
-    return cmm
-  }
-  const cmMap = md.columnIds.reduce(extendCMap, {})
-  const schema = new Schema(md.columnIds, cmMap)
-  return { tableName: md.tableName, schema, md }
-}
-
 export class TableRep {
   schema: Schema
   rowData: Array<Row>

src/renderMain.js

@@ -31,15 +31,15 @@ const remoteErrorDialog = remote.getGlobal('errorDialog')
 
 const ipcRenderer = require('electron').ipcRenderer
 
-const initMainProcess = (targetPath, srcFile): Promise<reltab.FileMetadata> => {
+const initMainProcess = (targetPath, srcFile): Promise<reltab.TableInfo> => {
   return new Promise((resolve, reject) => {
-    remoteInitMain(targetPath, srcFile, (err, mdStr) => {
+    remoteInitMain(targetPath, srcFile, (err, tiStr) => {
       if (err) {
         console.error('initMain error: ', err)
         reject(err)
       } else {
-        const md = JSON.parse(mdStr)
-        resolve(md)
+        const ti = JSON.parse(tiStr)
+        resolve(ti)
       }
     })
   })
@@ -72,16 +72,16 @@ const init = () => {
 
   // and kick off main process initialization:
   initMainProcess(targetPath, srcFile)
-    .then(md => {
-      const tableName = md.tableName
+    .then(ti => {
+      const tableName = ti.tableName
       const baseQuery = reltab.tableQuery(tableName)
 
       const rtc = reltabElectron.init()
 
       // module local to keep alive:
       var pivotRequester: ?PivotRequester = null  // eslint-disable-line
 
-      actions.initAppState(rtc, md.tableName, baseQuery, viewParams, updater)
+      actions.initAppState(rtc, ti.tableName, baseQuery, viewParams, updater)
         .then(() => {
           pivotRequester = new PivotRequester(stateRef) // eslint-disable-line
 
@@ -96,12 +96,12 @@ const init = () => {
               { requestId, contents: serState })
           })
           ipcRenderer.on('set-show-hidden-cols', (event, val) => {
-            console.log('got set-show-hidden-cols: ', val)
             actions.setShowHiddenCols(val, updater)
           })
         })
     })
     .catch(err => {
+      console.error('renderMain: caught error during initialization: ', err.message, err.stack)
       remoteErrorDialog('Error initializing Tad', err.message, true)
     })
 }

test/reltabSqliteTests.js

@@ -21,9 +21,10 @@ const sqliteTestSetup = (htest) => {
   htest('sqlite test setup', async (t) => {
     try {
       const showQueries = global.showQueries
-      const rtc = await reltabSqlite.getContext({showQueries})
+      const rtc = await reltabSqlite.getContext(':memory:', {showQueries})
       const md = await csvimport.importSqlite(testPath)
-      rtc.addImportedTable(md)
+      const ti = csvimport.mkTableInfo(md)
+      rtc.registerTable(ti)
       sharedRtc = rtc
       console.log('set rtc: ', sharedRtc)
       t.ok(true, 'setup and import complete')