refactor(cli): replace better-sqlite3 with sql.js (pure WASM)

- Remove native C++ addon dependency, no more pnpm approve-builds - sql.js loads SQLite as WASM, zero compilation required - WASM init is singleton (once per process) - Add queryAsObjects() adapter for sql.js columnar → row format - Tests migrated to sql.js (16 passing) Implements RFC #63
2026-04-23 07:25:08 +00:00
parent 9b93c4a4d9
commit c8bf4bf547
6 changed files with 130 additions and 70 deletions
@@ -20,13 +20,13 @@
  },
  "dependencies": {
    "@uncaged/nerve-core": "workspace:*",
-    "better-sqlite3": "^11.10.0",
-    "citty": "^0.1.6"
+    "citty": "^0.1.6",
+    "sql.js": "^1.14.1"
  },
  "devDependencies": {
-    "@uncaged/nerve-daemon": "workspace:*",
    "@types/better-sqlite3": "^7.6.13",
    "@types/node": "^22.0.0",
+    "@uncaged/nerve-daemon": "workspace:*",
    "vitest": "^4.1.5"
  }
 }
@@ -2,12 +2,12 @@
 * Tests for sense SQLite helpers used by `nerve sense schema` / `nerve sense query`.
 */

-import { mkdirSync, rmSync } from "node:fs";
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";

-import Database from "better-sqlite3";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import initSqlJs, { type Database } from "sql.js";
+import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";

 import {
  assertSenseDbExists,
@@ -17,11 +17,17 @@ import {
  listTableSqlStatements,
  parseSenseQueryArgs,
  pickDefaultPreviewTable,
+  queryAsObjects,
  senseDbPath,
 } from "../sense-sqlite.js";

+let SQL: Awaited<ReturnType<typeof initSqlJs>>;
 let tmpDir: string;

+beforeAll(async () => {
+  SQL = await initSqlJs();
+});
+
 beforeEach(() => {
  tmpDir = join(
    tmpdir(),
@@ -34,6 +40,22 @@ afterEach(() => {
  rmSync(tmpDir, { recursive: true, force: true });
 });

+/** Helper: create a SQLite db file with the given setup SQL. */
+function createDb(name: string, setupSql: string): void {
+  const db = new SQL.Database();
+  db.run(setupSql);
+  const data = db.export();
+  db.close();
+  writeFileSync(join(tmpDir, "data", "senses", `${name}.db`), Buffer.from(data));
+}
+
+/** Helper: open an in-memory db with setup SQL for unit tests. */
+function memDb(setupSql?: string): Database {
+  const db = new SQL.Database();
+  if (setupSql) db.run(setupSql);
+  return db;
+}
+
 describe("senseDbPath", () => {
  it("points at data/senses/<name>.db under the given root", () => {
    expect(senseDbPath("/root", "cpu-usage")).toBe(join("/root", "data", "senses", "cpu-usage.db"));
@@ -46,18 +68,14 @@ describe("assertSenseDbExists", () => {
  });

  it("returns the path when the file exists", () => {
-    const p = join(tmpDir, "data", "senses", "x.db");
-    new Database(p).close();
-    expect(assertSenseDbExists(tmpDir, "x")).toBe(p);
+    createDb("x", "SELECT 1");
+    expect(assertSenseDbExists(tmpDir, "x")).toBe(join(tmpDir, "data", "senses", "x.db"));
  });
 });

 describe("listTableSqlStatements", () => {
  it("returns CREATE statements ordered by tbl_name", () => {
-    const p = join(tmpDir, "data", "senses", "t.db");
-    const db = new Database(p);
-    db.exec("CREATE TABLE zebra (id INTEGER);");
-    db.exec("CREATE TABLE alpha (id INTEGER);");
+    const db = memDb("CREATE TABLE zebra (id INTEGER); CREATE TABLE alpha (id INTEGER);");
    const stmts = listTableSqlStatements(db);
    db.close();
    expect(stmts).toHaveLength(2);
@@ -68,18 +86,16 @@ describe("listTableSqlStatements", () => {

 describe("pickDefaultPreviewTable", () => {
  it("prefers non-_migrations tables when both exist", () => {
-    const p = join(tmpDir, "data", "senses", "t.db");
-    const db = new Database(p);
-    db.exec(`CREATE TABLE _migrations (name TEXT PRIMARY KEY);
-             CREATE TABLE readings (id INTEGER);`);
+    const db = memDb(
+      `CREATE TABLE _migrations (name TEXT PRIMARY KEY);
+       CREATE TABLE readings (id INTEGER);`,
+    );
    expect(pickDefaultPreviewTable(db)).toBe("readings");
    db.close();
  });

  it("uses _migrations when it is the only table", () => {
-    const p = join(tmpDir, "data", "senses", "t.db");
-    const db = new Database(p);
-    db.exec("CREATE TABLE _migrations (name TEXT PRIMARY KEY);");
+    const db = memDb("CREATE TABLE _migrations (name TEXT PRIMARY KEY);");
    expect(pickDefaultPreviewTable(db)).toBe("_migrations");
    db.close();
  });
@@ -137,22 +153,29 @@ describe("collectColumnKeys", () => {
  });
 });

-describe("readonly query integration", () => {
-  it("runs default preview SQL on a real db", () => {
-    const p = join(tmpDir, "data", "senses", "demo.db");
-    const rw = new Database(p);
-    rw.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, v TEXT);");
-    rw.exec(`INSERT INTO items (v) VALUES ('a'), ('b');`);
-    rw.close();
+describe("queryAsObjects", () => {
+  it("converts columnar sql.js results to row objects", () => {
+    const db = memDb("CREATE TABLE t (x INTEGER, y TEXT); INSERT INTO t VALUES (1, 'a'), (2, 'b');");
+    const rows = queryAsObjects(db, "SELECT * FROM t ORDER BY x");
+    db.close();
+    expect(rows).toEqual([
+      { x: 1, y: "a" },
+      { x: 2, y: "b" },
+    ]);
+  });
+});

-    const db = new Database(p, { readonly: true, fileMustExist: true });
+describe("readonly query integration", () => {
+  it("runs default preview SQL on a real db file", () => {
+    createDb("demo", "CREATE TABLE items (id INTEGER PRIMARY KEY, v TEXT); INSERT INTO items (v) VALUES ('a'), ('b');");
+
+    const buffer = require("node:fs").readFileSync(join(tmpDir, "data", "senses", "demo.db"));
+    const db = new SQL.Database(buffer);
    const table = pickDefaultPreviewTable(db);
    expect(table).toBe("items");
-    if (table === null) {
-      throw new Error("expected items table");
-    }
+    if (table === null) throw new Error("expected items table");
    const sql = defaultPreviewSql(table);
-    const rows = db.prepare(sql).all() as Record<string, unknown>[];
+    const rows = queryAsObjects(db, sql);
    db.close();
    expect(rows.length).toBeGreaterThanOrEqual(1);
  });
@@ -2,7 +2,6 @@ import { readFileSync } from "node:fs";
 import { join } from "node:path";

 import { type SenseInfo, parseNerveConfig } from "@uncaged/nerve-core";
-import Database from "better-sqlite3";
 import { defineCommand } from "citty";

 import { listSensesViaDaemon, triggerSenseViaDaemon } from "../daemon-client.js";
@@ -11,8 +10,10 @@ import {
  defaultPreviewSql,
  formatRowsAsAlignedTable,
  listTableSqlStatements,
+  openSenseDb,
  parseSenseQueryArgs,
  pickDefaultPreviewTable,
+  queryAsObjects,
 } from "../sense-sqlite.js";
 import { getNerveRoot, getSocketPath, isRunning } from "../workspace.js";

@@ -170,10 +171,9 @@ const senseSchemaCommand = defineCommand({
  },
  async run({ args }) {
    const nerveRoot = getNerveRoot();
-    let db: Database.Database | undefined;
+    let db: ReturnType<Awaited<ReturnType<typeof import("sql.js")>>["Database"]> | undefined;
    try {
-      const path = assertSenseDbExists(nerveRoot, args.name);
-      db = new Database(path, { readonly: true, fileMustExist: true });
+      db = await openSenseDb(nerveRoot, args.name);
      const statements = listTableSqlStatements(db);
      if (args.json) {
        process.stdout.write(`${JSON.stringify(statements, null, 2)}\n`);
@@ -217,7 +217,7 @@ const senseQueryCommand = defineCommand({
  },
  async run({ args, rawArgs }) {
    const nerveRoot = getNerveRoot();
-    let db: Database.Database | undefined;
+    let db: ReturnType<Awaited<ReturnType<typeof import("sql.js")>>["Database"]> | undefined;
    try {
      let parsed: { name: string; sql: string | undefined };
      try {
@@ -228,8 +228,7 @@ const senseQueryCommand = defineCommand({
        process.exit(1);
      }

-      const path = assertSenseDbExists(nerveRoot, args.name);
-      db = new Database(path, { readonly: true, fileMustExist: true });
+      db = await openSenseDb(nerveRoot, args.name);

      let sql = parsed.sql?.trim();
      if (!sql) {
@@ -242,8 +241,7 @@ const senseQueryCommand = defineCommand({
        }
      }

-      const stmt = db.prepare(sql);
-      const rows = stmt.all() as Record<string, unknown>[];
+      const rows = queryAsObjects(db, sql);

      if (args.json) {
        process.stdout.write(`${JSON.stringify(rows, null, 2)}\n`);
@@ -1,7 +1,25 @@
-import { existsSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";

-import type Database from "better-sqlite3";
+import initSqlJs, { type Database } from "sql.js";
+
+// ── WASM singleton ──────────────────────────────────────────────────────────
+let _SQL: Awaited<ReturnType<typeof initSqlJs>> | null = null;
+
+async function getSQL() {
+  if (!_SQL) {
+    _SQL = await initSqlJs();
+  }
+  return _SQL;
+}
+
+/** Open a sense SQLite database (readonly, loaded into memory via sql.js). */
+export async function openSenseDb(nerveRoot: string, senseName: string): Promise<Database> {
+  const path = assertSenseDbExists(nerveRoot, senseName);
+  const SQL = await getSQL();
+  const buffer = readFileSync(path);
+  return new SQL.Database(buffer);
+}

 /** SQLite path for a sense under the nerve workspace root. */
 export function senseDbPath(nerveRoot: string, senseName: string): string {
@@ -17,32 +35,30 @@ export function assertSenseDbExists(nerveRoot: string, senseName: string): strin
 }

 /** `SELECT sql FROM sqlite_master WHERE type='table'` (non-null sql only). */
-export function listTableSqlStatements(db: Database.Database): string[] {
-  const rows = db
-    .prepare(
-      `SELECT sql FROM sqlite_master WHERE type = 'table' AND sql IS NOT NULL ORDER BY tbl_name`,
-    )
-    .all() as { sql: string }[];
-  return rows.map((r) => r.sql);
+export function listTableSqlStatements(db: Database): string[] {
+  const results = db.exec(
+    `SELECT sql FROM sqlite_master WHERE type = 'table' AND sql IS NOT NULL ORDER BY tbl_name`,
+  );
+  if (results.length === 0) return [];
+  return results[0].values.map((row) => row[0] as string);
 }

 /**
 * Table used for `nerve sense query <name>` with no SQL.
 * Prefers real data tables over `_migrations`, then lexicographic by name.
 */
-export function pickDefaultPreviewTable(db: Database.Database): string | null {
-  const row = db
-    .prepare(
-      `SELECT name FROM sqlite_master
-       WHERE type = 'table' AND sql IS NOT NULL
-         AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'
-       ORDER BY
-         CASE WHEN name = '_migrations' THEN 1 ELSE 0 END,
-         name
-       LIMIT 1`,
-    )
-    .get() as { name: string } | undefined;
-  return row?.name ?? null;
+export function pickDefaultPreviewTable(db: Database): string | null {
+  const results = db.exec(
+    `SELECT name FROM sqlite_master
+     WHERE type = 'table' AND sql IS NOT NULL
+       AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'
+     ORDER BY
+       CASE WHEN name = '_migrations' THEN 1 ELSE 0 END,
+       name
+     LIMIT 1`,
+  );
+  if (results.length === 0 || results[0].values.length === 0) return null;
+  return results[0].values[0][0] as string;
 }

 export function defaultPreviewSql(table: string): string {
@@ -77,7 +93,7 @@ function stringifyCell(value: unknown): string {
  if (typeof value === "bigint") return value.toString();
  if (typeof value === "number" || typeof value === "boolean") return String(value);
  if (typeof value === "string") return value;
-  if (Buffer.isBuffer(value)) return value.toString("hex");
+  if (value instanceof Uint8Array) return Buffer.from(value).toString("hex");
  try {
    return JSON.stringify(value);
  } catch {
@@ -120,3 +136,20 @@ export function formatRowsAsAlignedTable(rows: Record<string, unknown>[]): strin
  const body = cells.map((r) => r.map((cell, j) => cell.padEnd(widths[j])).join(" | ")).join("\n");
  return `${header}\n${sep}\n${body}\n`;
 }
+
+/**
+ * Run a SQL query via sql.js and return rows as key-value objects.
+ * sql.js returns columnar data; this converts to the familiar row format.
+ */
+export function queryAsObjects(db: Database, sql: string): Record<string, unknown>[] {
+  const results = db.exec(sql);
+  if (results.length === 0) return [];
+  const { columns, values } = results[0];
+  return values.map((row) => {
+    const obj: Record<string, unknown> = {};
+    for (let i = 0; i < columns.length; i++) {
+      obj[columns[i]] = row[i];
+    }
+    return obj;
+  });
+}
@@ -9,5 +9,5 @@ export default defineConfig({
    js: "#!/usr/bin/env node",
  },
  /** Daemon is loaded from workspace node_modules at runtime — never bundle it. */
-  external: ["@uncaged/nerve-daemon", "better-sqlite3"],
+  external: ["@uncaged/nerve-daemon", "sql.js"],
 });
@@ -23,12 +23,12 @@ importers:
      '@uncaged/nerve-core':
        specifier: workspace:*
        version: link:../core
-      better-sqlite3:
-        specifier: ^11.10.0
-        version: 11.10.0
      citty:
        specifier: ^0.1.6
        version: 0.1.6
+      sql.js:
+        specifier: ^1.14.1
+        version: 1.14.1
    devDependencies:
      '@types/better-sqlite3':
        specifier: ^7.6.13
@@ -63,7 +63,7 @@ importers:
        version: 11.10.0
      drizzle-orm:
        specifier: ^0.43.1
-        version: 0.43.1(@types/better-sqlite3@7.6.13)(better-sqlite3@11.10.0)
+        version: 0.43.1(@types/better-sqlite3@7.6.13)(better-sqlite3@11.10.0)(sql.js@1.14.1)
      yaml:
        specifier: ^2.8.3
        version: 2.8.3
@@ -1074,6 +1074,9 @@ packages:
    resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==}
    engines: {node: '>= 12'}

+  sql.js@1.14.1:
+    resolution: {integrity: sha512-gcj8zBWU5cFsi9WUP+4bFNXAyF1iRpA3LLyS/DP5xlrNzGmPIizUeBggKa8DbDwdqaKwUcTEnChtd2grWo/x/A==}
+
  stackback@0.0.2:
    resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==}

@@ -1695,10 +1698,11 @@ snapshots:

  detect-libc@2.1.2: {}

-  drizzle-orm@0.43.1(@types/better-sqlite3@7.6.13)(better-sqlite3@11.10.0):
+  drizzle-orm@0.43.1(@types/better-sqlite3@7.6.13)(better-sqlite3@11.10.0)(sql.js@1.14.1):
    optionalDependencies:
      '@types/better-sqlite3': 7.6.13
      better-sqlite3: 11.10.0
+      sql.js: 1.14.1

  end-of-stream@1.4.5:
    dependencies:
@@ -2000,6 +2004,8 @@ snapshots:

  source-map@0.7.6: {}

+  sql.js@1.14.1: {}
+
  stackback@0.0.2: {}

  std-env@4.1.0: {}