Compare commits

...

2 Commits

Author SHA1 Message Date
xiaoju 418d8ee0c8 refactor(cli): replace sql.js with node:sqlite
Drop the sql.js WASM dependency in favour of Node 22's built-in
node:sqlite (DatabaseSync). This eliminates the ~2 MB WASM binary,
removes the async init ceremony, and lets us open databases in
readonly mode directly on disk instead of loading them into memory.

Breaking: requires Node >= 22.5.0 (sqlite support).

- Remove sql.js from cli dependencies
- Rewrite sense-sqlite.ts to use DatabaseSync
- Update sense command (schema/query) — sync API, no more queryAsObjects
- Update tests to use node:sqlite directly
- Remove sql.js from tsup externals

小橘 🍊(NEKO Team)
2026-04-23 08:43:39 +00:00
xiaomo 719c4c1449 Merge pull request 'refactor(cli): replace better-sqlite3 with sql.js (pure WASM) — implements RFC #63' (#64) from refactor/sql-js-migration into main 2026-04-23 07:32:38 +00:00
6 changed files with 67 additions and 122 deletions
+1 -2
View File
@@ -20,8 +20,7 @@
}, },
"dependencies": { "dependencies": {
"@uncaged/nerve-core": "workspace:*", "@uncaged/nerve-core": "workspace:*",
"citty": "^0.1.6", "citty": "^0.1.6"
"sql.js": "^1.14.1"
}, },
"devDependencies": { "devDependencies": {
"@types/better-sqlite3": "^7.6.13", "@types/better-sqlite3": "^7.6.13",
+28 -51
View File
@@ -2,12 +2,12 @@
* Tests for sense SQLite helpers used by `nerve sense schema` / `nerve sense query`. * Tests for sense SQLite helpers used by `nerve sense schema` / `nerve sense query`.
*/ */
import { mkdirSync, rmSync, writeFileSync } from "node:fs"; import { mkdirSync, rmSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import initSqlJs, { type Database } from "sql.js"; import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { import {
assertSenseDbExists, assertSenseDbExists,
@@ -17,17 +17,11 @@ import {
listTableSqlStatements, listTableSqlStatements,
parseSenseQueryArgs, parseSenseQueryArgs,
pickDefaultPreviewTable, pickDefaultPreviewTable,
queryAsObjects,
senseDbPath, senseDbPath,
} from "../sense-sqlite.js"; } from "../sense-sqlite.js";
let SQL: Awaited<ReturnType<typeof initSqlJs>>;
let tmpDir: string; let tmpDir: string;
beforeAll(async () => {
SQL = await initSqlJs();
});
beforeEach(() => { beforeEach(() => {
tmpDir = join( tmpDir = join(
tmpdir(), tmpdir(),
@@ -40,22 +34,6 @@ afterEach(() => {
rmSync(tmpDir, { recursive: true, force: true }); rmSync(tmpDir, { recursive: true, force: true });
}); });
/** Helper: create a SQLite db file with the given setup SQL. */
function createDb(name: string, setupSql: string): void {
const db = new SQL.Database();
db.run(setupSql);
const data = db.export();
db.close();
writeFileSync(join(tmpDir, "data", "senses", `${name}.db`), Buffer.from(data));
}
/** Helper: open an in-memory db with setup SQL for unit tests. */
function memDb(setupSql?: string): Database {
const db = new SQL.Database();
if (setupSql) db.run(setupSql);
return db;
}
describe("senseDbPath", () => { describe("senseDbPath", () => {
it("points at data/senses/<name>.db under the given root", () => { it("points at data/senses/<name>.db under the given root", () => {
expect(senseDbPath("/root", "cpu-usage")).toBe(join("/root", "data", "senses", "cpu-usage.db")); expect(senseDbPath("/root", "cpu-usage")).toBe(join("/root", "data", "senses", "cpu-usage.db"));
@@ -68,14 +46,18 @@ describe("assertSenseDbExists", () => {
}); });
it("returns the path when the file exists", () => { it("returns the path when the file exists", () => {
createDb("x", "SELECT 1"); const p = join(tmpDir, "data", "senses", "x.db");
expect(assertSenseDbExists(tmpDir, "x")).toBe(join(tmpDir, "data", "senses", "x.db")); new DatabaseSync(p).close();
expect(assertSenseDbExists(tmpDir, "x")).toBe(p);
}); });
}); });
describe("listTableSqlStatements", () => { describe("listTableSqlStatements", () => {
it("returns CREATE statements ordered by tbl_name", () => { it("returns CREATE statements ordered by tbl_name", () => {
const db = memDb("CREATE TABLE zebra (id INTEGER); CREATE TABLE alpha (id INTEGER);"); const p = join(tmpDir, "data", "senses", "t.db");
const db = new DatabaseSync(p);
db.exec("CREATE TABLE zebra (id INTEGER)");
db.exec("CREATE TABLE alpha (id INTEGER)");
const stmts = listTableSqlStatements(db); const stmts = listTableSqlStatements(db);
db.close(); db.close();
expect(stmts).toHaveLength(2); expect(stmts).toHaveLength(2);
@@ -86,16 +68,18 @@ describe("listTableSqlStatements", () => {
describe("pickDefaultPreviewTable", () => { describe("pickDefaultPreviewTable", () => {
it("prefers non-_migrations tables when both exist", () => { it("prefers non-_migrations tables when both exist", () => {
const db = memDb( const p = join(tmpDir, "data", "senses", "t.db");
`CREATE TABLE _migrations (name TEXT PRIMARY KEY); const db = new DatabaseSync(p);
CREATE TABLE readings (id INTEGER);`, db.exec("CREATE TABLE _migrations (name TEXT PRIMARY KEY)");
); db.exec("CREATE TABLE readings (id INTEGER)");
expect(pickDefaultPreviewTable(db)).toBe("readings"); expect(pickDefaultPreviewTable(db)).toBe("readings");
db.close(); db.close();
}); });
it("uses _migrations when it is the only table", () => { it("uses _migrations when it is the only table", () => {
const db = memDb("CREATE TABLE _migrations (name TEXT PRIMARY KEY);"); const p = join(tmpDir, "data", "senses", "t.db");
const db = new DatabaseSync(p);
db.exec("CREATE TABLE _migrations (name TEXT PRIMARY KEY)");
expect(pickDefaultPreviewTable(db)).toBe("_migrations"); expect(pickDefaultPreviewTable(db)).toBe("_migrations");
db.close(); db.close();
}); });
@@ -153,29 +137,22 @@ describe("collectColumnKeys", () => {
}); });
}); });
describe("queryAsObjects", () => {
it("converts columnar sql.js results to row objects", () => {
const db = memDb("CREATE TABLE t (x INTEGER, y TEXT); INSERT INTO t VALUES (1, 'a'), (2, 'b');");
const rows = queryAsObjects(db, "SELECT * FROM t ORDER BY x");
db.close();
expect(rows).toEqual([
{ x: 1, y: "a" },
{ x: 2, y: "b" },
]);
});
});
describe("readonly query integration", () => { describe("readonly query integration", () => {
it("runs default preview SQL on a real db file", () => { it("runs default preview SQL on a real db", () => {
createDb("demo", "CREATE TABLE items (id INTEGER PRIMARY KEY, v TEXT); INSERT INTO items (v) VALUES ('a'), ('b');"); const p = join(tmpDir, "data", "senses", "demo.db");
const rw = new DatabaseSync(p);
rw.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, v TEXT)");
rw.exec("INSERT INTO items (v) VALUES ('a'), ('b')");
rw.close();
const buffer = require("node:fs").readFileSync(join(tmpDir, "data", "senses", "demo.db")); const db = new DatabaseSync(p, { readOnly: true });
const db = new SQL.Database(buffer);
const table = pickDefaultPreviewTable(db); const table = pickDefaultPreviewTable(db);
expect(table).toBe("items"); expect(table).toBe("items");
if (table === null) throw new Error("expected items table"); if (table === null) {
throw new Error("expected items table");
}
const sql = defaultPreviewSql(table); const sql = defaultPreviewSql(table);
const rows = queryAsObjects(db, sql); const rows = db.prepare(sql).all() as Record<string, unknown>[];
db.close(); db.close();
expect(rows.length).toBeGreaterThanOrEqual(1); expect(rows.length).toBeGreaterThanOrEqual(1);
}); });
+6 -7
View File
@@ -1,5 +1,6 @@
import { readFileSync } from "node:fs"; import { readFileSync } from "node:fs";
import { join } from "node:path"; import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { type SenseInfo, parseNerveConfig } from "@uncaged/nerve-core"; import { type SenseInfo, parseNerveConfig } from "@uncaged/nerve-core";
import { defineCommand } from "citty"; import { defineCommand } from "citty";
@@ -13,7 +14,6 @@ import {
openSenseDb, openSenseDb,
parseSenseQueryArgs, parseSenseQueryArgs,
pickDefaultPreviewTable, pickDefaultPreviewTable,
queryAsObjects,
} from "../sense-sqlite.js"; } from "../sense-sqlite.js";
import { getNerveRoot, getSocketPath, isRunning } from "../workspace.js"; import { getNerveRoot, getSocketPath, isRunning } from "../workspace.js";
@@ -80,7 +80,6 @@ const senseListCommand = defineCommand({
}, },
async run() { async run() {
if (!isRunning()) { if (!isRunning()) {
// Daemon not running — show static info from nerve.yaml
process.stderr.write( process.stderr.write(
"⚠️ Daemon is not running — showing static config only (no last signal time).\n\n", "⚠️ Daemon is not running — showing static config only (no last signal time).\n\n",
); );
@@ -171,9 +170,9 @@ const senseSchemaCommand = defineCommand({
}, },
async run({ args }) { async run({ args }) {
const nerveRoot = getNerveRoot(); const nerveRoot = getNerveRoot();
let db: ReturnType<Awaited<ReturnType<typeof import("sql.js")>>["Database"]> | undefined; let db: DatabaseSync | undefined;
try { try {
db = await openSenseDb(nerveRoot, args.name); db = openSenseDb(nerveRoot, args.name);
const statements = listTableSqlStatements(db); const statements = listTableSqlStatements(db);
if (args.json) { if (args.json) {
process.stdout.write(`${JSON.stringify(statements, null, 2)}\n`); process.stdout.write(`${JSON.stringify(statements, null, 2)}\n`);
@@ -217,7 +216,7 @@ const senseQueryCommand = defineCommand({
}, },
async run({ args, rawArgs }) { async run({ args, rawArgs }) {
const nerveRoot = getNerveRoot(); const nerveRoot = getNerveRoot();
let db: ReturnType<Awaited<ReturnType<typeof import("sql.js")>>["Database"]> | undefined; let db: DatabaseSync | undefined;
try { try {
let parsed: { name: string; sql: string | undefined }; let parsed: { name: string; sql: string | undefined };
try { try {
@@ -228,7 +227,7 @@ const senseQueryCommand = defineCommand({
process.exit(1); process.exit(1);
} }
db = await openSenseDb(nerveRoot, args.name); db = openSenseDb(nerveRoot, args.name);
let sql = parsed.sql?.trim(); let sql = parsed.sql?.trim();
if (!sql) { if (!sql) {
@@ -241,7 +240,7 @@ const senseQueryCommand = defineCommand({
} }
} }
const rows = queryAsObjects(db, sql); const rows = db.prepare(sql).all() as Record<string, unknown>[];
if (args.json) { if (args.json) {
process.stdout.write(`${JSON.stringify(rows, null, 2)}\n`); process.stdout.write(`${JSON.stringify(rows, null, 2)}\n`);
+29 -57
View File
@@ -1,25 +1,6 @@
import { existsSync, readFileSync } from "node:fs"; import { existsSync } from "node:fs";
import { join } from "node:path"; import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import initSqlJs, { type Database } from "sql.js";
// ── WASM singleton ──────────────────────────────────────────────────────────
let _SQL: Awaited<ReturnType<typeof initSqlJs>> | null = null;
async function getSQL() {
if (!_SQL) {
_SQL = await initSqlJs();
}
return _SQL;
}
/** Open a sense SQLite database (readonly, loaded into memory via sql.js). */
export async function openSenseDb(nerveRoot: string, senseName: string): Promise<Database> {
const path = assertSenseDbExists(nerveRoot, senseName);
const SQL = await getSQL();
const buffer = readFileSync(path);
return new SQL.Database(buffer);
}
/** SQLite path for a sense under the nerve workspace root. */ /** SQLite path for a sense under the nerve workspace root. */
export function senseDbPath(nerveRoot: string, senseName: string): string { export function senseDbPath(nerveRoot: string, senseName: string): string {
@@ -34,31 +15,39 @@ export function assertSenseDbExists(nerveRoot: string, senseName: string): strin
return path; return path;
} }
/** Open a sense SQLite database in readonly mode using node:sqlite. */
export function openSenseDb(nerveRoot: string, senseName: string): DatabaseSync {
const path = assertSenseDbExists(nerveRoot, senseName);
return new DatabaseSync(path, { readOnly: true });
}
/** `SELECT sql FROM sqlite_master WHERE type='table'` (non-null sql only). */ /** `SELECT sql FROM sqlite_master WHERE type='table'` (non-null sql only). */
export function listTableSqlStatements(db: Database): string[] { export function listTableSqlStatements(db: DatabaseSync): string[] {
const results = db.exec( const rows = db
`SELECT sql FROM sqlite_master WHERE type = 'table' AND sql IS NOT NULL ORDER BY tbl_name`, .prepare(
); `SELECT sql FROM sqlite_master WHERE type = 'table' AND sql IS NOT NULL ORDER BY tbl_name`,
if (results.length === 0) return []; )
return results[0].values.map((row) => row[0] as string); .all() as { sql: string }[];
return rows.map((r) => r.sql);
} }
/** /**
* Table used for `nerve sense query <name>` with no SQL. * Table used for `nerve sense query <name>` with no SQL.
* Prefers real data tables over `_migrations`, then lexicographic by name. * Prefers real data tables over `_migrations`, then lexicographic by name.
*/ */
export function pickDefaultPreviewTable(db: Database): string | null { export function pickDefaultPreviewTable(db: DatabaseSync): string | null {
const results = db.exec( const row = db
`SELECT name FROM sqlite_master .prepare(
WHERE type = 'table' AND sql IS NOT NULL `SELECT name FROM sqlite_master
AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' WHERE type = 'table' AND sql IS NOT NULL
ORDER BY AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'
CASE WHEN name = '_migrations' THEN 1 ELSE 0 END, ORDER BY
name CASE WHEN name = '_migrations' THEN 1 ELSE 0 END,
LIMIT 1`, name
); LIMIT 1`,
if (results.length === 0 || results[0].values.length === 0) return null; )
return results[0].values[0][0] as string; .get() as { name: string } | undefined;
return row?.name ?? null;
} }
export function defaultPreviewSql(table: string): string { export function defaultPreviewSql(table: string): string {
@@ -93,7 +82,7 @@ function stringifyCell(value: unknown): string {
if (typeof value === "bigint") return value.toString(); if (typeof value === "bigint") return value.toString();
if (typeof value === "number" || typeof value === "boolean") return String(value); if (typeof value === "number" || typeof value === "boolean") return String(value);
if (typeof value === "string") return value; if (typeof value === "string") return value;
if (value instanceof Uint8Array) return Buffer.from(value).toString("hex"); if (Buffer.isBuffer(value)) return value.toString("hex");
try { try {
return JSON.stringify(value); return JSON.stringify(value);
} catch { } catch {
@@ -136,20 +125,3 @@ export function formatRowsAsAlignedTable(rows: Record<string, unknown>[]): strin
const body = cells.map((r) => r.map((cell, j) => cell.padEnd(widths[j])).join(" | ")).join("\n"); const body = cells.map((r) => r.map((cell, j) => cell.padEnd(widths[j])).join(" | ")).join("\n");
return `${header}\n${sep}\n${body}\n`; return `${header}\n${sep}\n${body}\n`;
} }
/**
* Run a SQL query via sql.js and return rows as key-value objects.
* sql.js returns columnar data; this converts to the familiar row format.
*/
export function queryAsObjects(db: Database, sql: string): Record<string, unknown>[] {
const results = db.exec(sql);
if (results.length === 0) return [];
const { columns, values } = results[0];
return values.map((row) => {
const obj: Record<string, unknown> = {};
for (let i = 0; i < columns.length; i++) {
obj[columns[i]] = row[i];
}
return obj;
});
}
+1 -1
View File
@@ -9,5 +9,5 @@ export default defineConfig({
js: "#!/usr/bin/env node", js: "#!/usr/bin/env node",
}, },
/** Daemon is loaded from workspace node_modules at runtime — never bundle it. */ /** Daemon is loaded from workspace node_modules at runtime — never bundle it. */
external: ["@uncaged/nerve-daemon", "sql.js"], external: ["@uncaged/nerve-daemon"],
}); });
+2 -4
View File
@@ -26,9 +26,6 @@ importers:
citty: citty:
specifier: ^0.1.6 specifier: ^0.1.6
version: 0.1.6 version: 0.1.6
sql.js:
specifier: ^1.14.1
version: 1.14.1
devDependencies: devDependencies:
'@types/better-sqlite3': '@types/better-sqlite3':
specifier: ^7.6.13 specifier: ^7.6.13
@@ -2004,7 +2001,8 @@ snapshots:
source-map@0.7.6: {} source-map@0.7.6: {}
sql.js@1.14.1: {} sql.js@1.14.1:
optional: true
stackback@0.0.2: {} stackback@0.0.2: {}