Compare commits

...

4 Commits

Author SHA1 Message Date
xiaoju 7f780f0642 chore: walkthrough cleanup — engines, types, mock fixes
- Add engines >= 22.5.0 to root and cli package.json (node:sqlite requirement)
- Remove unused @types/better-sqlite3 from cli devDeps (leftover from sql.js migration)
- Add files/publishConfig to core package.json (parity with other packages)
- Fix daemon test type errors: add getAllWorkflowRuns to mock LogStore,
  fix array destructuring on mock.calls, fix sense-runtime callback signatures

All 356 tests pass across all packages.

小橘 🍊(NEKO Team)
2026-04-23 09:08:24 +00:00
xiaomo 33e0d9a705 Merge pull request 'refactor(cli): replace sql.js with node:sqlite' (#66) from refactor/node-sqlite into main 2026-04-23 08:51:01 +00:00
xiaoju 418d8ee0c8 refactor(cli): replace sql.js with node:sqlite
Drop the sql.js WASM dependency in favour of Node 22's built-in
node:sqlite (DatabaseSync). This eliminates the ~2 MB WASM binary,
removes the async init ceremony, and lets us open databases in
readonly mode directly on disk instead of loading them into memory.

Breaking: requires Node >= 22.5.0 (sqlite support).

- Remove sql.js from cli dependencies
- Rewrite sense-sqlite.ts to use DatabaseSync
- Update sense command (schema/query) — sync API, no more queryAsObjects
- Update tests to use node:sqlite directly
- Remove sql.js from tsup externals

小橘 🍊(NEKO Team)
2026-04-23 08:43:39 +00:00
xiaomo 719c4c1449 Merge pull request 'refactor(cli): replace better-sqlite3 with sql.js (pure WASM) — implements RFC #63' (#64) from refactor/sql-js-migration into main 2026-04-23 07:32:38 +00:00
13 changed files with 110 additions and 155 deletions
+3
View File
@@ -1,6 +1,9 @@
{
"name": "nerve",
"private": true,
"engines": {
"node": ">=22.5.0"
},
"scripts": {
"build": "pnpm -r run build",
"check": "biome check .",
+4 -3
View File
@@ -1,5 +1,8 @@
{
"name": "@uncaged/nerve-cli",
"engines": {
"node": ">=22.5.0"
},
"version": "0.1.8",
"type": "module",
"bin": {
@@ -20,11 +23,9 @@
},
"dependencies": {
"@uncaged/nerve-core": "workspace:*",
"citty": "^0.1.6",
"sql.js": "^1.14.1"
"citty": "^0.1.6"
},
"devDependencies": {
"@types/better-sqlite3": "^7.6.13",
"@types/node": "^22.0.0",
"@uncaged/nerve-daemon": "workspace:*",
"vitest": "^4.1.5"
+28 -51
View File
@@ -2,12 +2,12 @@
* Tests for sense SQLite helpers used by `nerve sense schema` / `nerve sense query`.
*/
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
import { mkdirSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import initSqlJs, { type Database } from "sql.js";
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import {
assertSenseDbExists,
@@ -17,17 +17,11 @@ import {
listTableSqlStatements,
parseSenseQueryArgs,
pickDefaultPreviewTable,
queryAsObjects,
senseDbPath,
} from "../sense-sqlite.js";
let SQL: Awaited<ReturnType<typeof initSqlJs>>;
let tmpDir: string;
beforeAll(async () => {
SQL = await initSqlJs();
});
beforeEach(() => {
tmpDir = join(
tmpdir(),
@@ -40,22 +34,6 @@ afterEach(() => {
rmSync(tmpDir, { recursive: true, force: true });
});
/** Helper: create a SQLite db file with the given setup SQL. */
function createDb(name: string, setupSql: string): void {
const db = new SQL.Database();
db.run(setupSql);
const data = db.export();
db.close();
writeFileSync(join(tmpDir, "data", "senses", `${name}.db`), Buffer.from(data));
}
/** Helper: open an in-memory db with setup SQL for unit tests. */
function memDb(setupSql?: string): Database {
const db = new SQL.Database();
if (setupSql) db.run(setupSql);
return db;
}
describe("senseDbPath", () => {
it("points at data/senses/<name>.db under the given root", () => {
expect(senseDbPath("/root", "cpu-usage")).toBe(join("/root", "data", "senses", "cpu-usage.db"));
@@ -68,14 +46,18 @@ describe("assertSenseDbExists", () => {
});
it("returns the path when the file exists", () => {
createDb("x", "SELECT 1");
expect(assertSenseDbExists(tmpDir, "x")).toBe(join(tmpDir, "data", "senses", "x.db"));
const p = join(tmpDir, "data", "senses", "x.db");
new DatabaseSync(p).close();
expect(assertSenseDbExists(tmpDir, "x")).toBe(p);
});
});
describe("listTableSqlStatements", () => {
it("returns CREATE statements ordered by tbl_name", () => {
const db = memDb("CREATE TABLE zebra (id INTEGER); CREATE TABLE alpha (id INTEGER);");
const p = join(tmpDir, "data", "senses", "t.db");
const db = new DatabaseSync(p);
db.exec("CREATE TABLE zebra (id INTEGER)");
db.exec("CREATE TABLE alpha (id INTEGER)");
const stmts = listTableSqlStatements(db);
db.close();
expect(stmts).toHaveLength(2);
@@ -86,16 +68,18 @@ describe("listTableSqlStatements", () => {
describe("pickDefaultPreviewTable", () => {
it("prefers non-_migrations tables when both exist", () => {
const db = memDb(
`CREATE TABLE _migrations (name TEXT PRIMARY KEY);
CREATE TABLE readings (id INTEGER);`,
);
const p = join(tmpDir, "data", "senses", "t.db");
const db = new DatabaseSync(p);
db.exec("CREATE TABLE _migrations (name TEXT PRIMARY KEY)");
db.exec("CREATE TABLE readings (id INTEGER)");
expect(pickDefaultPreviewTable(db)).toBe("readings");
db.close();
});
it("uses _migrations when it is the only table", () => {
const db = memDb("CREATE TABLE _migrations (name TEXT PRIMARY KEY);");
const p = join(tmpDir, "data", "senses", "t.db");
const db = new DatabaseSync(p);
db.exec("CREATE TABLE _migrations (name TEXT PRIMARY KEY)");
expect(pickDefaultPreviewTable(db)).toBe("_migrations");
db.close();
});
@@ -153,29 +137,22 @@ describe("collectColumnKeys", () => {
});
});
describe("queryAsObjects", () => {
it("converts columnar sql.js results to row objects", () => {
const db = memDb("CREATE TABLE t (x INTEGER, y TEXT); INSERT INTO t VALUES (1, 'a'), (2, 'b');");
const rows = queryAsObjects(db, "SELECT * FROM t ORDER BY x");
db.close();
expect(rows).toEqual([
{ x: 1, y: "a" },
{ x: 2, y: "b" },
]);
});
});
describe("readonly query integration", () => {
it("runs default preview SQL on a real db file", () => {
createDb("demo", "CREATE TABLE items (id INTEGER PRIMARY KEY, v TEXT); INSERT INTO items (v) VALUES ('a'), ('b');");
it("runs default preview SQL on a real db", () => {
const p = join(tmpDir, "data", "senses", "demo.db");
const rw = new DatabaseSync(p);
rw.exec("CREATE TABLE items (id INTEGER PRIMARY KEY, v TEXT)");
rw.exec("INSERT INTO items (v) VALUES ('a'), ('b')");
rw.close();
const buffer = require("node:fs").readFileSync(join(tmpDir, "data", "senses", "demo.db"));
const db = new SQL.Database(buffer);
const db = new DatabaseSync(p, { readOnly: true });
const table = pickDefaultPreviewTable(db);
expect(table).toBe("items");
if (table === null) throw new Error("expected items table");
if (table === null) {
throw new Error("expected items table");
}
const sql = defaultPreviewSql(table);
const rows = queryAsObjects(db, sql);
const rows = db.prepare(sql).all() as Record<string, unknown>[];
db.close();
expect(rows.length).toBeGreaterThanOrEqual(1);
});
+6 -7
View File
@@ -1,5 +1,6 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { type SenseInfo, parseNerveConfig } from "@uncaged/nerve-core";
import { defineCommand } from "citty";
@@ -13,7 +14,6 @@ import {
openSenseDb,
parseSenseQueryArgs,
pickDefaultPreviewTable,
queryAsObjects,
} from "../sense-sqlite.js";
import { getNerveRoot, getSocketPath, isRunning } from "../workspace.js";
@@ -80,7 +80,6 @@ const senseListCommand = defineCommand({
},
async run() {
if (!isRunning()) {
// Daemon not running — show static info from nerve.yaml
process.stderr.write(
"⚠️ Daemon is not running — showing static config only (no last signal time).\n\n",
);
@@ -171,9 +170,9 @@ const senseSchemaCommand = defineCommand({
},
async run({ args }) {
const nerveRoot = getNerveRoot();
let db: ReturnType<Awaited<ReturnType<typeof import("sql.js")>>["Database"]> | undefined;
let db: DatabaseSync | undefined;
try {
db = await openSenseDb(nerveRoot, args.name);
db = openSenseDb(nerveRoot, args.name);
const statements = listTableSqlStatements(db);
if (args.json) {
process.stdout.write(`${JSON.stringify(statements, null, 2)}\n`);
@@ -217,7 +216,7 @@ const senseQueryCommand = defineCommand({
},
async run({ args, rawArgs }) {
const nerveRoot = getNerveRoot();
let db: ReturnType<Awaited<ReturnType<typeof import("sql.js")>>["Database"]> | undefined;
let db: DatabaseSync | undefined;
try {
let parsed: { name: string; sql: string | undefined };
try {
@@ -228,7 +227,7 @@ const senseQueryCommand = defineCommand({
process.exit(1);
}
db = await openSenseDb(nerveRoot, args.name);
db = openSenseDb(nerveRoot, args.name);
let sql = parsed.sql?.trim();
if (!sql) {
@@ -241,7 +240,7 @@ const senseQueryCommand = defineCommand({
}
}
const rows = queryAsObjects(db, sql);
const rows = db.prepare(sql).all() as Record<string, unknown>[];
if (args.json) {
process.stdout.write(`${JSON.stringify(rows, null, 2)}\n`);
+29 -57
View File
@@ -1,25 +1,6 @@
import { existsSync, readFileSync } from "node:fs";
import { existsSync } from "node:fs";
import { join } from "node:path";
import initSqlJs, { type Database } from "sql.js";
// ── WASM singleton ──────────────────────────────────────────────────────────
let _SQL: Awaited<ReturnType<typeof initSqlJs>> | null = null;
async function getSQL() {
if (!_SQL) {
_SQL = await initSqlJs();
}
return _SQL;
}
/** Open a sense SQLite database (readonly, loaded into memory via sql.js). */
export async function openSenseDb(nerveRoot: string, senseName: string): Promise<Database> {
const path = assertSenseDbExists(nerveRoot, senseName);
const SQL = await getSQL();
const buffer = readFileSync(path);
return new SQL.Database(buffer);
}
import { DatabaseSync } from "node:sqlite";
/** SQLite path for a sense under the nerve workspace root. */
export function senseDbPath(nerveRoot: string, senseName: string): string {
@@ -34,31 +15,39 @@ export function assertSenseDbExists(nerveRoot: string, senseName: string): strin
return path;
}
/** Open a sense SQLite database in readonly mode using node:sqlite. */
export function openSenseDb(nerveRoot: string, senseName: string): DatabaseSync {
const path = assertSenseDbExists(nerveRoot, senseName);
return new DatabaseSync(path, { readOnly: true });
}
/** `SELECT sql FROM sqlite_master WHERE type='table'` (non-null sql only). */
export function listTableSqlStatements(db: Database): string[] {
const results = db.exec(
`SELECT sql FROM sqlite_master WHERE type = 'table' AND sql IS NOT NULL ORDER BY tbl_name`,
);
if (results.length === 0) return [];
return results[0].values.map((row) => row[0] as string);
export function listTableSqlStatements(db: DatabaseSync): string[] {
const rows = db
.prepare(
`SELECT sql FROM sqlite_master WHERE type = 'table' AND sql IS NOT NULL ORDER BY tbl_name`,
)
.all() as { sql: string }[];
return rows.map((r) => r.sql);
}
/**
* Table used for `nerve sense query <name>` with no SQL.
* Prefers real data tables over `_migrations`, then lexicographic by name.
*/
export function pickDefaultPreviewTable(db: Database): string | null {
const results = db.exec(
`SELECT name FROM sqlite_master
WHERE type = 'table' AND sql IS NOT NULL
AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'
ORDER BY
CASE WHEN name = '_migrations' THEN 1 ELSE 0 END,
name
LIMIT 1`,
);
if (results.length === 0 || results[0].values.length === 0) return null;
return results[0].values[0][0] as string;
export function pickDefaultPreviewTable(db: DatabaseSync): string | null {
const row = db
.prepare(
`SELECT name FROM sqlite_master
WHERE type = 'table' AND sql IS NOT NULL
AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'
ORDER BY
CASE WHEN name = '_migrations' THEN 1 ELSE 0 END,
name
LIMIT 1`,
)
.get() as { name: string } | undefined;
return row?.name ?? null;
}
export function defaultPreviewSql(table: string): string {
@@ -93,7 +82,7 @@ function stringifyCell(value: unknown): string {
if (typeof value === "bigint") return value.toString();
if (typeof value === "number" || typeof value === "boolean") return String(value);
if (typeof value === "string") return value;
if (value instanceof Uint8Array) return Buffer.from(value).toString("hex");
if (Buffer.isBuffer(value)) return value.toString("hex");
try {
return JSON.stringify(value);
} catch {
@@ -136,20 +125,3 @@ export function formatRowsAsAlignedTable(rows: Record<string, unknown>[]): strin
const body = cells.map((r) => r.map((cell, j) => cell.padEnd(widths[j])).join(" | ")).join("\n");
return `${header}\n${sep}\n${body}\n`;
}
/**
* Run a SQL query via sql.js and return rows as key-value objects.
* sql.js returns columnar data; this converts to the familiar row format.
*/
export function queryAsObjects(db: Database, sql: string): Record<string, unknown>[] {
const results = db.exec(sql);
if (results.length === 0) return [];
const { columns, values } = results[0];
return values.map((row) => {
const obj: Record<string, unknown> = {};
for (let i = 0; i < columns.length; i++) {
obj[columns[i]] = row[i];
}
return obj;
});
}
+1 -1
View File
@@ -9,5 +9,5 @@ export default defineConfig({
js: "#!/usr/bin/env node",
},
/** Daemon is loaded from workspace node_modules at runtime — never bundle it. */
external: ["@uncaged/nerve-daemon", "sql.js"],
external: ["@uncaged/nerve-daemon"],
});
+4
View File
@@ -3,6 +3,10 @@
"version": "0.1.4",
"type": "module",
"main": "dist/index.js",
"files": ["dist"],
"publishConfig": {
"access": "public"
},
"types": "dist/index.d.ts",
"scripts": {
"prepublishOnly": "bash ../../scripts/prepublish-check.sh",
@@ -89,10 +89,11 @@ function makeLogStore(
}
return activeRuns;
}),
getTriggerPayload: vi.fn(() => ({ value: 42 })),
getThreadEvents: vi.fn(() => [{ type: "thread_start", triggerPayload: {} }]),
getTriggerPayload: vi.fn((): unknown => ({ value: 42 })),
getThreadEvents: vi.fn((): Array<{ type: string; [key: string]: unknown }> => [{ type: "thread_start", triggerPayload: {} }]),
archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })),
close: vi.fn(),
getAllWorkflowRuns: vi.fn(() => []),
};
return store;
}
@@ -127,7 +128,7 @@ describe("WorkflowManager — crash recovery (Phase 3)", () => {
child.emit("exit", 1, null);
const crashedCalls = logStore.upsertWorkflowRun.mock.calls.filter(
([entry]: [{ type: string }]) => entry.type === "crashed",
(args: any[]) => (args[0] as { type: string }).type === "crashed",
);
expect(crashedCalls).toHaveLength(2);
@@ -216,10 +217,10 @@ describe("WorkflowManager — crash recovery (Phase 3)", () => {
// resume-thread should have been sent
const resumeCalls = (secondChild.send as ReturnType<typeof vi.fn>).mock.calls.filter(
([msg]: [unknown]) =>
msg !== null &&
typeof msg === "object" &&
(msg as Record<string, unknown>).type === "resume-thread",
(args: any[]) =>
args[0] !== null &&
typeof args[0] === "object" &&
(args[0] as Record<string, unknown>).type === "resume-thread",
);
expect(resumeCalls).toHaveLength(1);
expect(resumeCalls[0][0]).toMatchObject({
@@ -286,7 +287,7 @@ describe("WorkflowManager — crash recovery (Phase 3)", () => {
});
const appendCalls = logStore.append.mock.calls.filter(
([entry]: [{ type: string }]) => entry.type === "thread_command_event",
(args: any[]) => (args[0] as { type: string }).type === "thread_command_event",
);
expect(appendCalls).toHaveLength(1);
expect(appendCalls[0][0]).toMatchObject({
@@ -313,7 +314,7 @@ describe("WorkflowManager — crash recovery (Phase 3)", () => {
mgr.startWorkflow("my-wf", payload);
const startedCall = logStore.upsertWorkflowRun.mock.calls.find(
([entry]: [{ type: string }]) => entry.type === "started",
(args: any[]) => (args[0] as { type: string }).type === "started",
);
expect(startedCall).toBeDefined();
const logEntry = startedCall?.[0] as { payload: string | null };
@@ -79,6 +79,7 @@ function makeLogStore() {
getThreadEvents: vi.fn(() => []),
archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })),
close: vi.fn(),
getAllWorkflowRuns: vi.fn(() => []),
};
}
@@ -126,7 +127,7 @@ describe("WorkflowManager — drainAndRespawn (Phase 3 hot reload)", () => {
await drainPromise;
const interruptedCalls = logStore.upsertWorkflowRun.mock.calls.filter(
([entry]: [{ type: string }]) => entry.type === "interrupted",
(args: any[]) => (args[0] as { type: string }).type === "interrupted",
);
expect(interruptedCalls).toHaveLength(2);
@@ -190,10 +191,10 @@ describe("WorkflowManager — drainAndRespawn (Phase 3 hot reload)", () => {
const newChild = mockChildren[1];
const resumeCalls = (newChild.send as ReturnType<typeof vi.fn>).mock.calls.filter(
([msg]: [unknown]) =>
msg !== null &&
typeof msg === "object" &&
(msg as Record<string, unknown>).type === "resume-thread",
(args: any[]) =>
args[0] !== null &&
typeof args[0] === "object" &&
(args[0] as Record<string, unknown>).type === "resume-thread",
);
expect(resumeCalls).toHaveLength(0);
@@ -218,10 +219,10 @@ describe("WorkflowManager — drainAndRespawn (Phase 3 hot reload)", () => {
const newChild = mockChildren[1];
const startCalls = (newChild.send as ReturnType<typeof vi.fn>).mock.calls.filter(
([msg]: [unknown]) =>
msg !== null &&
typeof msg === "object" &&
(msg as Record<string, unknown>).type === "start-thread",
(args: any[]) =>
args[0] !== null &&
typeof args[0] === "object" &&
(args[0] as Record<string, unknown>).type === "start-thread",
);
expect(startCalls).toHaveLength(1);
@@ -266,7 +267,7 @@ describe("Kernel — workflow hot reload via file-watcher (Phase 3)", () => {
// Kernel's handleWorkflowFileChange should log a workflow_reload event
// We test this via the kernel itself
const appendCalls = logStore.append.mock.calls;
const startCall = appendCalls.find(([e]: [{ type: string }]) => e.type === "start");
const startCall = appendCalls.find((args: any[]) => (args[0] as { type: string }).type === "start");
expect(startCall).toBeDefined();
const stopPromise = kernel.stop();
@@ -78,6 +78,7 @@ function makeLogStore() {
appendWithWorkflowUpdate: vi.fn(),
getWorkflowRun: vi.fn(() => null),
getActiveWorkflowRuns: vi.fn(() => []),
getAllWorkflowRuns: vi.fn(() => []),
getTriggerPayload: vi.fn(() => null),
getThreadEvents: vi.fn(() => []),
archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })),
@@ -137,10 +138,10 @@ describe("kernel + workflowManager integration", () => {
// We need to check that a start-thread message was sent to the workflow worker
const workflowWorker = mockChildren.find((c) =>
(c.send as ReturnType<typeof vi.fn>).mock.calls.some(
([msg]: [unknown]) =>
msg !== null &&
typeof msg === "object" &&
(msg as Record<string, unknown>).type === "start-thread",
(args: unknown[]) =>
args[0] !== null &&
typeof args[0] === "object" &&
(args[0] as Record<string, unknown>).type === "start-thread",
),
);
expect(workflowWorker).toBeDefined();
@@ -212,10 +213,10 @@ describe("kernel + workflowManager integration", () => {
// No workflow worker should have been spawned (only the sense group worker)
const workflowWorkerSpawned = mockChildren.some((c) =>
(c.send as ReturnType<typeof vi.fn>).mock.calls.some(
([msg]: [unknown]) =>
msg !== null &&
typeof msg === "object" &&
(msg as Record<string, unknown>).type === "start-thread",
(args: unknown[]) =>
args[0] !== null &&
typeof args[0] === "object" &&
(args[0] as Record<string, unknown>).type === "start-thread",
),
);
expect(workflowWorkerSpawned).toBe(false);
@@ -10,7 +10,7 @@ import { describe, expect, it } from "vitest";
import { createBlobStore } from "../blob-store.js";
import { parseParentMessage } from "../ipc.js";
import { executeCompute, openPeerDb, openSenseDb, runMigrations } from "../sense-runtime.js";
import type { DrizzleDB, PeerMap, SenseRuntime } from "../sense-runtime.js";
import type { ComputeFn, DrizzleDB, PeerMap, SenseRuntime } from "../sense-runtime.js";
// ---------------------------------------------------------------------------
// Helpers
@@ -168,7 +168,7 @@ describe("openPeerDb", () => {
// ---------------------------------------------------------------------------
describe("executeCompute", () => {
function makeRuntime(computeFn: (db: DrizzleDB, peers: PeerMap) => Promise<unknown | null>): {
function makeRuntime(computeFn: ComputeFn): {
runtime: SenseRuntime;
sqlite: Database.Database;
} {
@@ -76,6 +76,7 @@ function makeLogStore() {
getThreadEvents: vi.fn(() => []),
archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })),
close: vi.fn(),
getAllWorkflowRuns: vi.fn(() => []),
};
}
+2 -7
View File
@@ -26,13 +26,7 @@ importers:
citty:
specifier: ^0.1.6
version: 0.1.6
sql.js:
specifier: ^1.14.1
version: 1.14.1
devDependencies:
'@types/better-sqlite3':
specifier: ^7.6.13
version: 7.6.13
'@types/node':
specifier: ^22.0.0
version: 22.19.17
@@ -2004,7 +1998,8 @@ snapshots:
source-map@0.7.6: {}
sql.js@1.14.1: {}
sql.js@1.14.1:
optional: true
stackback@0.0.2: {}