Compare commits

...

4 Commits

Author SHA1 Message Date
xiaoju 85dd11c84d refactor(daemon): upgrade Drizzle v1.0-beta + migrate better-sqlite3 → node:sqlite
- Upgrade drizzle-orm from 0.43.1 to 1.0.0-beta.23
- Replace better-sqlite3 with node:sqlite (DatabaseSync) in:
  - sense-runtime.ts (Drizzle driver)
  - log-store.ts (raw SQL)
  - all test files
- Replace sqlite.pragma() with sqlite.exec('PRAGMA ...')
- Replace sqlite.transaction() with manual BEGIN/COMMIT/ROLLBACK
- Update CLI init command to verify node:sqlite instead of better-sqlite3
- Remove better-sqlite3 and @types/better-sqlite3 from dependencies
- Zero native addons remaining in the monorepo 🎉

Closes #67

小橘 <xiaoju@shazhou.work>
2026-04-23 09:18:44 +00:00
xiaomo d80a414530 Merge pull request 'chore: walkthrough cleanup — engines, types, mock fixes' (#68) from fix/walkthrough-cleanup into main 2026-04-23 09:10:09 +00:00
xiaoju 7f780f0642 chore: walkthrough cleanup — engines, types, mock fixes
- Add engines >= 22.5.0 to root and cli package.json (node:sqlite requirement)
- Remove unused @types/better-sqlite3 from cli devDeps (leftover from sql.js migration)
- Add files/publishConfig to core package.json (parity with other packages)
- Fix daemon test type errors: add getAllWorkflowRuns to mock LogStore,
  fix array destructuring on mock.calls, fix sense-runtime callback signatures

All 356 tests pass across all packages.

小橘 🍊(NEKO Team)
2026-04-23 09:08:24 +00:00
xiaomo 33e0d9a705 Merge pull request 'refactor(cli): replace sql.js with node:sqlite' (#66) from refactor/node-sqlite into main 2026-04-23 08:51:01 +00:00
15 changed files with 928 additions and 169 deletions
+3
View File
@@ -1,6 +1,9 @@
{ {
"name": "nerve", "name": "nerve",
"private": true, "private": true,
"engines": {
"node": ">=22.5.0"
},
"scripts": { "scripts": {
"build": "pnpm -r run build", "build": "pnpm -r run build",
"check": "biome check .", "check": "biome check .",
+3 -1
View File
@@ -1,5 +1,8 @@
{ {
"name": "@uncaged/nerve-cli", "name": "@uncaged/nerve-cli",
"engines": {
"node": ">=22.5.0"
},
"version": "0.1.8", "version": "0.1.8",
"type": "module", "type": "module",
"bin": { "bin": {
@@ -23,7 +26,6 @@
"citty": "^0.1.6" "citty": "^0.1.6"
}, },
"devDependencies": { "devDependencies": {
"@types/better-sqlite3": "^7.6.13",
"@types/node": "^22.0.0", "@types/node": "^22.0.0",
"@uncaged/nerve-daemon": "workspace:*", "@uncaged/nerve-daemon": "workspace:*",
"vitest": "^4.1.5" "vitest": "^4.1.5"
+16 -28
View File
@@ -1,4 +1,4 @@
import { spawn, execFile } from "node:child_process"; import { execFile, spawn } from "node:child_process";
import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path"; import { dirname, join } from "node:path";
import { promisify } from "node:util"; import { promisify } from "node:util";
@@ -35,7 +35,7 @@ const PACKAGE_JSON = `{
"drizzle-kit": "latest" "drizzle-kit": "latest"
}, },
"pnpm": { "pnpm": {
"onlyBuiltDependencies": ["better-sqlite3", "esbuild"] "onlyBuiltDependencies": ["esbuild"]
} }
} }
`; `;
@@ -218,14 +218,18 @@ const initWorkspaceCommand = defineCommand({
}, },
}); });
async function tryRequireSqlite(nerveRoot: string): Promise<boolean> { /** Verify built-in `node:sqlite` (Node.js ≥22.5) loads in a child process. */
async function verifyNodeSqlite(): Promise<boolean> {
try { try {
const modulePath = join(nerveRoot, "node_modules", "better-sqlite3"); await execFileAsync(
// Use a child process to test if the native module loads "node",
await execFileAsync("node", ["-e", `require(${JSON.stringify(modulePath)})`], { [
cwd: nerveRoot, "--input-type=module",
timeout: 10_000, "-e",
}); "import { DatabaseSync } from 'node:sqlite'; new DatabaseSync(':memory:').exec('SELECT 1');",
],
{ timeout: 10_000 },
);
return true; return true;
} catch { } catch {
return false; return false;
@@ -264,27 +268,11 @@ async function runInitWorkspace(force: boolean): Promise<void> {
); );
} }
// Verify better-sqlite3 native module — rebuild up to 2 times if broken if (!(await verifyNodeSqlite())) {
const sqlitePath = join(nerveRoot, "node_modules", "better-sqlite3");
if (existsSync(sqlitePath)) {
for (let attempt = 1; attempt <= 2; attempt++) {
if (await tryRequireSqlite(nerveRoot)) break;
process.stdout.write( process.stdout.write(
`${attempt === 1 ? "Building" : "Retrying build of"} native module better-sqlite3 (attempt ${attempt}/2)…\n`, "⚠️ Built-in SQLite (node:sqlite) is not available in this Node.js build. " +
"The daemon requires Node.js 22.5 or newer with SQLite enabled.\n",
); );
try {
await runCommand(cmd, ["rebuild", "better-sqlite3"], nerveRoot);
} catch {
// will be caught by the verify below
}
}
if (!(await tryRequireSqlite(nerveRoot))) {
process.stdout.write(
`⚠️ better-sqlite3 native module is not working. The daemon will fail to start.\n` +
` Fix: cd ${nerveRoot} && ${cmd} rebuild better-sqlite3\n` +
` Or: npm install --build-from-source better-sqlite3\n`,
);
}
} }
if (!existsSync(join(nerveRoot, ".git"))) { if (!existsSync(join(nerveRoot, ".git"))) {
+4
View File
@@ -3,6 +3,10 @@
"version": "0.1.4", "version": "0.1.4",
"type": "module", "type": "module",
"main": "dist/index.js", "main": "dist/index.js",
"files": ["dist"],
"publishConfig": {
"access": "public"
},
"types": "dist/index.d.ts", "types": "dist/index.d.ts",
"scripts": { "scripts": {
"prepublishOnly": "bash ../../scripts/prepublish-check.sh", "prepublishOnly": "bash ../../scripts/prepublish-check.sh",
+2 -3
View File
@@ -17,12 +17,11 @@
}, },
"dependencies": { "dependencies": {
"@uncaged/nerve-core": "workspace:*", "@uncaged/nerve-core": "workspace:*",
"better-sqlite3": "^11.10.0", "drizzle-orm": "1.0.0-beta.23-c10d10c",
"drizzle-orm": "^0.43.1",
"yaml": "^2.8.3" "yaml": "^2.8.3"
}, },
"devDependencies": { "devDependencies": {
"@types/better-sqlite3": "^7.6.13", "@types/node": "^22.0.0",
"vitest": "^4.1.5" "vitest": "^4.1.5"
} }
} }
@@ -89,10 +89,11 @@ function makeLogStore(
} }
return activeRuns; return activeRuns;
}), }),
getTriggerPayload: vi.fn(() => ({ value: 42 })), getTriggerPayload: vi.fn((): unknown => ({ value: 42 })),
getThreadEvents: vi.fn(() => [{ type: "thread_start", triggerPayload: {} }]), getThreadEvents: vi.fn((): Array<{ type: string; [key: string]: unknown }> => [{ type: "thread_start", triggerPayload: {} }]),
archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })), archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })),
close: vi.fn(), close: vi.fn(),
getAllWorkflowRuns: vi.fn(() => []),
}; };
return store; return store;
} }
@@ -127,7 +128,7 @@ describe("WorkflowManager — crash recovery (Phase 3)", () => {
child.emit("exit", 1, null); child.emit("exit", 1, null);
const crashedCalls = logStore.upsertWorkflowRun.mock.calls.filter( const crashedCalls = logStore.upsertWorkflowRun.mock.calls.filter(
([entry]: [{ type: string }]) => entry.type === "crashed", (args: any[]) => (args[0] as { type: string }).type === "crashed",
); );
expect(crashedCalls).toHaveLength(2); expect(crashedCalls).toHaveLength(2);
@@ -216,10 +217,10 @@ describe("WorkflowManager — crash recovery (Phase 3)", () => {
// resume-thread should have been sent // resume-thread should have been sent
const resumeCalls = (secondChild.send as ReturnType<typeof vi.fn>).mock.calls.filter( const resumeCalls = (secondChild.send as ReturnType<typeof vi.fn>).mock.calls.filter(
([msg]: [unknown]) => (args: any[]) =>
msg !== null && args[0] !== null &&
typeof msg === "object" && typeof args[0] === "object" &&
(msg as Record<string, unknown>).type === "resume-thread", (args[0] as Record<string, unknown>).type === "resume-thread",
); );
expect(resumeCalls).toHaveLength(1); expect(resumeCalls).toHaveLength(1);
expect(resumeCalls[0][0]).toMatchObject({ expect(resumeCalls[0][0]).toMatchObject({
@@ -286,7 +287,7 @@ describe("WorkflowManager — crash recovery (Phase 3)", () => {
}); });
const appendCalls = logStore.append.mock.calls.filter( const appendCalls = logStore.append.mock.calls.filter(
([entry]: [{ type: string }]) => entry.type === "thread_command_event", (args: any[]) => (args[0] as { type: string }).type === "thread_command_event",
); );
expect(appendCalls).toHaveLength(1); expect(appendCalls).toHaveLength(1);
expect(appendCalls[0][0]).toMatchObject({ expect(appendCalls[0][0]).toMatchObject({
@@ -313,7 +314,7 @@ describe("WorkflowManager — crash recovery (Phase 3)", () => {
mgr.startWorkflow("my-wf", payload); mgr.startWorkflow("my-wf", payload);
const startedCall = logStore.upsertWorkflowRun.mock.calls.find( const startedCall = logStore.upsertWorkflowRun.mock.calls.find(
([entry]: [{ type: string }]) => entry.type === "started", (args: any[]) => (args[0] as { type: string }).type === "started",
); );
expect(startedCall).toBeDefined(); expect(startedCall).toBeDefined();
const logEntry = startedCall?.[0] as { payload: string | null }; const logEntry = startedCall?.[0] as { payload: string | null };
@@ -79,6 +79,7 @@ function makeLogStore() {
getThreadEvents: vi.fn(() => []), getThreadEvents: vi.fn(() => []),
archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })), archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })),
close: vi.fn(), close: vi.fn(),
getAllWorkflowRuns: vi.fn(() => []),
}; };
} }
@@ -126,7 +127,7 @@ describe("WorkflowManager — drainAndRespawn (Phase 3 hot reload)", () => {
await drainPromise; await drainPromise;
const interruptedCalls = logStore.upsertWorkflowRun.mock.calls.filter( const interruptedCalls = logStore.upsertWorkflowRun.mock.calls.filter(
([entry]: [{ type: string }]) => entry.type === "interrupted", (args: any[]) => (args[0] as { type: string }).type === "interrupted",
); );
expect(interruptedCalls).toHaveLength(2); expect(interruptedCalls).toHaveLength(2);
@@ -190,10 +191,10 @@ describe("WorkflowManager — drainAndRespawn (Phase 3 hot reload)", () => {
const newChild = mockChildren[1]; const newChild = mockChildren[1];
const resumeCalls = (newChild.send as ReturnType<typeof vi.fn>).mock.calls.filter( const resumeCalls = (newChild.send as ReturnType<typeof vi.fn>).mock.calls.filter(
([msg]: [unknown]) => (args: any[]) =>
msg !== null && args[0] !== null &&
typeof msg === "object" && typeof args[0] === "object" &&
(msg as Record<string, unknown>).type === "resume-thread", (args[0] as Record<string, unknown>).type === "resume-thread",
); );
expect(resumeCalls).toHaveLength(0); expect(resumeCalls).toHaveLength(0);
@@ -218,10 +219,10 @@ describe("WorkflowManager — drainAndRespawn (Phase 3 hot reload)", () => {
const newChild = mockChildren[1]; const newChild = mockChildren[1];
const startCalls = (newChild.send as ReturnType<typeof vi.fn>).mock.calls.filter( const startCalls = (newChild.send as ReturnType<typeof vi.fn>).mock.calls.filter(
([msg]: [unknown]) => (args: any[]) =>
msg !== null && args[0] !== null &&
typeof msg === "object" && typeof args[0] === "object" &&
(msg as Record<string, unknown>).type === "start-thread", (args[0] as Record<string, unknown>).type === "start-thread",
); );
expect(startCalls).toHaveLength(1); expect(startCalls).toHaveLength(1);
@@ -266,7 +267,7 @@ describe("Kernel — workflow hot reload via file-watcher (Phase 3)", () => {
// Kernel's handleWorkflowFileChange should log a workflow_reload event // Kernel's handleWorkflowFileChange should log a workflow_reload event
// We test this via the kernel itself // We test this via the kernel itself
const appendCalls = logStore.append.mock.calls; const appendCalls = logStore.append.mock.calls;
const startCall = appendCalls.find(([e]: [{ type: string }]) => e.type === "start"); const startCall = appendCalls.find((args: any[]) => (args[0] as { type: string }).type === "start");
expect(startCall).toBeDefined(); expect(startCall).toBeDefined();
const stopPromise = kernel.stop(); const stopPromise = kernel.stop();
@@ -2,7 +2,7 @@
* Unit tests for kernel.triggerSense() — IPC issue #36. * Unit tests for kernel.triggerSense() — IPC issue #36.
* *
* These tests use a mock child_process and a mock LogStore so they do NOT * These tests use a mock child_process and a mock LogStore so they do NOT
* require better-sqlite3 to be present in the test environment. * require a real LogStore (node:sqlite) in integration tests.
*/ */
import { EventEmitter } from "node:events"; import { EventEmitter } from "node:events";
@@ -58,7 +58,7 @@ vi.mock("node:child_process", () => ({
const { createKernel } = await import("../kernel.js"); const { createKernel } = await import("../kernel.js");
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Mock LogStore factory (avoids better-sqlite3 dependency) // Mock LogStore factory (avoids SQLite I/O in this unit test)
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function makeMockLogStore() { function makeMockLogStore() {
@@ -78,6 +78,7 @@ function makeLogStore() {
appendWithWorkflowUpdate: vi.fn(), appendWithWorkflowUpdate: vi.fn(),
getWorkflowRun: vi.fn(() => null), getWorkflowRun: vi.fn(() => null),
getActiveWorkflowRuns: vi.fn(() => []), getActiveWorkflowRuns: vi.fn(() => []),
getAllWorkflowRuns: vi.fn(() => []),
getTriggerPayload: vi.fn(() => null), getTriggerPayload: vi.fn(() => null),
getThreadEvents: vi.fn(() => []), getThreadEvents: vi.fn(() => []),
archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })), archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })),
@@ -137,10 +138,10 @@ describe("kernel + workflowManager integration", () => {
// We need to check that a start-thread message was sent to the workflow worker // We need to check that a start-thread message was sent to the workflow worker
const workflowWorker = mockChildren.find((c) => const workflowWorker = mockChildren.find((c) =>
(c.send as ReturnType<typeof vi.fn>).mock.calls.some( (c.send as ReturnType<typeof vi.fn>).mock.calls.some(
([msg]: [unknown]) => (args: unknown[]) =>
msg !== null && args[0] !== null &&
typeof msg === "object" && typeof args[0] === "object" &&
(msg as Record<string, unknown>).type === "start-thread", (args[0] as Record<string, unknown>).type === "start-thread",
), ),
); );
expect(workflowWorker).toBeDefined(); expect(workflowWorker).toBeDefined();
@@ -212,10 +213,10 @@ describe("kernel + workflowManager integration", () => {
// No workflow worker should have been spawned (only the sense group worker) // No workflow worker should have been spawned (only the sense group worker)
const workflowWorkerSpawned = mockChildren.some((c) => const workflowWorkerSpawned = mockChildren.some((c) =>
(c.send as ReturnType<typeof vi.fn>).mock.calls.some( (c.send as ReturnType<typeof vi.fn>).mock.calls.some(
([msg]: [unknown]) => (args: unknown[]) =>
msg !== null && args[0] !== null &&
typeof msg === "object" && typeof args[0] === "object" &&
(msg as Record<string, unknown>).type === "start-thread", (args[0] as Record<string, unknown>).type === "start-thread",
), ),
); );
expect(workflowWorkerSpawned).toBe(false); expect(workflowWorkerSpawned).toBe(false);
@@ -2,15 +2,15 @@ import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
import Database from "better-sqlite3"; import { DatabaseSync } from "node:sqlite";
import { drizzle } from "drizzle-orm/better-sqlite3"; import { drizzle } from "drizzle-orm/node-sqlite";
import { integer, real, sqliteTable } from "drizzle-orm/sqlite-core"; import { integer, real, sqliteTable } from "drizzle-orm/sqlite-core";
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { createBlobStore } from "../blob-store.js"; import { createBlobStore } from "../blob-store.js";
import { parseParentMessage } from "../ipc.js"; import { parseParentMessage } from "../ipc.js";
import { executeCompute, openPeerDb, openSenseDb, runMigrations } from "../sense-runtime.js"; import { executeCompute, openPeerDb, openSenseDb, runMigrations } from "../sense-runtime.js";
import type { DrizzleDB, PeerMap, SenseRuntime } from "../sense-runtime.js"; import type { ComputeFn, DrizzleDB, PeerMap, SenseRuntime } from "../sense-runtime.js";
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Helpers // Helpers
@@ -49,7 +49,7 @@ const samples = sqliteTable("samples", {
describe("runMigrations", () => { describe("runMigrations", () => {
it("creates table via SQL migration file", () => { it("creates table via SQL migration file", () => {
const sqlite = new Database(":memory:"); const sqlite = new DatabaseSync(":memory:");
const migrationsDir = makeTempMigrationsDir(INIT_SQL); const migrationsDir = makeTempMigrationsDir(INIT_SQL);
const result = runMigrations(sqlite, migrationsDir); const result = runMigrations(sqlite, migrationsDir);
@@ -64,7 +64,7 @@ describe("runMigrations", () => {
}); });
it("runs multiple migrations in lexicographic order", () => { it("runs multiple migrations in lexicographic order", () => {
const sqlite = new Database(":memory:"); const sqlite = new DatabaseSync(":memory:");
const dir = mkdtempSync(join(tmpdir(), "nerve-multi-")); const dir = mkdtempSync(join(tmpdir(), "nerve-multi-"));
writeFileSync(join(dir, "0001_init.sql"), INIT_SQL); writeFileSync(join(dir, "0001_init.sql"), INIT_SQL);
@@ -81,7 +81,7 @@ describe("runMigrations", () => {
}); });
it("returns ok when migrations directory is empty", () => { it("returns ok when migrations directory is empty", () => {
const sqlite = new Database(":memory:"); const sqlite = new DatabaseSync(":memory:");
const dir = makeTempMigrationsDirEmpty(); const dir = makeTempMigrationsDirEmpty();
const result = runMigrations(sqlite, dir); const result = runMigrations(sqlite, dir);
expect(result.ok).toBe(true); expect(result.ok).toBe(true);
@@ -89,14 +89,14 @@ describe("runMigrations", () => {
}); });
it("returns err when migrations directory does not exist", () => { it("returns err when migrations directory does not exist", () => {
const sqlite = new Database(":memory:"); const sqlite = new DatabaseSync(":memory:");
const result = runMigrations(sqlite, "/nonexistent/path/migrations"); const result = runMigrations(sqlite, "/nonexistent/path/migrations");
expect(result.ok).toBe(false); expect(result.ok).toBe(false);
sqlite.close(); sqlite.close();
}); });
it("returns err when a migration SQL is invalid", () => { it("returns err when a migration SQL is invalid", () => {
const sqlite = new Database(":memory:"); const sqlite = new DatabaseSync(":memory:");
const dir = mkdtempSync(join(tmpdir(), "nerve-bad-sql-")); const dir = mkdtempSync(join(tmpdir(), "nerve-bad-sql-"));
writeFileSync(join(dir, "0001_bad.sql"), "NOT VALID SQL !!!;"); writeFileSync(join(dir, "0001_bad.sql"), "NOT VALID SQL !!!;");
const result = runMigrations(sqlite, dir); const result = runMigrations(sqlite, dir);
@@ -141,7 +141,7 @@ describe("openPeerDb", () => {
it("opens an existing db in read-only mode", () => { it("opens an existing db in read-only mode", () => {
// Create a writable db first // Create a writable db first
const dbPath = makeTempDbPath(); const dbPath = makeTempDbPath();
const sqlite = new Database(dbPath); const sqlite = new DatabaseSync(dbPath);
sqlite.exec(INIT_SQL); sqlite.exec(INIT_SQL);
sqlite.prepare("INSERT INTO samples (ts, value) VALUES (1, 42.0)").run(); sqlite.prepare("INSERT INTO samples (ts, value) VALUES (1, 42.0)").run();
sqlite.close(); sqlite.close();
@@ -168,13 +168,13 @@ describe("openPeerDb", () => {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
describe("executeCompute", () => { describe("executeCompute", () => {
function makeRuntime(computeFn: (db: DrizzleDB, peers: PeerMap) => Promise<unknown | null>): { function makeRuntime(computeFn: ComputeFn): {
runtime: SenseRuntime; runtime: SenseRuntime;
sqlite: Database.Database; sqlite: DatabaseSync;
} { } {
const sqlite = new Database(":memory:"); const sqlite = new DatabaseSync(":memory:");
sqlite.exec(INIT_SQL); sqlite.exec(INIT_SQL);
const db = drizzle(sqlite) as DrizzleDB; const db = drizzle({ client: sqlite }) as DrizzleDB;
return { return {
runtime: { name: "test-sense", db, compute: computeFn }, runtime: { name: "test-sense", db, compute: computeFn },
sqlite, sqlite,
@@ -226,10 +226,10 @@ describe("executeCompute", () => {
it("compute can read from peers", async () => { it("compute can read from peers", async () => {
// Set up a peer db with data // Set up a peer db with data
const peerSqlite = new Database(":memory:"); const peerSqlite = new DatabaseSync(":memory:");
peerSqlite.exec(INIT_SQL); peerSqlite.exec(INIT_SQL);
peerSqlite.prepare("INSERT INTO samples (ts, value) VALUES (100, 3.14)").run(); peerSqlite.prepare("INSERT INTO samples (ts, value) VALUES (100, 3.14)").run();
const peerDb = drizzle(peerSqlite) as DrizzleDB; const peerDb = drizzle({ client: peerSqlite }) as DrizzleDB;
const peers: PeerMap = { "other-sense": peerDb }; const peers: PeerMap = { "other-sense": peerDb };
@@ -248,9 +248,9 @@ describe("executeCompute", () => {
}); });
it("write to own db does not affect peer db (isolation)", async () => { it("write to own db does not affect peer db (isolation)", async () => {
const peerSqlite = new Database(":memory:"); const peerSqlite = new DatabaseSync(":memory:");
peerSqlite.exec(INIT_SQL); peerSqlite.exec(INIT_SQL);
const peerDb = drizzle(peerSqlite) as DrizzleDB; const peerDb = drizzle({ client: peerSqlite }) as DrizzleDB;
const peers: PeerMap = { "peer-sense": peerDb }; const peers: PeerMap = { "peer-sense": peerDb };
const { runtime, sqlite } = makeRuntime(async (db) => { const { runtime, sqlite } = makeRuntime(async (db) => {
@@ -403,7 +403,7 @@ describe("parseParentMessage", () => {
describe("runMigrations journal", () => { describe("runMigrations journal", () => {
it("does not re-run an already-applied migration", () => { it("does not re-run an already-applied migration", () => {
const sqlite = new Database(":memory:"); const sqlite = new DatabaseSync(":memory:");
const dir = mkdtempSync(join(tmpdir(), "nerve-journal-")); const dir = mkdtempSync(join(tmpdir(), "nerve-journal-"));
writeFileSync(join(dir, "0001_init.sql"), INIT_SQL); writeFileSync(join(dir, "0001_init.sql"), INIT_SQL);
@@ -430,7 +430,7 @@ describe("runMigrations journal", () => {
}); });
it("tracks migrations in _migrations table", () => { it("tracks migrations in _migrations table", () => {
const sqlite = new Database(":memory:"); const sqlite = new DatabaseSync(":memory:");
const dir = mkdtempSync(join(tmpdir(), "nerve-journal2-")); const dir = mkdtempSync(join(tmpdir(), "nerve-journal2-"));
writeFileSync(join(dir, "0001_init.sql"), INIT_SQL); writeFileSync(join(dir, "0001_init.sql"), INIT_SQL);
@@ -76,6 +76,7 @@ function makeLogStore() {
getThreadEvents: vi.fn(() => []), getThreadEvents: vi.fn(() => []),
archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })), archiveLogs: vi.fn(() => ({ days: [], vacuumed: false })),
close: vi.fn(), close: vi.fn(),
getAllWorkflowRuns: vi.fn(() => []),
}; };
} }
+30 -13
View File
@@ -9,8 +9,7 @@
import { mkdirSync, writeFileSync } from "node:fs"; import { mkdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path"; import { dirname, join } from "node:path";
import Database from "better-sqlite3"; import { DatabaseSync, type StatementSync } from "node:sqlite";
import type BetterSqlite3 from "better-sqlite3";
import { import {
DEFAULT_LOG_RETENTION_MS, DEFAULT_LOG_RETENTION_MS,
@@ -184,7 +183,23 @@ function buildJsonlBody(rows: SqlLogRow[]): string {
return `${lines.join("\n")}\n`; return `${lines.join("\n")}\n`;
} }
function runOptionalVacuum(sqlite: BetterSqlite3.Database, vacuum?: boolean): boolean { function runInTransaction<T>(db: DatabaseSync, fn: () => T): T {
db.exec("BEGIN IMMEDIATE");
try {
const out = fn();
db.exec("COMMIT");
return out;
} catch (e) {
try {
db.exec("ROLLBACK");
} catch {
// ignore rollback errors
}
throw e;
}
}
function runOptionalVacuum(sqlite: DatabaseSync, vacuum?: boolean): boolean {
if (vacuum !== true) return false; if (vacuum !== true) return false;
sqlite.exec("VACUUM"); sqlite.exec("VACUUM");
return true; return true;
@@ -199,7 +214,7 @@ function resolveArchiveStartDay(watermark: string | null, minDay: string): strin
function runArchiveDayLoop( function runArchiveDayLoop(
dbPath: string, dbPath: string,
options: ArchiveLogsOptions, options: ArchiveLogsOptions,
selectLogsForDayStmt: BetterSqlite3.Statement, selectLogsForDayStmt: StatementSync,
archiveDayTx: (day: string, start: number, endExclusive: number) => void, archiveDayTx: (day: string, start: number, endExclusive: number) => void,
startDay: string, startDay: string,
lastDay: string, lastDay: string,
@@ -235,8 +250,8 @@ function runArchiveDayLoop(
export function createLogStore(dbPath: string): LogStore { export function createLogStore(dbPath: string): LogStore {
mkdirSync(dirname(dbPath), { recursive: true }); mkdirSync(dirname(dbPath), { recursive: true });
const sqlite: BetterSqlite3.Database = new Database(dbPath); const sqlite = new DatabaseSync(dbPath);
sqlite.pragma("journal_mode = WAL"); sqlite.exec("PRAGMA journal_mode=WAL");
sqlite.exec(SCHEMA_SQL); sqlite.exec(SCHEMA_SQL);
const insertStmt = sqlite.prepare( const insertStmt = sqlite.prepare(
@@ -288,8 +303,8 @@ export function createLogStore(dbPath: string): LogStore {
"DELETE FROM logs WHERE ts >= @start AND ts < @endExclusive", "DELETE FROM logs WHERE ts >= @start AND ts < @endExclusive",
); );
const upsertWorkflowRunTx = sqlite.transaction( function upsertWorkflowRunTx(entry: Omit<LogEntry, "id">, run: WorkflowRun): LogEntry {
(entry: Omit<LogEntry, "id">, run: WorkflowRun) => { return runInTransaction(sqlite, () => {
const info = insertStmt.run({ const info = insertStmt.run({
source: entry.source, source: entry.source,
type: entry.type, type: entry.type,
@@ -304,8 +319,8 @@ export function createLogStore(dbPath: string): LogStore {
ts: run.ts, ts: run.ts,
}); });
return { ...entry, id: Number(info.lastInsertRowid) }; return { ...entry, id: Number(info.lastInsertRowid) };
}, });
); }
function append(entry: Omit<LogEntry, "id">): LogEntry { function append(entry: Omit<LogEntry, "id">): LogEntry {
const info = insertStmt.run({ const info = insertStmt.run({
@@ -376,11 +391,11 @@ export function createLogStore(dbPath: string): LogStore {
} }
function upsertWorkflowRun(entry: Omit<LogEntry, "id">, run: WorkflowRun): LogEntry { function upsertWorkflowRun(entry: Omit<LogEntry, "id">, run: WorkflowRun): LogEntry {
return upsertWorkflowRunTx(entry, run) as LogEntry; return upsertWorkflowRunTx(entry, run);
} }
function appendWithWorkflowUpdate(entry: Omit<LogEntry, "id">, run: WorkflowRun): LogEntry { function appendWithWorkflowUpdate(entry: Omit<LogEntry, "id">, run: WorkflowRun): LogEntry {
return upsertWorkflowRunTx(entry, run) as LogEntry; return upsertWorkflowRunTx(entry, run);
} }
function getWorkflowRun(runId: string): WorkflowRun | null { function getWorkflowRun(runId: string): WorkflowRun | null {
@@ -460,10 +475,12 @@ export function createLogStore(dbPath: string): LogStore {
return result; return result;
} }
const archiveDayTx = sqlite.transaction((day: string, start: number, endExclusive: number) => { function archiveDayTx(day: string, start: number, endExclusive: number): void {
runInTransaction(sqlite, () => {
deleteLogsForDayStmt.run({ start, endExclusive }); deleteLogsForDayStmt.run({ start, endExclusive });
setMetaStmt.run({ key: LOG_ARCHIVE_META_KEY, value: day }); setMetaStmt.run({ key: LOG_ARCHIVE_META_KEY, value: day });
}); });
}
function readWatermark(): string | null { function readWatermark(): string | null {
const raw = getMeta(LOG_ARCHIVE_META_KEY); const raw = getMeta(LOG_ARCHIVE_META_KEY);
+23 -23
View File
@@ -1,9 +1,9 @@
import { mkdirSync, readFileSync, readdirSync } from "node:fs"; import { mkdirSync, readFileSync, readdirSync } from "node:fs";
import { dirname, join } from "node:path"; import { dirname, join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import Database from "better-sqlite3"; import { drizzle } from "drizzle-orm/node-sqlite";
import { drizzle } from "drizzle-orm/better-sqlite3"; import type { NodeSQLiteDatabase } from "drizzle-orm/node-sqlite";
import type { BetterSQLite3Database } from "drizzle-orm/better-sqlite3";
import type { Result } from "@uncaged/nerve-core"; import type { Result } from "@uncaged/nerve-core";
import { err, ok } from "@uncaged/nerve-core"; import { err, ok } from "@uncaged/nerve-core";
@@ -11,7 +11,7 @@ import { err, ok } from "@uncaged/nerve-core";
import type { BlobStore } from "./blob-store.js"; import type { BlobStore } from "./blob-store.js";
/** A Drizzle DB instance (schema-generic) */ /** A Drizzle DB instance (schema-generic) */
export type DrizzleDB = BetterSQLite3Database<Record<string, never>>; export type DrizzleDB = NodeSQLiteDatabase<Record<string, never>>;
/** Read-only map of peer sense name → their Drizzle DB */ /** Read-only map of peer sense name → their Drizzle DB */
export type PeerMap = Readonly<Record<string, DrizzleDB>>; export type PeerMap = Readonly<Record<string, DrizzleDB>>;
@@ -42,7 +42,7 @@ export type SenseRuntime = {
compute: ComputeFn; compute: ComputeFn;
}; };
function ensureMigrationsTable(sqlite: Database.Database): Result<void> { function ensureMigrationsTable(sqlite: DatabaseSync): Result<void> {
try { try {
sqlite.exec( sqlite.exec(
`CREATE TABLE IF NOT EXISTS _migrations ( `CREATE TABLE IF NOT EXISTS _migrations (
@@ -69,11 +69,7 @@ function listMigrationFiles(migrationsDir: string): Result<string[]> {
} }
} }
function applyMigrationFile( function applyMigrationFile(sqlite: DatabaseSync, file: string, filePath: string): Result<void> {
sqlite: Database.Database,
file: string,
filePath: string,
): Result<void> {
let sql: string; let sql: string;
try { try {
sql = readFileSync(filePath, "utf8"); sql = readFileSync(filePath, "utf8");
@@ -83,13 +79,18 @@ function applyMigrationFile(
} }
const insertJournal = sqlite.prepare("INSERT INTO _migrations (name, applied_at) VALUES (?, ?)"); const insertJournal = sqlite.prepare("INSERT INTO _migrations (name, applied_at) VALUES (?, ?)");
sqlite.exec("BEGIN IMMEDIATE");
try { try {
sqlite.transaction(() => {
sqlite.exec(sql); sqlite.exec(sql);
insertJournal.run(file, Date.now()); insertJournal.run(file, Date.now());
})(); sqlite.exec("COMMIT");
return ok(undefined); return ok(undefined);
} catch (e) { } catch (e) {
try {
sqlite.exec("ROLLBACK");
} catch {
// ignore secondary errors during rollback
}
const msg = e instanceof Error ? e.message : String(e); const msg = e instanceof Error ? e.message : String(e);
return err(new Error(`Migration "${file}" failed: ${msg}`)); return err(new Error(`Migration "${file}" failed: ${msg}`));
} }
@@ -97,10 +98,10 @@ function applyMigrationFile(
/** /**
* Run all *.sql migration files in the given directory against a * Run all *.sql migration files in the given directory against a
* better-sqlite3 Database, in lexicographic order. * `node:sqlite` DatabaseSync, in lexicographic order.
* Tracks applied migrations in _migrations table to avoid re-running. * Tracks applied migrations in _migrations table to avoid re-running.
*/ */
export function runMigrations(sqlite: Database.Database, migrationsDir: string): Result<void> { export function runMigrations(sqlite: DatabaseSync, migrationsDir: string): Result<void> {
const tableResult = ensureMigrationsTable(sqlite); const tableResult = ensureMigrationsTable(sqlite);
if (!tableResult.ok) return tableResult; if (!tableResult.ok) return tableResult;
@@ -129,14 +130,13 @@ export function runMigrations(sqlite: Database.Database, migrationsDir: string):
export function openSenseDb( export function openSenseDb(
dbPath: string, dbPath: string,
migrationsDir: string, migrationsDir: string,
): Result<{ sqlite: Database.Database; db: DrizzleDB }> { ): Result<{ sqlite: DatabaseSync; db: DrizzleDB }> {
let sqlite: Database.Database; let sqlite: DatabaseSync;
try { try {
mkdirSync(dirname(dbPath), { recursive: true }); mkdirSync(dirname(dbPath), { recursive: true });
sqlite = new Database(dbPath); sqlite = new DatabaseSync(dbPath);
// WAL mode for better concurrent read performance sqlite.exec("PRAGMA journal_mode=WAL");
sqlite.pragma("journal_mode = WAL");
} catch (e) { } catch (e) {
const msg = e instanceof Error ? e.message : String(e); const msg = e instanceof Error ? e.message : String(e);
return err(new Error(`Failed to open database "${dbPath}": ${msg}`)); return err(new Error(`Failed to open database "${dbPath}": ${msg}`));
@@ -145,7 +145,7 @@ export function openSenseDb(
const migResult = runMigrations(sqlite, migrationsDir); const migResult = runMigrations(sqlite, migrationsDir);
if (!migResult.ok) return migResult; if (!migResult.ok) return migResult;
const db = drizzle(sqlite) as DrizzleDB; const db = drizzle({ client: sqlite }) as DrizzleDB;
return ok({ sqlite, db }); return ok({ sqlite, db });
} }
@@ -153,16 +153,16 @@ export function openSenseDb(
* Open a peer sense DB in read-only mode (no migrations). * Open a peer sense DB in read-only mode (no migrations).
*/ */
export function openPeerDb(dbPath: string): Result<DrizzleDB> { export function openPeerDb(dbPath: string): Result<DrizzleDB> {
let sqlite: Database.Database; let sqlite: DatabaseSync;
try { try {
sqlite = new Database(dbPath, { readonly: true }); sqlite = new DatabaseSync(dbPath, { readOnly: true });
} catch (e) { } catch (e) {
const msg = e instanceof Error ? e.message : String(e); const msg = e instanceof Error ? e.message : String(e);
return err(new Error(`Failed to open peer database "${dbPath}" (readonly): ${msg}`)); return err(new Error(`Failed to open peer database "${dbPath}" (readonly): ${msg}`));
} }
return ok(drizzle(sqlite) as DrizzleDB); return ok(drizzle({ client: sqlite }) as DrizzleDB);
} }
/** /**
+788 -45
View File
File diff suppressed because it is too large Load Diff
-1
View File
@@ -3,5 +3,4 @@ packages:
onlyBuiltDependencies: onlyBuiltDependencies:
- "@biomejs/biome" - "@biomejs/biome"
- better-sqlite3
- esbuild - esbuild