feat: Merkle node format + content → CAS

- MerkleNode type: { type, payload, children } serialized as YAML
- RoleOutput.content → contentHash (CAS hash of Merkle content node)
- Engine stores content in global CAS before writing to .data.jsonl
- create-workflow puts content as Merkle node, merges contentHash into refs
- fork/parse adapted for contentHash format
- buildAgentPrompt now async, reads content from CAS
- Bundle validator allows @uncaged/workflow import
- 150 tests passing

BREAKING: .data.jsonl no longer contains inline content

Fixes #41

packages/cli-workflow/__tests__/commands.test.ts

@@ -16,6 +16,9 @@ import { addCliArgs } from "./bundle-fixture.js";
 const fixtureDescriptor = `export const descriptor = { description: "fixture", roles: {} };
 `;
 
+const wfPutImport = `import { putContentMerkleNode } from "@uncaged/workflow";
+`;
+
 describe("cli workflow commands", () => {
   let prevEnv: string | undefined;
   let storageRoot: string;
@@ -41,11 +44,13 @@ describe("cli workflow commands", () => {
     const bundlePath = join(bundleDir, "demo.esm.js");
     await writeFile(
       bundlePath,
-      `${fixtureDescriptor}import fs from "node:fs";
+      `${fixtureDescriptor}${wfPutImport}import fs from "node:fs";
 
-export const run = async function* (input) {
+export const run = async function* (input, options) {
   fs.existsSync(".");
-  yield { role: "noop", content: input.prompt, meta: { done: true } };
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, input.prompt);
+  yield { role: "noop", contentHash: h, meta: { done: true }, refs: [h] };
   return { returnCode: 0, summary: "done" };
 }
 `,
@@ -112,8 +117,8 @@ export const run = async function* (input) { return { returnCode: 0, summary: in
     const bundlePath = join(storageRoot, "solo.esm.js");
     await writeFile(
       bundlePath,
-      `export const run = async function* (input) {
-  yield { role: "x", content: input.prompt, meta: {} };
+      `export const run = async function* () {
+  yield { role: "x", contentHash: "STUBHASH00000000000000001", meta: {}, refs: [] };
   return { returnCode: 0, summary: "ok" };
 }
 `,
@@ -141,8 +146,11 @@ export const run = async function* (input) { return { returnCode: 0, summary: in
     },
   },
 };
-export const run = async function* (input) {
-  yield { role: "greeter", content: input.prompt, meta: { greeting: "hi" } };
+${wfPutImport}
+export const run = async function* (input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, input.prompt);
+  yield { role: "greeter", contentHash: h, meta: { greeting: "hi" }, refs: [h] };
   return { returnCode: 0, summary: "ok" };
 };
 `,
@@ -180,8 +188,10 @@ export const run = async function* (input) {
     const bundlePath = join(bundleDir, "demo.esm.js");
     await writeFile(
       bundlePath,
-      `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "x", meta: {} };
+      `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "x");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "x" };
 }
 `,
@@ -209,8 +219,10 @@ export const run = async function* (input) {
     const dtsPath = join(bundleDir, "types.d.ts");
     await writeFile(
       bundlePath,
-      `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "x", meta: {} };
+      `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "x");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "x" };
 }
 `,
@@ -240,8 +252,10 @@ export const run = async function* (input) {
     const bundlePath = join(bundleDir, "demo.esm.js");
     await writeFile(
       bundlePath,
-      `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "x", meta: {} };
+      `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "x");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "x" };
 }
 `,
@@ -261,13 +275,17 @@ export const run = async function* (input) {
     const bundleDir = join(storageRoot, "src");
     await mkdir(bundleDir, { recursive: true });
     const bundlePath = join(bundleDir, "demo.esm.js");
-    const v1 = `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "v1", meta: {} };
+    const v1 = `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "v1");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "v1" };
 }
 `;
-    const v2 = `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "v2", meta: {} };
+    const v2 = `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "v2");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "v2" };
 }
 `;
@@ -299,13 +317,17 @@ export const run = async function* (input) {
     const bundleDir = join(storageRoot, "src");
     await mkdir(bundleDir, { recursive: true });
     const bundlePath = join(bundleDir, "demo.esm.js");
-    const v1 = `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "v1", meta: {} };
+    const v1 = `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "v1");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "v1" };
 }
 `;
-    const v2 = `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "v2", meta: {} };
+    const v2 = `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "v2");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "v2" };
 }
 `;
@@ -347,8 +369,10 @@ export const run = async function* (input) {
     const bundlePath = join(bundleDir, "demo.esm.js");
     await writeFile(
       bundlePath,
-      `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "x", meta: {} };
+      `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "x");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "x" };
 }
 `,
@@ -358,8 +382,10 @@ export const run = async function* (input) {
     expect(add1.ok).toBe(true);
     await writeFile(
       bundlePath,
-      `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "y", meta: {} };
+      `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "y");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "y" };
 }
 `,
@@ -409,8 +435,10 @@ export const run = async function* (input) {
     const bundlePath = join(bundleDir, "demo.esm.js");
     await writeFile(
       bundlePath,
-      `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "x", meta: {} };
+      `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "x");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "x" };
 }
 `,
@@ -424,8 +452,10 @@ export const run = async function* (input) {
     const hash1 = add1.value.hash;
     await writeFile(
       bundlePath,
-      `${fixtureDescriptor}export const run = async function* (input) {
-  yield { role: "a", content: "y", meta: {} };
+      `${fixtureDescriptor}${wfPutImport}export const run = async function* (_input, options) {
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "y");
+  yield { role: "a", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "y" };
 }
 `,

packages/cli-workflow/__tests__/fork-cli.test.ts

@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { createCasStore, getContentMerklePayload, getGlobalCasDir } from "@uncaged/workflow";
 import { cmdAdd } from "../src/cmd-add.js";
 import { cmdFork } from "../src/cmd-fork.js";
 import { cmdRun } from "../src/cmd-run.js";
@@ -9,7 +10,9 @@ import { pathExists } from "../src/fs-utils.js";
 import { addCliArgs } from "./bundle-fixture.js";
 
 /** Three-role workflow that respects `input.steps` for fork/resume. */
-const threeRoleBundleSource = `export const descriptor = {
+const threeRoleBundleSource = `import { putContentMerkleNode } from "@uncaged/workflow";
+
+export const descriptor = {
   description: "fork-cli",
   roles: {
     planner: { description: "planner", schema: {} },
@@ -17,20 +20,21 @@ const threeRoleBundleSource = `export const descriptor = {
     reviewer: { description: "reviewer", schema: {} },
   },
 };
-export const run = async function* (input) {
+export const run = async function* (input, options) {
+  const cas = options.cas;
   const has = (r) => input.steps.some((s) => s.role === r);
   if (!has("planner")) {
-    yield { role: "planner", content: "p1", meta: { k: "planner" } };
+    const h = await putContentMerkleNode(cas, "p1");
+    yield { role: "planner", contentHash: h, meta: { k: "planner" }, refs: [h] };
   }
   if (!has("coder")) {
-    yield { role: "coder", content: "c1", meta: { k: "coder" } };
+    const h = await putContentMerkleNode(cas, "c1");
+    yield { role: "coder", contentHash: h, meta: { k: "coder" }, refs: [h] };
   }
   if (!has("reviewer")) {
-    yield {
-      role: "reviewer",
-      content: "rev-" + String(input.steps.length),
-      meta: { k: "reviewer" },
-    };
+    const body = "rev-" + String(input.steps.length);
+    const h = await putContentMerkleNode(cas, body);
+    yield { role: "reviewer", contentHash: h, meta: { k: "reviewer" }, refs: [h] };
   }
   return { returnCode: 0, summary: "done" };
 };
@@ -132,7 +136,8 @@ describe("cli fork", () => {
 
     const last = JSON.parse(lines[lines.length - 1] ?? "{}") as Record<string, unknown>;
     expect(last.role).toBe("reviewer");
-    expect(last.content).toBe("rev-1");
+    const cas = createCasStore(getGlobalCasDir(storageRoot));
+    expect(await getContentMerklePayload(cas, String(last.contentHash))).toBe("rev-1");
   });
 
   test("fork without --from-role retries last role", async () => {
@@ -179,11 +184,12 @@ describe("cli fork", () => {
 
     const replayCoder = JSON.parse(lines[2] ?? "{}") as Record<string, unknown>;
     expect(replayCoder.role).toBe("coder");
-    expect(replayCoder.content).toBe("c1");
+    const cas = createCasStore(getGlobalCasDir(storageRoot));
+    expect(await getContentMerklePayload(cas, String(replayCoder.contentHash))).toBe("c1");
 
     const last = JSON.parse(lines[lines.length - 1] ?? "{}") as Record<string, unknown>;
     expect(last.role).toBe("reviewer");
-    expect(last.content).toBe("rev-2");
+    expect(await getContentMerklePayload(cas, String(last.contentHash))).toBe("rev-2");
   });
 
   test("fork rejects unknown role with available names", async () => {

packages/cli-workflow/__tests__/gc-cli.test.ts

@@ -4,31 +4,43 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { createCasStore, garbageCollectCas, getGlobalCasDir } from "@uncaged/workflow";
+import {
+  createCasStore,
+  garbageCollectCas,
+  getGlobalCasDir,
+  putContentMerkleNode,
+} from "@uncaged/workflow";
 import { cmdThreadRemove } from "../src/cmd-thread.js";
 import { pathExists } from "../src/fs-utils.js";
 
 const cliEntryPath = fileURLToPath(new URL("../src/cli.ts", import.meta.url));
 
-/** Minimal valid `.data.jsonl` with one role step referencing `activeHash` in `refs`. */
-function makeDataJsonl(threadId: string, bundleHash: string, activeHash: string): string {
-  return [
+async function writeDemoDataJsonl(params: {
+  path: string;
+  threadId: string;
+  bundleHash: string;
+  cas: ReturnType<typeof createCasStore>;
+  activeHash: string;
+}): Promise<void> {
+  const bodyHash = await putContentMerkleNode(params.cas, "p");
+  const text = [
     JSON.stringify({
       name: "demo",
-      hash: bundleHash,
-      threadId,
+      hash: params.bundleHash,
+      threadId: params.threadId,
       parameters: { prompt: "hi", options: { maxRounds: 5 } },
       timestamp: 100,
     }),
     JSON.stringify({
       role: "planner",
-      content: "p",
+      contentHash: bodyHash,
       meta: {},
-      refs: [activeHash],
+      refs: [params.activeHash, bodyHash],
       timestamp: 101,
     }),
     "",
   ].join("\n");
+  await writeFile(params.path, text, "utf8");
 }
 
 describe("gc cli and garbageCollectCas", () => {
@@ -60,11 +72,13 @@ describe("gc cli and garbageCollectCas", () => {
     const activeHash = await cas.put("active-blob");
     const orphanHash = await cas.put("orphan-blob");
 
-    await writeFile(
-      join(logsDir, `${threadId}.data.jsonl`),
-      makeDataJsonl(threadId, bundleHash, activeHash),
-      "utf8",
-    );
+    await writeDemoDataJsonl({
+      path: join(logsDir, `${threadId}.data.jsonl`),
+      threadId,
+      bundleHash,
+      cas,
+      activeHash,
+    });
 
     const gc = await garbageCollectCas(storageRoot);
     expect(gc.ok).toBe(true);
@@ -72,7 +86,7 @@ describe("gc cli and garbageCollectCas", () => {
       return;
     }
     expect(gc.value.scannedThreads).toBe(1);
-    expect(gc.value.activeRefs).toBe(1);
+    expect(gc.value.activeRefs).toBe(2);
     expect(gc.value.deletedEntries).toBe(1);
     expect(gc.value.deletedHashes).toEqual([orphanHash]);
 
@@ -106,16 +120,18 @@ describe("gc cli and garbageCollectCas", () => {
     const activeHash = await cas.put("keep-me");
     await cas.put("drop-me");
 
-    await writeFile(
-      join(logsDir, `${threadId}.data.jsonl`),
-      makeDataJsonl(threadId, bundleHash, activeHash),
-      "utf8",
-    );
+    await writeDemoDataJsonl({
+      path: join(logsDir, `${threadId}.data.jsonl`),
+      threadId,
+      bundleHash,
+      cas,
+      activeHash,
+    });
 
     const env = { ...process.env, UNCAGED_WORKFLOW_STORAGE_ROOT: storageRoot };
     const proc = spawnSync(process.execPath, [cliEntryPath, "gc"], { env, encoding: "utf8" });
     expect(proc.status).toBe(0);
-    expect(String(proc.stdout).trim()).toBe("scanned 1 threads, 1 active refs, deleted 1 entries");
+    expect(String(proc.stdout).trim()).toBe("scanned 1 threads, 2 active refs, deleted 1 entries");
   });
 
   test("thread rm triggers gc so unreferenced CAS is removed", async () => {
@@ -126,11 +142,13 @@ describe("gc cli and garbageCollectCas", () => {
 
     const cas = createCasStore(getGlobalCasDir(storageRoot));
     const activeHash = await cas.put("pinned-by-ref");
-    await writeFile(
-      join(logsDir, `${threadId}.data.jsonl`),
-      makeDataJsonl(threadId, bundleHash, activeHash),
-      "utf8",
-    );
+    await writeDemoDataJsonl({
+      path: join(logsDir, `${threadId}.data.jsonl`),
+      threadId,
+      bundleHash,
+      cas,
+      activeHash,
+    });
 
     const orphanHash = await cas.put("orphan-after-rm");
     const orphanPath = join(getGlobalCasDir(storageRoot), `${orphanHash}.txt`);

packages/cli-workflow/__tests__/thread-cli.test.ts

@@ -17,6 +17,9 @@ import { cmdThreads } from "../src/cmd-threads.js";
 import { pathExists, readTextFileIfExists } from "../src/fs-utils.js";
 import { addCliArgs } from "./bundle-fixture.js";
 
+const wfPutImport = `import { putContentMerkleNode } from "@uncaged/workflow";
+`;
+
 const threadFixtureDescriptor = `export const descriptor = {
   description: "thread-cli",
   roles: {
@@ -31,18 +34,26 @@ const threadFixtureDescriptor = `export const descriptor = {
 `;
 
 const fastBundleSource = `${threadFixtureDescriptor}
-export const run = async function* (input) {
-  yield { role: "planner", content: "plan", meta: { plan: input.prompt } };
-  yield { role: "coder", content: "code", meta: { diff: "y" } };
+${wfPutImport}
+export const run = async function* (input, options) {
+  const cas = options.cas;
+  let h = await putContentMerkleNode(cas, "plan");
+  yield { role: "planner", contentHash: h, meta: { plan: input.prompt }, refs: [h] };
+  h = await putContentMerkleNode(cas, "code");
+  yield { role: "coder", contentHash: h, meta: { diff: "y" }, refs: [h] };
   return { returnCode: 0, summary: "done" };
 };
 `;
 
 const slowPlannerBundleSource = `${threadFixtureDescriptor}
-export const run = async function* (input) {
+${wfPutImport}
+export const run = async function* (input, options) {
   await new Promise((r) => setTimeout(r, 400));
-  yield { role: "planner", content: "plan", meta: { plan: input.prompt } };
-  yield { role: "coder", content: "code", meta: { diff: "y" } };
+  const cas = options.cas;
+  let h = await putContentMerkleNode(cas, "plan");
+  yield { role: "planner", contentHash: h, meta: { plan: input.prompt }, refs: [h] };
+  h = await putContentMerkleNode(cas, "code");
+  yield { role: "coder", contentHash: h, meta: { diff: "y" }, refs: [h] };
   return { returnCode: 0, summary: "done" };
 };
 `;
@@ -50,27 +61,38 @@ export const run = async function* (input) {
 const cliEntryPath = fileURLToPath(new URL("../src/cli.ts", import.meta.url));
 
 const abortablePlannerBundleSource = `${threadFixtureDescriptor}
-export const run = async function* (input) {
+${wfPutImport}
+export const run = async function* (input, options) {
   await new Promise((r) => setTimeout(r, 600));
-  yield { role: "planner", content: "plan", meta: { plan: input.prompt } };
-  yield { role: "coder", content: "code", meta: { diff: "y" } };
+  const cas = options.cas;
+  let h = await putContentMerkleNode(cas, "plan");
+  yield { role: "planner", contentHash: h, meta: { plan: input.prompt }, refs: [h] };
+  h = await putContentMerkleNode(cas, "code");
+  yield { role: "coder", contentHash: h, meta: { diff: "y" }, refs: [h] };
   return { returnCode: 0, summary: "done" };
 };
 `;
 
 const pauseResumeBundleSource = `${threadFixtureDescriptor}
-export const run = async function* (input) {
-  yield { role: "first", content: "f", meta: {} };
+${wfPutImport}
+export const run = async function* (_input, options) {
+  const cas = options.cas;
+  let h = await putContentMerkleNode(cas, "f");
+  yield { role: "first", contentHash: h, meta: {}, refs: [h] };
   await new Promise((r) => setTimeout(r, 1500));
-  yield { role: "second", content: "s", meta: {} };
+  h = await putContentMerkleNode(cas, "s");
+  yield { role: "second", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "done" };
 };
 `;
 
 const delayedFirstYieldBundleSource = `${threadFixtureDescriptor}
-export const run = async function* (input) {
+${wfPutImport}
+export const run = async function* (_input, options) {
   await new Promise((r) => setTimeout(r, 900));
-  yield { role: "only", content: "x", meta: {} };
+  const cas = options.cas;
+  const h = await putContentMerkleNode(cas, "x");
+  yield { role: "only", contentHash: h, meta: {}, refs: [h] };
   return { returnCode: 0, summary: "done" };
 };
 `;

packages/cli-workflow/src/cmd-add.ts

@@ -192,7 +192,7 @@ export async function cmdAdd(
     return validated;
   }
 
-  const extracted = await extractBundleExports(resolvedPath);
+  const extracted = await extractBundleExports(resolvedPath, { storageRoot });
   if (!extracted.ok) {
     return extracted;
   }

packages/cli-workflow/src/cmd-fork.ts

@@ -65,8 +65,9 @@ export async function cmdFork(
   const newThreadId = generateUlid(Date.now());
   const stepsOnWire = plan.value.historicalSteps.map((s) => ({
     role: s.role,
-    content: s.content,
+    contentHash: s.contentHash,
     meta: s.meta,
+    refs: s.refs,
     timestamp: s.timestamp,
   }));