debug: dump raw NDJSON for issue #439 investigation

Saves Claude Code stdout to /tmp/uwf-ndjson-dump/ before parsing.
Temporary — remove after root cause confirmed.

Refs #439

package.json

@@ -9,7 +9,7 @@
     "check": "bunx tsc --build && biome check . && bash scripts/lint-log-tags.sh",
     "typecheck": "bunx tsc --build",
     "format": "biome format --write .",
-    "test": "bun run --filter './packages/*' test",
+    "test": "bun run --filter '*' test",
     "changeset": "bunx changeset",
     "version": "bunx changeset version",
     "release": "bun run build && bun test && node scripts/publish-all.mjs"

packages/cli-workflow/src/__tests__/thread.test.ts

@@ -266,7 +266,12 @@ describe("cmdThreadRead ### Content section", () => {
 
     expect(markdown).toContain("### Content");
     expect(markdown).toContain("The assistant response text");
-    expect(markdown).not.toContain("### Output");
+
+    const contentIdx = markdown.indexOf("### Content");
+    const outputIdx = markdown.indexOf("### Output");
+    expect(contentIdx).toBeGreaterThanOrEqual(0);
+    expect(outputIdx).toBeGreaterThanOrEqual(0);
+    expect(contentIdx).toBeLessThan(outputIdx);
   });
 
   test("omits ### Content when detail has no matching assistant turns", async () => {
@@ -309,7 +314,7 @@ describe("cmdThreadRead ### Content section", () => {
     const markdown = await cmdThreadRead(tmpDir, threadId, THREAD_READ_DEFAULT_QUOTA, null, false);
 
     expect(markdown).not.toContain("### Content");
-    expect(markdown).not.toContain("### Output");
+    expect(markdown).toContain("### Output");
   });
 });
 
@@ -387,87 +392,3 @@ describe("cmdThreadStepDetails", () => {
     await expect(cmdThreadStepDetails(tmpDir, "nonexistenth0" as CasRef)).rejects.toThrow();
   });
 });
-
-// ── cmdThreadRead: ### Prompt deduplication ───────────────────────────────────
-
-describe("cmdThreadRead ### Prompt deduplication", () => {
-  async function makeThreadWithRoles(uwf: UwfStore, roles: string[]): Promise<string> {
-    const roleMap: Record<string, unknown> = {};
-    for (const r of [...new Set(roles)]) {
-      roleMap[r] = {
-        description: r,
-        goal: `Goal for ${r}`,
-        capabilities: [],
-        procedure: "Do stuff.",
-        output: "Output.",
-        meta: "placeholder00" as CasRef,
-      };
-    }
-    const workflowHash = await uwf.store.put(uwf.schemas.workflow, {
-      name: "dedup-wf",
-      description: "desc",
-      roles: roleMap,
-      conditions: {},
-      graph: {},
-    });
-    const startHash = await uwf.store.put(uwf.schemas.startNode, {
-      workflow: workflowHash,
-      prompt: "Start",
-    });
-    const outputHash = await uwf.store.put(uwf.schemas.workflow, {
-      name: "out",
-      description: "",
-      roles: {},
-      conditions: {},
-      graph: {},
-    });
-
-    let prev: string | null = null;
-    let stepHash = "";
-    for (const role of roles) {
-      stepHash = await uwf.store.put(uwf.schemas.stepNode, {
-        start: startHash,
-        prev: prev as CasRef | null,
-        role,
-        output: outputHash,
-        detail: null,
-        agent: "uwf-test",
-      });
-      prev = stepHash;
-    }
-    return stepHash;
-  }
-
-  test("same consecutive role shows ### Prompt once", async () => {
-    const uwf = await makeUwfStore(tmpDir);
-    const headHash = await makeThreadWithRoles(uwf, ["writer", "writer"]);
-    const threadId = "01JTEST0000000000000003" as ThreadId;
-    await saveThreadsIndex(tmpDir, { [threadId]: headHash });
-
-    const markdown = await cmdThreadRead(tmpDir, threadId, THREAD_READ_DEFAULT_QUOTA, null, false);
-    const count = (markdown.match(/### Prompt/g) ?? []).length;
-    expect(count).toBe(1);
-  });
-
-  test("different consecutive roles each show ### Prompt", async () => {
-    const uwf = await makeUwfStore(tmpDir);
-    const headHash = await makeThreadWithRoles(uwf, ["planner", "coder"]);
-    const threadId = "01JTEST0000000000000004" as ThreadId;
-    await saveThreadsIndex(tmpDir, { [threadId]: headHash });
-
-    const markdown = await cmdThreadRead(tmpDir, threadId, THREAD_READ_DEFAULT_QUOTA, null, false);
-    const count = (markdown.match(/### Prompt/g) ?? []).length;
-    expect(count).toBe(2);
-  });
-
-  test("non-consecutive same role shows ### Prompt twice", async () => {
-    const uwf = await makeUwfStore(tmpDir);
-    const headHash = await makeThreadWithRoles(uwf, ["roleA", "roleB", "roleA"]);
-    const threadId = "01JTEST0000000000000005" as ThreadId;
-    await saveThreadsIndex(tmpDir, { [threadId]: headHash });
-
-    const markdown = await cmdThreadRead(tmpDir, threadId, THREAD_READ_DEFAULT_QUOTA, null, false);
-    const count = (markdown.match(/### Prompt/g) ?? []).length;
-    expect(count).toBe(2);
-  });
-});

packages/cli-workflow/src/commands/thread.ts

@@ -655,11 +655,11 @@ function formatThreadReadMarkdown(options: {
 
   // Step blocks
   const startIndex = candidates.length - selected.length;
-  const shownPromptRoles = new Set<string>();
   for (let i = 0; i < selected.length; i++) {
     const item = selected[i];
     if (item === undefined) continue;
     const stepNum = startIndex + i + 1;
+    const outputYaml = formatYaml(expandOutput(uwf, item.payload.output));
     const ts = new Date(item.timestamp)
       .toISOString()
       .replace("T", " ")
@@ -669,10 +669,9 @@ function formatThreadReadMarkdown(options: {
       `**Agent:** ${item.payload.agent} | **Time:** ${ts}`,
     ];
     const roleDef = workflow.roles[item.payload.role];
-    if (roleDef && !shownPromptRoles.has(item.payload.role)) {
+    if (roleDef) {
       const prompt = roleDef.goal;
       stepLines.push("", "### Prompt", "", prompt);
-      shownPromptRoles.add(item.payload.role);
     }
     if (item.payload.detail) {
       const content = extractLastAssistantContent(uwf, item.payload.detail);
@@ -680,6 +679,7 @@ function formatThreadReadMarkdown(options: {
         stepLines.push("", "### Content", "", content);
       }
     }
+    stepLines.push("", "### Output", "", "```yaml", outputYaml, "```");
     parts.push(stepLines.join("\n"));
   }
 

packages/workflow-agent-builtin/src/agent.ts

@@ -13,28 +13,10 @@ import { storeBuiltinDetail } from "./detail.js";
 import type { ChatMessage } from "./llm/index.js";
 import { BUILTIN_CONTINUE_MAX_TURNS, BUILTIN_MAX_TURNS, runBuiltinLoop } from "./loop.js";
 import { buildBuiltinMessages } from "./prompt.js";
-import { initSessionDir } from "./session.js";
+import { initSessionDir, removeSession } from "./session.js";
 
 const log = createLogger({ sink: { kind: "stderr" } });
 
-const FRONTMATTER_FENCE = "---";
-
-/**
- * Strip any text before the first `---` fence.
- * LLMs sometimes emit preamble text before the frontmatter block.
- */
-function stripPreamble(text: string): string {
-  if (text.startsWith(FRONTMATTER_FENCE)) {
-    return text;
-  }
-  const idx = text.indexOf(`\n${FRONTMATTER_FENCE}\n`);
-  if (idx !== -1) {
-    log("6GWRP3QX", `stripped ${idx + 1} chars of preamble before frontmatter`);
-    return text.slice(idx + 1);
-  }
-  return text;
-}
-
 type SessionRecord = {
   sessionId: string;
   model: string;
@@ -66,7 +48,6 @@ async function runBuiltinWithMessages(
   session: SessionRecord,
   store: Store,
   maxTurns: number,
-  noTools: boolean,
 ): Promise<AgentRunResult> {
   const loopResult = await runBuiltinLoop({
     provider,
@@ -75,13 +56,13 @@ async function runBuiltinWithMessages(
     maxTurns,
     storageRoot,
     sessionId: session.sessionId,
-    noTools,
   });
 
   session.messages = loopResult.messages;
 
   if (loopResult.turnCount === 0) {
     log("5RWTK9NB", "no turns produced, returning empty output");
+    await removeSession(storageRoot, session.sessionId);
     return { output: "", detailHash: "", sessionId: session.sessionId };
   }
 
@@ -94,7 +75,10 @@ async function runBuiltinWithMessages(
     session.startedAtMs,
   );
 
-  return { output: stripPreamble(loopResult.finalText), detailHash, sessionId: session.sessionId };
+  // Clean up session jsonl
+  await removeSession(storageRoot, session.sessionId);
+
+  return { output: loopResult.finalText, detailHash, sessionId: session.sessionId };
 }
 
 async function runBuiltin(ctx: AgentContext): Promise<AgentRunResult> {
@@ -121,7 +105,6 @@ async function runBuiltin(ctx: AgentContext): Promise<AgentRunResult> {
     session,
     ctx.store,
     BUILTIN_MAX_TURNS,
-    false,
   );
 }
 
@@ -144,7 +127,6 @@ async function continueBuiltin(
     session,
     store,
     BUILTIN_CONTINUE_MAX_TURNS,
-    true,
   );
 }
 

packages/workflow-agent-builtin/src/llm/llm.ts

@@ -96,17 +96,8 @@ function serializeMessage(message: ChatMessage): Record<string, unknown> {
 export async function chatCompletionWithTools(
   provider: ResolvedLlmProvider,
   messages: ChatMessage[],
-  tools: OpenAiToolDefinition[] | null,
+  tools: OpenAiToolDefinition[],
 ): Promise<LlmAssistantResponse> {
-  const body: Record<string, unknown> = {
-    model: provider.model,
-    messages: messages.map(serializeMessage),
-  };
-  if (tools !== null && tools.length > 0) {
-    body.tools = tools;
-    body.tool_choice = "auto";
-  }
-
   let response: Response;
   try {
     response = await fetch(chatUrl(provider.baseUrl), {
@@ -115,7 +106,12 @@ export async function chatCompletionWithTools(
         Authorization: `Bearer ${provider.apiKey}`,
         "Content-Type": "application/json",
       },
-      body: JSON.stringify(body),
+      body: JSON.stringify({
+        model: provider.model,
+        messages: messages.map(serializeMessage),
+        tools,
+        tool_choice: "auto",
+      }),
     });
   } catch (cause) {
     const message = cause instanceof Error ? cause.message : String(cause);

packages/workflow-agent-builtin/src/loop.ts

@@ -23,8 +23,6 @@ export type RunBuiltinLoopOptions = {
   maxTurns: number;
   storageRoot: string;
   sessionId: string;
-  /** When true, do not provide tools — force LLM to emit text only. */
-  noTools: boolean;
 };
 
 export type RunBuiltinLoopResult = {
@@ -75,17 +73,13 @@ export async function runBuiltinLoop(
   options: RunBuiltinLoopOptions,
 ): Promise<RunBuiltinLoopResult> {
   const messages = [...options.messages];
-  const openAiTools = options.noTools ? [] : builtinToolsToOpenAi(getBuiltinTools());
+  const openAiTools = builtinToolsToOpenAi(getBuiltinTools());
   let finalText = "";
   let turnCount = 0;
 
   for (let turn = 0; turn < options.maxTurns; turn++) {
     log("8K2M4N7P", `builtin loop turn ${turn + 1}/${options.maxTurns}`);
-    const response = await chatCompletionWithTools(
-      options.provider,
-      messages,
-      openAiTools.length > 0 ? openAiTools : null,
-    );
+    const response = await chatCompletionWithTools(options.provider, messages, openAiTools);
 
     const assistantMessage: ChatMessage = {
       role: "assistant",
@@ -95,27 +89,14 @@ export async function runBuiltinLoop(
     messages.push(assistantMessage);
 
     if (response.toolCalls === null || response.toolCalls.length === 0) {
-      const text = response.content ?? "";
+      finalText = response.content ?? "";
       await appendTurn(options.storageRoot, options.sessionId, {
         role: "assistant",
-        content: text,
+        content: response.content ?? "",
         toolCalls: null,
         reasoning: null,
       });
       turnCount += 1;
-
-      // If tools are available but LLM stopped calling them without producing
-      // frontmatter, nudge it to continue working or output frontmatter.
-      if (!options.noTools && !text.trimStart().startsWith("---") && turn < options.maxTurns - 1) {
-        log("7FXQM2KN", "text-only turn without frontmatter, nudging LLM to continue");
-        const nudge =
-          "You stopped calling tools but your response does not start with the required `---` YAML frontmatter. " +
-          "Either continue using tools to complete your work, or output your final response starting with `---`.";
-        messages.push({ role: "user", content: nudge });
-        continue;
-      }
-
-      finalText = text;
       break;
     }
 

packages/workflow-agent-builtin/src/prompt.ts

@@ -59,21 +59,6 @@ export function buildBuiltinMessages(ctx: AgentContext): ChatMessage[] {
   }
   systemParts.push(rolePrompt);
 
-  systemParts.push(
-    "",
-    "## Workflow",
-    "",
-    `Your working directory is: ${process.cwd()}`,
-    "",
-    "You have tools available (read_file, write_file, run_command). " +
-      "Use them to complete your task — read files, run commands, make changes as needed. " +
-      "Your task is described in the user message below — do NOT use uwf or workflow CLI commands to discover your task. " +
-      "When you are done, output your final response with the YAML frontmatter block as specified above. " +
-      "Do NOT output the frontmatter until you have completed all necessary work. " +
-      "CRITICAL: Your final output MUST start with the `---` fence on the very first line — " +
-      "no preamble text, no explanation before it. The parser requires `---` at position 0.",
-  );
-
   const messages: ChatMessage[] = [{ role: "system", content: systemParts.join("\n") }];
 
   const roleVisitIndices: number[] = [];

packages/workflow-agent-claude-code/src/claude-code.ts

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
 import type { Store } from "@uncaged/json-cas";
 import {
   type AgentContext,
@@ -117,7 +118,17 @@ function spawnClaudeResume(
   ]);
 }
 
+const NDJSON_DUMP_DIR = "/tmp/uwf-ndjson-dump";
+
 async function processClaudeOutput(stdout: string, store: Store): Promise<AgentRunResult> {
+  // Debug dump: save raw NDJSON for issue #439 investigation
+  try {
+    mkdirSync(NDJSON_DUMP_DIR, { recursive: true });
+    writeFileSync(`${NDJSON_DUMP_DIR}/${Date.now()}.ndjson`, stdout);
+  } catch {
+    // ignore dump failures
+  }
+
   const parsed = parseClaudeCodeStreamOutput(stdout);
 
   if (parsed !== null) {