fix: dynamic frontmatter field extraction from role schema

Replace hardcoded 5-field candidate with schema-driven extraction.
Now reads outputSchema properties and picks matching fields from
parsed frontmatter, supporting role-specific fields like plan,
approved, success.

Falls back to standard 5 fields when schema has no properties.

Fixes #388

小橘 <xiaoju@shazhou.work>

.workflows/solve-issue.yaml

@@ -21,7 +21,7 @@ roles:
 
       After producing the test spec:
       1. Store it via `uwf cas put-text "<markdown content>"` and capture the returned hash
-      2. Put the hash in meta.plan (required when status=ready)
+      2. Put the hash in frontmatter.plan (required when status=ready)
     output: "Output a brief summary of the test spec. Frontmatter must include: status (ready or insufficient_info) and plan (CAS hash of the test spec, required when status=ready)."
     frontmatter:
       type: object

packages/workflow-agent-hermes/src/hermes.ts

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import type { Store } from "@uncaged/json-cas";
 
 import {
   type AgentContext,
@@ -10,7 +11,6 @@ import {
 import {
   loadHermesSession,
   parseSessionIdFromStdout,
-  storeHermesRawOutput,
   storeHermesSessionDetail,
 } from "./session-detail.js";
 
@@ -52,17 +52,8 @@ export function buildHermesPrompt(ctx: AgentContext): string {
   return parts.join("\n");
 }
 
-function spawnHermesChat(prompt: string): Promise<{ stdout: string; stderr: string }> {
+function spawnHermes(args: string[]): Promise<{ stdout: string; stderr: string }> {
   return new Promise((resolve, reject) => {
-    const args = [
-      "chat",
-      "-q",
-      prompt,
-      "--yolo",
-      "--max-turns",
-      String(HERMES_MAX_TURNS),
-      "--quiet",
-    ];
     const child = spawn(HERMES_COMMAND, args, {
       env: process.env,
       shell: false,
@@ -94,23 +85,73 @@ function spawnHermesChat(prompt: string): Promise<{ stdout: string; stderr: stri
   });
 }
 
+function spawnHermesChat(prompt: string): Promise<{ stdout: string; stderr: string }> {
+  return spawnHermes([
+    "chat",
+    "-q",
+    prompt,
+    "--yolo",
+    "--max-turns",
+    String(HERMES_MAX_TURNS),
+    "--quiet",
+  ]);
+}
+
+function spawnHermesResume(
+  sessionId: string,
+  message: string,
+): Promise<{ stdout: string; stderr: string }> {
+  return spawnHermes([
+    "chat",
+    "--resume",
+    sessionId,
+    "-q",
+    message,
+    "--yolo",
+    "--max-turns",
+    String(HERMES_MAX_TURNS),
+    "--quiet",
+  ]);
+}
+
+function parseSessionId(stdout: string, stderr: string): string {
+  const sessionId = parseSessionIdFromStdout(stderr) ?? parseSessionIdFromStdout(stdout);
+  if (sessionId === null) {
+    throw new Error(
+      "Failed to parse session_id from hermes output.\n" +
+        `stderr (first 200 chars): ${stderr.slice(0, 200)}\n` +
+        `stdout (first 200 chars): ${stdout.slice(0, 200)}`,
+    );
+  }
+  return sessionId;
+}
+
+async function buildResultFromSession(sessionId: string, store: Store): Promise<AgentRunResult> {
+  const session = await loadHermesSession(sessionId);
+  if (session === null) {
+    throw new Error(`Failed to load hermes session file for session_id: ${sessionId}`);
+  }
+  const { detailHash, output } = await storeHermesSessionDetail(store, session);
+  return { output, detailHash, sessionId };
+}
+
 async function runHermes(ctx: AgentContext): Promise<AgentRunResult> {
   const fullPrompt = buildHermesPrompt(ctx);
   const { stdout, stderr } = await spawnHermesChat(fullPrompt);
-  const { store } = ctx;
+  const sessionId = parseSessionId(stdout, stderr);
+  return buildResultFromSession(sessionId, ctx.store);
+}
 
-  // --quiet mode: session_id may be on stdout or stderr
-  const sessionId = parseSessionIdFromStdout(stderr) ?? parseSessionIdFromStdout(stdout);
-  if (sessionId !== null) {
-    const session = await loadHermesSession(sessionId);
-    if (session !== null) {
-      const { detailHash, output } = await storeHermesSessionDetail(store, session);
-      return { output, detailHash };
-    }
-  }
-
-  const detailHash = await storeHermesRawOutput(store, stdout);
-  return { output: stdout, detailHash };
+async function continueHermes(
+  sessionId: string,
+  message: string,
+  store: Store,
+): Promise<AgentRunResult> {
+  const { stdout, stderr } = await spawnHermesResume(sessionId, message);
+  // Resume may return a new session_id
+  const newSessionId = parseSessionIdFromStdout(stderr) ?? parseSessionIdFromStdout(stdout);
+  const resolvedId = newSessionId ?? sessionId;
+  return buildResultFromSession(resolvedId, store);
 }
 
 /** Agent CLI factory: parses argv, runs Hermes, extracts output, writes StepNode. */
@@ -118,5 +159,6 @@ export function createHermesAgent(): () => Promise<void> {
   return createAgent({
     name: "hermes",
     run: runHermes,
+    continue: continueHermes,
   });
 }

packages/workflow-agent-kit/__tests__/frontmatter-fast-path.test.ts

@@ -29,6 +29,27 @@ const STRICT_SCHEMA = {
   additionalProperties: false,
 };
 
+/** Role-specific schema (reviewer) — only approved, no standard agent fields. */
+const REVIEWER_SCHEMA = {
+  type: "object",
+  properties: {
+    approved: { type: "boolean" },
+  },
+  required: ["approved"],
+  additionalProperties: false,
+};
+
+/** Role-specific schema (planner) — custom status enum + plan hash. */
+const PLANNER_SCHEMA = {
+  type: "object",
+  properties: {
+    status: { type: "string", enum: ["ready", "insufficient_info"] },
+    plan: { type: "string" },
+  },
+  required: ["status"],
+  additionalProperties: false,
+};
+
 async function makeStoreWithSchema(schema: Record<string, unknown>) {
   const store = createMemoryStore();
   const schemaHash = await putSchema(store, schema);
@@ -134,3 +155,48 @@ describe("tryFrontmatterFastPath — fallback: schema mismatch", () => {
     expect(result).toBeNull();
   });
 });
+
+// ── Role-specific schema fields ───────────────────────────────────────────────
+
+describe("tryFrontmatterFastPath — role-specific fields", () => {
+  test("extracts approved only for reviewer schema (no extra standard fields)", async () => {
+    const { store, schemaHash } = await makeStoreWithSchema(REVIEWER_SCHEMA);
+
+    const raw = "---\napproved: true\n---\n\nReview passed.";
+
+    const result = await tryFrontmatterFastPath(raw, schemaHash, store);
+    expect(result).not.toBeNull();
+
+    const node = store.get(result!.outputHash);
+    expect(node).not.toBeNull();
+    const payload = node!.payload as Record<string, unknown>;
+    expect(payload).toEqual({ approved: true });
+    expect(payload.status).toBeUndefined();
+    expect(payload.scope).toBeUndefined();
+  });
+
+  test("extracts plan and role-specific status for planner schema", async () => {
+    const { store, schemaHash } = await makeStoreWithSchema(PLANNER_SCHEMA);
+
+    const raw = "---\nstatus: ready\nplan: 01HASHPLANNER0001\n---\n\nSpec summary.";
+
+    const result = await tryFrontmatterFastPath(raw, schemaHash, store);
+    expect(result).not.toBeNull();
+
+    const node = store.get(result!.outputHash);
+    expect(node).not.toBeNull();
+    const payload = node!.payload as Record<string, unknown>;
+    expect(payload.status).toBe("ready");
+    expect(payload.plan).toBe("01HASHPLANNER0001");
+    expect(payload.scope).toBeUndefined();
+  });
+
+  test("returns null when required role-specific field is missing", async () => {
+    const { store, schemaHash } = await makeStoreWithSchema(REVIEWER_SCHEMA);
+
+    const raw = "---\nstatus: done\nscope: role\n---\n\nBody.";
+
+    const result = await tryFrontmatterFastPath(raw, schemaHash, store);
+    expect(result).toBeNull();
+  });
+});

packages/workflow-agent-kit/src/build-output-format-instruction.ts

@@ -9,7 +9,7 @@ import type { JSONSchema } from "@uncaged/json-cas";
  *
  * Returns an empty array for schemas with no inspectable property definitions.
  */
-function extractSchemaFields(schema: JSONSchema): string[] {
+export function extractSchemaFields(schema: JSONSchema): string[] {
   if (typeof schema.properties === "object" && schema.properties !== null) {
     return Object.keys(schema.properties as Record<string, unknown>);
   }

packages/workflow-agent-kit/src/frontmatter.ts

@@ -1,13 +1,139 @@
 import type { Store } from "@uncaged/json-cas";
-import { validate } from "@uncaged/json-cas";
+import { getSchema, validate } from "@uncaged/json-cas";
 import type { CasRef } from "@uncaged/workflow-protocol";
-import { parseFrontmatterMarkdown, validateFrontmatter } from "@uncaged/workflow-util";
+import {
+  type AgentFrontmatter,
+  createLogger,
+  parseFrontmatterMarkdown,
+  validateFrontmatter,
+} from "@uncaged/workflow-util";
+import { parse as parseYaml } from "yaml";
+
+import { extractSchemaFields } from "./build-output-format-instruction.js";
+
+const log = createLogger({ sink: { kind: "stderr" } });
+
+const STANDARD_KEYS = ["status", "next", "confidence", "artifacts", "scope"] as const;
+
+type StandardKey = (typeof STANDARD_KEYS)[number];
 
 export type FrontmatterFastPathResult = {
   body: string;
   outputHash: CasRef;
 };
 
+function extractYamlBlock(raw: string): string | null {
+  const fence = "---";
+  if (!raw.startsWith(fence)) {
+    return null;
+  }
+
+  const rest = raw.slice(fence.length);
+  if (rest.length > 0 && rest[0] !== "\n" && rest[0] !== "\r") {
+    return null;
+  }
+
+  const afterOpen = rest.startsWith("\n") ? rest.slice(1) : rest;
+  const closeIndex = afterOpen.indexOf(`\n${fence}`);
+  if (closeIndex === -1) {
+    return null;
+  }
+
+  return afterOpen.slice(0, closeIndex);
+}
+
+function parseRawFrontmatterFields(raw: string): Record<string, unknown> {
+  const yamlText = extractYamlBlock(raw);
+  if (yamlText === null) {
+    return {};
+  }
+
+  try {
+    const parsed = parseYaml(yamlText);
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      return {};
+    }
+    return parsed as Record<string, unknown>;
+  } catch {
+    return {};
+  }
+}
+
+function defaultCandidate(frontmatter: AgentFrontmatter): Record<string, unknown> {
+  return {
+    status: frontmatter.status,
+    next: frontmatter.next,
+    confidence: frontmatter.confidence,
+    artifacts: [...frontmatter.artifacts],
+    scope: frontmatter.scope,
+  };
+}
+
+function pickStandardField(frontmatter: AgentFrontmatter, key: StandardKey): unknown {
+  switch (key) {
+    case "status":
+      return frontmatter.status;
+    case "next":
+      return frontmatter.next;
+    case "confidence":
+      return frontmatter.confidence;
+    case "artifacts":
+      return [...frontmatter.artifacts];
+    case "scope":
+      return frontmatter.scope;
+  }
+}
+
+function isStandardKey(key: string): key is StandardKey {
+  return (STANDARD_KEYS as readonly string[]).includes(key);
+}
+
+function pickFieldValue(
+  field: string,
+  frontmatter: AgentFrontmatter,
+  rawFields: Record<string, unknown>,
+): unknown | undefined {
+  if (!isStandardKey(field)) {
+    return Object.hasOwn(rawFields, field) ? rawFields[field] : undefined;
+  }
+
+  const coerced = pickStandardField(frontmatter, field);
+  if (field === "artifacts" || field === "scope") {
+    return coerced;
+  }
+  if (coerced !== null) {
+    return coerced;
+  }
+  return Object.hasOwn(rawFields, field) ? rawFields[field] : coerced;
+}
+
+/**
+ * Build a CAS candidate object from schema property keys and parsed frontmatter.
+ *
+ * When the schema has no inspectable properties, falls back to the five standard
+ * agent frontmatter fields for backward compatibility.
+ */
+function buildCandidate(
+  frontmatter: AgentFrontmatter,
+  rawFields: Record<string, unknown>,
+  schemaFields: string[],
+): Record<string, unknown> {
+  if (schemaFields.length === 0) {
+    return defaultCandidate(frontmatter);
+  }
+
+  const candidate: Record<string, unknown> = {};
+
+  for (const field of schemaFields) {
+    const value = pickFieldValue(field, frontmatter, rawFields);
+    if (value !== undefined) {
+      candidate[field] = value;
+    }
+  }
+
+  return candidate;
+}
+
 /**
  * Try to satisfy `outputSchema` from frontmatter fields alone.
  *
@@ -32,16 +158,22 @@ export async function tryFrontmatterFastPath(
 
   const validationErrors = validateFrontmatter(frontmatter);
   if (validationErrors.length > 0) {
+    log(
+      "9GNPS4WY",
+      `frontmatter validation errors: ${validationErrors.map((e) => e.message).join("; ")}`,
+    );
     return null;
   }
 
-  const candidate: Record<string, unknown> = {
-    status: frontmatter.status,
-    next: frontmatter.next,
-    confidence: frontmatter.confidence,
-    artifacts: [...frontmatter.artifacts],
-    scope: frontmatter.scope,
-  };
+  const schema = getSchema(store, outputSchema);
+  if (schema === null) {
+    log("8FHMR2QX", `output schema not found in CAS: ${outputSchema}`);
+    return null;
+  }
+
+  const schemaFields = extractSchemaFields(schema);
+  const rawFields = parseRawFrontmatterFields(raw);
+  const candidate = buildCandidate(frontmatter, rawFields, schemaFields);
 
   let outputHash: CasRef;
   let node: ReturnType<Store["get"]>;
@@ -50,10 +182,12 @@ export async function tryFrontmatterFastPath(
     outputHash = await store.put(outputSchema, candidate);
     node = store.get(outputHash);
   } catch {
+    log("2KMQT7NR", "failed to store frontmatter candidate in CAS");
     return null;
   }
 
   if (node === null || !validate(store, node)) {
+    log("2KMQT7NR", "stored frontmatter candidate failed schema validation");
     return null;
   }
 

packages/workflow-agent-kit/src/index.ts

@@ -12,4 +12,10 @@ export type { FrontmatterFastPathResult } from "./frontmatter.js";
 export { tryFrontmatterFastPath } from "./frontmatter.js";
 export { createAgent } from "./run.js";
 export { getConfigPath, getEnvPath, loadWorkflowConfig } from "./storage.js";
-export type { AgentContext, AgentOptions, AgentRunFn, AgentRunResult } from "./types.js";
+export type {
+  AgentContext,
+  AgentContinueFn,
+  AgentOptions,
+  AgentRunFn,
+  AgentRunResult,
+} from "./types.js";

packages/workflow-agent-kit/src/run.ts

@@ -3,11 +3,12 @@ import type { CasRef, StepNodePayload, ThreadId } from "@uncaged/workflow-protoc
 import { config as loadDotenv } from "dotenv";
 import { buildOutputFormatInstruction } from "./build-output-format-instruction.js";
 import { buildContextWithMeta } from "./context.js";
-import { extract } from "./extract.js";
 import { tryFrontmatterFastPath } from "./frontmatter.js";
 import type { AgentStore } from "./storage.js";
-import { getEnvPath, loadWorkflowConfig, resolveStorageRoot } from "./storage.js";
-import type { AgentContext, AgentOptions, AgentRunResult } from "./types.js";
+import { getEnvPath, resolveStorageRoot } from "./storage.js";
+import type { AgentOptions } from "./types.js";
+
+const MAX_FRONTMATTER_RETRIES = 2;
 
 function fail(message: string): never {
   process.stderr.write(`${message}\n`);
@@ -66,31 +67,16 @@ async function writeStepNode(options: {
   return hash;
 }
 
-async function runAgent(options: AgentOptions, ctx: AgentContext): Promise<AgentRunResult> {
-  return runWithMessage("agent run failed", () => options.run(ctx));
-}
-
-async function extractOutput(
+async function tryExtractOutput(
   rawOutput: string,
   outputSchema: CasRef,
-  storageRoot: string,
   ctx: Awaited<ReturnType<typeof buildContextWithMeta>>,
-): Promise<CasRef> {
-  const fastPath = await runWithMessage("frontmatter fast path", () =>
-    tryFrontmatterFastPath(rawOutput, outputSchema, ctx.meta.store),
-  ).catch(() => null);
-
+): Promise<CasRef | null> {
+  const fastPath = await tryFrontmatterFastPath(rawOutput, outputSchema, ctx.meta.store);
   if (fastPath !== null) {
     return fastPath.outputHash;
   }
-
-  const config = await runWithMessage("failed to load config", () =>
-    loadWorkflowConfig(storageRoot),
-  );
-  const extracted = await runWithMessage("extract failed", () =>
-    extract(rawOutput, outputSchema, config),
-  );
-  return extracted.hash;
+  return null;
 }
 
 async function persistStep(options: {
@@ -112,11 +98,6 @@ async function persistStep(options: {
   });
 }
 
-/**
- * Create an agent CLI entrypoint.
- * Parses argv (`<thread-id> <role>`), runs the agent, extracts structured output,
- * writes StepNode to CAS, and prints the new node hash to stdout.
- */
 export function createAgent(options: AgentOptions): () => Promise<void> {
   return async function main(): Promise<void> {
     const { threadId, role } = parseArgv(process.argv);
@@ -135,13 +116,31 @@ export function createAgent(options: AgentOptions): () => Promise<void> {
       ctx.outputFormatInstruction = buildOutputFormatInstruction(frontmatterSchema);
     }
 
-    const agentResult = await runAgent(options, ctx);
-    const outputHash = await extractOutput(
-      agentResult.output,
-      roleDef.frontmatter,
-      storageRoot,
-      ctx,
-    );
+    let agentResult = await runWithMessage("agent run failed", () => options.run(ctx));
+
+    // Try to extract frontmatter; retry via continue if it fails
+    let outputHash = await tryExtractOutput(agentResult.output, roleDef.frontmatter, ctx);
+
+    for (let retry = 0; retry < MAX_FRONTMATTER_RETRIES && outputHash === null; retry++) {
+      const correctionMessage =
+        "Your previous response did not contain valid YAML frontmatter matching the role schema.\n" +
+        "You MUST begin your response with a YAML frontmatter block (--- delimited).\n" +
+        "Please output ONLY the corrected frontmatter block followed by your work.";
+
+      agentResult = await runWithMessage("agent continue failed", () =>
+        options.continue(agentResult.sessionId, correctionMessage, ctx.meta.store),
+      );
+      outputHash = await tryExtractOutput(agentResult.output, roleDef.frontmatter, ctx);
+    }
+
+    if (outputHash === null) {
+      fail(
+        "Agent output does not contain valid YAML frontmatter matching the role schema " +
+          `after ${MAX_FRONTMATTER_RETRIES} retries.\n` +
+          `Raw output (first 500 chars): ${agentResult.output.slice(0, 500)}`,
+      );
+    }
+
     const stepHash = await persistStep({
       ctx,
       outputHash,

packages/workflow-agent-kit/src/types.ts

@@ -17,11 +17,19 @@ export type AgentContext = ModeratorContext & {
 export type AgentRunResult = {
   output: string;
   detailHash: string;
+  sessionId: string;
 };
 
+export type AgentContinueFn = (
+  sessionId: string,
+  message: string,
+  store: AgentContext["store"],
+) => Promise<AgentRunResult>;
+
 export type AgentRunFn = (ctx: AgentContext) => Promise<AgentRunResult>;
 
 export type AgentOptions = {
   name: string;
   run: AgentRunFn;
+  continue: AgentContinueFn;
 };