Merge pull request 'feat: !include YAML tag and folder-based workflow layout' (#584) from feat/include-and-folder-workflow into main

biome.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://biomejs.dev/schemas/2.4.15/schema.json",
+  "$schema": "https://biomejs.dev/schemas/2.4.14/schema.json",
   "files": {
     "includes": [
       "**",

packages/cli-workflow/src/__tests__/include-tag.test.ts

@@ -0,0 +1,84 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "vitest";
+import { parse } from "yaml";
+import { createIncludeTag } from "../include.js";
+
+let tmpDir: string;
+
+beforeEach(async () => {
+  tmpDir = await mkdtemp(join(tmpdir(), "include-tag-test-"));
+});
+
+afterEach(async () => {
+  await rm(tmpDir, { recursive: true, force: true });
+});
+
+describe("!include tag", () => {
+  test("includes .md file as string", async () => {
+    await writeFile(join(tmpDir, "prompt.md"), "You are an analyst.");
+    const yaml = "system: !include prompt.md";
+    const result = parse(yaml, { customTags: [createIncludeTag(tmpDir)] });
+    expect(result.system).toBe("You are an analyst.");
+  });
+
+  test("includes .json file as parsed object", async () => {
+    await writeFile(join(tmpDir, "schema.json"), '{"type":"object","properties":{}}');
+    const yaml = "outputSchema: !include schema.json";
+    const result = parse(yaml, { customTags: [createIncludeTag(tmpDir)] });
+    expect(result.outputSchema).toEqual({ type: "object", properties: {} });
+  });
+
+  test("includes .yaml file as parsed object", async () => {
+    await writeFile(join(tmpDir, "config.yaml"), "key: value\nlist:\n  - a\n  - b");
+    const yaml = "config: !include config.yaml";
+    const result = parse(yaml, { customTags: [createIncludeTag(tmpDir)] });
+    expect(result.config).toEqual({ key: "value", list: ["a", "b"] });
+  });
+
+  test("resolves relative subdirectory paths", async () => {
+    const subdir = join(tmpDir, "roles");
+    await mkdir(subdir, { recursive: true });
+    await writeFile(join(subdir, "analyst.md"), "Analyze data.");
+    const yaml = "system: !include roles/analyst.md";
+    const result = parse(yaml, { customTags: [createIncludeTag(tmpDir)] });
+    expect(result.system).toBe("Analyze data.");
+  });
+
+  test("throws on missing file", () => {
+    const yaml = "system: !include nonexistent.md";
+    expect(() => parse(yaml, { customTags: [createIncludeTag(tmpDir)] })).toThrow();
+  });
+
+  test("includes .txt file as string", async () => {
+    await writeFile(join(tmpDir, "note.txt"), "Hello world");
+    const yaml = "note: !include note.txt";
+    const result = parse(yaml, { customTags: [createIncludeTag(tmpDir)] });
+    expect(result.note).toBe("Hello world");
+  });
+
+  test("blocks path traversal with ../", async () => {
+    const yaml = "secret: !include ../../etc/passwd";
+    expect(() => parse(yaml, { customTags: [createIncludeTag(tmpDir)] })).toThrow(
+      /path traversal blocked/,
+    );
+  });
+
+  test("blocks absolute path traversal", async () => {
+    const yaml = "secret: !include /etc/passwd";
+    expect(() => parse(yaml, { customTags: [createIncludeTag(tmpDir)] })).toThrow(
+      /path traversal blocked/,
+    );
+  });
+
+  test("supports nested !include in yaml files", async () => {
+    const subdir = join(tmpDir, "parts");
+    await mkdir(subdir, { recursive: true });
+    await writeFile(join(subdir, "inner.md"), "nested content");
+    await writeFile(join(tmpDir, "outer.yaml"), "value: !include parts/inner.md");
+    const yaml = "config: !include outer.yaml";
+    const result = parse(yaml, { customTags: [createIncludeTag(tmpDir)] });
+    expect(result.config).toEqual({ value: "nested content" });
+  });
+});

packages/cli-workflow/src/__tests__/workflow-resolution.test.ts

@@ -257,6 +257,49 @@ describe("Strategy 3: Local Discovery", () => {
 
     expect(result.workflow).toMatch(/^[0-9A-HJKMNP-TV-Z]{13}$/);
   });
+
+  test("should find workflow in folder-based layout (name/index.yaml)", async () => {
+    await makeUwfStore(storageRoot);
+    const workflowDir = join(projectRoot, ".workflow", "solve-issue");
+    await mkdir(workflowDir, { recursive: true });
+    await writeFile(join(workflowDir, "index.yaml"), await createWorkflowYaml("solve-issue"));
+
+    const result = await cmdThreadStart(storageRoot, "solve-issue", "prompt", projectRoot);
+
+    expect(result.workflow).toMatch(/^[0-9A-HJKMNP-TV-Z]{13}$/);
+    const uwf = await makeUwfStore(storageRoot);
+    const node = uwf.store.get(result.workflow);
+    expect(node).not.toBeNull();
+    if (node !== null) {
+      expect((node.payload as WorkflowPayload).name).toBe("solve-issue");
+    }
+  });
+
+  test("should prefer flat file over folder-based layout", async () => {
+    await makeUwfStore(storageRoot);
+    const workflowDir = join(projectRoot, ".workflow");
+    await mkdir(workflowDir, { recursive: true });
+    await writeFile(
+      join(workflowDir, "solve-issue.yaml"),
+      await createWorkflowYaml("solve-issue", "flat"),
+    );
+
+    const folderDir = join(workflowDir, "solve-issue");
+    await mkdir(folderDir, { recursive: true });
+    await writeFile(
+      join(folderDir, "index.yaml"),
+      await createWorkflowYaml("solve-issue", "folder"),
+    );
+
+    const result = await cmdThreadStart(storageRoot, "solve-issue", "prompt", projectRoot);
+
+    const uwf = await makeUwfStore(storageRoot);
+    const node = uwf.store.get(result.workflow);
+    expect(node).not.toBeNull();
+    if (node !== null) {
+      expect((node.payload as WorkflowPayload).description).toBe("Test workflow (flat)");
+    }
+  });
 });
 
 // ── Strategy 4: Global Registry Fallback ──────────────────────────────────────

packages/cli-workflow/src/commands/thread.ts

@@ -28,6 +28,7 @@ import { getEnvPath, loadWorkflowConfig } from "@uncaged/workflow-util-agent";
 import { config as loadDotenv } from "dotenv";
 import { parse } from "yaml";
 import { createMarker, deleteMarker, isThreadRunning } from "../background/index.js";
+import { createIncludeTag } from "../include.js";
 import { evaluate } from "../moderator/index.js";
 import {
   appendThreadHistory,
@@ -118,6 +119,15 @@ async function findWorkflowInDir(dir: string, name: string): Promise<string | nu
       return result;
     }
   }
+  for (const indexName of ["index.yaml", "index.yml"]) {
+    const candidate = resolvePath(dir, ".workflow", name, indexName);
+    try {
+      await access(candidate);
+      return candidate;
+    } catch {
+      /* not found */
+    }
+  }
 
   // Check .workflows/ directory as fallback (legacy)
   for (const ext of [".yaml", ".yml"]) {
@@ -126,6 +136,15 @@ async function findWorkflowInDir(dir: string, name: string): Promise<string | nu
       return result;
     }
   }
+  for (const indexName of ["index.yaml", "index.yml"]) {
+    const candidate = resolvePath(dir, ".workflows", name, indexName);
+    try {
+      await access(candidate);
+      return candidate;
+    } catch {
+      /* not found */
+    }
+  }
 
   return null;
 }
@@ -172,7 +191,7 @@ async function materializeLocalWorkflow(uwf: UwfStore, filePath: string): Promis
 
   let raw: unknown;
   try {
-    raw = parse(text) as unknown;
+    raw = parse(text, { customTags: [createIncludeTag(dirname(filePath))] }) as unknown;
   } catch (e) {
     fail(`invalid YAML in ${filePath}: ${e instanceof Error ? e.message : String(e)}`);
   }

packages/cli-workflow/src/commands/workflow.ts

@@ -1,9 +1,11 @@
 import { readFile } from "node:fs/promises";
+import { dirname, resolve as resolvePath } from "node:path";
 
 import type { JSONSchema } from "@uncaged/json-cas";
 import { putSchema, validate } from "@uncaged/json-cas";
 import type { CasRef, RoleDefinition, Target, WorkflowPayload } from "@uncaged/workflow-protocol";
 import { parse } from "yaml";
+import { createIncludeTag } from "../include.js";
 
 import {
   createUwfStore,
@@ -123,7 +125,9 @@ export async function cmdWorkflowAdd(
 
   let raw: unknown;
   try {
-    raw = parse(text) as unknown;
+    raw = parse(text, {
+      customTags: [createIncludeTag(dirname(resolvePath(filePath)))],
+    }) as unknown;
   } catch (e) {
     fail(`invalid YAML: ${e instanceof Error ? e.message : String(e)}`);
   }

packages/cli-workflow/src/include.ts

@@ -0,0 +1,37 @@
+import { readFileSync } from "node:fs";
+import { dirname, extname, resolve } from "node:path";
+import { parse as parseYaml } from "yaml";
+
+/**
+ * Create a YAML customTags entry for !include that resolves file paths
+ * relative to the given base directory.
+ *
+ * Security: resolved paths must stay within baseDir (path traversal prevention).
+ * Nested !include in .yaml/.yml files is supported (customTags passed recursively).
+ */
+export function createIncludeTag(baseDir: string) {
+  const resolvedBase = resolve(baseDir);
+  return {
+    tag: "!include",
+    resolve(str: string) {
+      const filePath = resolve(resolvedBase, str);
+      // Path traversal guard: resolved path must be inside baseDir
+      if (!filePath.startsWith(`${resolvedBase}/`) && filePath !== resolvedBase) {
+        throw new Error(
+          `!include path traversal blocked: "${str}" resolves outside base directory`,
+        );
+      }
+      const content = readFileSync(filePath, "utf8");
+      const ext = extname(filePath).toLowerCase();
+      if (ext === ".json") {
+        return JSON.parse(content);
+      }
+      if (ext === ".yaml" || ext === ".yml") {
+        // Pass customTags recursively so nested !include works,
+        // scoped to the included file's directory
+        return parseYaml(content, { customTags: [createIncludeTag(dirname(filePath))] });
+      }
+      return content;
+    },
+  };
+}

packages/cli-workflow/src/store.ts

@@ -1,4 +1,5 @@
-import { appendFile, mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import type { Dirent } from "node:fs";
+import { access, appendFile, mkdir, readdir, readFile, writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
 import { join } from "node:path";
 
@@ -19,17 +20,38 @@ export type ProjectWorkflowEntry = {
   filePath: string;
 };
 
+/** Extract workflow name from a YAML filename (strip .yaml/.yml extension). */
+function stemFromYaml(name: string): string {
+  if (name.endsWith(".yaml")) return name.slice(0, -5);
+  if (name.endsWith(".yml")) return name.slice(0, -4);
+  return name;
+}
+
+/** Check if a directory contains an index.yaml or index.yml workflow file. */
+async function findIndexWorkflow(
+  dir: string,
+  dirName: string,
+): Promise<ProjectWorkflowEntry | null> {
+  for (const indexName of ["index.yaml", "index.yml"]) {
+    const indexPath = join(dir, dirName, indexName);
+    try {
+      await access(indexPath);
+      return { name: dirName, filePath: indexPath };
+    } catch {
+      // not found, try next
+    }
+  }
+  return null;
+}
+
 /**
- * Scan `<projectRoot>/.workflows/*.yaml` (non-recursive) and return discovered entries.
- * Returns an empty array if the directory does not exist.
+ * Scan a single directory for workflow entries (flat YAML files + folder/index.yaml).
+ * Returns discovered entries. Returns empty array if directory does not exist.
  */
-export async function discoverProjectWorkflows(
-  projectRoot: string,
-): Promise<ProjectWorkflowEntry[]> {
-  const dir = join(projectRoot, ".workflows");
-  let entries: string[];
+async function scanWorkflowDir(dir: string): Promise<ProjectWorkflowEntry[]> {
+  let dirents: Dirent[];
   try {
-    entries = await readdir(dir);
+    dirents = await readdir(dir, { withFileTypes: true });
   } catch (e) {
     const err = e as NodeJS.ErrnoException;
     if (err.code === "ENOENT" || err.code === "ENOTDIR") {
@@ -39,16 +61,39 @@ export async function discoverProjectWorkflows(
   }
 
   const result: ProjectWorkflowEntry[] = [];
-  for (const entry of entries) {
-    if (!entry.endsWith(".yaml") && !entry.endsWith(".yml")) {
-      continue;
+  for (const entry of dirents) {
+    if (entry.isFile() && (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml"))) {
+      result.push({ name: stemFromYaml(entry.name), filePath: join(dir, entry.name) });
+    } else if (entry.isDirectory()) {
+      const found = await findIndexWorkflow(dir, entry.name);
+      if (found !== null) {
+        result.push(found);
+      }
     }
-    const stem = entry.endsWith(".yaml") ? entry.slice(0, -5) : entry.slice(0, -4);
-    result.push({ name: stem, filePath: join(dir, entry) });
   }
   return result;
 }
 
+/**
+ * Scan `<projectRoot>/.workflow/` (preferred) and `.workflows/` (legacy) for workflow entries.
+ * .workflow/ takes priority: if a name is found in both, .workflow/ wins.
+ * Returns an empty array if neither directory exists.
+ */
+export async function discoverProjectWorkflows(
+  projectRoot: string,
+): Promise<ProjectWorkflowEntry[]> {
+  const primary = await scanWorkflowDir(join(projectRoot, ".workflow"));
+  const legacy = await scanWorkflowDir(join(projectRoot, ".workflows"));
+  const seen = new Set(primary.map((e) => e.name));
+  const merged = [...primary];
+  for (const entry of legacy) {
+    if (!seen.has(entry.name)) {
+      merged.push(entry);
+    }
+  }
+  return merged;
+}
+
 /** Default filesystem root for uwf data (`~/.uncaged/workflow`). */
 export function getDefaultStorageRoot(): string {
   return join(homedir(), ".uncaged", "workflow");

packages/cli-workflow/src/validate.ts

@@ -1,4 +1,4 @@
-import { basename } from "node:path";
+import { basename, dirname } from "node:path";
 import type { CasRef, WorkflowPayload } from "@uncaged/workflow-protocol";
 
 const CAS_REF_PATTERN = /^[0-9A-HJKMNP-TV-Z]{13}$/;
@@ -68,9 +68,15 @@ function isGraph(value: unknown): boolean {
  */
 export function workflowNameFromPath(filePath: string): string {
   const base = basename(filePath);
-  if (base.endsWith(".yaml")) return base.slice(0, -5);
-  if (base.endsWith(".yml")) return base.slice(0, -4);
-  return base;
+  const stem = base.endsWith(".yaml")
+    ? base.slice(0, -5)
+    : base.endsWith(".yml")
+      ? base.slice(0, -4)
+      : base;
+  if (stem === "index") {
+    return basename(dirname(filePath));
+  }
+  return stem;
 }
 
 /**