feat(cli): add workflow semantic validation before execution

Implements validateWorkflow() that performs deep semantic checks on
parsed WorkflowPayload before registration or execution:

- Role reference integrity (unknown roles, orphans, reserved names)
- Graph structure (/ constraints, reachability, edge targets)
- Status-edge consistency (single/multi-exit matching)
- Mustache template variable existence
- oneOf discriminant validity ( const check)

All errors collected (not fail-fast). Integrated into:
- uwf workflow add (before CAS registration)
- uwf thread start (local workflow materialization)

Closes #506
This commit is contained in:
2026-05-25 07:17:49 +00:00
parent dfb6fda06d
commit 9b2460633c
5 changed files with 680 additions and 10 deletions
@@ -0,0 +1,366 @@
import type { WorkflowPayload } from "@uncaged/workflow-protocol";
import { describe, expect, test } from "vitest";
import { validateWorkflow } from "../validate-semantic.js";
/** Build a valid two-role workflow that passes all checks. */
function makeWorkflow(overrides?: Partial<WorkflowPayload>): WorkflowPayload {
const base: WorkflowPayload = {
name: "test-workflow",
description: "A test workflow",
roles: {
writer: {
description: "Writes content",
goal: "Write content",
capabilities: ["writing"],
procedure: "Write it",
output: "The content",
frontmatter: {
type: "object",
properties: {
$status: { enum: ["_"] },
plan: { type: "string" },
},
required: ["$status", "plan"],
} as unknown as string,
},
reviewer: {
description: "Reviews content",
goal: "Review content",
capabilities: ["reviewing"],
procedure: "Review it",
output: "The review",
frontmatter: {
type: "object",
oneOf: [
{
properties: {
$status: { const: "approved" },
summary: { type: "string" },
},
required: ["$status", "summary"],
},
{
properties: {
$status: { const: "rejected" },
reason: { type: "string" },
},
required: ["$status", "reason"],
},
],
} as unknown as string,
},
},
graph: {
$START: { _: { role: "writer", prompt: "Begin writing" } },
writer: { _: { role: "reviewer", prompt: "Review this: {{{plan}}}" } },
reviewer: {
approved: { role: "$END", prompt: "Done: {{{summary}}}" },
rejected: { role: "writer", prompt: "Fix: {{{reason}}}" },
},
},
};
if (!overrides) return base;
return { ...base, ...overrides };
}
describe("Suite 1: Role Reference Integrity", () => {
test("1.1 graph references unknown role", () => {
const wf = makeWorkflow();
wf.graph.nonexistent = { _: { role: "$END", prompt: "done" } };
const errors = validateWorkflow(wf);
expect(errors.some((e) => e.includes('unknown role "nonexistent"'))).toBe(true);
});
test("1.2 orphan role not in graph", () => {
const wf = makeWorkflow();
wf.roles.orphan = {
description: "Orphan",
goal: "Nothing",
capabilities: [],
procedure: "None",
output: "None",
frontmatter: {
type: "object",
properties: { $status: { enum: ["_"] } },
required: ["$status"],
} as unknown as string,
};
const errors = validateWorkflow(wf);
expect(
errors.some((e) => e.includes('role "orphan" is defined but not referenced in graph')),
).toBe(true);
});
test("1.3 $START in roles", () => {
const wf = makeWorkflow();
(wf.roles as Record<string, unknown>).$START = {
description: "Bad",
goal: "Bad",
capabilities: [],
procedure: "Bad",
output: "Bad",
frontmatter: { type: "object", properties: {}, required: [] },
};
const errors = validateWorkflow(wf);
expect(errors.some((e) => e.includes('reserved name "$START"'))).toBe(true);
});
test("1.4 $END in roles", () => {
const wf = makeWorkflow();
(wf.roles as Record<string, unknown>).$END = {
description: "Bad",
goal: "Bad",
capabilities: [],
procedure: "Bad",
output: "Bad",
frontmatter: { type: "object", properties: {}, required: [] },
};
const errors = validateWorkflow(wf);
expect(errors.some((e) => e.includes('reserved name "$END"'))).toBe(true);
});
test("1.5 valid workflow returns no errors", () => {
const wf = makeWorkflow();
const errors = validateWorkflow(wf);
expect(errors).toEqual([]);
});
});
describe("Suite 2: Graph Structure", () => {
test("2.1 $START missing from graph", () => {
const wf = makeWorkflow();
delete wf.graph.$START;
const errors = validateWorkflow(wf);
expect(errors.some((e) => e.includes("$START must be defined in graph"))).toBe(true);
});
test("2.2 $START has multiple status keys", () => {
const wf = makeWorkflow();
wf.graph.$START = {
_: { role: "writer", prompt: "Begin" },
other: { role: "reviewer", prompt: "Also" },
};
const errors = validateWorkflow(wf);
expect(
errors.some((e) => e.includes('$START must have exactly one edge with status "_"')),
).toBe(true);
});
test("2.3 $START edge uses non-_ status", () => {
const wf = makeWorkflow();
wf.graph.$START = { ready: { role: "writer", prompt: "Begin" } };
const errors = validateWorkflow(wf);
expect(
errors.some((e) => e.includes('$START must have exactly one edge with status "_"')),
).toBe(true);
});
test("2.4 $END has outgoing edges", () => {
const wf = makeWorkflow();
wf.graph.$END = { _: { role: "writer", prompt: "Loop" } };
const errors = validateWorkflow(wf);
expect(errors.some((e) => e.includes("$END must not have outgoing edges"))).toBe(true);
});
test("2.5 unreachable role", () => {
const wf = makeWorkflow();
wf.roles.isolated = {
description: "Isolated",
goal: "Isolated",
capabilities: [],
procedure: "Isolated",
output: "Isolated",
frontmatter: {
type: "object",
properties: { $status: { enum: ["_"] } },
required: ["$status"],
} as unknown as string,
};
wf.graph.isolated = { _: { role: "$END", prompt: "done" } };
const errors = validateWorkflow(wf);
expect(errors.some((e) => e.includes('role "isolated" is not reachable from $START'))).toBe(
true,
);
});
test("2.6 edge target references invalid role", () => {
const wf = makeWorkflow();
wf.graph.writer = { _: { role: "ghost", prompt: "Go to ghost" } };
const errors = validateWorkflow(wf);
expect(errors.some((e) => e.includes('unknown target role "ghost"'))).toBe(true);
});
});
describe("Suite 3: Status-Edge Consistency", () => {
test("3.1 single-exit role with multiple graph keys", () => {
const wf = makeWorkflow();
wf.graph.writer = {
_: { role: "reviewer", prompt: "Review" },
extra: { role: "$END", prompt: "Done" },
};
const errors = validateWorkflow(wf);
expect(
errors.some((e) =>
e.includes('role "writer" is single-exit but has status keys other than "_"'),
),
).toBe(true);
});
test("3.2 single-exit role missing _ key", () => {
const wf = makeWorkflow();
wf.graph.writer = { done: { role: "reviewer", prompt: "Review" } };
const errors = validateWorkflow(wf);
expect(
errors.some((e) => e.includes('role "writer" is single-exit but graph has no "_" key')),
).toBe(true);
});
test("3.3 multi-exit role with extra statuses", () => {
const wf = makeWorkflow();
wf.graph.reviewer = {
approved: { role: "$END", prompt: "Done" },
rejected: { role: "writer", prompt: "Fix" },
timeout: { role: "$END", prompt: "Timed out" },
};
const errors = validateWorkflow(wf);
expect(
errors.some((e) => e.includes('role "reviewer" graph has extra status keys: timeout')),
).toBe(true);
});
test("3.4 multi-exit role missing a status", () => {
const wf = makeWorkflow();
wf.graph.reviewer = {
approved: { role: "$END", prompt: "Done" },
};
const errors = validateWorkflow(wf);
expect(
errors.some((e) => e.includes('role "reviewer" graph is missing status keys: rejected')),
).toBe(true);
});
test("3.5 multi-exit role with _ key", () => {
const wf = makeWorkflow();
wf.graph.reviewer = { _: { role: "$END", prompt: "Done" } };
const errors = validateWorkflow(wf);
expect(errors.some((e) => e.includes('role "reviewer" is multi-exit but graph uses "_"'))).toBe(
true,
);
});
});
describe("Suite 4: Mustache Template Variable Existence", () => {
test("4.1 prompt references nonexistent variable (single-exit)", () => {
const wf = makeWorkflow();
wf.graph.writer = { _: { role: "reviewer", prompt: "Review: {{{branch}}}" } };
const errors = validateWorkflow(wf);
expect(
errors.some((e) =>
e.includes('prompt variable "branch" not found in role "writer" frontmatter'),
),
).toBe(true);
});
test("4.2 prompt references nonexistent variable (multi-exit)", () => {
const wf = makeWorkflow();
wf.graph.reviewer = {
approved: { role: "$END", prompt: "Done: {{{branch}}}" },
rejected: { role: "writer", prompt: "Fix: {{{reason}}}" },
};
const errors = validateWorkflow(wf);
expect(
errors.some((e) =>
e.includes('prompt variable "branch" not found in role "reviewer" variant "approved"'),
),
).toBe(true);
});
test("4.3 valid mustache variables pass", () => {
const wf = makeWorkflow();
const errors = validateWorkflow(wf);
expect(errors).toEqual([]);
});
test("4.4 $status variable is always valid", () => {
const wf = makeWorkflow();
wf.graph.writer = { _: { role: "reviewer", prompt: "Status: {{$status}}" } };
const errors = validateWorkflow(wf);
expect(errors).toEqual([]);
});
});
describe("Suite 5: oneOf Discriminant Validity", () => {
test("5.1 oneOf without $status const", () => {
const wf = makeWorkflow();
wf.roles.reviewer = {
...wf.roles.reviewer,
frontmatter: {
type: "object",
oneOf: [
{ properties: { summary: { type: "string" } }, required: ["summary"] },
{ properties: { reason: { type: "string" } }, required: ["reason"] },
],
} as unknown as string,
};
const errors = validateWorkflow(wf);
expect(
errors.some((e) => e.includes('oneOf variants must have "$status" as const discriminant')),
).toBe(true);
});
test("5.2 oneOf with non-const $status", () => {
const wf = makeWorkflow();
wf.roles.reviewer = {
...wf.roles.reviewer,
frontmatter: {
type: "object",
oneOf: [
{
properties: { $status: { type: "string" }, summary: { type: "string" } },
required: ["$status", "summary"],
},
{
properties: { $status: { type: "string" }, reason: { type: "string" } },
required: ["$status", "reason"],
},
],
} as unknown as string,
};
const errors = validateWorkflow(wf);
expect(errors.some((e) => e.includes("oneOf variant $status must be a const value"))).toBe(
true,
);
});
test("5.3 valid oneOf passes", () => {
const wf = makeWorkflow();
const errors = validateWorkflow(wf);
expect(errors).toEqual([]);
});
});
describe("Suite 6: Multiple Errors Collection", () => {
test("6.1 multiple errors collected", () => {
const wf = makeWorkflow();
// orphan role
wf.roles.orphan = {
description: "Orphan",
goal: "Nothing",
capabilities: [],
procedure: "None",
output: "None",
frontmatter: {
type: "object",
properties: { $status: { enum: ["_"] } },
required: ["$status"],
} as unknown as string,
};
// unknown graph reference
wf.graph.nonexistent = { _: { role: "$END", prompt: "done" } };
// bad mustache var
wf.graph.writer = { _: { role: "reviewer", prompt: "{{{badvar}}}" } };
const errors = validateWorkflow(wf);
expect(errors.length).toBeGreaterThanOrEqual(3);
});
});
@@ -20,23 +20,37 @@ async function makeUwfStore(storageRoot: string): Promise<UwfStore> {
return { storageRoot, store, schemas }; return { storageRoot, store, schemas };
} }
async function storeWorkflow(uwf: UwfStore, name: string): Promise<CasRef> { function makeMinimalPayload(name: string, description: string): WorkflowPayload {
const payload: WorkflowPayload = { return {
name, name,
description: "Test workflow", description,
roles: {}, roles: {
graph: {}, worker: {
description: "worker role",
goal: "do work",
capabilities: [],
procedure: "",
output: "",
frontmatter: { type: "0000000000000" } as unknown as CasRef,
},
},
graph: {
$START: { _: { role: "worker", prompt: "start working" } },
worker: { _: { role: "$END", prompt: "done" } },
},
}; };
}
async function storeWorkflow(uwf: UwfStore, name: string): Promise<CasRef> {
const payload = makeMinimalPayload(name, "Test workflow");
return await uwf.store.put(uwf.schemas.workflow, payload); return await uwf.store.put(uwf.schemas.workflow, payload);
} }
async function createWorkflowYaml(name: string, version: string | null = null): Promise<string> { async function createWorkflowYaml(name: string, version: string | null = null): Promise<string> {
const payload: WorkflowPayload = { const payload = makeMinimalPayload(
name, name,
description: version !== null ? `Test workflow (${version})` : "Test workflow", version !== null ? `Test workflow (${version})` : "Test workflow",
roles: {}, );
graph: {},
};
const yaml = stringify(payload); const yaml = stringify(payload);
return yaml; return yaml;
} }
@@ -40,6 +40,7 @@ import {
type UwfStore, type UwfStore,
} from "../store.js"; } from "../store.js";
import { checkWorkflowFilenameConsistency, isCasRef, parseWorkflowPayload } from "../validate.js"; import { checkWorkflowFilenameConsistency, isCasRef, parseWorkflowPayload } from "../validate.js";
import { validateWorkflow } from "../validate-semantic.js";
import { import {
type ChainState, type ChainState,
collectOrderedSteps, collectOrderedSteps,
@@ -169,6 +170,11 @@ async function materializeLocalWorkflow(uwf: UwfStore, filePath: string): Promis
fail(filenameError); fail(filenameError);
} }
const semanticErrors = validateWorkflow(payload);
if (semanticErrors.length > 0) {
fail(`workflow validation failed:\n${semanticErrors.map((e) => ` - ${e}`).join("\n")}`);
}
const materialized = await materializeWorkflowPayload(uwf, payload); const materialized = await materializeWorkflowPayload(uwf, payload);
const hash = await uwf.store.put(uwf.schemas.workflow, materialized); const hash = await uwf.store.put(uwf.schemas.workflow, materialized);
const stored = uwf.store.get(hash); const stored = uwf.store.get(hash);
@@ -15,6 +15,7 @@ import {
type UwfStore, type UwfStore,
} from "../store.js"; } from "../store.js";
import { checkWorkflowFilenameConsistency, parseWorkflowPayload } from "../validate.js"; import { checkWorkflowFilenameConsistency, parseWorkflowPayload } from "../validate.js";
import { validateWorkflow } from "../validate-semantic.js";
export type WorkflowOrigin = "local" | "global"; export type WorkflowOrigin = "local" | "global";
@@ -136,6 +137,11 @@ export async function cmdWorkflowAdd(
fail(filenameError); fail(filenameError);
} }
const semanticErrors = validateWorkflow(payload);
if (semanticErrors.length > 0) {
fail(`workflow validation failed:\n${semanticErrors.map((e) => ` - ${e}`).join("\n")}`);
}
const uwf = await createUwfStore(storageRoot); const uwf = await createUwfStore(storageRoot);
const materialized = await materializeWorkflowPayload(uwf, payload); const materialized = await materializeWorkflowPayload(uwf, payload);
@@ -0,0 +1,278 @@
import type { WorkflowPayload } from "@uncaged/workflow-protocol";
type SchemaObj = Record<string, unknown>;
const RESERVED_NAMES = new Set(["$START", "$END"]);
/** Extract mustache variable names from a prompt string. */
function extractMustacheVars(prompt: string): string[] {
const vars: string[] = [];
const re = /\{\{\{?([^}]+)\}\}\}?/g;
let m: RegExpExecArray | null = re.exec(prompt);
while (m !== null) {
vars.push(m[1]);
m = re.exec(prompt);
}
return vars;
}
/** Check if a frontmatter schema is a oneOf (multi-exit) type. */
function isOneOfSchema(fm: unknown): fm is SchemaObj & { oneOf: SchemaObj[] } {
if (typeof fm !== "object" || fm === null) return false;
const obj = fm as SchemaObj;
return Array.isArray(obj.oneOf);
}
/** Get property names from a schema object. */
function getPropertyNames(schema: SchemaObj): Set<string> {
const props = schema.properties;
if (typeof props !== "object" || props === null) return new Set();
return new Set(Object.keys(props as Record<string, unknown>));
}
/** Extract $status const values from oneOf variants. */
function getOneOfStatuses(variants: SchemaObj[]): string[] {
const statuses: string[] = [];
for (const variant of variants) {
const props = variant.properties as Record<string, SchemaObj> | undefined;
if (props?.$status) {
const statusDef = props.$status;
if (typeof statusDef.const === "string") {
statuses.push(statusDef.const);
}
}
}
return statuses;
}
/** Check reserved names and role/graph reference integrity. */
function checkRoleReferences(payload: WorkflowPayload, errors: string[]): void {
const roleNames = new Set(Object.keys(payload.roles));
const graphNodes = new Set(Object.keys(payload.graph));
for (const name of roleNames) {
if (RESERVED_NAMES.has(name)) {
errors.push(`reserved name "${name}" must not appear in roles`);
}
}
for (const node of graphNodes) {
if (!RESERVED_NAMES.has(node) && !roleNames.has(node)) {
errors.push(`graph references unknown role "${node}"`);
}
}
for (const name of roleNames) {
if (RESERVED_NAMES.has(name)) continue;
if (!graphNodes.has(name)) {
errors.push(`role "${name}" is defined but not referenced in graph`);
}
}
}
/** Check $START/$END constraints, edge targets, and reachability. */
function checkGraphStructure(payload: WorkflowPayload, errors: string[]): void {
const roleNames = new Set(Object.keys(payload.roles));
const graphNodes = new Set(Object.keys(payload.graph));
if (!graphNodes.has("$START")) {
errors.push("$START must be defined in graph");
} else {
const startKeys = Object.keys(payload.graph.$START);
if (startKeys.length !== 1 || startKeys[0] !== "_") {
errors.push('$START must have exactly one edge with status "_"');
}
}
if (graphNodes.has("$END")) {
errors.push("$END must not have outgoing edges");
}
for (const [node, statusMap] of Object.entries(payload.graph)) {
for (const [status, target] of Object.entries(statusMap)) {
if (target.role !== "$END" && !roleNames.has(target.role)) {
errors.push(`edge ${node}${status}: unknown target role "${target.role}"`);
}
}
}
checkReachability(roleNames, collectReachableRoles(payload.graph), errors);
}
/** BFS to collect all roles reachable from $START. */
function collectReachableRoles(graph: WorkflowPayload["graph"]): Set<string> {
const reachable = new Set<string>();
const startEdges = graph.$START;
if (!startEdges) return reachable;
const queue: string[] = [];
for (const target of Object.values(startEdges)) {
if (target.role !== "$END" && !reachable.has(target.role)) {
reachable.add(target.role);
queue.push(target.role);
}
}
while (queue.length > 0) {
const current = queue.shift() as string;
const edges = graph[current];
if (!edges) continue;
for (const target of Object.values(edges)) {
if (target.role !== "$END" && !reachable.has(target.role)) {
reachable.add(target.role);
queue.push(target.role);
}
}
}
return reachable;
}
/** Check that all defined roles are reachable from $START. */
function checkReachability(roleNames: Set<string>, reachable: Set<string>, errors: string[]): void {
for (const name of roleNames) {
if (RESERVED_NAMES.has(name)) continue;
if (!reachable.has(name)) {
errors.push(`role "${name}" is not reachable from $START`);
}
}
}
/** Check oneOf discriminant validity for a role. */
function checkOneOfDiscriminant(
roleName: string,
variants: SchemaObj[],
statuses: string[],
errors: string[],
): void {
if (statuses.length === variants.length) return;
let foundMissing = false;
for (const variant of variants) {
const props = variant.properties as Record<string, SchemaObj> | undefined;
if (!props?.$status) {
errors.push(`role "${roleName}": oneOf variants must have "$status" as const discriminant`);
foundMissing = true;
break;
}
if (typeof props.$status.const !== "string") {
errors.push(`role "${roleName}": oneOf variant $status must be a const value`);
foundMissing = true;
break;
}
}
if (!foundMissing) {
errors.push(`role "${roleName}": oneOf variant $status must be a const value`);
}
}
/** Check status-edge consistency for a multi-exit role. */
function checkMultiExitEdges(
roleName: string,
graphKeys: Set<string>,
statusSet: Set<string>,
errors: string[],
): void {
if (graphKeys.has("_")) {
errors.push(`role "${roleName}" is multi-exit but graph uses "_"`);
return;
}
const extraKeys = [...graphKeys].filter((k) => !statusSet.has(k));
const missingKeys = [...statusSet].filter((k) => !graphKeys.has(k));
if (extraKeys.length > 0) {
errors.push(`role "${roleName}" graph has extra status keys: ${extraKeys.join(", ")}`);
}
if (missingKeys.length > 0) {
errors.push(`role "${roleName}" graph is missing status keys: ${missingKeys.join(", ")}`);
}
}
/** Check mustache variables for multi-exit role. */
function checkMultiExitMustache(
roleName: string,
graphEntry: Record<string, { role: string; prompt: string }>,
variants: SchemaObj[],
errors: string[],
): void {
for (const [status, target] of Object.entries(graphEntry)) {
const vars = extractMustacheVars(target.prompt);
const variant = variants.find((v) => {
const props = v.properties as Record<string, SchemaObj> | undefined;
return props?.$status?.const === status;
});
if (!variant) continue;
const propNames = getPropertyNames(variant);
for (const v of vars) {
if (v === "$status") continue;
if (!propNames.has(v)) {
errors.push(`prompt variable "${v}" not found in role "${roleName}" variant "${status}"`);
}
}
}
}
/** Check status-edge consistency and mustache for each role. */
function checkRoleConsistency(payload: WorkflowPayload, errors: string[]): void {
for (const [roleName, role] of Object.entries(payload.roles)) {
if (RESERVED_NAMES.has(roleName)) continue;
const graphEntry = payload.graph[roleName];
if (!graphEntry) continue;
const fm = role.frontmatter as unknown;
const graphKeys = new Set(Object.keys(graphEntry));
if (isOneOfSchema(fm)) {
const variants = fm.oneOf as SchemaObj[];
const statuses = getOneOfStatuses(variants);
checkOneOfDiscriminant(roleName, variants, statuses, errors);
checkMultiExitEdges(roleName, graphKeys, new Set(statuses), errors);
checkMultiExitMustache(roleName, graphEntry, variants, errors);
} else {
checkSingleExitRole(roleName, graphKeys, graphEntry, fm as SchemaObj | null, errors);
}
}
}
/** Check single-exit role status and mustache. */
function checkSingleExitRole(
roleName: string,
graphKeys: Set<string>,
graphEntry: Record<string, { role: string; prompt: string }>,
fm: SchemaObj | null,
errors: string[],
): void {
if (graphKeys.size > 1 || (graphKeys.size === 1 && !graphKeys.has("_"))) {
if (!graphKeys.has("_")) {
errors.push(`role "${roleName}" is single-exit but graph has no "_" key`);
} else {
errors.push(`role "${roleName}" is single-exit but has status keys other than "_"`);
}
}
const singleTarget = graphEntry._;
if (!singleTarget) return;
const vars = extractMustacheVars(singleTarget.prompt);
const propNames = fm ? getPropertyNames(fm) : new Set<string>();
for (const v of vars) {
if (v === "$status") continue;
if (!propNames.has(v)) {
errors.push(`prompt variable "${v}" not found in role "${roleName}" frontmatter`);
}
}
}
/**
* Validate a parsed WorkflowPayload for semantic correctness.
* Returns an array of error messages. Empty array = valid.
*/
export function validateWorkflow(payload: WorkflowPayload): string[] {
const errors: string[] = [];
checkRoleReferences(payload, errors);
checkGraphStructure(payload, errors);
checkRoleConsistency(payload, errors);
return errors;
}