diff --git a/docs/rfc-001-workflow-engine.md b/docs/rfc-001-workflow-engine.md index ff2804d..1ba54a0 100644 --- a/docs/rfc-001-workflow-engine.md +++ b/docs/rfc-001-workflow-engine.md @@ -202,7 +202,99 @@ No concurrency control or timeout settings in the registry — those belong to e |---------|-------------| | `uncaged-workflow fork [--from-role ]` | Fork from a historical thread state | -## 8. Design Decisions & Rationale +## 8. Workflow Execution Model: Role, Moderator, Agent + +A workflow is a finite-state automaton driven by three concepts: + +### Core Types + +```typescript +/** Sentinel values for automaton control flow. */ +const START = "__start__" as const; +const END = "__end__" as const; + +/** Maps role names → their meta types. Single generic drives all inference. */ +type RoleMeta = Record>; + +/** Typed output of a Role execution. */ +type RoleResult = { content: string; meta: Meta }; + +/** Engine start frame: initial prompt + thread identity. */ +type StartStep = { + role: START; + content: string; // the user prompt + meta: { maxRounds: number; threadId: string }; + timestamp: number; +}; + +/** A completed role step in the thread. */ +type RoleStep = { + [K in keyof M & string]: { role: K; meta: M[K]; content: string; timestamp: number }; +}[keyof M & string]; + +/** Thread-scoped context passed to roles and moderator. */ +type ThreadContext = { + threadId: string; + start: StartStep; + steps: RoleStep[]; +}; + +/** + * A Role — receives full thread context, returns typed content + meta. + * Implementation can be an agent, LLM call, script, HTTP request, etc. + */ +type Role = (ctx: ThreadContext) => Promise>; + +/** + * An Agent — raw string output interface for LLM/CLI adapters. + * Structured meta is extracted by the role's extract layer. + */ +type AgentFn = (ctx: ThreadContext, systemPrompt: string) => Promise; + +/** + * The Moderator — a pure routing function. + * Receives the full thread context (start + all prior steps). + * On initial call, `steps` is empty. + * Returns the next role name or END to terminate. + */ +type Moderator = (ctx: ThreadContext) => (keyof M & string) | END; + +/** Complete workflow definition as authored by users. */ +type WorkflowDefinition = { + name: string; + roles: { [K in keyof M & string]: Role }; + moderator: Moderator; +}; +``` + +### Execution Flow + +``` +START (prompt) → Moderator → Role A → Moderator → Role B → ... → Moderator → END +``` + +1. Engine creates a `StartStep` with the user prompt and maxRounds +2. Moderator is called with `steps = []`, returns the first role name +3. Role executes, appends a `RoleStep` to the thread +4. Moderator is called again with updated steps, returns next role or END +5. Repeat until END or maxRounds reached + +### Responsibilities + +| Component | Responsibility | Purity | +|-----------|---------------|--------| +| **Moderator** | Route to next role based on thread state | Pure function, no side effects | +| **Role** | Execute a step (call LLM, run script, etc.) | Async, may have side effects | +| **AgentFn** | Low-level LLM/CLI invocation adapter | Async, side effects | + +### Key Constraints + +- Moderator is **synchronous and pure** — no I/O, no state mutation +- Roles receive the **full thread context** (not just the last message) +- Round count = `steps.length`; max rounds in `start.meta.maxRounds` +- The `meta` field on each step is **typed per role** via the `RoleMeta` generic + +## 9. Design Decisions & Rationale ### Why single-file ESM? - Hash = version. No ambiguity. diff --git a/packages/cli-workflow/tsconfig.json b/packages/cli-workflow/tsconfig.json new file mode 100644 index 0000000..3160acf --- /dev/null +++ b/packages/cli-workflow/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "exactOptionalPropertyTypes": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "composite": true, + "outDir": "dist", + "rootDir": "src" + }, + "references": [{ "path": "../workflow" }], + "include": ["src/**/*.ts"] +} diff --git a/packages/workflow/package.json b/packages/workflow/package.json index 3d4529f..f82b3c5 100644 --- a/packages/workflow/package.json +++ b/packages/workflow/package.json @@ -7,5 +7,13 @@ "scripts": { "build": "echo 'TODO'", "test": "bun test" + }, + "dependencies": { + "acorn": "^8.16.0", + "xxhashjs": "^0.2.2", + "yaml": "^2.8.4" + }, + "devDependencies": { + "@types/acorn": "^6.0.4" } } diff --git a/packages/workflow/src/base32.ts b/packages/workflow/src/base32.ts new file mode 100644 index 0000000..683abe7 --- /dev/null +++ b/packages/workflow/src/base32.ts @@ -0,0 +1,77 @@ +import { err, ok, type Result } from "./result.js"; + +/** Crockford Base32 alphabet (no I, L, O, U). */ +export const CROCKFORD_BASE32_ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXZ"; + +const DECODE_MAP: Record = (() => { + const map: Record = {}; + for (let i = 0; i < CROCKFORD_BASE32_ALPHABET.length; i++) { + map[CROCKFORD_BASE32_ALPHABET[i]] = i; + } + return map; +})(); + +function padBitCount(bitLength: number): number { + const r = bitLength % 5; + return r === 0 ? 0 : 5 - r; +} + +/** + * Encode an integer using exactly `bitLength` significant bits, MSB-first, + * with the minimum number of leading zero bits so the total is a multiple of 5. + */ +export function encodeCrockfordBase32Bits(value: bigint, bitLength: number): string { + if (bitLength <= 0) { + throw new Error("bitLength must be positive"); + } + const padBits = padBitCount(bitLength); + const totalBits = bitLength + padBits; + const charCount = totalBits / 5; + const shifted = value << BigInt(padBits); + let result = ""; + for (let i = 0; i < charCount; i++) { + const shift = totalBits - 5 * (i + 1); + const quintet = Number((shifted >> BigInt(shift)) & 0x1fn); + result += CROCKFORD_BASE32_ALPHABET[quintet]; + } + return result; +} + +export function decodeCrockfordBase32Bits(encoded: string, bitLength: number): Result { + if (bitLength <= 0) { + return err(new Error("bitLength must be positive")); + } + const padBits = padBitCount(bitLength); + const totalBits = encoded.length * 5; + if (totalBits !== bitLength + padBits) { + return err(new Error("encoded length does not match bitLength")); + } + let shifted = 0n; + for (let i = 0; i < encoded.length; i++) { + const ch = encoded[i]; + if (ch === undefined) { + return err(new Error("invalid encoded string")); + } + const upper = ch.toUpperCase(); + const val = DECODE_MAP[upper]; + if (val === undefined) { + return err(new Error(`invalid Crockford Base32 character: ${ch}`)); + } + shifted = (shifted << 5n) | BigInt(val); + } + return ok(shifted >> BigInt(padBits)); +} + +/** XXH64-sized value (13 Crockford chars). */ +export function encodeUint64AsCrockford(value: bigint): string { + const masked = value & 0xffff_ffff_ffff_ffffn; + return encodeCrockfordBase32Bits(masked, 64); +} + +export function decodeCrockfordToUint64(encoded: string): Result { + const decoded = decodeCrockfordBase32Bits(encoded, 64); + if (!decoded.ok) { + return decoded; + } + return ok(decoded.value & 0xffff_ffff_ffff_ffffn); +} diff --git a/packages/workflow/src/bundle-validator.ts b/packages/workflow/src/bundle-validator.ts new file mode 100644 index 0000000..7f16a36 --- /dev/null +++ b/packages/workflow/src/bundle-validator.ts @@ -0,0 +1,142 @@ +import { isBuiltin } from "node:module"; + +import * as acorn from "acorn"; +import type { Node, Program } from "acorn"; + +import { err, ok, type Result } from "./result.js"; + +export type WorkflowBundleValidationInput = { + /** Absolute or relative path (used for `.esm.js` suffix checks). */ + filePath: string; + /** UTF-8 source of the bundle. */ + source: string; +}; + +function endsWithEsmJs(path: string): boolean { + return path.endsWith(".esm.js"); +} + +function isAllowedImportSpecifier(spec: string): boolean { + if (spec.length === 0) { + return false; + } + if (spec.startsWith(".") || spec.startsWith("/")) { + return false; + } + return isBuiltin(spec); +} + +function walk(node: Node, visit: (n: Node) => void): void { + visit(node); + for (const key of Object.keys(node)) { + const val = (node as Record)[key]; + if (val === null || val === undefined) { + continue; + } + if (Array.isArray(val)) { + for (const item of val) { + if (item !== null && typeof item === "object" && "type" in item) { + walk(item as Node, visit); + } + } + } else if (typeof val === "object" && "type" in val) { + walk(val as Node, visit); + } + } +} + +function programHasDefaultExport(body: readonly Node[]): boolean { + for (const stmt of body) { + if (stmt.type === "ExportDefaultDeclaration") { + return true; + } + } + return false; +} + +/** + * Validate RFC-001 bundle rules: single-file ESM shape, default export, + * no dynamic `import()`, static imports restricted to Node builtins. + */ +export function validateWorkflowBundle(input: WorkflowBundleValidationInput): Result { + if (!endsWithEsmJs(input.filePath)) { + return err('workflow bundle file must use the ".esm.js" suffix'); + } + + let ast: Node; + try { + ast = acorn.parse(input.source, { + ecmaVersion: 2022, + sourceType: "module", + locations: false, + }) as Node; + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + return err(`failed to parse module: ${message}`); + } + + if (ast.type !== "Program") { + return err("internal error: expected Program root"); + } + + const program = ast as Program; + if (!programHasDefaultExport(program.body)) { + return err("workflow bundle must have a default export"); + } + + let walkError: string | null = null; + walk(ast, (n) => { + if (walkError !== null) { + return; + } + if (n.type === "ImportExpression") { + walkError = "dynamic import() is not allowed in workflow bundles"; + return; + } + if (n.type === "ImportDeclaration") { + const src = n.source; + if (src.type !== "Literal" || typeof src.value !== "string") { + walkError = "only static string import specifiers are allowed"; + return; + } + if (!isAllowedImportSpecifier(src.value)) { + walkError = `disallowed import specifier "${src.value}" (only Node built-ins are allowed)`; + } + return; + } + if (n.type === "ExportNamedDeclaration" && n.source !== null && n.source !== undefined) { + const src = n.source; + if (src.type !== "Literal" || typeof src.value !== "string") { + walkError = "only static string re-export specifiers are allowed"; + return; + } + if (!isAllowedImportSpecifier(src.value)) { + walkError = `disallowed re-export specifier "${src.value}" (only Node built-ins are allowed)`; + } + return; + } + if (n.type === "ExportAllDeclaration") { + const src = n.source; + if (src.type !== "Literal" || typeof src.value !== "string") { + walkError = "only static string export-all specifiers are allowed"; + return; + } + if (!isAllowedImportSpecifier(src.value)) { + walkError = `disallowed export-all specifier "${src.value}" (only Node built-ins are allowed)`; + } + return; + } + if (n.type === "CallExpression") { + const c = n.callee; + if (c.type === "Identifier" && c.name === "require") { + walkError = "require() is not allowed in workflow bundles"; + } + } + }); + + if (walkError !== null) { + return err(walkError); + } + + return ok(undefined); +} diff --git a/packages/workflow/src/hash.ts b/packages/workflow/src/hash.ts new file mode 100644 index 0000000..7d2c51c --- /dev/null +++ b/packages/workflow/src/hash.ts @@ -0,0 +1,17 @@ +import { Buffer } from "node:buffer"; + +import XXH from "xxhashjs"; + +import { encodeUint64AsCrockford } from "./base32.js"; + +function digestToUint64(digest: { toString(radix?: number): string }): bigint { + const hex = digest.toString(16).padStart(16, "0"); + return BigInt(`0x${hex}`); +} + +/** XXH64 (seed 0) over bundle bytes, encoded as 13-char Crockford Base32. */ +export function hashWorkflowBundleBytes(data: Uint8Array): string { + const buf = Buffer.from(data.buffer, data.byteOffset, data.byteLength); + const digest = XXH.h64(0).update(buf).digest(); + return encodeUint64AsCrockford(digestToUint64(digest)); +} diff --git a/packages/workflow/src/index.ts b/packages/workflow/src/index.ts index 322c6a9..93e00a4 100644 --- a/packages/workflow/src/index.ts +++ b/packages/workflow/src/index.ts @@ -1,2 +1,32 @@ -// @uncaged/workflow - core library -export {}; +export { + CROCKFORD_BASE32_ALPHABET, + decodeCrockfordBase32Bits, + decodeCrockfordToUint64, + encodeCrockfordBase32Bits, + encodeUint64AsCrockford, +} from "./base32.js"; +export { validateWorkflowBundle, type WorkflowBundleValidationInput } from "./bundle-validator.js"; +export { hashWorkflowBundleBytes } from "./hash.js"; +export { + createLogger, + type CreateLoggerOptions, + type LogFn, + type LoggerSink, +} from "./logger.js"; +export { + getRegisteredWorkflow, + listRegisteredWorkflowNames, + parseWorkflowRegistryYaml, + readWorkflowRegistry, + registerWorkflowVersion, + stringifyWorkflowRegistryYaml, + unregisterWorkflow, + workflowRegistryPath, + writeWorkflowRegistry, + type WorkflowHistoryEntry, + type WorkflowRegistryEntry, + type WorkflowRegistryFile, +} from "./registry.js"; +export { err, ok, type Result } from "./result.js"; +export { getDefaultWorkflowStorageRoot } from "./storage-root.js"; +export { generateUlid } from "./ulid.js"; diff --git a/packages/workflow/src/logger.ts b/packages/workflow/src/logger.ts new file mode 100644 index 0000000..7f70316 --- /dev/null +++ b/packages/workflow/src/logger.ts @@ -0,0 +1,57 @@ +import { appendFileSync } from "node:fs"; + +const TAG_LENGTH = 8; + +function assertValidLogTag(tag: string): void { + if (tag.length !== TAG_LENGTH) { + throw new Error(`log tag must be exactly ${TAG_LENGTH} characters`); + } + for (let i = 0; i < tag.length; i++) { + const ch = tag[i]; + if (ch === undefined) { + throw new Error("log tag validation failed"); + } + const upper = ch.toUpperCase(); + if (!/[0-9A-HJKMNP-TV-Z]/.test(upper)) { + throw new Error(`invalid Crockford Base32 character in log tag: ${ch}`); + } + } +} + +export type LoggerSink = + | { kind: "stderr" } + | { kind: "file"; path: string }; + +export type CreateLoggerOptions = { + sink: LoggerSink; +}; + +export type LogFn = (tag: string, content: string) => void; + +/** Append one JSONL log record: `{ tag, content, timestamp }` per RFC-001. */ +export function createLogger(options: CreateLoggerOptions): LogFn { + if (options.sink.kind === "stderr") { + return (tag: string, content: string) => { + assertValidLogTag(tag); + const line = + `${JSON.stringify({ + tag: tag.toUpperCase(), + content, + timestamp: Date.now(), + })}\n`; + process.stderr.write(line); + }; + } + + const filePath = options.sink.path; + return (tag: string, content: string) => { + assertValidLogTag(tag); + const line = + `${JSON.stringify({ + tag: tag.toUpperCase(), + content, + timestamp: Date.now(), + })}\n`; + appendFileSync(filePath, line, "utf8"); + }; +} diff --git a/packages/workflow/src/registry.ts b/packages/workflow/src/registry.ts new file mode 100644 index 0000000..d9f073b --- /dev/null +++ b/packages/workflow/src/registry.ts @@ -0,0 +1,164 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import { parseDocument, stringify } from "yaml"; + +import { err, ok, type Result } from "./result.js"; + +export type WorkflowHistoryEntry = { + hash: string; + timestamp: number; +}; + +export type WorkflowRegistryEntry = { + hash: string; + timestamp: number; + history: WorkflowHistoryEntry[]; +}; + +export type WorkflowRegistryFile = { + workflows: Record; +}; + +export function workflowRegistryPath(storageRoot: string): string { + return join(storageRoot, "workflow.yaml"); +} + +function emptyRegistry(): WorkflowRegistryFile { + return { workflows: {} }; +} + +function normalizeRegistry(raw: unknown): Result { + if (raw === null || typeof raw !== "object") { + return err(new Error("registry root must be a mapping")); + } + const root = raw as Record; + const workflowsRaw = root.workflows; + if (workflowsRaw === null || workflowsRaw === undefined || typeof workflowsRaw !== "object") { + return err(new Error('registry must contain a "workflows" mapping')); + } + const workflows: Record = {}; + for (const [name, entryRaw] of Object.entries(workflowsRaw)) { + if (entryRaw === null || typeof entryRaw !== "object") { + return err(new Error(`workflow "${name}" must be a mapping`)); + } + const e = entryRaw as Record; + const hash = e.hash; + const timestamp = e.timestamp; + const historyRaw = e.history; + if (typeof hash !== "string") { + return err(new Error(`workflow "${name}" must have a string hash`)); + } + if (typeof timestamp !== "number" || !Number.isFinite(timestamp)) { + return err(new Error(`workflow "${name}" must have a finite numeric timestamp`)); + } + if (!Array.isArray(historyRaw)) { + return err(new Error(`workflow "${name}" must have a history array`)); + } + const history: WorkflowHistoryEntry[] = []; + for (let i = 0; i < historyRaw.length; i++) { + const h = historyRaw[i]; + if (h === null || typeof h !== "object") { + return err(new Error(`workflow "${name}" history[${i}] must be a mapping`)); + } + const he = h as Record; + if (typeof he.hash !== "string" || typeof he.timestamp !== "number" || !Number.isFinite(he.timestamp)) { + return err(new Error(`workflow "${name}" history[${i}] must have hash and timestamp`)); + } + history.push({ hash: he.hash, timestamp: he.timestamp }); + } + workflows[name] = { hash, timestamp, history }; + } + return ok({ workflows }); +} + +export function parseWorkflowRegistryYaml(text: string): Result { + if (text.trim() === "") { + return ok(emptyRegistry()); + } + let doc: unknown; + try { + doc = parseDocument(text).toJSON(); + } catch (e) { + return err(e instanceof Error ? e : new Error(String(e))); + } + return normalizeRegistry(doc); +} + +export function stringifyWorkflowRegistryYaml(registry: WorkflowRegistryFile): string { + return `${stringify(registry, { indent: 2 })}\n`; +} + +export async function readWorkflowRegistry(storageRoot: string): Promise> { + const path = workflowRegistryPath(storageRoot); + let text: string; + try { + text = await readFile(path, "utf8"); + } catch (e) { + const errObj = e as NodeJS.ErrnoException; + if (errObj.code === "ENOENT") { + return ok(emptyRegistry()); + } + return err(errObj instanceof Error ? errObj : new Error(String(e))); + } + return parseWorkflowRegistryYaml(text); +} + +export async function writeWorkflowRegistry( + storageRoot: string, + registry: WorkflowRegistryFile, +): Promise> { + const path = workflowRegistryPath(storageRoot); + try { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, stringifyWorkflowRegistryYaml(registry), "utf8"); + } catch (e) { + return err(e instanceof Error ? e : new Error(String(e))); + } + return ok(undefined); +} + +export function listRegisteredWorkflowNames(registry: WorkflowRegistryFile): string[] { + return Object.keys(registry.workflows).sort(); +} + +export function getRegisteredWorkflow( + registry: WorkflowRegistryFile, + name: string, +): WorkflowRegistryEntry | null { + const entry = registry.workflows[name]; + if (entry === undefined) { + return null; + } + return entry; +} + +/** Register or upgrade a workflow version, moving the previous head into `history`. */ +export function registerWorkflowVersion( + registry: WorkflowRegistryFile, + name: string, + hash: string, + timestamp: number, +): WorkflowRegistryFile { + const prev = registry.workflows[name]; + const baseHistory = prev === undefined ? [] : prev.history; + const history: WorkflowHistoryEntry[] = + prev === undefined + ? baseHistory + : [{ hash: prev.hash, timestamp: prev.timestamp }, ...baseHistory]; + const next: WorkflowRegistryEntry = { hash, timestamp, history }; + return { + workflows: { ...registry.workflows, [name]: next }, + }; +} + +export function unregisterWorkflow( + registry: WorkflowRegistryFile, + name: string, +): Result { + if (registry.workflows[name] === undefined) { + return err(new Error(`workflow not registered: ${name}`)); + } + const { [name]: _removed, ...rest } = registry.workflows; + return ok({ workflows: rest }); +} diff --git a/packages/workflow/src/result.ts b/packages/workflow/src/result.ts new file mode 100644 index 0000000..39e40a2 --- /dev/null +++ b/packages/workflow/src/result.ts @@ -0,0 +1,9 @@ +export type Result = { ok: true; value: T } | { ok: false; error: E }; + +export function ok(value: T): Result { + return { ok: true, value }; +} + +export function err(error: E): Result { + return { ok: false, error }; +} diff --git a/packages/workflow/src/storage-root.ts b/packages/workflow/src/storage-root.ts new file mode 100644 index 0000000..81c20d8 --- /dev/null +++ b/packages/workflow/src/storage-root.ts @@ -0,0 +1,7 @@ +import { homedir } from "node:os"; +import { join } from "node:path"; + +/** Default filesystem root for workflow data (`~/.uncaged/workflow`). */ +export function getDefaultWorkflowStorageRoot(): string { + return join(homedir(), ".uncaged", "workflow"); +} diff --git a/packages/workflow/src/ulid.ts b/packages/workflow/src/ulid.ts new file mode 100644 index 0000000..60938cf --- /dev/null +++ b/packages/workflow/src/ulid.ts @@ -0,0 +1,28 @@ +import { encodeCrockfordBase32Bits } from "./base32.js"; + +const ULID_TIME_BITS = 48; +const ULID_RANDOM_BITS = 80; + +function readRandomUint80(): bigint { + const bytes = new Uint8Array(10); + crypto.getRandomValues(bytes); + let x = 0n; + for (let i = 0; i < bytes.length; i++) { + x = (x << 8n) | BigInt(bytes[i]); + } + return x & ((1n << 80n) - 1n); +} + +/** + * Generate a ULID using Crockford Base32: 10 timestamp chars + 16 random chars. + * Timestamp uses 48 bits of Unix time in milliseconds. + */ +export function generateUlid(nowMs: number): string { + if (!Number.isFinite(nowMs) || nowMs < 0 || nowMs >= 2 ** ULID_TIME_BITS) { + throw new Error("nowMs must be a finite number in [0, 2^48)"); + } + const time = BigInt(Math.floor(nowMs)); + const rand = readRandomUint80(); + const payload = (time << BigInt(ULID_RANDOM_BITS)) | rand; + return encodeCrockfordBase32Bits(payload, ULID_TIME_BITS + ULID_RANDOM_BITS); +} diff --git a/packages/workflow/tsconfig.json b/packages/workflow/tsconfig.json new file mode 100644 index 0000000..944ec14 --- /dev/null +++ b/packages/workflow/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "exactOptionalPropertyTypes": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "composite": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts", "xxhashjs.d.ts"] +} diff --git a/packages/workflow/xxhashjs.d.ts b/packages/workflow/xxhashjs.d.ts new file mode 100644 index 0000000..beee6d4 --- /dev/null +++ b/packages/workflow/xxhashjs.d.ts @@ -0,0 +1,17 @@ +declare module "xxhashjs" { + type Digest = { + toString(radix?: number): string; + }; + + type Hasher64 = { + update(data: Buffer): Hasher64; + digest(): Digest; + }; + + type XXH = { + h64(seed: number): Hasher64; + }; + + const XXH: XXH; + export default XXH; +}