feat: tester e2e verification — dynamic import + tick lifecycle + commit/push
CI / test (push) Has been cancelled
CI / test (push) Has been cancelled
Tester no longer duplicates build+test (that's coder's job). Instead: dynamic-import workflows from engine dir, spin temp store, tick to __end__, verify lifecycle completes. Workflow-agnostic.
This commit is contained in:
@@ -1,92 +1,211 @@
|
||||
/**
|
||||
* Meta Tester role — build + test + commit/push on pass.
|
||||
* No LLM needed, just exit codes and git.
|
||||
* Meta Tester role — e2e verification.
|
||||
*
|
||||
* Dynamic-imports the workflow coder just wrote, spins up a temp store,
|
||||
* ticks through a full lifecycle, and checks it reaches __end__.
|
||||
* On pass: git commit + push. On fail: back to coder.
|
||||
*
|
||||
* 小橘 🍊 (NEKO Team)
|
||||
*/
|
||||
|
||||
import { execSync } from 'node:child_process';
|
||||
import { existsSync, mkdtempSync, readdirSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { MetaTesterMeta } from '../meta.js';
|
||||
import type { Role, RoleResult, WorkflowMessage } from '../workflow-type.js';
|
||||
import type { Role, RoleResult, WorkflowMessage, WorkflowType } from '../workflow-type.js';
|
||||
import { createStore } from '../../store.js';
|
||||
import { createWorkflowRule } from '../workflow-rule-adapter.js';
|
||||
|
||||
export function createMetaTesterRole(opts: {
|
||||
repoDir: string;
|
||||
/** cwd for build command (defaults to repoDir) */
|
||||
buildDir?: string;
|
||||
/** cwd for test command (defaults to repoDir) */
|
||||
testDir?: string;
|
||||
/** git remote to push to (auto-detect if omitted) */
|
||||
/** git remote (auto-detect if omitted) */
|
||||
remote?: string;
|
||||
/** git branch (default: main) */
|
||||
branch?: string;
|
||||
/** max ticks for e2e run (default: 50) */
|
||||
maxTicks?: number;
|
||||
}): Role<MetaTesterMeta> {
|
||||
const branch = opts.branch ?? 'main';
|
||||
const maxTicks = opts.maxTicks ?? 50;
|
||||
|
||||
return async (
|
||||
chain: WorkflowMessage[],
|
||||
): Promise<RoleResult<MetaTesterMeta>> => {
|
||||
const cwd = opts.repoDir;
|
||||
const buildCwd = opts.buildDir ?? cwd;
|
||||
const testCwd = opts.testDir ?? cwd;
|
||||
|
||||
// Step 1: Build
|
||||
let buildOk = false;
|
||||
// Step 0: Build first (coder should have done this, but verify)
|
||||
let buildOutput = '';
|
||||
try {
|
||||
buildOutput = execSync('bun run build 2>&1', {
|
||||
cwd: buildCwd,
|
||||
cwd,
|
||||
timeout: 60_000,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
buildOk = true;
|
||||
} catch (err: any) {
|
||||
buildOutput = err.stdout ?? err.message;
|
||||
return {
|
||||
content: `编译失败\n\n---\n${(err.stdout ?? err.message).slice(0, 2000)}`,
|
||||
meta: { pass: false, reason: '编译失败' },
|
||||
};
|
||||
}
|
||||
|
||||
// Step 2: Test (only if build succeeded)
|
||||
let testOk = false;
|
||||
let testOutput = '';
|
||||
if (buildOk) {
|
||||
// Step 1: Discover workflow files in engine src/workflows/
|
||||
const workflowDir = join(cwd, 'src', 'workflows');
|
||||
if (!existsSync(workflowDir)) {
|
||||
return {
|
||||
content: 'src/workflows/ 目录不存在',
|
||||
meta: { pass: false, reason: 'src/workflows/ not found' },
|
||||
};
|
||||
}
|
||||
|
||||
const files = readdirSync(workflowDir).filter(
|
||||
(f) => f.endsWith('.ts') && !f.endsWith('.test.ts'),
|
||||
);
|
||||
|
||||
// Step 2: Dynamic import each workflow, find WorkflowType exports
|
||||
const workflows: WorkflowType<any>[] = [];
|
||||
const importErrors: string[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const fullPath = join(workflowDir, file);
|
||||
try {
|
||||
testOutput = execSync('bun test src/workflows/ 2>&1', {
|
||||
cwd: testCwd,
|
||||
timeout: 60_000,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
testOk = true;
|
||||
const mod = await import(fullPath);
|
||||
for (const [key, val] of Object.entries(mod)) {
|
||||
// Direct WorkflowType export (e.g. `export const pingPong: WorkflowType`)
|
||||
if (val && typeof val === 'object' && 'name' in (val as any) && 'roles' in (val as any) && 'moderator' in (val as any)) {
|
||||
workflows.push(val as WorkflowType<any>);
|
||||
}
|
||||
// Factory function (e.g. `export function createWerewolfWorkflow()`)
|
||||
if (typeof val === 'function' && key.startsWith('create') && key.endsWith('Workflow')) {
|
||||
try {
|
||||
// Call with no args first (mock mode)
|
||||
const wf = val();
|
||||
if (wf && typeof wf === 'object' && 'name' in wf && 'roles' in wf && 'moderator' in wf) {
|
||||
workflows.push(wf as WorkflowType<any>);
|
||||
}
|
||||
} catch {
|
||||
// Factory needs args — skip (can't auto-test)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
testOutput = err.stdout ?? err.message;
|
||||
importErrors.push(`${file}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const pass = buildOk && testOk;
|
||||
const output = buildOk ? testOutput : buildOutput;
|
||||
const reason = pass
|
||||
? '编译通过,测试全绿'
|
||||
: buildOk
|
||||
? '测试失败'
|
||||
: '编译失败';
|
||||
if (workflows.length === 0) {
|
||||
return {
|
||||
content: `没有找到可测试的 WorkflowType\n\nimport errors:\n${importErrors.join('\n')}`,
|
||||
meta: { pass: false, reason: 'no testable workflows found' },
|
||||
};
|
||||
}
|
||||
|
||||
// Step 3: If pass, commit + push
|
||||
let commitHash: string | undefined;
|
||||
let pushed: boolean | undefined;
|
||||
if (pass) {
|
||||
const exec = (cmd: string) =>
|
||||
execSync(cmd, { cwd, encoding: 'utf-8', timeout: 30_000 }).trim();
|
||||
// Step 3: E2E — for each workflow, create temp store, tick through lifecycle
|
||||
const results: { name: string; pass: boolean; detail: string }[] = [];
|
||||
|
||||
for (const wf of workflows) {
|
||||
const tmpDir = mkdtempSync(join(tmpdir(), `pulse-tester-${wf.name}-`));
|
||||
const testStore = createStore({
|
||||
eventsDbPath: join(tmpDir, 'events.db'),
|
||||
objectsDir: join(tmpDir, 'objects'),
|
||||
});
|
||||
|
||||
try {
|
||||
// Extract task name from __start__ for commit message
|
||||
const startMsg = chain.find((m) => m.role === '__start__');
|
||||
const firstLine = (startMsg?.content ?? '').split('\n')[0].slice(0, 60);
|
||||
const commitMsg = firstLine || 'meta workflow auto-commit';
|
||||
const rule = createWorkflowRule(wf, testStore);
|
||||
|
||||
exec('git add -A');
|
||||
// Seed __start__
|
||||
const taskContent = `e2e test for ${wf.name}`;
|
||||
const hash = await testStore.putObject(taskContent);
|
||||
await testStore.appendEvent({
|
||||
occurredAt: Date.now(),
|
||||
kind: `${wf.name}.__start__`,
|
||||
key: `e2e-test-${Date.now()}`,
|
||||
hash,
|
||||
});
|
||||
|
||||
// Tick until __end__ or maxTicks
|
||||
let reachedEnd = false;
|
||||
let tickCount = 0;
|
||||
let lastError: string | null = null;
|
||||
|
||||
for (let i = 0; i < maxTicks; i++) {
|
||||
tickCount++;
|
||||
try {
|
||||
const r = await rule.tick();
|
||||
if (r.executed.length === 0) {
|
||||
// Check if we reached __end__
|
||||
const events = await testStore.getAfter(0);
|
||||
const endEvt = events.find((ev: any) => ev.kind === `${wf.name}.__end__`);
|
||||
if (endEvt) {
|
||||
reachedEnd = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
// Check for __end__ in executed
|
||||
const events2 = await testStore.getAfter(0);
|
||||
const endEvt2 = events2.find((ev: any) => ev.kind === `${wf.name}.__end__`);
|
||||
if (endEvt2) {
|
||||
reachedEnd = true;
|
||||
break;
|
||||
}
|
||||
} catch (err: any) {
|
||||
lastError = err.message;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (reachedEnd) {
|
||||
results.push({ name: wf.name, pass: true, detail: `completed in ${tickCount} ticks` });
|
||||
} else if (lastError) {
|
||||
results.push({ name: wf.name, pass: false, detail: `error: ${lastError}` });
|
||||
} else {
|
||||
results.push({ name: wf.name, pass: false, detail: `did not reach __end__ in ${tickCount} ticks` });
|
||||
}
|
||||
} finally {
|
||||
await testStore.close();
|
||||
}
|
||||
}
|
||||
|
||||
const allPass = results.every((r) => r.pass);
|
||||
const summary = results
|
||||
.map((r) => `${r.pass ? '✅' : '❌'} ${r.name}: ${r.detail}`)
|
||||
.join('\n');
|
||||
|
||||
if (!allPass) {
|
||||
return {
|
||||
content: `e2e 验证失败\n\n${summary}`,
|
||||
meta: { pass: false, reason: 'e2e verification failed' },
|
||||
};
|
||||
}
|
||||
|
||||
// Step 4: All pass — commit + push
|
||||
let commitHash: string | undefined;
|
||||
let pushed: boolean | undefined;
|
||||
|
||||
const exec = (cmd: string) =>
|
||||
execSync(cmd, { cwd, encoding: 'utf-8', timeout: 30_000 }).trim();
|
||||
|
||||
try {
|
||||
const startMsg = chain.find((m) => m.role === '__start__');
|
||||
const firstLine = (startMsg?.content ?? '').split('\n')[0].slice(0, 60);
|
||||
const commitMsg = firstLine || 'meta workflow auto-commit';
|
||||
|
||||
exec('git add -A');
|
||||
|
||||
// Check if there's anything to commit
|
||||
try {
|
||||
exec('git diff --cached --quiet');
|
||||
// No changes — still pass, just no commit needed
|
||||
commitHash = exec('git rev-parse --short HEAD');
|
||||
pushed = false;
|
||||
} catch {
|
||||
// There are staged changes
|
||||
exec(
|
||||
`git commit -m "${commitMsg}" --author="小橘 <xiaoju@shazhou.work>" --allow-empty`,
|
||||
`git commit -m "${commitMsg}" --author="小橘 <xiaoju@shazhou.work>"`,
|
||||
);
|
||||
commitHash = exec('git rev-parse --short HEAD');
|
||||
|
||||
// Try push — auto-detect remote
|
||||
// Auto-detect remote
|
||||
const remote = opts.remote ?? (() => {
|
||||
try {
|
||||
const remotes = exec('git remote').split('\n').filter(Boolean);
|
||||
@@ -104,20 +223,15 @@ export function createMetaTesterRole(opts: {
|
||||
} else {
|
||||
pushed = false;
|
||||
}
|
||||
} catch (err: any) {
|
||||
// Commit failed — still pass, just note it
|
||||
commitHash = undefined;
|
||||
pushed = false;
|
||||
}
|
||||
} catch (err: any) {
|
||||
commitHash = undefined;
|
||||
pushed = false;
|
||||
}
|
||||
|
||||
const summary = pass
|
||||
? `${reason}\nCommit: ${commitHash ?? 'none'}\nPushed: ${pushed ? 'yes' : 'no'}`
|
||||
: `${reason}\n\n---\n${output.slice(0, 2000)}`;
|
||||
|
||||
return {
|
||||
content: summary,
|
||||
meta: { pass, reason, commitHash, pushed },
|
||||
content: `e2e 验证通过\n\n${summary}\n\nCommit: ${commitHash ?? 'none'}\nPushed: ${pushed ? 'yes' : 'no'}`,
|
||||
meta: { pass: true, reason: 'e2e verification passed', commitHash, pushed },
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user