feat: add checker role to meta workflow (coder→checker→tester→END)
CI / test (push) Has been cancelled

- checker validates file scope, build, unit tests before e2e
- tester enhanced with diagnostic info on failure (event chain, last role, meta)
- tester no longer duplicates build (checker handles it)
- 6 meta workflow tests, 78 total pass
This commit is contained in:
2026-04-18 09:39:21 +00:00
parent 695dac8106
commit a3db066808
5 changed files with 295 additions and 71 deletions
@@ -20,6 +20,7 @@ import { createMetaWorkflow } from '../workflows/meta.js';
import { createCursorRunner } from '../workflows/roles/agent-executor.js';
import { createMetaCoderRole } from '../workflows/roles/meta-coder-cursor.js';
import { createMetaTesterRole } from '../workflows/roles/meta-tester.js';
import { createMetaCheckerRole } from '../workflows/roles/meta-checker.js';
import { createReportWorkflow } from '../workflows/report.js';
import { createAnalystRole } from '../workflows/roles/analyst-llm.js';
import { createRendererRole } from '../workflows/roles/renderer-template.js';
@@ -80,6 +81,7 @@ const codingRule = createWorkflowRule(codingWf, store, logStore);
// 2. Meta workflow (simplified: coder → tester → promoter)
const metaWf = createMetaWorkflow({
coder: createMetaCoderRole(cursorRunner, llm, ENGINE_DIR),
checker: createMetaCheckerRole({ engineDir: ENGINE_DIR }),
tester: createMetaTesterRole({ repoDir: ENGINE_DIR }),
});
const metaRule = createWorkflowRule(metaWf, store, logStore);
+115 -50
View File
@@ -10,11 +10,12 @@ import { join } from 'node:path';
import { createStore } from '../store.js';
import {
createMetaWorkflow,
type MetaCheckerMeta,
type MetaCoderMeta,
type MetaTesterMeta,
} from './meta.js';
import { createWorkflowRule } from './workflow-rule-adapter.js';
import { END, START, type WorkflowMessage } from './workflow-type.js';
import { END, START } from './workflow-type.js';
function mockStore() {
const dir = mkdtempSync(join(tmpdir(), 'meta-wf-test-'));
@@ -24,17 +25,37 @@ function mockStore() {
});
}
const mockCoder = async () => ({
content: 'ok',
meta: { filesChanged: ['a.ts'], testsPassed: true },
});
const mockCheckerPass = async () => ({
content: 'pass',
meta: { pass: true, reason: '约束检查通过' },
});
const mockCheckerFail = async () => ({
content: 'fail',
meta: { pass: false, reason: '文件范围越界', violations: ['越界文件: package.json'] },
});
const mockTesterPass = async () => ({
content: 'pass',
meta: { pass: true, reason: 'e2e verification passed' },
});
const mockTesterFail = async () => ({
content: 'fail',
meta: { pass: false, reason: 'e2e verification failed' },
});
describe('Meta Workflow', () => {
test('moderator routes: START→coder→tester(pass)→END', () => {
test('moderator routes: START→coder→checker(pass)→tester(pass)→END', () => {
const wf = createMetaWorkflow({
coder: async () => ({
content: 'ok',
meta: { filesChanged: ['a.js'], testsPassed: true },
}),
tester: async () => ({
content: 'pass',
meta: { pass: true, reason: '编译通过,测试全绿' },
}),
coder: mockCoder,
checker: mockCheckerPass,
tester: mockTesterPass,
});
expect(wf.moderator({ role: START, meta: null }, 'x')).toBe('coder');
@@ -43,36 +64,60 @@ describe('Meta Workflow', () => {
{ role: 'coder', meta: { filesChanged: [], testsPassed: true } },
'x',
),
).toBe('checker');
expect(
wf.moderator(
{ role: 'checker', meta: { pass: true, reason: 'ok' } },
'x',
),
).toBe('tester');
expect(
wf.moderator(
{ role: 'tester', meta: { pass: true, reason: '通过' } },
{ role: 'tester', meta: { pass: true, reason: 'ok' } },
'x',
),
).toBe(END);
});
test('checker fail → back to coder', () => {
const wf = createMetaWorkflow({
coder: mockCoder,
checker: mockCheckerPass,
tester: mockTesterPass,
});
expect(
wf.moderator(
{ role: 'tester', meta: { pass: false, reason: '失败' } },
{ role: 'checker', meta: { pass: false, reason: '越界' } },
'x',
),
).toBe('coder');
});
test('mock: full happy path via adapter ticks', async () => {
test('tester fail → back to coder', () => {
const wf = createMetaWorkflow({
coder: mockCoder,
checker: mockCheckerPass,
tester: mockTesterPass,
});
expect(
wf.moderator(
{ role: 'tester', meta: { pass: false, reason: 'e2e fail' } },
'x',
),
).toBe('coder');
});
test('happy path: coder→checker→tester via adapter ticks', async () => {
const store = mockStore();
const wf = createMetaWorkflow({
coder: async () => ({
content: 'files changed',
meta: { filesChanged: ['demo.js'], testsPassed: true },
}),
tester: async () => ({
content: 'all pass',
meta: { pass: true, reason: '编译通过,测试全绿' },
}),
coder: mockCoder,
checker: mockCheckerPass,
tester: mockTesterPass,
});
const rule = createWorkflowRule(wf, store);
const hash = await store.putObject('build a demo workflow');
await store.appendEvent({
occurredAt: Date.now(),
@@ -82,60 +127,80 @@ describe('Meta Workflow', () => {
});
const roles: string[] = [];
for (let i = 0; i < 5; i++) {
for (let i = 0; i < 6; i++) {
const r = await rule.tick();
if (r.executed.length === 0) break;
roles.push(...r.executed.map((a) => a.role));
}
expect(roles).toEqual(['coder', 'tester']);
const events = await store.getAfter(0);
expect(events.length).toBe(3); // __start__ + coder + tester
expect(roles).toEqual(['coder', 'checker', 'tester']);
await store.close();
});
test('mock: tester failure → retry coder', async () => {
test('checker fail → retry: coder→checker(fail)→coder→checker(pass)→tester', async () => {
const store = mockStore();
let testCount = 0;
let checkerCount = 0;
const wf = createMetaWorkflow({
coder: async () => ({
content: 'code',
meta: { filesChanged: [], testsPassed: true },
}),
tester: async () => {
testCount++;
if (testCount === 1)
return {
content: 'build failed',
meta: { pass: false, reason: '编译失败' },
};
return {
content: 'ok',
meta: { pass: true, reason: '编译通过,测试全绿' },
};
coder: mockCoder,
checker: async () => {
checkerCount++;
if (checkerCount === 1) return mockCheckerFail();
return mockCheckerPass();
},
tester: mockTesterPass,
});
const rule = createWorkflowRule(wf, store);
const hash = await store.putObject('test retry');
const hash = await store.putObject('test checker retry');
await store.appendEvent({
occurredAt: Date.now(),
kind: 'meta.__start__',
key: 'retry-1',
key: 'retry-checker-1',
hash,
});
const roles: string[] = [];
for (let i = 0; i < 8; i++) {
for (let i = 0; i < 10; i++) {
const r = await rule.tick();
if (r.executed.length === 0) break;
roles.push(...r.executed.map((a) => a.role));
}
// coder → tester(fail) → coder → tester(pass) → END
expect(roles).toEqual(['coder', 'tester', 'coder', 'tester']);
expect(roles).toEqual(['coder', 'checker', 'coder', 'checker', 'tester']);
await store.close();
});
test('tester fail → retry: coder→checker→tester(fail)→coder→checker→tester(pass)', async () => {
const store = mockStore();
let testCount = 0;
const wf = createMetaWorkflow({
coder: mockCoder,
checker: mockCheckerPass,
tester: async () => {
testCount++;
if (testCount === 1) return mockTesterFail();
return mockTesterPass();
},
});
const rule = createWorkflowRule(wf, store);
const hash = await store.putObject('test tester retry');
await store.appendEvent({
occurredAt: Date.now(),
kind: 'meta.__start__',
key: 'retry-tester-1',
hash,
});
const roles: string[] = [];
for (let i = 0; i < 12; i++) {
const r = await rule.tick();
if (r.executed.length === 0) break;
roles.push(...r.executed.map((a) => a.role));
}
// coder→checker→tester(fail)→coder→checker→tester(pass)
expect(roles).toEqual(['coder', 'checker', 'tester', 'coder', 'checker', 'tester']);
await store.close();
});
});
+20 -5
View File
@@ -2,13 +2,16 @@
* Meta Workflow — workflow for developing workflows.
*
* Roles:
* coder (Agent) → implement + self-test
* tester (code) → build + test + commit/push on pass
* coder (Agent) → implement code
* checker (code) → validate file scope + build + unit test
* tester (code) → e2e lifecycle verification + commit/push on pass
*
* Flow:
* START → coder → tester
* START → coder → checker
* → fail → coder (file scope violation or build/test fail)
* checker pass → tester
* → pass → END (with commit + push)
* → fail → coder (retry)
* → fail → coder (e2e fail, with diagnostic)
*
* 小橘 🍊 (NEKO Team)
*/
@@ -29,6 +32,13 @@ export interface MetaCoderMeta {
testsPassed: boolean;
}
export interface MetaCheckerMeta {
[key: string]: unknown;
pass: boolean;
reason: string;
violations?: string[];
}
export interface MetaTesterMeta {
[key: string]: unknown;
pass: boolean;
@@ -40,6 +50,7 @@ export interface MetaTesterMeta {
export type MetaWorkflowRoles = {
coder: Role<MetaCoderMeta>;
checker: Role<MetaCheckerMeta>;
tester: Role<MetaTesterMeta>;
};
@@ -53,7 +64,11 @@ function metaModerator(
case START:
return 'coder';
case 'coder':
return 'tester';
return 'checker';
case 'checker': {
const meta = input.meta as MetaCheckerMeta | null;
return meta?.pass ? 'tester' : 'coder';
}
case 'tester': {
const meta = input.meta as MetaTesterMeta | null;
return meta?.pass ? END : 'coder';
@@ -0,0 +1,132 @@
/**
* Meta Checker role — validates coder output before e2e testing.
*
* Checks:
* 1. All changed files are within the allowed directory (engine dir)
* 2. No modifications to pulse core files
* 3. Build succeeds
* 4. Unit tests pass
*
* Pure code, no LLM.
*
* 小橘 🍊 (NEKO Team)
*/
import { execSync } from 'node:child_process';
import type { Role, RoleResult, WorkflowMessage } from '../workflow-type.js';
export interface MetaCheckerMeta {
[key: string]: unknown;
pass: boolean;
reason: string;
violations?: string[];
}
export function createMetaCheckerRole(opts: {
/** Engine repo directory (allowed file scope) */
engineDir: string;
/** Extra allowed path prefixes (optional) */
allowedPrefixes?: string[];
}): Role<MetaCheckerMeta> {
return async (
_chain: WorkflowMessage[],
): Promise<RoleResult<MetaCheckerMeta>> => {
const cwd = opts.engineDir;
const exec = (cmd: string) =>
execSync(cmd, { cwd, encoding: 'utf-8', timeout: 30_000 }).trim();
const violations: string[] = [];
// 1. Check git diff — what files did coder change?
let changedFiles: string[] = [];
try {
// Get all uncommitted changes + last commit changes
const diffOutput = exec('git diff --name-only HEAD~1 HEAD 2>/dev/null || git diff --name-only');
changedFiles = diffOutput.split('\n').filter(Boolean);
} catch {
// No git history — check working tree
try {
const statusOutput = exec('git status --porcelain');
changedFiles = statusOutput
.split('\n')
.filter(Boolean)
.map((line) => line.slice(3).trim());
} catch {
// Can't check — skip file validation
}
}
if (changedFiles.length > 0) {
// Allowed: src/workflows/*, docs/*, tests under src/
const allowedPrefixes = [
'src/',
'docs/',
'test/',
'tests/',
...(opts.allowedPrefixes ?? []),
];
for (const file of changedFiles) {
const allowed = allowedPrefixes.some((prefix) =>
file.startsWith(prefix),
);
if (!allowed) {
violations.push(`越界文件: ${file}(只允许修改 ${allowedPrefixes.join(', ')} 下的文件)`);
}
}
// Blacklist: never touch these even if under src/
const blacklist = [
'package.json',
'tsconfig.json',
'bun.lockb',
'.gitignore',
];
for (const file of changedFiles) {
const basename = file.split('/').pop() ?? file;
if (blacklist.includes(basename) && !file.startsWith('src/workflows/')) {
violations.push(`禁止修改: ${file}`);
}
}
}
if (violations.length > 0) {
return {
content: `约束检查失败\n\n${violations.map((v) => `${v}`).join('\n')}\n\n修改的文件:\n${changedFiles.map((f) => ` ${f}`).join('\n')}`,
meta: { pass: false, reason: '文件范围越界', violations },
};
}
// 2. Build check
try {
exec('bun run build 2>&1');
} catch (err: any) {
return {
content: `编译失败\n\n---\n${(err.stdout ?? err.message).slice(0, 2000)}`,
meta: { pass: false, reason: '编译失败' },
};
}
// 3. Unit test check
try {
const testOutput = exec('bun test src/workflows/ 2>&1');
// Check for failures
if (testOutput.includes('fail') && !testOutput.includes('0 fail')) {
return {
content: `单元测试失败\n\n---\n${testOutput.slice(-2000)}`,
meta: { pass: false, reason: '单元测试失败' },
};
}
} catch (err: any) {
return {
content: `单元测试失败\n\n---\n${(err.stdout ?? err.message).slice(0, 2000)}`,
meta: { pass: false, reason: '单元测试失败' },
};
}
return {
content: `约束检查通过\n\n修改文件: ${changedFiles.length}\n编译: ✅\n单元测试: ✅`,
meta: { pass: true, reason: '约束检查通过' },
};
};
}
@@ -34,20 +34,8 @@ export function createMetaTesterRole(opts: {
): Promise<RoleResult<MetaTesterMeta>> => {
const cwd = opts.repoDir;
// Step 0: Build first (coder should have done this, but verify)
let buildOutput = '';
try {
buildOutput = execSync('bun run build 2>&1', {
cwd,
timeout: 60_000,
encoding: 'utf-8',
});
} catch (err: any) {
return {
content: `编译失败\n\n---\n${(err.stdout ?? err.message).slice(0, 2000)}`,
meta: { pass: false, reason: '编译失败' },
};
}
// Step 0: Skip build — checker already verified build + unit tests
// Go straight to e2e verification
// Step 1: Discover workflow files in engine src/workflows/
const workflowDir = join(cwd, 'src', 'workflows');
@@ -147,12 +135,34 @@ export function createMetaTesterRole(opts: {
}
}
// Gather diagnostic info on failure
let diagnostic = '';
if (!completed) {
try {
const allEvents = await testStore.getAfter(0);
const wfEvents = allEvents.filter((ev: any) => ev.kind.startsWith(`${wf.name}.`));
const lastEvent = wfEvents[wfEvents.length - 1];
const roles = wfEvents.map((ev: any) => ev.kind.replace(`${wf.name}.`, '')).join(' → ');
diagnostic = `\n 事件链: ${roles}`;
if (lastEvent) {
diagnostic += `\n 最后事件: ${lastEvent.kind} (id=${lastEvent.id})`;
if (lastEvent.meta) {
try {
const meta = JSON.parse(lastEvent.meta);
diagnostic += `\n 最后 meta: ${JSON.stringify(meta)}`;
} catch {}
}
}
diagnostic += `\n 总事件数: ${wfEvents.length}`;
} catch {}
}
if (completed) {
results.push({ name: wf.name, pass: true, detail: `completed in ${tickCount} ticks` });
} else if (lastError) {
results.push({ name: wf.name, pass: false, detail: `error: ${lastError}` });
results.push({ name: wf.name, pass: false, detail: `error: ${lastError}${diagnostic}` });
} else {
results.push({ name: wf.name, pass: false, detail: `did not complete in ${tickCount} ticks` });
results.push({ name: wf.name, pass: false, detail: `did not complete in ${tickCount} ticks${diagnostic}` });
}
} finally {
await testStore.close();