feat: add checker role to meta workflow (coder→checker→tester→END)
CI / test (push) Has been cancelled
CI / test (push) Has been cancelled
- checker validates file scope, build, unit tests before e2e - tester enhanced with diagnostic info on failure (event chain, last role, meta) - tester no longer duplicates build (checker handles it) - 6 meta workflow tests, 78 total pass
This commit is contained in:
@@ -20,6 +20,7 @@ import { createMetaWorkflow } from '../workflows/meta.js';
|
||||
import { createCursorRunner } from '../workflows/roles/agent-executor.js';
|
||||
import { createMetaCoderRole } from '../workflows/roles/meta-coder-cursor.js';
|
||||
import { createMetaTesterRole } from '../workflows/roles/meta-tester.js';
|
||||
import { createMetaCheckerRole } from '../workflows/roles/meta-checker.js';
|
||||
import { createReportWorkflow } from '../workflows/report.js';
|
||||
import { createAnalystRole } from '../workflows/roles/analyst-llm.js';
|
||||
import { createRendererRole } from '../workflows/roles/renderer-template.js';
|
||||
@@ -80,6 +81,7 @@ const codingRule = createWorkflowRule(codingWf, store, logStore);
|
||||
// 2. Meta workflow (simplified: coder → tester → promoter)
|
||||
const metaWf = createMetaWorkflow({
|
||||
coder: createMetaCoderRole(cursorRunner, llm, ENGINE_DIR),
|
||||
checker: createMetaCheckerRole({ engineDir: ENGINE_DIR }),
|
||||
tester: createMetaTesterRole({ repoDir: ENGINE_DIR }),
|
||||
});
|
||||
const metaRule = createWorkflowRule(metaWf, store, logStore);
|
||||
|
||||
@@ -10,11 +10,12 @@ import { join } from 'node:path';
|
||||
import { createStore } from '../store.js';
|
||||
import {
|
||||
createMetaWorkflow,
|
||||
type MetaCheckerMeta,
|
||||
type MetaCoderMeta,
|
||||
type MetaTesterMeta,
|
||||
} from './meta.js';
|
||||
import { createWorkflowRule } from './workflow-rule-adapter.js';
|
||||
import { END, START, type WorkflowMessage } from './workflow-type.js';
|
||||
import { END, START } from './workflow-type.js';
|
||||
|
||||
function mockStore() {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'meta-wf-test-'));
|
||||
@@ -24,17 +25,37 @@ function mockStore() {
|
||||
});
|
||||
}
|
||||
|
||||
describe('Meta Workflow', () => {
|
||||
test('moderator routes: START→coder→tester(pass)→END', () => {
|
||||
const wf = createMetaWorkflow({
|
||||
coder: async () => ({
|
||||
const mockCoder = async () => ({
|
||||
content: 'ok',
|
||||
meta: { filesChanged: ['a.js'], testsPassed: true },
|
||||
}),
|
||||
tester: async () => ({
|
||||
meta: { filesChanged: ['a.ts'], testsPassed: true },
|
||||
});
|
||||
|
||||
const mockCheckerPass = async () => ({
|
||||
content: 'pass',
|
||||
meta: { pass: true, reason: '编译通过,测试全绿' },
|
||||
}),
|
||||
meta: { pass: true, reason: '约束检查通过' },
|
||||
});
|
||||
|
||||
const mockCheckerFail = async () => ({
|
||||
content: 'fail',
|
||||
meta: { pass: false, reason: '文件范围越界', violations: ['越界文件: package.json'] },
|
||||
});
|
||||
|
||||
const mockTesterPass = async () => ({
|
||||
content: 'pass',
|
||||
meta: { pass: true, reason: 'e2e verification passed' },
|
||||
});
|
||||
|
||||
const mockTesterFail = async () => ({
|
||||
content: 'fail',
|
||||
meta: { pass: false, reason: 'e2e verification failed' },
|
||||
});
|
||||
|
||||
describe('Meta Workflow', () => {
|
||||
test('moderator routes: START→coder→checker(pass)→tester(pass)→END', () => {
|
||||
const wf = createMetaWorkflow({
|
||||
coder: mockCoder,
|
||||
checker: mockCheckerPass,
|
||||
tester: mockTesterPass,
|
||||
});
|
||||
|
||||
expect(wf.moderator({ role: START, meta: null }, 'x')).toBe('coder');
|
||||
@@ -43,36 +64,60 @@ describe('Meta Workflow', () => {
|
||||
{ role: 'coder', meta: { filesChanged: [], testsPassed: true } },
|
||||
'x',
|
||||
),
|
||||
).toBe('checker');
|
||||
expect(
|
||||
wf.moderator(
|
||||
{ role: 'checker', meta: { pass: true, reason: 'ok' } },
|
||||
'x',
|
||||
),
|
||||
).toBe('tester');
|
||||
expect(
|
||||
wf.moderator(
|
||||
{ role: 'tester', meta: { pass: true, reason: '通过' } },
|
||||
{ role: 'tester', meta: { pass: true, reason: 'ok' } },
|
||||
'x',
|
||||
),
|
||||
).toBe(END);
|
||||
});
|
||||
|
||||
test('checker fail → back to coder', () => {
|
||||
const wf = createMetaWorkflow({
|
||||
coder: mockCoder,
|
||||
checker: mockCheckerPass,
|
||||
tester: mockTesterPass,
|
||||
});
|
||||
|
||||
expect(
|
||||
wf.moderator(
|
||||
{ role: 'tester', meta: { pass: false, reason: '失败' } },
|
||||
{ role: 'checker', meta: { pass: false, reason: '越界' } },
|
||||
'x',
|
||||
),
|
||||
).toBe('coder');
|
||||
});
|
||||
|
||||
test('mock: full happy path via adapter ticks', async () => {
|
||||
test('tester fail → back to coder', () => {
|
||||
const wf = createMetaWorkflow({
|
||||
coder: mockCoder,
|
||||
checker: mockCheckerPass,
|
||||
tester: mockTesterPass,
|
||||
});
|
||||
|
||||
expect(
|
||||
wf.moderator(
|
||||
{ role: 'tester', meta: { pass: false, reason: 'e2e fail' } },
|
||||
'x',
|
||||
),
|
||||
).toBe('coder');
|
||||
});
|
||||
|
||||
test('happy path: coder→checker→tester via adapter ticks', async () => {
|
||||
const store = mockStore();
|
||||
const wf = createMetaWorkflow({
|
||||
coder: async () => ({
|
||||
content: 'files changed',
|
||||
meta: { filesChanged: ['demo.js'], testsPassed: true },
|
||||
}),
|
||||
tester: async () => ({
|
||||
content: 'all pass',
|
||||
meta: { pass: true, reason: '编译通过,测试全绿' },
|
||||
}),
|
||||
coder: mockCoder,
|
||||
checker: mockCheckerPass,
|
||||
tester: mockTesterPass,
|
||||
});
|
||||
|
||||
const rule = createWorkflowRule(wf, store);
|
||||
|
||||
const hash = await store.putObject('build a demo workflow');
|
||||
await store.appendEvent({
|
||||
occurredAt: Date.now(),
|
||||
@@ -82,60 +127,80 @@ describe('Meta Workflow', () => {
|
||||
});
|
||||
|
||||
const roles: string[] = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
for (let i = 0; i < 6; i++) {
|
||||
const r = await rule.tick();
|
||||
if (r.executed.length === 0) break;
|
||||
roles.push(...r.executed.map((a) => a.role));
|
||||
}
|
||||
|
||||
expect(roles).toEqual(['coder', 'tester']);
|
||||
|
||||
const events = await store.getAfter(0);
|
||||
expect(events.length).toBe(3); // __start__ + coder + tester
|
||||
|
||||
expect(roles).toEqual(['coder', 'checker', 'tester']);
|
||||
await store.close();
|
||||
});
|
||||
|
||||
test('mock: tester failure → retry coder', async () => {
|
||||
test('checker fail → retry: coder→checker(fail)→coder→checker(pass)→tester', async () => {
|
||||
const store = mockStore();
|
||||
let testCount = 0;
|
||||
let checkerCount = 0;
|
||||
const wf = createMetaWorkflow({
|
||||
coder: async () => ({
|
||||
content: 'code',
|
||||
meta: { filesChanged: [], testsPassed: true },
|
||||
}),
|
||||
tester: async () => {
|
||||
testCount++;
|
||||
if (testCount === 1)
|
||||
return {
|
||||
content: 'build failed',
|
||||
meta: { pass: false, reason: '编译失败' },
|
||||
};
|
||||
return {
|
||||
content: 'ok',
|
||||
meta: { pass: true, reason: '编译通过,测试全绿' },
|
||||
};
|
||||
coder: mockCoder,
|
||||
checker: async () => {
|
||||
checkerCount++;
|
||||
if (checkerCount === 1) return mockCheckerFail();
|
||||
return mockCheckerPass();
|
||||
},
|
||||
tester: mockTesterPass,
|
||||
});
|
||||
|
||||
const rule = createWorkflowRule(wf, store);
|
||||
const hash = await store.putObject('test retry');
|
||||
const hash = await store.putObject('test checker retry');
|
||||
await store.appendEvent({
|
||||
occurredAt: Date.now(),
|
||||
kind: 'meta.__start__',
|
||||
key: 'retry-1',
|
||||
key: 'retry-checker-1',
|
||||
hash,
|
||||
});
|
||||
|
||||
const roles: string[] = [];
|
||||
for (let i = 0; i < 8; i++) {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const r = await rule.tick();
|
||||
if (r.executed.length === 0) break;
|
||||
roles.push(...r.executed.map((a) => a.role));
|
||||
}
|
||||
|
||||
// coder → tester(fail) → coder → tester(pass) → END
|
||||
expect(roles).toEqual(['coder', 'tester', 'coder', 'tester']);
|
||||
expect(roles).toEqual(['coder', 'checker', 'coder', 'checker', 'tester']);
|
||||
await store.close();
|
||||
});
|
||||
|
||||
test('tester fail → retry: coder→checker→tester(fail)→coder→checker→tester(pass)', async () => {
|
||||
const store = mockStore();
|
||||
let testCount = 0;
|
||||
const wf = createMetaWorkflow({
|
||||
coder: mockCoder,
|
||||
checker: mockCheckerPass,
|
||||
tester: async () => {
|
||||
testCount++;
|
||||
if (testCount === 1) return mockTesterFail();
|
||||
return mockTesterPass();
|
||||
},
|
||||
});
|
||||
|
||||
const rule = createWorkflowRule(wf, store);
|
||||
const hash = await store.putObject('test tester retry');
|
||||
await store.appendEvent({
|
||||
occurredAt: Date.now(),
|
||||
kind: 'meta.__start__',
|
||||
key: 'retry-tester-1',
|
||||
hash,
|
||||
});
|
||||
|
||||
const roles: string[] = [];
|
||||
for (let i = 0; i < 12; i++) {
|
||||
const r = await rule.tick();
|
||||
if (r.executed.length === 0) break;
|
||||
roles.push(...r.executed.map((a) => a.role));
|
||||
}
|
||||
|
||||
// coder→checker→tester(fail)→coder→checker→tester(pass)
|
||||
expect(roles).toEqual(['coder', 'checker', 'tester', 'coder', 'checker', 'tester']);
|
||||
await store.close();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,13 +2,16 @@
|
||||
* Meta Workflow — workflow for developing workflows.
|
||||
*
|
||||
* Roles:
|
||||
* coder (Agent) → implement + self-test
|
||||
* tester (code) → build + test + commit/push on pass
|
||||
* coder (Agent) → implement code
|
||||
* checker (code) → validate file scope + build + unit test
|
||||
* tester (code) → e2e lifecycle verification + commit/push on pass
|
||||
*
|
||||
* Flow:
|
||||
* START → coder → tester
|
||||
* START → coder → checker
|
||||
* → fail → coder (file scope violation or build/test fail)
|
||||
* checker pass → tester
|
||||
* → pass → END (with commit + push)
|
||||
* → fail → coder (retry)
|
||||
* → fail → coder (e2e fail, with diagnostic)
|
||||
*
|
||||
* 小橘 🍊 (NEKO Team)
|
||||
*/
|
||||
@@ -29,6 +32,13 @@ export interface MetaCoderMeta {
|
||||
testsPassed: boolean;
|
||||
}
|
||||
|
||||
export interface MetaCheckerMeta {
|
||||
[key: string]: unknown;
|
||||
pass: boolean;
|
||||
reason: string;
|
||||
violations?: string[];
|
||||
}
|
||||
|
||||
export interface MetaTesterMeta {
|
||||
[key: string]: unknown;
|
||||
pass: boolean;
|
||||
@@ -40,6 +50,7 @@ export interface MetaTesterMeta {
|
||||
|
||||
export type MetaWorkflowRoles = {
|
||||
coder: Role<MetaCoderMeta>;
|
||||
checker: Role<MetaCheckerMeta>;
|
||||
tester: Role<MetaTesterMeta>;
|
||||
};
|
||||
|
||||
@@ -53,7 +64,11 @@ function metaModerator(
|
||||
case START:
|
||||
return 'coder';
|
||||
case 'coder':
|
||||
return 'tester';
|
||||
return 'checker';
|
||||
case 'checker': {
|
||||
const meta = input.meta as MetaCheckerMeta | null;
|
||||
return meta?.pass ? 'tester' : 'coder';
|
||||
}
|
||||
case 'tester': {
|
||||
const meta = input.meta as MetaTesterMeta | null;
|
||||
return meta?.pass ? END : 'coder';
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* Meta Checker role — validates coder output before e2e testing.
|
||||
*
|
||||
* Checks:
|
||||
* 1. All changed files are within the allowed directory (engine dir)
|
||||
* 2. No modifications to pulse core files
|
||||
* 3. Build succeeds
|
||||
* 4. Unit tests pass
|
||||
*
|
||||
* Pure code, no LLM.
|
||||
*
|
||||
* 小橘 🍊 (NEKO Team)
|
||||
*/
|
||||
|
||||
import { execSync } from 'node:child_process';
|
||||
import type { Role, RoleResult, WorkflowMessage } from '../workflow-type.js';
|
||||
|
||||
export interface MetaCheckerMeta {
|
||||
[key: string]: unknown;
|
||||
pass: boolean;
|
||||
reason: string;
|
||||
violations?: string[];
|
||||
}
|
||||
|
||||
export function createMetaCheckerRole(opts: {
|
||||
/** Engine repo directory (allowed file scope) */
|
||||
engineDir: string;
|
||||
/** Extra allowed path prefixes (optional) */
|
||||
allowedPrefixes?: string[];
|
||||
}): Role<MetaCheckerMeta> {
|
||||
return async (
|
||||
_chain: WorkflowMessage[],
|
||||
): Promise<RoleResult<MetaCheckerMeta>> => {
|
||||
const cwd = opts.engineDir;
|
||||
const exec = (cmd: string) =>
|
||||
execSync(cmd, { cwd, encoding: 'utf-8', timeout: 30_000 }).trim();
|
||||
|
||||
const violations: string[] = [];
|
||||
|
||||
// 1. Check git diff — what files did coder change?
|
||||
let changedFiles: string[] = [];
|
||||
try {
|
||||
// Get all uncommitted changes + last commit changes
|
||||
const diffOutput = exec('git diff --name-only HEAD~1 HEAD 2>/dev/null || git diff --name-only');
|
||||
changedFiles = diffOutput.split('\n').filter(Boolean);
|
||||
} catch {
|
||||
// No git history — check working tree
|
||||
try {
|
||||
const statusOutput = exec('git status --porcelain');
|
||||
changedFiles = statusOutput
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => line.slice(3).trim());
|
||||
} catch {
|
||||
// Can't check — skip file validation
|
||||
}
|
||||
}
|
||||
|
||||
if (changedFiles.length > 0) {
|
||||
// Allowed: src/workflows/*, docs/*, tests under src/
|
||||
const allowedPrefixes = [
|
||||
'src/',
|
||||
'docs/',
|
||||
'test/',
|
||||
'tests/',
|
||||
...(opts.allowedPrefixes ?? []),
|
||||
];
|
||||
|
||||
for (const file of changedFiles) {
|
||||
const allowed = allowedPrefixes.some((prefix) =>
|
||||
file.startsWith(prefix),
|
||||
);
|
||||
if (!allowed) {
|
||||
violations.push(`越界文件: ${file}(只允许修改 ${allowedPrefixes.join(', ')} 下的文件)`);
|
||||
}
|
||||
}
|
||||
|
||||
// Blacklist: never touch these even if under src/
|
||||
const blacklist = [
|
||||
'package.json',
|
||||
'tsconfig.json',
|
||||
'bun.lockb',
|
||||
'.gitignore',
|
||||
];
|
||||
for (const file of changedFiles) {
|
||||
const basename = file.split('/').pop() ?? file;
|
||||
if (blacklist.includes(basename) && !file.startsWith('src/workflows/')) {
|
||||
violations.push(`禁止修改: ${file}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (violations.length > 0) {
|
||||
return {
|
||||
content: `约束检查失败\n\n${violations.map((v) => `❌ ${v}`).join('\n')}\n\n修改的文件:\n${changedFiles.map((f) => ` ${f}`).join('\n')}`,
|
||||
meta: { pass: false, reason: '文件范围越界', violations },
|
||||
};
|
||||
}
|
||||
|
||||
// 2. Build check
|
||||
try {
|
||||
exec('bun run build 2>&1');
|
||||
} catch (err: any) {
|
||||
return {
|
||||
content: `编译失败\n\n---\n${(err.stdout ?? err.message).slice(0, 2000)}`,
|
||||
meta: { pass: false, reason: '编译失败' },
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Unit test check
|
||||
try {
|
||||
const testOutput = exec('bun test src/workflows/ 2>&1');
|
||||
// Check for failures
|
||||
if (testOutput.includes('fail') && !testOutput.includes('0 fail')) {
|
||||
return {
|
||||
content: `单元测试失败\n\n---\n${testOutput.slice(-2000)}`,
|
||||
meta: { pass: false, reason: '单元测试失败' },
|
||||
};
|
||||
}
|
||||
} catch (err: any) {
|
||||
return {
|
||||
content: `单元测试失败\n\n---\n${(err.stdout ?? err.message).slice(0, 2000)}`,
|
||||
meta: { pass: false, reason: '单元测试失败' },
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: `约束检查通过\n\n修改文件: ${changedFiles.length} 个\n编译: ✅\n单元测试: ✅`,
|
||||
meta: { pass: true, reason: '约束检查通过' },
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -34,20 +34,8 @@ export function createMetaTesterRole(opts: {
|
||||
): Promise<RoleResult<MetaTesterMeta>> => {
|
||||
const cwd = opts.repoDir;
|
||||
|
||||
// Step 0: Build first (coder should have done this, but verify)
|
||||
let buildOutput = '';
|
||||
try {
|
||||
buildOutput = execSync('bun run build 2>&1', {
|
||||
cwd,
|
||||
timeout: 60_000,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
} catch (err: any) {
|
||||
return {
|
||||
content: `编译失败\n\n---\n${(err.stdout ?? err.message).slice(0, 2000)}`,
|
||||
meta: { pass: false, reason: '编译失败' },
|
||||
};
|
||||
}
|
||||
// Step 0: Skip build — checker already verified build + unit tests
|
||||
// Go straight to e2e verification
|
||||
|
||||
// Step 1: Discover workflow files in engine src/workflows/
|
||||
const workflowDir = join(cwd, 'src', 'workflows');
|
||||
@@ -147,12 +135,34 @@ export function createMetaTesterRole(opts: {
|
||||
}
|
||||
}
|
||||
|
||||
// Gather diagnostic info on failure
|
||||
let diagnostic = '';
|
||||
if (!completed) {
|
||||
try {
|
||||
const allEvents = await testStore.getAfter(0);
|
||||
const wfEvents = allEvents.filter((ev: any) => ev.kind.startsWith(`${wf.name}.`));
|
||||
const lastEvent = wfEvents[wfEvents.length - 1];
|
||||
const roles = wfEvents.map((ev: any) => ev.kind.replace(`${wf.name}.`, '')).join(' → ');
|
||||
diagnostic = `\n 事件链: ${roles}`;
|
||||
if (lastEvent) {
|
||||
diagnostic += `\n 最后事件: ${lastEvent.kind} (id=${lastEvent.id})`;
|
||||
if (lastEvent.meta) {
|
||||
try {
|
||||
const meta = JSON.parse(lastEvent.meta);
|
||||
diagnostic += `\n 最后 meta: ${JSON.stringify(meta)}`;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
diagnostic += `\n 总事件数: ${wfEvents.length}`;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
if (completed) {
|
||||
results.push({ name: wf.name, pass: true, detail: `completed in ${tickCount} ticks` });
|
||||
} else if (lastError) {
|
||||
results.push({ name: wf.name, pass: false, detail: `error: ${lastError}` });
|
||||
results.push({ name: wf.name, pass: false, detail: `error: ${lastError}${diagnostic}` });
|
||||
} else {
|
||||
results.push({ name: wf.name, pass: false, detail: `did not complete in ${tickCount} ticks` });
|
||||
results.push({ name: wf.name, pass: false, detail: `did not complete in ${tickCount} ticks${diagnostic}` });
|
||||
}
|
||||
} finally {
|
||||
await testStore.close();
|
||||
|
||||
Reference in New Issue
Block a user