diff --git a/packages/agent-hermes/__tests__/usage-delta.test.ts b/packages/agent-hermes/__tests__/usage-delta.test.ts new file mode 100644 index 0000000..12bceb0 --- /dev/null +++ b/packages/agent-hermes/__tests__/usage-delta.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "vitest"; +import { computeUsageDelta, snapshotUsage } from "../src/hermes.js"; +import type { HermesSessionJson } from "../src/types.js"; + +function makeSession(overrides: Partial = {}): HermesSessionJson { + return { + session_id: "test-session", + model: "test-model", + session_start: "2026-01-01T00:00:00Z", + messages: [], + inputTokens: 0, + outputTokens: 0, + ...overrides, + }; +} + +describe("snapshotUsage", () => { + test("returns zero snapshot for null session", () => { + const result = snapshotUsage(null); + expect(result).toEqual({ turns: 0, inputTokens: 0, outputTokens: 0 }); + }); + + test("returns zero snapshot for empty session", () => { + const result = snapshotUsage(makeSession()); + expect(result).toEqual({ turns: 0, inputTokens: 0, outputTokens: 0 }); + }); + + test("counts assistant messages as turns", () => { + const result = snapshotUsage( + makeSession({ + messages: [ + { role: "user", content: "hello", reasoning: null, tool_calls: null }, + { role: "assistant", content: "hi", reasoning: null, tool_calls: null }, + { role: "user", content: "do X", reasoning: null, tool_calls: null }, + { role: "tool", content: "result", reasoning: null, tool_calls: null }, + { role: "assistant", content: "done", reasoning: null, tool_calls: null }, + ], + inputTokens: 1000, + outputTokens: 500, + }), + ); + expect(result).toEqual({ turns: 2, inputTokens: 1000, outputTokens: 500 }); + }); + + test("ignores non-assistant messages for turn count", () => { + const result = snapshotUsage( + makeSession({ + messages: [ + { role: "user", content: "hello", reasoning: null, tool_calls: null }, + { role: "tool", content: "result", reasoning: null, tool_calls: null }, + ], + }), + ); + expect(result.turns).toBe(0); + }); +}); + +describe("computeUsageDelta", () => { + test("first visit: before is zero, after has all values", () => { + const before = { turns: 0, inputTokens: 0, outputTokens: 0 }; + const after = { turns: 3, inputTokens: 5000, outputTokens: 2000 }; + const result = computeUsageDelta(before, after, 12.5); + expect(result).toEqual({ + turns: 3, + inputTokens: 5000, + outputTokens: 2000, + duration: 13, + }); + }); + + test("re-entry: computes delta correctly", () => { + const before = { turns: 2, inputTokens: 3000, outputTokens: 1000 }; + const after = { turns: 4, inputTokens: 8000, outputTokens: 3500 }; + const result = computeUsageDelta(before, after, 7.3); + expect(result).toEqual({ + turns: 2, + inputTokens: 5000, + outputTokens: 2500, + duration: 7, + }); + }); + + test("floors negative deltas at 0 (defensive)", () => { + const before = { turns: 5, inputTokens: 10000, outputTokens: 5000 }; + const after = { turns: 3, inputTokens: 8000, outputTokens: 4000 }; + const result = computeUsageDelta(before, after, 1.0); + // turns would be negative (-2), floored to 0, then || 1 gives 1 + expect(result.turns).toBe(1); + expect(result.inputTokens).toBe(0); + expect(result.outputTokens).toBe(0); + }); + + test("zero turns delta defaults to 1 (at least one turn happened)", () => { + const before = { turns: 3, inputTokens: 1000, outputTokens: 500 }; + const after = { turns: 3, inputTokens: 2000, outputTokens: 1000 }; + const result = computeUsageDelta(before, after, 5.0); + // turns delta is 0, || 1 gives 1 + expect(result.turns).toBe(1); + expect(result.inputTokens).toBe(1000); + expect(result.outputTokens).toBe(500); + }); + + test("duration is rounded", () => { + const before = { turns: 0, inputTokens: 0, outputTokens: 0 }; + const after = { turns: 1, inputTokens: 100, outputTokens: 50 }; + expect(computeUsageDelta(before, after, 3.7).duration).toBe(4); + expect(computeUsageDelta(before, after, 3.2).duration).toBe(3); + expect(computeUsageDelta(before, after, 0.0).duration).toBe(0); + }); +}); diff --git a/packages/agent-hermes/src/acp-client.ts b/packages/agent-hermes/src/acp-client.ts index 1624f52..307f7b7 100644 --- a/packages/agent-hermes/src/acp-client.ts +++ b/packages/agent-hermes/src/acp-client.ts @@ -72,6 +72,11 @@ export class HermesAcpClient { return sessionId; } + /** Return the current session ID, or null if not connected. */ + getSessionId(): string | null { + return this.sessionId; + } + /** Send prompt and collect final assistant text from ACP stream chunks. */ async prompt(text: string): Promise { if (this.sessionId === null) { diff --git a/packages/agent-hermes/src/hermes.ts b/packages/agent-hermes/src/hermes.ts index 76a700a..a525e32 100644 --- a/packages/agent-hermes/src/hermes.ts +++ b/packages/agent-hermes/src/hermes.ts @@ -12,9 +12,45 @@ import { import { HermesAcpClient } from "./acp-client.js"; import { getCachedSessionId, setCachedSessionId } from "./session-cache.js"; import { loadHermesSession, storeHermesSessionDetail } from "./session-detail.js"; +import type { HermesSessionJson } from "./types.js"; const log = createLogger({ sink: { kind: "stderr" } }); +/** Snapshot of session metrics taken before and after a prompt call. */ +type UsageSnapshot = { + turns: number; + inputTokens: number; + outputTokens: number; +}; + +const ZERO_SNAPSHOT: UsageSnapshot = { turns: 0, inputTokens: 0, outputTokens: 0 }; + +/** Extract usage metrics from a session. Returns zeros for null sessions. */ +export function snapshotUsage(session: HermesSessionJson | null): UsageSnapshot { + if (session === null) { + return ZERO_SNAPSHOT; + } + return { + turns: session.messages.filter((m) => m.role === "assistant").length, + inputTokens: session.inputTokens, + outputTokens: session.outputTokens, + }; +} + +/** Compute the delta between two snapshots (after minus before). Floors at 0. */ +export function computeUsageDelta( + before: UsageSnapshot, + after: UsageSnapshot, + durationSec: number, +): Usage { + return { + turns: Math.max(0, after.turns - before.turns) || 1, + inputTokens: Math.max(0, after.inputTokens - before.inputTokens), + outputTokens: Math.max(0, after.outputTokens - before.outputTokens), + duration: Math.round(durationSec), + }; +} + /** Assemble system prompt, task, and prior step outputs for Hermes. */ export function buildHermesPrompt(ctx: AgentContext): string { const parts: string[] = []; @@ -109,7 +145,11 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise void client.close(); }); - async function runPrompt(ctx: AgentContext, useContinuation: boolean): Promise { + async function runPrompt( + ctx: AgentContext, + useContinuation: boolean, + beforeSnapshot: UsageSnapshot, + ): Promise { const effectiveCtx = useContinuation ? ctx : { ...ctx, isFirstVisit: true }; const fullPrompt = buildHermesPrompt(effectiveCtx); const startMs = Date.now(); @@ -121,16 +161,9 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise await setCachedSessionId(ctx.threadId, ctx.role, sessionId, ctx.storageRoot); } - const session = await loadHermesSession(sessionId); - const turns = - session !== null ? session.messages.filter((m) => m.role === "assistant").length : 1; - - const usage: Usage = { - turns, - inputTokens: session !== null ? session.inputTokens : 0, - outputTokens: session !== null ? session.outputTokens : 0, - duration: Math.round(durationSec), - }; + const afterSession = await loadHermesSession(sessionId); + const afterSnapshot = snapshotUsage(afterSession); + const usage = computeUsageDelta(beforeSnapshot, afterSnapshot, durationSec); return { output: text, detailHash, sessionId, assembledPrompt: fullPrompt, usage }; } @@ -139,8 +172,17 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise const cwd = process.cwd(); const attempt = await prepareSession(client, ctx, cwd, resumeDisabled); + // Snapshot before prompt: for resumed sessions, captures cumulative state + // so we can compute the delta. For new sessions, this is ZERO_SNAPSHOT. + const currentSessionId = client.getSessionId(); + const beforeSession = + attempt.resumed && currentSessionId !== null + ? await loadHermesSession(currentSessionId) + : null; + const beforeSnapshot = snapshotUsage(beforeSession); + try { - return await runPrompt(ctx, attempt.useContinuation); + return await runPrompt(ctx, attempt.useContinuation, beforeSnapshot); } catch (error) { if (!attempt.resumed) { throw error; @@ -150,7 +192,8 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise log("8FQW2R6N", `continuation prompt failed, retrying with initial prompt: ${message}`); await client.close(); await client.connect(cwd); - return runPrompt(ctx, false); + // Fresh session after retry — reset snapshot to zero + return runPrompt(ctx, false, ZERO_SNAPSHOT); } } @@ -161,17 +204,20 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise ): Promise { // Client is already connected from runHermes — same ACP session, // so the agent sees the full conversation history (crucial for retries). + // Snapshot before the continuation prompt for delta computation. + const currentSessionId = client.getSessionId(); + const beforeSession = + currentSessionId !== null ? await loadHermesSession(currentSessionId) : null; + const beforeSnapshot = snapshotUsage(beforeSession); + const startMs = Date.now(); const { text, sessionId } = await client.prompt(message); const durationSec = (Date.now() - startMs) / 1000; const { detailHash } = await storePromptResult(store, sessionId); - const usage: Usage = { - turns: 1, - inputTokens: 0, - outputTokens: 0, - duration: Math.round(durationSec), - }; + const afterSession = await loadHermesSession(sessionId); + const afterSnapshot = snapshotUsage(afterSession); + const usage = computeUsageDelta(beforeSnapshot, afterSnapshot, durationSec); return { output: text, detailHash, sessionId, assembledPrompt: "", usage }; } diff --git a/packages/agent-hermes/src/index.ts b/packages/agent-hermes/src/index.ts index cf147d7..7de0bde 100644 --- a/packages/agent-hermes/src/index.ts +++ b/packages/agent-hermes/src/index.ts @@ -1,2 +1,7 @@ export { HermesAcpClient } from "./acp-client.js"; -export { buildHermesPrompt, createHermesAgent } from "./hermes.js"; +export { + buildHermesPrompt, + computeUsageDelta, + createHermesAgent, + snapshotUsage, +} from "./hermes.js";