feat: agent-hermes reads real token counts from session DB #84

Merged
xiaomo merged 2 commits from feat/76-hermes-real-tokens into main 2026-06-04 23:31:09 +00:00
7 changed files with 199 additions and 23 deletions
@@ -140,7 +140,9 @@ function createTestDb(dbPath: string): TestDb {
db.exec(`CREATE TABLE sessions (
id TEXT PRIMARY KEY,
model TEXT NOT NULL,
started_at INTEGER NOT NULL
started_at INTEGER NOT NULL,
input_tokens INTEGER DEFAULT 0,
output_tokens INTEGER DEFAULT 0
)`);
db.exec(`CREATE TABLE messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -0,0 +1,110 @@
import { describe, expect, test } from "vitest";
import { computeUsageDelta, snapshotUsage } from "../src/hermes.js";
import type { HermesSessionJson } from "../src/types.js";
function makeSession(overrides: Partial<HermesSessionJson> = {}): HermesSessionJson {
return {
session_id: "test-session",
model: "test-model",
session_start: "2026-01-01T00:00:00Z",
messages: [],
inputTokens: 0,
outputTokens: 0,
...overrides,
};
}
describe("snapshotUsage", () => {
test("returns zero snapshot for null session", () => {
const result = snapshotUsage(null);
expect(result).toEqual({ turns: 0, inputTokens: 0, outputTokens: 0 });
});
test("returns zero snapshot for empty session", () => {
const result = snapshotUsage(makeSession());
expect(result).toEqual({ turns: 0, inputTokens: 0, outputTokens: 0 });
});
test("counts assistant messages as turns", () => {
const result = snapshotUsage(
makeSession({
messages: [
{ role: "user", content: "hello", reasoning: null, tool_calls: null },
{ role: "assistant", content: "hi", reasoning: null, tool_calls: null },
{ role: "user", content: "do X", reasoning: null, tool_calls: null },
{ role: "tool", content: "result", reasoning: null, tool_calls: null },
{ role: "assistant", content: "done", reasoning: null, tool_calls: null },
],
inputTokens: 1000,
outputTokens: 500,
}),
);
expect(result).toEqual({ turns: 2, inputTokens: 1000, outputTokens: 500 });
});
test("ignores non-assistant messages for turn count", () => {
const result = snapshotUsage(
makeSession({
messages: [
{ role: "user", content: "hello", reasoning: null, tool_calls: null },
{ role: "tool", content: "result", reasoning: null, tool_calls: null },
],
}),
);
expect(result.turns).toBe(0);
});
});
describe("computeUsageDelta", () => {
test("first visit: before is zero, after has all values", () => {
const before = { turns: 0, inputTokens: 0, outputTokens: 0 };
const after = { turns: 3, inputTokens: 5000, outputTokens: 2000 };
const result = computeUsageDelta(before, after, 12.5);
expect(result).toEqual({
turns: 3,
inputTokens: 5000,
outputTokens: 2000,
duration: 13,
});
});
test("re-entry: computes delta correctly", () => {
const before = { turns: 2, inputTokens: 3000, outputTokens: 1000 };
const after = { turns: 4, inputTokens: 8000, outputTokens: 3500 };
const result = computeUsageDelta(before, after, 7.3);
expect(result).toEqual({
turns: 2,
inputTokens: 5000,
outputTokens: 2500,
duration: 7,
});
});
test("floors negative deltas at 0 (defensive)", () => {
const before = { turns: 5, inputTokens: 10000, outputTokens: 5000 };
const after = { turns: 3, inputTokens: 8000, outputTokens: 4000 };
const result = computeUsageDelta(before, after, 1.0);
// turns would be negative (-2), floored to 0, then || 1 gives 1
expect(result.turns).toBe(1);
expect(result.inputTokens).toBe(0);
expect(result.outputTokens).toBe(0);
});
test("zero turns delta defaults to 1 (at least one turn happened)", () => {
const before = { turns: 3, inputTokens: 1000, outputTokens: 500 };
const after = { turns: 3, inputTokens: 2000, outputTokens: 1000 };
const result = computeUsageDelta(before, after, 5.0);
// turns delta is 0, || 1 gives 1
expect(result.turns).toBe(1);
expect(result.inputTokens).toBe(1000);
expect(result.outputTokens).toBe(500);
});
test("duration is rounded", () => {
const before = { turns: 0, inputTokens: 0, outputTokens: 0 };
const after = { turns: 1, inputTokens: 100, outputTokens: 50 };
expect(computeUsageDelta(before, after, 3.7).duration).toBe(4);
expect(computeUsageDelta(before, after, 3.2).duration).toBe(3);
expect(computeUsageDelta(before, after, 0.0).duration).toBe(0);
});
});
+5
View File
@@ -72,6 +72,11 @@ export class HermesAcpClient {
return sessionId;
}
/** Return the current session ID, or null if not connected. */
getSessionId(): string | null {
return this.sessionId;
}
/** Send prompt and collect final assistant text from ACP stream chunks. */
async prompt(text: string): Promise<AcpPromptResult> {
if (this.sessionId === null) {
+65 -19
View File
@@ -12,9 +12,45 @@ import {
import { HermesAcpClient } from "./acp-client.js";
import { getCachedSessionId, setCachedSessionId } from "./session-cache.js";
import { loadHermesSession, storeHermesSessionDetail } from "./session-detail.js";
import type { HermesSessionJson } from "./types.js";
const log = createLogger({ sink: { kind: "stderr" } });
/** Snapshot of session metrics taken before and after a prompt call. */
type UsageSnapshot = {
turns: number;
inputTokens: number;
outputTokens: number;
};
const ZERO_SNAPSHOT: UsageSnapshot = { turns: 0, inputTokens: 0, outputTokens: 0 };
/** Extract usage metrics from a session. Returns zeros for null sessions. */
export function snapshotUsage(session: HermesSessionJson | null): UsageSnapshot {
if (session === null) {
return ZERO_SNAPSHOT;
}
return {
turns: session.messages.filter((m) => m.role === "assistant").length,
inputTokens: session.inputTokens,
outputTokens: session.outputTokens,
};
}
/** Compute the delta between two snapshots (after minus before). Floors at 0. */
export function computeUsageDelta(
before: UsageSnapshot,
after: UsageSnapshot,
durationSec: number,
): Usage {
return {
turns: Math.max(0, after.turns - before.turns) || 1,
inputTokens: Math.max(0, after.inputTokens - before.inputTokens),
outputTokens: Math.max(0, after.outputTokens - before.outputTokens),
duration: Math.round(durationSec),
};
}
/** Assemble system prompt, task, and prior step outputs for Hermes. */
export function buildHermesPrompt(ctx: AgentContext): string {
const parts: string[] = [];
@@ -109,7 +145,11 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
void client.close();
});
async function runPrompt(ctx: AgentContext, useContinuation: boolean): Promise<AgentRunResult> {
async function runPrompt(
ctx: AgentContext,
useContinuation: boolean,
beforeSnapshot: UsageSnapshot,
): Promise<AgentRunResult> {
const effectiveCtx = useContinuation ? ctx : { ...ctx, isFirstVisit: true };
const fullPrompt = buildHermesPrompt(effectiveCtx);
const startMs = Date.now();
@@ -121,16 +161,9 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
await setCachedSessionId(ctx.threadId, ctx.role, sessionId, ctx.storageRoot);
}
const session = await loadHermesSession(sessionId);
const turns =
session !== null ? session.messages.filter((m) => m.role === "assistant").length : 1;
const usage: Usage = {
turns,
inputTokens: 0,
outputTokens: 0,
duration: Math.round(durationSec),
};
const afterSession = await loadHermesSession(sessionId);
const afterSnapshot = snapshotUsage(afterSession);
const usage = computeUsageDelta(beforeSnapshot, afterSnapshot, durationSec);
return { output: text, detailHash, sessionId, assembledPrompt: fullPrompt, usage };
}
@@ -139,8 +172,17 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
const cwd = process.cwd();
const attempt = await prepareSession(client, ctx, cwd, resumeDisabled);
// Snapshot before prompt: for resumed sessions, captures cumulative state
// so we can compute the delta. For new sessions, this is ZERO_SNAPSHOT.
const currentSessionId = client.getSessionId();
const beforeSession =
attempt.resumed && currentSessionId !== null
? await loadHermesSession(currentSessionId)
: null;
const beforeSnapshot = snapshotUsage(beforeSession);
try {
return await runPrompt(ctx, attempt.useContinuation);
return await runPrompt(ctx, attempt.useContinuation, beforeSnapshot);
} catch (error) {
if (!attempt.resumed) {
throw error;
@@ -150,7 +192,8 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
log("8FQW2R6N", `continuation prompt failed, retrying with initial prompt: ${message}`);
await client.close();
await client.connect(cwd);
return runPrompt(ctx, false);
// Fresh session after retry — reset snapshot to zero
return runPrompt(ctx, false, ZERO_SNAPSHOT);
}
}
@@ -161,17 +204,20 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
): Promise<AgentRunResult> {
// Client is already connected from runHermes — same ACP session,
// so the agent sees the full conversation history (crucial for retries).
// Snapshot before the continuation prompt for delta computation.
const currentSessionId = client.getSessionId();
const beforeSession =
currentSessionId !== null ? await loadHermesSession(currentSessionId) : null;
const beforeSnapshot = snapshotUsage(beforeSession);
const startMs = Date.now();
const { text, sessionId } = await client.prompt(message);
const durationSec = (Date.now() - startMs) / 1000;
const { detailHash } = await storePromptResult(store, sessionId);
const usage: Usage = {
turns: 1,
inputTokens: 0,
outputTokens: 0,
duration: Math.round(durationSec),
};
const afterSession = await loadHermesSession(sessionId);
const afterSnapshot = snapshotUsage(afterSession);
const usage = computeUsageDelta(beforeSnapshot, afterSnapshot, durationSec);
return { output: text, detailHash, sessionId, assembledPrompt: "", usage };
}
+6 -1
View File
@@ -1,2 +1,7 @@
export { HermesAcpClient } from "./acp-client.js";
export { buildHermesPrompt, createHermesAgent } from "./hermes.js";
export {
buildHermesPrompt,
computeUsageDelta,
createHermesAgent,
snapshotUsage,
} from "./hermes.js";
+8 -2
View File
@@ -106,7 +106,7 @@ function parseSessionJson(raw: unknown): HermesSessionJson | null {
messages.push(msg);
}
}
return { session_id, model, session_start, messages };
return { session_id, model, session_start, messages, inputTokens: 0, outputTokens: 0 };
}
export function getHermesDbPath(): string {
@@ -117,6 +117,8 @@ type DbSessionRow = {
id: string;
model: string;
started_at: number;
input_tokens: number;
output_tokens: number;
};
type DbMessageRow = {
@@ -156,7 +158,9 @@ export function loadHermesSessionFromDb(
try {
db = new DatabaseSync(resolvedPath, { readOnly: true });
const session = db
.prepare("SELECT id, model, started_at FROM sessions WHERE id = ?")
.prepare(
"SELECT id, model, started_at, input_tokens, output_tokens FROM sessions WHERE id = ?",
)
.get(sessionId) as DbSessionRow | null;
if (session === null) {
return null;
@@ -181,6 +185,8 @@ export function loadHermesSessionFromDb(
model: session.model,
session_start: new Date(session.started_at * 1000).toISOString(),
messages,
inputTokens: session.input_tokens ?? 0,
outputTokens: session.output_tokens ?? 0,
};
} catch {
return null;
+2
View File
@@ -40,4 +40,6 @@ export type HermesSessionJson = {
model: string;
session_start: string;
messages: HermesSessionMessage[];
inputTokens: number;
outputTokens: number;
};