feat: agent-hermes reads real token counts from session DB #84
@@ -140,7 +140,9 @@ function createTestDb(dbPath: string): TestDb {
|
||||
db.exec(`CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
model TEXT NOT NULL,
|
||||
started_at INTEGER NOT NULL
|
||||
started_at INTEGER NOT NULL,
|
||||
input_tokens INTEGER DEFAULT 0,
|
||||
output_tokens INTEGER DEFAULT 0
|
||||
)`);
|
||||
db.exec(`CREATE TABLE messages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { computeUsageDelta, snapshotUsage } from "../src/hermes.js";
|
||||
import type { HermesSessionJson } from "../src/types.js";
|
||||
|
||||
function makeSession(overrides: Partial<HermesSessionJson> = {}): HermesSessionJson {
|
||||
return {
|
||||
session_id: "test-session",
|
||||
model: "test-model",
|
||||
session_start: "2026-01-01T00:00:00Z",
|
||||
messages: [],
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("snapshotUsage", () => {
|
||||
test("returns zero snapshot for null session", () => {
|
||||
const result = snapshotUsage(null);
|
||||
expect(result).toEqual({ turns: 0, inputTokens: 0, outputTokens: 0 });
|
||||
});
|
||||
|
||||
test("returns zero snapshot for empty session", () => {
|
||||
const result = snapshotUsage(makeSession());
|
||||
expect(result).toEqual({ turns: 0, inputTokens: 0, outputTokens: 0 });
|
||||
});
|
||||
|
||||
test("counts assistant messages as turns", () => {
|
||||
const result = snapshotUsage(
|
||||
makeSession({
|
||||
messages: [
|
||||
{ role: "user", content: "hello", reasoning: null, tool_calls: null },
|
||||
{ role: "assistant", content: "hi", reasoning: null, tool_calls: null },
|
||||
{ role: "user", content: "do X", reasoning: null, tool_calls: null },
|
||||
{ role: "tool", content: "result", reasoning: null, tool_calls: null },
|
||||
{ role: "assistant", content: "done", reasoning: null, tool_calls: null },
|
||||
],
|
||||
inputTokens: 1000,
|
||||
outputTokens: 500,
|
||||
}),
|
||||
);
|
||||
expect(result).toEqual({ turns: 2, inputTokens: 1000, outputTokens: 500 });
|
||||
});
|
||||
|
||||
test("ignores non-assistant messages for turn count", () => {
|
||||
const result = snapshotUsage(
|
||||
makeSession({
|
||||
messages: [
|
||||
{ role: "user", content: "hello", reasoning: null, tool_calls: null },
|
||||
{ role: "tool", content: "result", reasoning: null, tool_calls: null },
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(result.turns).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeUsageDelta", () => {
|
||||
test("first visit: before is zero, after has all values", () => {
|
||||
const before = { turns: 0, inputTokens: 0, outputTokens: 0 };
|
||||
const after = { turns: 3, inputTokens: 5000, outputTokens: 2000 };
|
||||
const result = computeUsageDelta(before, after, 12.5);
|
||||
expect(result).toEqual({
|
||||
turns: 3,
|
||||
inputTokens: 5000,
|
||||
outputTokens: 2000,
|
||||
duration: 13,
|
||||
});
|
||||
});
|
||||
|
||||
test("re-entry: computes delta correctly", () => {
|
||||
const before = { turns: 2, inputTokens: 3000, outputTokens: 1000 };
|
||||
const after = { turns: 4, inputTokens: 8000, outputTokens: 3500 };
|
||||
const result = computeUsageDelta(before, after, 7.3);
|
||||
expect(result).toEqual({
|
||||
turns: 2,
|
||||
inputTokens: 5000,
|
||||
outputTokens: 2500,
|
||||
duration: 7,
|
||||
});
|
||||
});
|
||||
|
||||
test("floors negative deltas at 0 (defensive)", () => {
|
||||
const before = { turns: 5, inputTokens: 10000, outputTokens: 5000 };
|
||||
const after = { turns: 3, inputTokens: 8000, outputTokens: 4000 };
|
||||
const result = computeUsageDelta(before, after, 1.0);
|
||||
// turns would be negative (-2), floored to 0, then || 1 gives 1
|
||||
expect(result.turns).toBe(1);
|
||||
expect(result.inputTokens).toBe(0);
|
||||
expect(result.outputTokens).toBe(0);
|
||||
});
|
||||
|
||||
test("zero turns delta defaults to 1 (at least one turn happened)", () => {
|
||||
const before = { turns: 3, inputTokens: 1000, outputTokens: 500 };
|
||||
const after = { turns: 3, inputTokens: 2000, outputTokens: 1000 };
|
||||
const result = computeUsageDelta(before, after, 5.0);
|
||||
// turns delta is 0, || 1 gives 1
|
||||
expect(result.turns).toBe(1);
|
||||
expect(result.inputTokens).toBe(1000);
|
||||
expect(result.outputTokens).toBe(500);
|
||||
});
|
||||
|
||||
test("duration is rounded", () => {
|
||||
const before = { turns: 0, inputTokens: 0, outputTokens: 0 };
|
||||
const after = { turns: 1, inputTokens: 100, outputTokens: 50 };
|
||||
expect(computeUsageDelta(before, after, 3.7).duration).toBe(4);
|
||||
expect(computeUsageDelta(before, after, 3.2).duration).toBe(3);
|
||||
expect(computeUsageDelta(before, after, 0.0).duration).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -72,6 +72,11 @@ export class HermesAcpClient {
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
/** Return the current session ID, or null if not connected. */
|
||||
getSessionId(): string | null {
|
||||
return this.sessionId;
|
||||
}
|
||||
|
||||
/** Send prompt and collect final assistant text from ACP stream chunks. */
|
||||
async prompt(text: string): Promise<AcpPromptResult> {
|
||||
if (this.sessionId === null) {
|
||||
|
||||
@@ -12,9 +12,45 @@ import {
|
||||
import { HermesAcpClient } from "./acp-client.js";
|
||||
import { getCachedSessionId, setCachedSessionId } from "./session-cache.js";
|
||||
import { loadHermesSession, storeHermesSessionDetail } from "./session-detail.js";
|
||||
import type { HermesSessionJson } from "./types.js";
|
||||
|
||||
const log = createLogger({ sink: { kind: "stderr" } });
|
||||
|
||||
/** Snapshot of session metrics taken before and after a prompt call. */
|
||||
type UsageSnapshot = {
|
||||
turns: number;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
};
|
||||
|
||||
const ZERO_SNAPSHOT: UsageSnapshot = { turns: 0, inputTokens: 0, outputTokens: 0 };
|
||||
|
||||
/** Extract usage metrics from a session. Returns zeros for null sessions. */
|
||||
export function snapshotUsage(session: HermesSessionJson | null): UsageSnapshot {
|
||||
if (session === null) {
|
||||
return ZERO_SNAPSHOT;
|
||||
}
|
||||
return {
|
||||
turns: session.messages.filter((m) => m.role === "assistant").length,
|
||||
inputTokens: session.inputTokens,
|
||||
outputTokens: session.outputTokens,
|
||||
};
|
||||
}
|
||||
|
||||
/** Compute the delta between two snapshots (after minus before). Floors at 0. */
|
||||
export function computeUsageDelta(
|
||||
before: UsageSnapshot,
|
||||
after: UsageSnapshot,
|
||||
durationSec: number,
|
||||
): Usage {
|
||||
return {
|
||||
turns: Math.max(0, after.turns - before.turns) || 1,
|
||||
inputTokens: Math.max(0, after.inputTokens - before.inputTokens),
|
||||
outputTokens: Math.max(0, after.outputTokens - before.outputTokens),
|
||||
duration: Math.round(durationSec),
|
||||
};
|
||||
}
|
||||
|
||||
/** Assemble system prompt, task, and prior step outputs for Hermes. */
|
||||
export function buildHermesPrompt(ctx: AgentContext): string {
|
||||
const parts: string[] = [];
|
||||
@@ -109,7 +145,11 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
|
||||
void client.close();
|
||||
});
|
||||
|
||||
async function runPrompt(ctx: AgentContext, useContinuation: boolean): Promise<AgentRunResult> {
|
||||
async function runPrompt(
|
||||
ctx: AgentContext,
|
||||
useContinuation: boolean,
|
||||
beforeSnapshot: UsageSnapshot,
|
||||
): Promise<AgentRunResult> {
|
||||
const effectiveCtx = useContinuation ? ctx : { ...ctx, isFirstVisit: true };
|
||||
const fullPrompt = buildHermesPrompt(effectiveCtx);
|
||||
const startMs = Date.now();
|
||||
@@ -121,16 +161,9 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
|
||||
await setCachedSessionId(ctx.threadId, ctx.role, sessionId, ctx.storageRoot);
|
||||
}
|
||||
|
||||
const session = await loadHermesSession(sessionId);
|
||||
const turns =
|
||||
session !== null ? session.messages.filter((m) => m.role === "assistant").length : 1;
|
||||
|
||||
const usage: Usage = {
|
||||
turns,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
duration: Math.round(durationSec),
|
||||
};
|
||||
const afterSession = await loadHermesSession(sessionId);
|
||||
const afterSnapshot = snapshotUsage(afterSession);
|
||||
const usage = computeUsageDelta(beforeSnapshot, afterSnapshot, durationSec);
|
||||
|
||||
return { output: text, detailHash, sessionId, assembledPrompt: fullPrompt, usage };
|
||||
}
|
||||
@@ -139,8 +172,17 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
|
||||
const cwd = process.cwd();
|
||||
const attempt = await prepareSession(client, ctx, cwd, resumeDisabled);
|
||||
|
||||
// Snapshot before prompt: for resumed sessions, captures cumulative state
|
||||
// so we can compute the delta. For new sessions, this is ZERO_SNAPSHOT.
|
||||
const currentSessionId = client.getSessionId();
|
||||
const beforeSession =
|
||||
attempt.resumed && currentSessionId !== null
|
||||
? await loadHermesSession(currentSessionId)
|
||||
: null;
|
||||
const beforeSnapshot = snapshotUsage(beforeSession);
|
||||
|
||||
try {
|
||||
return await runPrompt(ctx, attempt.useContinuation);
|
||||
return await runPrompt(ctx, attempt.useContinuation, beforeSnapshot);
|
||||
} catch (error) {
|
||||
if (!attempt.resumed) {
|
||||
throw error;
|
||||
@@ -150,7 +192,8 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
|
||||
log("8FQW2R6N", `continuation prompt failed, retrying with initial prompt: ${message}`);
|
||||
await client.close();
|
||||
await client.connect(cwd);
|
||||
return runPrompt(ctx, false);
|
||||
// Fresh session after retry — reset snapshot to zero
|
||||
return runPrompt(ctx, false, ZERO_SNAPSHOT);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,17 +204,20 @@ export function createHermesAgent(resumeDisabled: boolean): () => Promise<void>
|
||||
): Promise<AgentRunResult> {
|
||||
// Client is already connected from runHermes — same ACP session,
|
||||
// so the agent sees the full conversation history (crucial for retries).
|
||||
// Snapshot before the continuation prompt for delta computation.
|
||||
const currentSessionId = client.getSessionId();
|
||||
const beforeSession =
|
||||
currentSessionId !== null ? await loadHermesSession(currentSessionId) : null;
|
||||
const beforeSnapshot = snapshotUsage(beforeSession);
|
||||
|
||||
const startMs = Date.now();
|
||||
const { text, sessionId } = await client.prompt(message);
|
||||
const durationSec = (Date.now() - startMs) / 1000;
|
||||
const { detailHash } = await storePromptResult(store, sessionId);
|
||||
|
||||
const usage: Usage = {
|
||||
turns: 1,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
duration: Math.round(durationSec),
|
||||
};
|
||||
const afterSession = await loadHermesSession(sessionId);
|
||||
const afterSnapshot = snapshotUsage(afterSession);
|
||||
const usage = computeUsageDelta(beforeSnapshot, afterSnapshot, durationSec);
|
||||
|
||||
return { output: text, detailHash, sessionId, assembledPrompt: "", usage };
|
||||
}
|
||||
|
||||
@@ -1,2 +1,7 @@
|
||||
export { HermesAcpClient } from "./acp-client.js";
|
||||
export { buildHermesPrompt, createHermesAgent } from "./hermes.js";
|
||||
export {
|
||||
buildHermesPrompt,
|
||||
computeUsageDelta,
|
||||
createHermesAgent,
|
||||
snapshotUsage,
|
||||
} from "./hermes.js";
|
||||
|
||||
@@ -106,7 +106,7 @@ function parseSessionJson(raw: unknown): HermesSessionJson | null {
|
||||
messages.push(msg);
|
||||
}
|
||||
}
|
||||
return { session_id, model, session_start, messages };
|
||||
return { session_id, model, session_start, messages, inputTokens: 0, outputTokens: 0 };
|
||||
}
|
||||
|
||||
export function getHermesDbPath(): string {
|
||||
@@ -117,6 +117,8 @@ type DbSessionRow = {
|
||||
id: string;
|
||||
model: string;
|
||||
started_at: number;
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
};
|
||||
|
||||
type DbMessageRow = {
|
||||
@@ -156,7 +158,9 @@ export function loadHermesSessionFromDb(
|
||||
try {
|
||||
db = new DatabaseSync(resolvedPath, { readOnly: true });
|
||||
const session = db
|
||||
.prepare("SELECT id, model, started_at FROM sessions WHERE id = ?")
|
||||
.prepare(
|
||||
"SELECT id, model, started_at, input_tokens, output_tokens FROM sessions WHERE id = ?",
|
||||
)
|
||||
.get(sessionId) as DbSessionRow | null;
|
||||
if (session === null) {
|
||||
return null;
|
||||
@@ -181,6 +185,8 @@ export function loadHermesSessionFromDb(
|
||||
model: session.model,
|
||||
session_start: new Date(session.started_at * 1000).toISOString(),
|
||||
messages,
|
||||
inputTokens: session.input_tokens ?? 0,
|
||||
outputTokens: session.output_tokens ?? 0,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
|
||||
@@ -40,4 +40,6 @@ export type HermesSessionJson = {
|
||||
model: string;
|
||||
session_start: string;
|
||||
messages: HermesSessionMessage[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user