diff --git a/nerve.yaml b/nerve.yaml index eed6ef4..b534ad2 100644 --- a/nerve.yaml +++ b/nerve.yaml @@ -1,24 +1,14 @@ # nerve.yaml — Nerve workspace configuration senses: - cpu-usage: - group: system - throttle: 5s - timeout: 10s - grace_period: null linux-system-health: group: system throttle: 10s timeout: 15s grace_period: null - linux-tcp-socket-stats: + hermes-gateway-health: group: system - throttle: 15s - timeout: 10s - grace_period: null - disk-usage-mounts: - group: system - throttle: 10s - timeout: 15s + throttle: 30s + timeout: 30s grace_period: null workflows: @@ -27,15 +17,9 @@ workflows: overflow: drop reflexes: - - kind: sense - sense: cpu-usage - interval: 10s - kind: sense sense: linux-system-health interval: 30s - kind: sense - sense: linux-tcp-socket-stats - interval: 1m - - kind: sense - sense: disk-usage-mounts - interval: 1m + sense: hermes-gateway-health + interval: 2m diff --git a/senses/cpu-usage/index.js b/senses/cpu-usage/index.js deleted file mode 100644 index 175df04..0000000 --- a/senses/cpu-usage/index.js +++ /dev/null @@ -1,22 +0,0 @@ -import { cpus } from "node:os"; - -export async function compute() { - const cpuList = cpus(); - - let totalIdle = 0; - let totalTick = 0; - for (const cpu of cpuList) { - for (const [, time] of Object.entries(cpu.times)) { - totalTick += time; - } - totalIdle += cpu.times.idle; - } - - const loadPercent = totalTick === 0 ? 0 : ((totalTick - totalIdle) / totalTick) * 100; - - return { - model: cpuList[0]?.model ?? "unknown", - loadPercent: Math.round(loadPercent * 100) / 100, - ts: Date.now(), - }; -} diff --git a/senses/cpu-usage/migrations/0001_init.sql b/senses/cpu-usage/migrations/0001_init.sql deleted file mode 100644 index e8a064c..0000000 --- a/senses/cpu-usage/migrations/0001_init.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE IF NOT EXISTS cpu_usage ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ts INTEGER NOT NULL, - model TEXT NOT NULL, - load_percent REAL NOT NULL -); diff --git a/senses/cpu-usage/schema.ts b/senses/cpu-usage/schema.ts deleted file mode 100644 index c0a405a..0000000 --- a/senses/cpu-usage/schema.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core"; - -export const cpuUsage = sqliteTable("cpu_usage", { - id: integer("id").primaryKey({ autoIncrement: true }), - ts: integer("ts").notNull(), - model: text("model").notNull(), - loadPercent: real("load_percent").notNull(), -}); diff --git a/senses/disk-usage-mounts/index.js b/senses/disk-usage-mounts/index.js deleted file mode 100644 index 9dfacc8..0000000 --- a/senses/disk-usage-mounts/index.js +++ /dev/null @@ -1,131 +0,0 @@ -import { execSync } from "node:child_process"; -import { diskUsageMounts } from "./schema.ts"; - -const DF_CMD = - "df -B1 --output=source,target,fstype,size,used,avail,pcent"; - -/** fstype-based exclusions to avoid pseudo / volatile filesystem noise */ -const EXCLUDED_FSTYPES = new Set([ - "tmpfs", - "devtmpfs", - "proc", - "sysfs", - "cgroup2", - "cgroup", -]); - -function round2(n) { - return Math.round(n * 100) / 100; -} - -function parseUIntField(s) { - if (s === "-") return null; - const n = Number.parseInt(String(s), 10); - if (!Number.isFinite(n) || n < 0) return null; - return n; -} - -function parsePcent(tok) { - if (!tok || typeof tok !== "string") return null; - const t = tok.trim(); - if (!/^[\d.]+%$/.test(t)) return null; - const raw = Number.parseFloat(t.replace("%", "")); - return Number.isFinite(raw) ? round2(raw) : null; -} - -/** - * Parse one `df --output=source,target,fstype,size,used,avail,pcent` data line. - * Mount may contain spaces; last five logical columns are fixed. - */ -function parseDfLine(line) { - const parts = line.trim().split(/\s+/); - if (parts.length < 7) return null; - - const pcentTok = parts[parts.length - 1]; - const availBytes = parseUIntField(parts[parts.length - 2]); - const usedBytes = parseUIntField(parts[parts.length - 3]); - const totalBytes = parseUIntField(parts[parts.length - 4]); - const fstype = parts[parts.length - 5]; - const device = parts[0]; - const mount = parts.slice(1, parts.length - 5).join(" "); - - if (!device || !mount || !fstype) return null; - if (totalBytes === null || usedBytes === null || availBytes === null) - return null; - - const pcentFromDf = parsePcent(pcentTok); - const computed = - totalBytes > 0 ? round2((usedBytes / totalBytes) * 100) : 0; - let usedPercent = computed; - if (pcentFromDf !== null) { - const diff = Math.abs(computed - pcentFromDf); - usedPercent = diff > 1 ? computed : pcentFromDf; - } - - return { - device, - mount, - fstype, - totalBytes, - usedBytes, - availBytes, - usedPercent, - }; -} - -function parseDfOutput(text) { - const rows = []; - const lines = text.split("\n"); - for (const line of lines) { - const trimmed = line.trim(); - if (!trimmed) continue; - if (/^Filesystem\s+/.test(trimmed) || trimmed.startsWith("Filesystem")) - continue; - const row = parseDfLine(line); - if (row && !EXCLUDED_FSTYPES.has(row.fstype)) rows.push(row); - } - return rows; -} - -export async function compute(db, _peers) { - const ts = Date.now(); - let mounts = []; - - try { - const out = execSync(DF_CMD, { - encoding: "utf-8", - maxBuffer: 10 * 1024 * 1024, - }); - mounts = parseDfOutput(out); - } catch { - mounts = []; - } - - if (mounts.length > 0) { - await db.insert(diskUsageMounts).values( - mounts.map((m) => ({ - ts, - device: m.device, - mount: m.mount, - fstype: m.fstype, - totalBytes: m.totalBytes, - usedBytes: m.usedBytes, - availBytes: m.availBytes, - usedPercent: m.usedPercent, - })), - ); - } - - return { - ts, - mounts: mounts.map((m) => ({ - device: m.device, - mount: m.mount, - fstype: m.fstype, - totalBytes: m.totalBytes, - usedBytes: m.usedBytes, - availBytes: m.availBytes, - usedPercent: m.usedPercent, - })), - }; -} diff --git a/senses/disk-usage-mounts/migrations/0001_init.sql b/senses/disk-usage-mounts/migrations/0001_init.sql deleted file mode 100644 index b853f99..0000000 --- a/senses/disk-usage-mounts/migrations/0001_init.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Migration: 0001_init --- Creates the disk_usage_mounts table for disk-usage-mounts sense. - -CREATE TABLE IF NOT EXISTS disk_usage_mounts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ts INTEGER NOT NULL, - device TEXT NOT NULL, - mount TEXT NOT NULL, - fstype TEXT NOT NULL, - total_bytes INTEGER NOT NULL, - used_bytes INTEGER NOT NULL, - avail_bytes INTEGER NOT NULL, - used_percent REAL NOT NULL -); diff --git a/senses/disk-usage-mounts/schema.ts b/senses/disk-usage-mounts/schema.ts deleted file mode 100644 index 177a545..0000000 --- a/senses/disk-usage-mounts/schema.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core"; - -export const diskUsageMounts = sqliteTable("disk_usage_mounts", { - id: integer("id").primaryKey({ autoIncrement: true }), - ts: integer("ts").notNull(), - device: text("device").notNull(), - mount: text("mount").notNull(), - fstype: text("fstype").notNull(), - totalBytes: integer("total_bytes").notNull(), - usedBytes: integer("used_bytes").notNull(), - availBytes: integer("avail_bytes").notNull(), - usedPercent: real("used_percent").notNull(), -}); diff --git a/senses/hermes-gateway-health/index.js b/senses/hermes-gateway-health/index.js new file mode 100644 index 0000000..dde4bb2 --- /dev/null +++ b/senses/hermes-gateway-health/index.js @@ -0,0 +1,321 @@ +import { execFile } from "node:child_process"; +import { hermesGatewayHealth } from "./schema.ts"; + +/** Keep subprocess deadlines slightly under typical sense timeout (30s). */ +const EXEC_TIMEOUT_MS = 25_000; + +/** + * When `ps` lacks `etimes` (wall-clock seconds since start), parse `etime` + * ([[dd-]hh:]mm:ss) into seconds. See ps(1) `etime` field description. + */ +function etimeToSeconds(etime) { + let s = String(etime).trim(); + if (!s) return 0; + let days = 0; + if (s.includes("-")) { + const idx = s.indexOf("-"); + const d = Number.parseInt(s.slice(0, idx), 10); + days = Number.isFinite(d) ? d : 0; + s = s.slice(idx + 1); + } + const parts = s.split(":").map((x) => Number.parseInt(String(x).trim(), 10)); + if (parts.some((n) => !Number.isFinite(n))) return 0; + if (parts.length === 3) { + return Math.trunc(days * 86_400 + parts[0] * 3600 + parts[1] * 60 + parts[2]); + } + if (parts.length === 2) { + return Math.trunc(days * 86_400 + parts[0] * 60 + parts[1]); + } + if (parts.length === 1) { + return Math.trunc(days * 86_400 + parts[0]); + } + return 0; +} + +function execFileUtf8(file, args, opts = {}) { + return new Promise((resolve) => { + execFile( + file, + args, + { + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + timeout: EXEC_TIMEOUT_MS, + ...opts, + }, + (err, stdout, stderr) => { + const exitCode = + err && typeof err.status === "number" ? err.status : err ? -1 : 0; + resolve({ + exitCode, + errCode: err?.code, + stdout: String(stdout ?? ""), + stderr: String(stderr ?? ""), + }); + }, + ); + }); +} + +function parseMainPidFromStatus(text) { + const m = text.match(/Main PID:\s*(\d+)/i); + return m ? Math.trunc(Number.parseInt(m[1], 10)) || 0 : 0; +} + +function parseActiveLineFromStatus(text) { + for (const line of text.split("\n")) { + if (/^\s*Active:/i.test(line)) { + const m = line.match(/Active:\s*(\S+)\s*\(([^)]*)\)/i); + if (m) { + return { + active: m[1].toLowerCase() === "active", + subRunning: m[2].toLowerCase().includes("running"), + }; + } + } + } + return { active: false, subRunning: false }; +} + +function parseSystemctlShow(text) { + let mainPid = 0; + let active = false; + let subRunning = false; + for (const line of text.split("\n")) { + const t = line.trim(); + if (t.startsWith("MainPID=")) { + mainPid = Math.trunc(Number.parseInt(t.slice("MainPID=".length), 10)) || 0; + } else if (t.startsWith("ActiveState=")) { + active = t.slice("ActiveState=".length).trim().toLowerCase() === "active"; + } else if (t.startsWith("SubState=")) { + subRunning = t.slice("SubState=".length).trim().toLowerCase() === "running"; + } + } + return { mainPid, active, subRunning }; +} + +async function readSystemdState() { + const status = await execFileUtf8("systemctl", [ + "--user", + "--no-pager", + "status", + "hermes-gateway", + ]); + const combined = `${status.stdout}\n${status.stderr}`.trim(); + let mainPid = parseMainPidFromStatus(combined); + let { active, subRunning } = parseActiveLineFromStatus(combined); + + const needShow = + mainPid <= 0 || !active || !subRunning; + + if (needShow) { + const show = await execFileUtf8("systemctl", [ + "--user", + "--no-pager", + "show", + "hermes-gateway", + "-p", + "MainPID", + "-p", + "ActiveState", + "-p", + "SubState", + ]); + const showText = `${show.stdout}\n${show.stderr}`; + const s = parseSystemctlShow(showText); + if (mainPid <= 0 && s.mainPid > 0) mainPid = s.mainPid; + if (!active) active = s.active; + if (!subRunning) subRunning = s.subRunning; + } + + return { mainPid, systemdActiveRunning: active && subRunning }; +} + +async function processExists(mainPid) { + if (mainPid <= 0) return false; + const r = await execFileUtf8("ps", ["-p", String(mainPid), "-o", "pid="]); + if (r.errCode === "ENOENT") return false; + return r.stdout.trim().length > 0; +} + +async function readPsMetrics(mainPid) { + if (mainPid <= 0) { + return { rssBytes: 0, cpuPercent: 0, uptimeSec: 0 }; + } + let r = await execFileUtf8("ps", [ + "-p", + String(mainPid), + "-o", + "rss=,%cpu=,etimes=", + ]); + let line = r.stdout.trim().replace(/\s+/g, " "); + if (r.errCode === "ENOENT" || !line) { + return { rssBytes: 0, cpuPercent: 0, uptimeSec: 0 }; + } + let parts = line.split(" ").filter(Boolean); + if (parts.length < 3) { + r = await execFileUtf8("ps", [ + "-p", + String(mainPid), + "-o", + "rss=,%cpu=,etime=", + ]); + line = r.stdout.trim().replace(/\s+/g, " "); + parts = line.split(" ").filter(Boolean); + if (parts.length < 3) { + return { rssBytes: 0, cpuPercent: 0, uptimeSec: 0 }; + } + const rssKiB = Number(parts[0]); + const cpu = Number(parts[1]); + const uptimeSec = etimeToSeconds(parts.slice(2).join(" ")); + const rssBytes = Number.isFinite(rssKiB) + ? Math.trunc(rssKiB * 1024) + : 0; + const cpuPercent = Number.isFinite(cpu) + ? Math.round(cpu * 100) / 100 + : 0; + return { rssBytes, cpuPercent, uptimeSec }; + } + const rssKiB = Number(parts[0]); + const cpu = Number(parts[1]); + const etimes = Number(parts[2]); + const rssBytes = Number.isFinite(rssKiB) ? Math.trunc(rssKiB * 1024) : 0; + const cpuPercent = Number.isFinite(cpu) ? Math.round(cpu * 100) / 100 : 0; + const uptimeSec = Number.isFinite(etimes) + ? Math.trunc(etimes) + : 0; + return { rssBytes, cpuPercent, uptimeSec }; +} + +function parseActiveSessionsFromHermesStats(text) { + const src = String(text); + const patterns = [ + /^\s*Active\s+sessions?:\s*(\d+)/gim, + /^\s*active\s+sessions?:\s*(\d+)/gim, + /^\s*Total\s+sessions?:\s*(\d+)/gim, + ]; + for (const re of patterns) { + re.lastIndex = 0; + const m = re.exec(src); + if (m) { + const n = Math.trunc(Number.parseInt(m[1], 10)); + return Number.isFinite(n) ? n : 0; + } + } + return 0; +} + +async function readActiveSessions() { + try { + const r = await execFileUtf8("hermes", ["sessions", "stats"]); + if (r.errCode === "ENOENT") return 0; + return parseActiveSessionsFromHermesStats(`${r.stdout}\n${r.stderr}`); + } catch { + return 0; + } +} + +async function countDirectChildren(mainPid) { + if (mainPid <= 0) return 0; + try { + const r = await execFileUtf8("ps", [ + "--no-headers", + "-o", + "pid", + "--ppid", + String(mainPid), + ]); + if (r.errCode === "ENOENT") return 0; + const lines = r.stdout + .split("\n") + .map((l) => l.trim()) + .filter(Boolean); + return lines.length; + } catch { + return 0; + } +} + +export async function compute(db, _peers) { + const ts = Date.now(); + + let mainPid = 0; + let systemdActiveRunning = false; + + try { + const st = await readSystemdState(); + mainPid = st.mainPid; + systemdActiveRunning = st.systemdActiveRunning; + } catch { + mainPid = 0; + systemdActiveRunning = false; + } + + let psOk = false; + try { + psOk = await processExists(mainPid); + } catch { + psOk = false; + } + + let rssBytes = 0; + let cpuPercent = 0; + let uptimeSec = 0; + if (psOk) { + try { + const m = await readPsMetrics(mainPid); + rssBytes = m.rssBytes; + cpuPercent = m.cpuPercent; + uptimeSec = m.uptimeSec; + } catch { + rssBytes = 0; + cpuPercent = 0; + uptimeSec = 0; + } + } + + const alive = + systemdActiveRunning && mainPid > 0 && psOk ? 1 : 0; + + let activeSessions = 0; + try { + activeSessions = await readActiveSessions(); + } catch { + activeSessions = 0; + } + + let childProcessCount = 0; + if (alive && mainPid > 0) { + try { + childProcessCount = await countDirectChildren(mainPid); + } catch { + childProcessCount = 0; + } + } + + const storedMainPid = mainPid > 0 ? mainPid : 0; + + const row = { + ts, + alive, + mainPid: storedMainPid, + rssBytes: alive ? rssBytes : 0, + cpuPercent: alive ? cpuPercent : 0, + uptimeSec: alive ? uptimeSec : 0, + activeSessions, + childProcessCount: alive ? childProcessCount : 0, + }; + + await db.insert(hermesGatewayHealth).values(row); + + return { + ts: row.ts, + alive: row.alive, + mainPid: row.mainPid, + rssBytes: row.rssBytes, + cpuPercent: row.cpuPercent, + uptimeSec: row.uptimeSec, + activeSessions: row.activeSessions, + childProcessCount: row.childProcessCount, + }; +} diff --git a/senses/hermes-gateway-health/migrations/0001_init.sql b/senses/hermes-gateway-health/migrations/0001_init.sql new file mode 100644 index 0000000..51ce31d --- /dev/null +++ b/senses/hermes-gateway-health/migrations/0001_init.sql @@ -0,0 +1,14 @@ +-- Migration: 0001_init +-- Creates the hermes_gateway_health table for hermes-gateway-health sense. + +CREATE TABLE IF NOT EXISTS hermes_gateway_health ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + alive INTEGER NOT NULL, + main_pid INTEGER NOT NULL, + rss_bytes INTEGER NOT NULL, + cpu_percent REAL NOT NULL, + uptime_sec INTEGER NOT NULL, + active_sessions INTEGER NOT NULL, + child_process_count INTEGER NOT NULL +); diff --git a/senses/hermes-gateway-health/schema.ts b/senses/hermes-gateway-health/schema.ts new file mode 100644 index 0000000..66469f2 --- /dev/null +++ b/senses/hermes-gateway-health/schema.ts @@ -0,0 +1,13 @@ +import { integer, real, sqliteTable } from "drizzle-orm/sqlite-core"; + +export const hermesGatewayHealth = sqliteTable("hermes_gateway_health", { + id: integer("id").primaryKey({ autoIncrement: true }), + ts: integer("ts").notNull(), + alive: integer("alive").notNull(), + mainPid: integer("main_pid").notNull(), + rssBytes: integer("rss_bytes").notNull(), + cpuPercent: real("cpu_percent").notNull(), + uptimeSec: integer("uptime_sec").notNull(), + activeSessions: integer("active_sessions").notNull(), + childProcessCount: integer("child_process_count").notNull(), +}); diff --git a/senses/linux-system-health/index.js b/senses/linux-system-health/index.js index f96fdb0..69de1a0 100644 --- a/senses/linux-system-health/index.js +++ b/senses/linux-system-health/index.js @@ -1,7 +1,38 @@ import { loadavg, totalmem, freemem, uptime } from "node:os"; import { execSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; import { snapshots } from "./schema.ts"; +const SOCKSTAT_PATH = "/proc/net/sockstat"; + +function parseSockstat(content) { + let socketsUsed = 0, tcpInuse = 0, tcpOrphan = 0, tcpTw = 0, tcpAlloc = 0, tcpMemPages = 0; + + for (const line of content.split("\n")) { + const trimmed = line.trim(); + if (trimmed.startsWith("sockets:")) { + const parts = trimmed.split(/\s+/); + const idx = parts.indexOf("used"); + if (idx !== -1 && idx + 1 < parts.length) { + socketsUsed = Number.parseInt(parts[idx + 1], 10) || 0; + } + } else if (trimmed.startsWith("TCP:")) { + const parts = trimmed.split(/\s+/); + const map = {}; + for (let i = 1; i + 1 < parts.length; i += 2) { + map[parts[i]] = Number.parseInt(parts[i + 1], 10) || 0; + } + tcpInuse = map.inuse ?? 0; + tcpOrphan = map.orphan ?? 0; + tcpTw = map.tw ?? 0; + tcpAlloc = map.alloc ?? 0; + tcpMemPages = map.mem ?? 0; + } + } + + return { socketsUsed, tcpInuse, tcpOrphan, tcpTw, tcpAlloc, tcpMemPages }; +} + export async function compute(db, _peers) { const [load1, load5, load15] = loadavg(); @@ -23,6 +54,13 @@ export async function compute(db, _peers) { diskUsedPct = total > 0 ? Math.round((used / total) * 10000) / 100 : 0; } catch {} + // TCP socket stats + let tcp = { socketsUsed: 0, tcpInuse: 0, tcpOrphan: 0, tcpTw: 0, tcpAlloc: 0, tcpMemPages: 0 }; + try { + const content = await readFile(SOCKSTAT_PATH, "utf8"); + tcp = parseSockstat(content); + } catch {} + const ts = Date.now(); const uptimeSec = Math.round(uptime()); @@ -31,12 +69,19 @@ export async function compute(db, _peers) { memTotalMB, memUsedMB, memUsedPct, diskTotalGB, diskUsedGB, diskUsedPct, uptimeSec, + socketsUsed: tcp.socketsUsed, + tcpInuse: tcp.tcpInuse, + tcpOrphan: tcp.tcpOrphan, + tcpTw: tcp.tcpTw, + tcpAlloc: tcp.tcpAlloc, + tcpMemPages: tcp.tcpMemPages, }); return { cpu: { load1m: load1, load5m: load5, load15m: load15 }, memory: { totalMB: memTotalMB, usedMB: memUsedMB, usedPct: memUsedPct }, disk: { totalGB: diskTotalGB, usedGB: diskUsedGB, usedPct: diskUsedPct }, + tcp: { socketsUsed: tcp.socketsUsed, inuse: tcp.tcpInuse, orphan: tcp.tcpOrphan, tw: tcp.tcpTw, alloc: tcp.tcpAlloc, memPages: tcp.tcpMemPages }, uptimeSec, }; } diff --git a/senses/linux-system-health/migrations/0002_add_tcp_stats.sql b/senses/linux-system-health/migrations/0002_add_tcp_stats.sql new file mode 100644 index 0000000..5efd2d5 --- /dev/null +++ b/senses/linux-system-health/migrations/0002_add_tcp_stats.sql @@ -0,0 +1,6 @@ +ALTER TABLE snapshots ADD COLUMN sockets_used INTEGER; +ALTER TABLE snapshots ADD COLUMN tcp_inuse INTEGER; +ALTER TABLE snapshots ADD COLUMN tcp_orphan INTEGER; +ALTER TABLE snapshots ADD COLUMN tcp_tw INTEGER; +ALTER TABLE snapshots ADD COLUMN tcp_alloc INTEGER; +ALTER TABLE snapshots ADD COLUMN tcp_mem_pages INTEGER; diff --git a/senses/linux-system-health/schema.ts b/senses/linux-system-health/schema.ts index 470102e..de382a2 100644 --- a/senses/linux-system-health/schema.ts +++ b/senses/linux-system-health/schema.ts @@ -1,4 +1,4 @@ -import { integer, real, sqliteTable } from "drizzle-orm/sqlite-core"; +import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core"; export const snapshots = sqliteTable("snapshots", { ts: integer("ts").primaryKey(), @@ -12,4 +12,11 @@ export const snapshots = sqliteTable("snapshots", { diskUsedGB: real("disk_used_gb").notNull(), diskUsedPct: real("disk_used_pct").notNull(), uptimeSec: integer("uptime_sec").notNull(), + // TCP socket stats (merged from linux-tcp-socket-stats) + socketsUsed: integer("sockets_used"), + tcpInuse: integer("tcp_inuse"), + tcpOrphan: integer("tcp_orphan"), + tcpTw: integer("tcp_tw"), + tcpAlloc: integer("tcp_alloc"), + tcpMemPages: integer("tcp_mem_pages"), }); diff --git a/senses/linux-tcp-socket-stats/index.js b/senses/linux-tcp-socket-stats/index.js deleted file mode 100644 index b51091c..0000000 --- a/senses/linux-tcp-socket-stats/index.js +++ /dev/null @@ -1,120 +0,0 @@ -import { readFile } from "node:fs/promises"; -import { linuxTcpSocketStats } from "./schema.ts"; - -const SOCKSTAT_PATH = "/proc/net/sockstat"; -const RAW_MAX = 4096; - -function parseInt10(s) { - const n = Number.parseInt(String(s), 10); - return Number.isFinite(n) ? Math.trunc(n) : NaN; -} - -function parseSockstat(content) { - let socketsUsed = 0; - let tcpInuse = 0; - let tcpOrphan = 0; - let tcpTw = 0; - let tcpAlloc = 0; - let tcpMemPages = 0; - let parseOk = 1; - - let socketsOk = false; - let tcpOk = false; - - for (const line of content.split("\n")) { - const trimmed = line.trim(); - if (trimmed.startsWith("sockets:")) { - const parts = trimmed.split(/\s+/); - const usedIdx = parts.indexOf("used"); - if (usedIdx !== -1 && usedIdx + 1 < parts.length) { - const v = parseInt10(parts[usedIdx + 1]); - if (Number.isFinite(v)) { - socketsUsed = v; - socketsOk = true; - } - } - } else if (trimmed.startsWith("TCP:")) { - const parts = trimmed.split(/\s+/); - const map = {}; - for (let i = 1; i + 1 < parts.length; i += 2) { - map[parts[i]] = parseInt10(parts[i + 1]); - } - const keys = ["inuse", "orphan", "tw", "alloc", "mem"]; - if (keys.every((k) => Number.isFinite(map[k]))) { - tcpInuse = map.inuse; - tcpOrphan = map.orphan; - tcpTw = map.tw; - tcpAlloc = map.alloc; - tcpMemPages = map.mem; - tcpOk = true; - } - } - } - - if (!socketsOk || !tcpOk) { - parseOk = 0; - socketsUsed = 0; - tcpInuse = 0; - tcpOrphan = 0; - tcpTw = 0; - tcpAlloc = 0; - tcpMemPages = 0; - } - - return { - socketsUsed, - tcpInuse, - tcpOrphan, - tcpTw, - tcpAlloc, - tcpMemPages, - parseOk, - }; -} - -export async function compute(db, _peers) { - const ts = Date.now(); - let rawSockstat = ""; - let row = { - socketsUsed: 0, - tcpInuse: 0, - tcpOrphan: 0, - tcpTw: 0, - tcpAlloc: 0, - tcpMemPages: 0, - parseOk: 0, - }; - - try { - const content = await readFile(SOCKSTAT_PATH, "utf8"); - rawSockstat = - content.length > RAW_MAX ? content.slice(0, RAW_MAX) : content; - row = parseSockstat(content); - } catch { - rawSockstat = ""; - } - - await db.insert(linuxTcpSocketStats).values({ - ts, - socketsUsed: row.socketsUsed, - tcpInuse: row.tcpInuse, - tcpOrphan: row.tcpOrphan, - tcpTw: row.tcpTw, - tcpAlloc: row.tcpAlloc, - tcpMemPages: row.tcpMemPages, - parseOk: row.parseOk, - rawSockstat, - }); - - return { - ts, - socketsUsed: row.socketsUsed, - tcpInuse: row.tcpInuse, - tcpOrphan: row.tcpOrphan, - tcpTw: row.tcpTw, - tcpAlloc: row.tcpAlloc, - tcpMemPages: row.tcpMemPages, - parseOk: row.parseOk, - rawSockstat, - }; -} diff --git a/senses/linux-tcp-socket-stats/migrations/0001_init.sql b/senses/linux-tcp-socket-stats/migrations/0001_init.sql deleted file mode 100644 index 524c32f..0000000 --- a/senses/linux-tcp-socket-stats/migrations/0001_init.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Migration: 0001_init --- Creates the linux_tcp_socket_stats table for linux-tcp-socket-stats sense. - -CREATE TABLE IF NOT EXISTS linux_tcp_socket_stats ( - ts INTEGER PRIMARY KEY, - sockets_used INTEGER NOT NULL, - tcp_inuse INTEGER NOT NULL, - tcp_orphan INTEGER NOT NULL, - tcp_tw INTEGER NOT NULL, - tcp_alloc INTEGER NOT NULL, - tcp_mem_pages INTEGER NOT NULL, - parse_ok INTEGER NOT NULL, - raw_sockstat TEXT NOT NULL -); diff --git a/senses/linux-tcp-socket-stats/schema.ts b/senses/linux-tcp-socket-stats/schema.ts deleted file mode 100644 index ba71561..0000000 --- a/senses/linux-tcp-socket-stats/schema.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; - -export const linuxTcpSocketStats = sqliteTable("linux_tcp_socket_stats", { - ts: integer("ts").primaryKey(), - socketsUsed: integer("sockets_used").notNull(), - tcpInuse: integer("tcp_inuse").notNull(), - tcpOrphan: integer("tcp_orphan").notNull(), - tcpTw: integer("tcp_tw").notNull(), - tcpAlloc: integer("tcp_alloc").notNull(), - tcpMemPages: integer("tcp_mem_pages").notNull(), - parseOk: integer("parse_ok").notNull(), - rawSockstat: text("raw_sockstat").notNull(), -}); diff --git a/workflows/sense-generator/index.ts b/workflows/sense-generator/index.ts index 3b92166..78381b7 100644 --- a/workflows/sense-generator/index.ts +++ b/workflows/sense-generator/index.ts @@ -3,26 +3,89 @@ import { execSync } from "node:child_process"; import { readFileSync, existsSync } from "node:fs"; import { join } from "node:path"; -const NERVE_ROOT = join(process.env.HOME ?? "/home/azureuser", ".uncaged-nerve"); +const HOME = process.env.HOME ?? "/home/azureuser"; +const NERVE_ROOT = join(HOME, ".uncaged-nerve"); const SENSES_DIR = join(NERVE_ROOT, "senses"); // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- +function nerveCommandEnv(): NodeJS.ProcessEnv { + const pnpmHome = join(HOME, ".local/share/pnpm"); + const npmUserBin = join(HOME, ".local/share/npm/bin"); + return { + ...process.env, + PNPM_HOME: pnpmHome, + PATH: `${npmUserBin}:${pnpmHome}:${process.env.PATH ?? ""}`, + }; +} + function run(cmd: string, cwd?: string): string { return execSync(cmd, { encoding: "utf-8", cwd: cwd ?? NERVE_ROOT, timeout: 300_000, - env: { - ...process.env, - PNPM_HOME: join(process.env.HOME ?? "/home/azureuser", ".local/share/pnpm"), - PATH: `${join(process.env.HOME ?? "/home/azureuser", ".local/share/pnpm")}:${process.env.PATH}`, - }, + env: nerveCommandEnv(), }).trim(); } +/** + * Run the same checks the workflow used to ask Hermes to perform, but locally. + * Hermes chat often returns UI prose instead of shell output, which caused false failures. + */ +function runSenseSmokeTest(senseName: string): { ok: boolean; log: string; reason: string } { + const logParts: string[] = []; + try { + const status = run("nerve status"); + logParts.push("=== nerve status ===\n" + status); + if (!status.includes(senseName)) { + return { + ok: false, + log: logParts.join("\n\n"), + reason: `Sense "${senseName}" not listed in \`nerve status\` output`, + }; + } + + const triggerOut = run(`nerve sense trigger ${senseName}`); + logParts.push("=== nerve sense trigger ===\n" + triggerOut); + + let lastQuery = ""; + for (let i = 0; i < 25; i++) { + run("sleep 1"); + try { + lastQuery = run(`nerve sense query ${senseName}`); + } catch (e) { + logParts.push(`=== nerve sense query (attempt ${i + 1}) ===\nERROR: ${String(e)}`); + continue; + } + logParts.push(`=== nerve sense query (attempt ${i + 1}) ===\n${lastQuery}`); + if (!lastQuery.includes("(0 rows)")) { + return { + ok: true, + log: logParts.join("\n\n"), + reason: "Trigger succeeded and query returned at least one row", + }; + } + } + + return { + ok: false, + log: logParts.join("\n\n"), + reason: lastQuery.includes("(0 rows)") + ? "Query still returned 0 rows after trigger (compute error, throttle drop, or DB not written)" + : "Timed out waiting for successful sense query", + }; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return { + ok: false, + log: logParts.join("\n\n"), + reason: `Smoke test command failed: ${msg}`, + }; + } +} + /** * Call a cheap LLM with tool_choice to extract structured metadata from text. * Uses DashScope (Alibaba Cloud, OpenAI-compatible) with qwen-plus. @@ -61,13 +124,6 @@ function cursorAgent(prompt: string, mode: "plan" | "ask" | "default", cwd: stri ); } -function hermesRun(prompt: string): string { - const escaped = prompt.replace(/'/g, "'\\''"); - return run( - `hermes chat -q '${escaped}' --model anthropic/claude-sonnet-4 -t terminal --yolo 2>&1 || true`, - ); -} - // Build context string with existing sense examples function buildSenseExamples(): string { const examples: string[] = []; @@ -263,50 +319,18 @@ Create all files now.`; }; } - // Reload daemon to pick up new sense, then trigger it - const testPrompt = `You need to test a newly created Nerve sense called "${senseName}". + const smoke = runSenseSmokeTest(senseName); + ctx.log(`tester: smoke — ok=${smoke.ok}, reason="${smoke.reason}"`); + ctx.log(`tester: log head — ${smoke.log.substring(0, 400)}`); -Steps: -1. Restart the nerve daemon: run "nerve stop && nerve start" (make sure PNPM_HOME and PATH are set) -2. Wait 2 seconds -3. Check status: run "nerve status" and verify "${senseName}" appears in senses list -4. Trigger the sense: run "nerve sense trigger ${senseName}" -5. Check the result: run "nerve sense query ${senseName} --limit 1" - -Report whether the sense works. If there are errors, include the full error output. -The nerve binary is at ~/.local/share/pnpm/nerve. - -Reply with either: -- "PASS: " if the sense works -- "FAIL: " if it doesn't`; - - const result = hermesRun(testPrompt); - ctx.log(`tester: raw result — ${result.substring(0, 300)}`); - - // Use LLM to judge pass/fail instead of fragile string matching - const verdict = llmExtract<{ passed: boolean; reason: string }>( - `Test output for sense "${senseName}":\n\n${result.substring(0, 4000)}`, - "judge_test_result", - "Determine whether the test passed or failed based on the output", - { - type: "object", - properties: { - passed: { type: "boolean", description: "true if the sense was successfully triggered and returned valid data" }, - reason: { type: "string", description: "Brief explanation of why it passed or failed" }, - }, - required: ["passed", "reason"], - }, - ); - ctx.log(`tester: verdict — passed=${verdict.passed}, reason="${verdict.reason}"`); - - if (verdict.passed) { - return { type: "test_passed", senseName, result: verdict.reason }; + if (smoke.ok) { + return { type: "test_passed", senseName, result: smoke.reason }; } return { type: "test_failed", senseName, - reason: verdict.reason, + reason: `${smoke.reason}\n\n--- smoke log ---\n${smoke.log}`, attempt, }; },