refactor: consolidate senses — merge tcp-socket-stats into system-health, remove cpu-usage
- Remove cpu-usage sense (redundant with system-health loadavg) - Remove linux-tcp-socket-stats (merged into linux-system-health) - Remove disk-usage-mounts (unused) - Add tcp socket fields to system-health schema + migration - Simplify nerve.yaml: 4 senses → 2 小橘 <xiaoju@shazhou.work>
This commit is contained in:
parent
d1a2ee876a
commit
9a3c50c257
26
nerve.yaml
26
nerve.yaml
@ -1,24 +1,14 @@
|
||||
# nerve.yaml — Nerve workspace configuration
|
||||
senses:
|
||||
cpu-usage:
|
||||
group: system
|
||||
throttle: 5s
|
||||
timeout: 10s
|
||||
grace_period: null
|
||||
linux-system-health:
|
||||
group: system
|
||||
throttle: 10s
|
||||
timeout: 15s
|
||||
grace_period: null
|
||||
linux-tcp-socket-stats:
|
||||
hermes-gateway-health:
|
||||
group: system
|
||||
throttle: 15s
|
||||
timeout: 10s
|
||||
grace_period: null
|
||||
disk-usage-mounts:
|
||||
group: system
|
||||
throttle: 10s
|
||||
timeout: 15s
|
||||
throttle: 30s
|
||||
timeout: 30s
|
||||
grace_period: null
|
||||
|
||||
workflows:
|
||||
@ -27,15 +17,9 @@ workflows:
|
||||
overflow: drop
|
||||
|
||||
reflexes:
|
||||
- kind: sense
|
||||
sense: cpu-usage
|
||||
interval: 10s
|
||||
- kind: sense
|
||||
sense: linux-system-health
|
||||
interval: 30s
|
||||
- kind: sense
|
||||
sense: linux-tcp-socket-stats
|
||||
interval: 1m
|
||||
- kind: sense
|
||||
sense: disk-usage-mounts
|
||||
interval: 1m
|
||||
sense: hermes-gateway-health
|
||||
interval: 2m
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
import { cpus } from "node:os";
|
||||
|
||||
export async function compute() {
|
||||
const cpuList = cpus();
|
||||
|
||||
let totalIdle = 0;
|
||||
let totalTick = 0;
|
||||
for (const cpu of cpuList) {
|
||||
for (const [, time] of Object.entries(cpu.times)) {
|
||||
totalTick += time;
|
||||
}
|
||||
totalIdle += cpu.times.idle;
|
||||
}
|
||||
|
||||
const loadPercent = totalTick === 0 ? 0 : ((totalTick - totalIdle) / totalTick) * 100;
|
||||
|
||||
return {
|
||||
model: cpuList[0]?.model ?? "unknown",
|
||||
loadPercent: Math.round(loadPercent * 100) / 100,
|
||||
ts: Date.now(),
|
||||
};
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
CREATE TABLE IF NOT EXISTS cpu_usage (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
load_percent REAL NOT NULL
|
||||
);
|
||||
@ -1,8 +0,0 @@
|
||||
import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
||||
|
||||
export const cpuUsage = sqliteTable("cpu_usage", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
ts: integer("ts").notNull(),
|
||||
model: text("model").notNull(),
|
||||
loadPercent: real("load_percent").notNull(),
|
||||
});
|
||||
@ -1,131 +0,0 @@
|
||||
import { execSync } from "node:child_process";
|
||||
import { diskUsageMounts } from "./schema.ts";
|
||||
|
||||
const DF_CMD =
|
||||
"df -B1 --output=source,target,fstype,size,used,avail,pcent";
|
||||
|
||||
/** fstype-based exclusions to avoid pseudo / volatile filesystem noise */
|
||||
const EXCLUDED_FSTYPES = new Set([
|
||||
"tmpfs",
|
||||
"devtmpfs",
|
||||
"proc",
|
||||
"sysfs",
|
||||
"cgroup2",
|
||||
"cgroup",
|
||||
]);
|
||||
|
||||
function round2(n) {
|
||||
return Math.round(n * 100) / 100;
|
||||
}
|
||||
|
||||
function parseUIntField(s) {
|
||||
if (s === "-") return null;
|
||||
const n = Number.parseInt(String(s), 10);
|
||||
if (!Number.isFinite(n) || n < 0) return null;
|
||||
return n;
|
||||
}
|
||||
|
||||
function parsePcent(tok) {
|
||||
if (!tok || typeof tok !== "string") return null;
|
||||
const t = tok.trim();
|
||||
if (!/^[\d.]+%$/.test(t)) return null;
|
||||
const raw = Number.parseFloat(t.replace("%", ""));
|
||||
return Number.isFinite(raw) ? round2(raw) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one `df --output=source,target,fstype,size,used,avail,pcent` data line.
|
||||
* Mount may contain spaces; last five logical columns are fixed.
|
||||
*/
|
||||
function parseDfLine(line) {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
if (parts.length < 7) return null;
|
||||
|
||||
const pcentTok = parts[parts.length - 1];
|
||||
const availBytes = parseUIntField(parts[parts.length - 2]);
|
||||
const usedBytes = parseUIntField(parts[parts.length - 3]);
|
||||
const totalBytes = parseUIntField(parts[parts.length - 4]);
|
||||
const fstype = parts[parts.length - 5];
|
||||
const device = parts[0];
|
||||
const mount = parts.slice(1, parts.length - 5).join(" ");
|
||||
|
||||
if (!device || !mount || !fstype) return null;
|
||||
if (totalBytes === null || usedBytes === null || availBytes === null)
|
||||
return null;
|
||||
|
||||
const pcentFromDf = parsePcent(pcentTok);
|
||||
const computed =
|
||||
totalBytes > 0 ? round2((usedBytes / totalBytes) * 100) : 0;
|
||||
let usedPercent = computed;
|
||||
if (pcentFromDf !== null) {
|
||||
const diff = Math.abs(computed - pcentFromDf);
|
||||
usedPercent = diff > 1 ? computed : pcentFromDf;
|
||||
}
|
||||
|
||||
return {
|
||||
device,
|
||||
mount,
|
||||
fstype,
|
||||
totalBytes,
|
||||
usedBytes,
|
||||
availBytes,
|
||||
usedPercent,
|
||||
};
|
||||
}
|
||||
|
||||
function parseDfOutput(text) {
|
||||
const rows = [];
|
||||
const lines = text.split("\n");
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
if (/^Filesystem\s+/.test(trimmed) || trimmed.startsWith("Filesystem"))
|
||||
continue;
|
||||
const row = parseDfLine(line);
|
||||
if (row && !EXCLUDED_FSTYPES.has(row.fstype)) rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
export async function compute(db, _peers) {
|
||||
const ts = Date.now();
|
||||
let mounts = [];
|
||||
|
||||
try {
|
||||
const out = execSync(DF_CMD, {
|
||||
encoding: "utf-8",
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
mounts = parseDfOutput(out);
|
||||
} catch {
|
||||
mounts = [];
|
||||
}
|
||||
|
||||
if (mounts.length > 0) {
|
||||
await db.insert(diskUsageMounts).values(
|
||||
mounts.map((m) => ({
|
||||
ts,
|
||||
device: m.device,
|
||||
mount: m.mount,
|
||||
fstype: m.fstype,
|
||||
totalBytes: m.totalBytes,
|
||||
usedBytes: m.usedBytes,
|
||||
availBytes: m.availBytes,
|
||||
usedPercent: m.usedPercent,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
ts,
|
||||
mounts: mounts.map((m) => ({
|
||||
device: m.device,
|
||||
mount: m.mount,
|
||||
fstype: m.fstype,
|
||||
totalBytes: m.totalBytes,
|
||||
usedBytes: m.usedBytes,
|
||||
availBytes: m.availBytes,
|
||||
usedPercent: m.usedPercent,
|
||||
})),
|
||||
};
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
-- Migration: 0001_init
|
||||
-- Creates the disk_usage_mounts table for disk-usage-mounts sense.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS disk_usage_mounts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER NOT NULL,
|
||||
device TEXT NOT NULL,
|
||||
mount TEXT NOT NULL,
|
||||
fstype TEXT NOT NULL,
|
||||
total_bytes INTEGER NOT NULL,
|
||||
used_bytes INTEGER NOT NULL,
|
||||
avail_bytes INTEGER NOT NULL,
|
||||
used_percent REAL NOT NULL
|
||||
);
|
||||
@ -1,13 +0,0 @@
|
||||
import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
||||
|
||||
export const diskUsageMounts = sqliteTable("disk_usage_mounts", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
ts: integer("ts").notNull(),
|
||||
device: text("device").notNull(),
|
||||
mount: text("mount").notNull(),
|
||||
fstype: text("fstype").notNull(),
|
||||
totalBytes: integer("total_bytes").notNull(),
|
||||
usedBytes: integer("used_bytes").notNull(),
|
||||
availBytes: integer("avail_bytes").notNull(),
|
||||
usedPercent: real("used_percent").notNull(),
|
||||
});
|
||||
321
senses/hermes-gateway-health/index.js
Normal file
321
senses/hermes-gateway-health/index.js
Normal file
@ -0,0 +1,321 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { hermesGatewayHealth } from "./schema.ts";
|
||||
|
||||
/** Keep subprocess deadlines slightly under typical sense timeout (30s). */
|
||||
const EXEC_TIMEOUT_MS = 25_000;
|
||||
|
||||
/**
|
||||
* When `ps` lacks `etimes` (wall-clock seconds since start), parse `etime`
|
||||
* ([[dd-]hh:]mm:ss) into seconds. See ps(1) `etime` field description.
|
||||
*/
|
||||
function etimeToSeconds(etime) {
|
||||
let s = String(etime).trim();
|
||||
if (!s) return 0;
|
||||
let days = 0;
|
||||
if (s.includes("-")) {
|
||||
const idx = s.indexOf("-");
|
||||
const d = Number.parseInt(s.slice(0, idx), 10);
|
||||
days = Number.isFinite(d) ? d : 0;
|
||||
s = s.slice(idx + 1);
|
||||
}
|
||||
const parts = s.split(":").map((x) => Number.parseInt(String(x).trim(), 10));
|
||||
if (parts.some((n) => !Number.isFinite(n))) return 0;
|
||||
if (parts.length === 3) {
|
||||
return Math.trunc(days * 86_400 + parts[0] * 3600 + parts[1] * 60 + parts[2]);
|
||||
}
|
||||
if (parts.length === 2) {
|
||||
return Math.trunc(days * 86_400 + parts[0] * 60 + parts[1]);
|
||||
}
|
||||
if (parts.length === 1) {
|
||||
return Math.trunc(days * 86_400 + parts[0]);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function execFileUtf8(file, args, opts = {}) {
|
||||
return new Promise((resolve) => {
|
||||
execFile(
|
||||
file,
|
||||
args,
|
||||
{
|
||||
encoding: "utf8",
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
timeout: EXEC_TIMEOUT_MS,
|
||||
...opts,
|
||||
},
|
||||
(err, stdout, stderr) => {
|
||||
const exitCode =
|
||||
err && typeof err.status === "number" ? err.status : err ? -1 : 0;
|
||||
resolve({
|
||||
exitCode,
|
||||
errCode: err?.code,
|
||||
stdout: String(stdout ?? ""),
|
||||
stderr: String(stderr ?? ""),
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function parseMainPidFromStatus(text) {
|
||||
const m = text.match(/Main PID:\s*(\d+)/i);
|
||||
return m ? Math.trunc(Number.parseInt(m[1], 10)) || 0 : 0;
|
||||
}
|
||||
|
||||
function parseActiveLineFromStatus(text) {
|
||||
for (const line of text.split("\n")) {
|
||||
if (/^\s*Active:/i.test(line)) {
|
||||
const m = line.match(/Active:\s*(\S+)\s*\(([^)]*)\)/i);
|
||||
if (m) {
|
||||
return {
|
||||
active: m[1].toLowerCase() === "active",
|
||||
subRunning: m[2].toLowerCase().includes("running"),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return { active: false, subRunning: false };
|
||||
}
|
||||
|
||||
function parseSystemctlShow(text) {
|
||||
let mainPid = 0;
|
||||
let active = false;
|
||||
let subRunning = false;
|
||||
for (const line of text.split("\n")) {
|
||||
const t = line.trim();
|
||||
if (t.startsWith("MainPID=")) {
|
||||
mainPid = Math.trunc(Number.parseInt(t.slice("MainPID=".length), 10)) || 0;
|
||||
} else if (t.startsWith("ActiveState=")) {
|
||||
active = t.slice("ActiveState=".length).trim().toLowerCase() === "active";
|
||||
} else if (t.startsWith("SubState=")) {
|
||||
subRunning = t.slice("SubState=".length).trim().toLowerCase() === "running";
|
||||
}
|
||||
}
|
||||
return { mainPid, active, subRunning };
|
||||
}
|
||||
|
||||
async function readSystemdState() {
|
||||
const status = await execFileUtf8("systemctl", [
|
||||
"--user",
|
||||
"--no-pager",
|
||||
"status",
|
||||
"hermes-gateway",
|
||||
]);
|
||||
const combined = `${status.stdout}\n${status.stderr}`.trim();
|
||||
let mainPid = parseMainPidFromStatus(combined);
|
||||
let { active, subRunning } = parseActiveLineFromStatus(combined);
|
||||
|
||||
const needShow =
|
||||
mainPid <= 0 || !active || !subRunning;
|
||||
|
||||
if (needShow) {
|
||||
const show = await execFileUtf8("systemctl", [
|
||||
"--user",
|
||||
"--no-pager",
|
||||
"show",
|
||||
"hermes-gateway",
|
||||
"-p",
|
||||
"MainPID",
|
||||
"-p",
|
||||
"ActiveState",
|
||||
"-p",
|
||||
"SubState",
|
||||
]);
|
||||
const showText = `${show.stdout}\n${show.stderr}`;
|
||||
const s = parseSystemctlShow(showText);
|
||||
if (mainPid <= 0 && s.mainPid > 0) mainPid = s.mainPid;
|
||||
if (!active) active = s.active;
|
||||
if (!subRunning) subRunning = s.subRunning;
|
||||
}
|
||||
|
||||
return { mainPid, systemdActiveRunning: active && subRunning };
|
||||
}
|
||||
|
||||
async function processExists(mainPid) {
|
||||
if (mainPid <= 0) return false;
|
||||
const r = await execFileUtf8("ps", ["-p", String(mainPid), "-o", "pid="]);
|
||||
if (r.errCode === "ENOENT") return false;
|
||||
return r.stdout.trim().length > 0;
|
||||
}
|
||||
|
||||
async function readPsMetrics(mainPid) {
|
||||
if (mainPid <= 0) {
|
||||
return { rssBytes: 0, cpuPercent: 0, uptimeSec: 0 };
|
||||
}
|
||||
let r = await execFileUtf8("ps", [
|
||||
"-p",
|
||||
String(mainPid),
|
||||
"-o",
|
||||
"rss=,%cpu=,etimes=",
|
||||
]);
|
||||
let line = r.stdout.trim().replace(/\s+/g, " ");
|
||||
if (r.errCode === "ENOENT" || !line) {
|
||||
return { rssBytes: 0, cpuPercent: 0, uptimeSec: 0 };
|
||||
}
|
||||
let parts = line.split(" ").filter(Boolean);
|
||||
if (parts.length < 3) {
|
||||
r = await execFileUtf8("ps", [
|
||||
"-p",
|
||||
String(mainPid),
|
||||
"-o",
|
||||
"rss=,%cpu=,etime=",
|
||||
]);
|
||||
line = r.stdout.trim().replace(/\s+/g, " ");
|
||||
parts = line.split(" ").filter(Boolean);
|
||||
if (parts.length < 3) {
|
||||
return { rssBytes: 0, cpuPercent: 0, uptimeSec: 0 };
|
||||
}
|
||||
const rssKiB = Number(parts[0]);
|
||||
const cpu = Number(parts[1]);
|
||||
const uptimeSec = etimeToSeconds(parts.slice(2).join(" "));
|
||||
const rssBytes = Number.isFinite(rssKiB)
|
||||
? Math.trunc(rssKiB * 1024)
|
||||
: 0;
|
||||
const cpuPercent = Number.isFinite(cpu)
|
||||
? Math.round(cpu * 100) / 100
|
||||
: 0;
|
||||
return { rssBytes, cpuPercent, uptimeSec };
|
||||
}
|
||||
const rssKiB = Number(parts[0]);
|
||||
const cpu = Number(parts[1]);
|
||||
const etimes = Number(parts[2]);
|
||||
const rssBytes = Number.isFinite(rssKiB) ? Math.trunc(rssKiB * 1024) : 0;
|
||||
const cpuPercent = Number.isFinite(cpu) ? Math.round(cpu * 100) / 100 : 0;
|
||||
const uptimeSec = Number.isFinite(etimes)
|
||||
? Math.trunc(etimes)
|
||||
: 0;
|
||||
return { rssBytes, cpuPercent, uptimeSec };
|
||||
}
|
||||
|
||||
function parseActiveSessionsFromHermesStats(text) {
|
||||
const src = String(text);
|
||||
const patterns = [
|
||||
/^\s*Active\s+sessions?:\s*(\d+)/gim,
|
||||
/^\s*active\s+sessions?:\s*(\d+)/gim,
|
||||
/^\s*Total\s+sessions?:\s*(\d+)/gim,
|
||||
];
|
||||
for (const re of patterns) {
|
||||
re.lastIndex = 0;
|
||||
const m = re.exec(src);
|
||||
if (m) {
|
||||
const n = Math.trunc(Number.parseInt(m[1], 10));
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function readActiveSessions() {
|
||||
try {
|
||||
const r = await execFileUtf8("hermes", ["sessions", "stats"]);
|
||||
if (r.errCode === "ENOENT") return 0;
|
||||
return parseActiveSessionsFromHermesStats(`${r.stdout}\n${r.stderr}`);
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function countDirectChildren(mainPid) {
|
||||
if (mainPid <= 0) return 0;
|
||||
try {
|
||||
const r = await execFileUtf8("ps", [
|
||||
"--no-headers",
|
||||
"-o",
|
||||
"pid",
|
||||
"--ppid",
|
||||
String(mainPid),
|
||||
]);
|
||||
if (r.errCode === "ENOENT") return 0;
|
||||
const lines = r.stdout
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
return lines.length;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
export async function compute(db, _peers) {
|
||||
const ts = Date.now();
|
||||
|
||||
let mainPid = 0;
|
||||
let systemdActiveRunning = false;
|
||||
|
||||
try {
|
||||
const st = await readSystemdState();
|
||||
mainPid = st.mainPid;
|
||||
systemdActiveRunning = st.systemdActiveRunning;
|
||||
} catch {
|
||||
mainPid = 0;
|
||||
systemdActiveRunning = false;
|
||||
}
|
||||
|
||||
let psOk = false;
|
||||
try {
|
||||
psOk = await processExists(mainPid);
|
||||
} catch {
|
||||
psOk = false;
|
||||
}
|
||||
|
||||
let rssBytes = 0;
|
||||
let cpuPercent = 0;
|
||||
let uptimeSec = 0;
|
||||
if (psOk) {
|
||||
try {
|
||||
const m = await readPsMetrics(mainPid);
|
||||
rssBytes = m.rssBytes;
|
||||
cpuPercent = m.cpuPercent;
|
||||
uptimeSec = m.uptimeSec;
|
||||
} catch {
|
||||
rssBytes = 0;
|
||||
cpuPercent = 0;
|
||||
uptimeSec = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const alive =
|
||||
systemdActiveRunning && mainPid > 0 && psOk ? 1 : 0;
|
||||
|
||||
let activeSessions = 0;
|
||||
try {
|
||||
activeSessions = await readActiveSessions();
|
||||
} catch {
|
||||
activeSessions = 0;
|
||||
}
|
||||
|
||||
let childProcessCount = 0;
|
||||
if (alive && mainPid > 0) {
|
||||
try {
|
||||
childProcessCount = await countDirectChildren(mainPid);
|
||||
} catch {
|
||||
childProcessCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const storedMainPid = mainPid > 0 ? mainPid : 0;
|
||||
|
||||
const row = {
|
||||
ts,
|
||||
alive,
|
||||
mainPid: storedMainPid,
|
||||
rssBytes: alive ? rssBytes : 0,
|
||||
cpuPercent: alive ? cpuPercent : 0,
|
||||
uptimeSec: alive ? uptimeSec : 0,
|
||||
activeSessions,
|
||||
childProcessCount: alive ? childProcessCount : 0,
|
||||
};
|
||||
|
||||
await db.insert(hermesGatewayHealth).values(row);
|
||||
|
||||
return {
|
||||
ts: row.ts,
|
||||
alive: row.alive,
|
||||
mainPid: row.mainPid,
|
||||
rssBytes: row.rssBytes,
|
||||
cpuPercent: row.cpuPercent,
|
||||
uptimeSec: row.uptimeSec,
|
||||
activeSessions: row.activeSessions,
|
||||
childProcessCount: row.childProcessCount,
|
||||
};
|
||||
}
|
||||
14
senses/hermes-gateway-health/migrations/0001_init.sql
Normal file
14
senses/hermes-gateway-health/migrations/0001_init.sql
Normal file
@ -0,0 +1,14 @@
|
||||
-- Migration: 0001_init
|
||||
-- Creates the hermes_gateway_health table for hermes-gateway-health sense.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS hermes_gateway_health (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER NOT NULL,
|
||||
alive INTEGER NOT NULL,
|
||||
main_pid INTEGER NOT NULL,
|
||||
rss_bytes INTEGER NOT NULL,
|
||||
cpu_percent REAL NOT NULL,
|
||||
uptime_sec INTEGER NOT NULL,
|
||||
active_sessions INTEGER NOT NULL,
|
||||
child_process_count INTEGER NOT NULL
|
||||
);
|
||||
13
senses/hermes-gateway-health/schema.ts
Normal file
13
senses/hermes-gateway-health/schema.ts
Normal file
@ -0,0 +1,13 @@
|
||||
import { integer, real, sqliteTable } from "drizzle-orm/sqlite-core";
|
||||
|
||||
export const hermesGatewayHealth = sqliteTable("hermes_gateway_health", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
ts: integer("ts").notNull(),
|
||||
alive: integer("alive").notNull(),
|
||||
mainPid: integer("main_pid").notNull(),
|
||||
rssBytes: integer("rss_bytes").notNull(),
|
||||
cpuPercent: real("cpu_percent").notNull(),
|
||||
uptimeSec: integer("uptime_sec").notNull(),
|
||||
activeSessions: integer("active_sessions").notNull(),
|
||||
childProcessCount: integer("child_process_count").notNull(),
|
||||
});
|
||||
@ -1,7 +1,38 @@
|
||||
import { loadavg, totalmem, freemem, uptime } from "node:os";
|
||||
import { execSync } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { snapshots } from "./schema.ts";
|
||||
|
||||
const SOCKSTAT_PATH = "/proc/net/sockstat";
|
||||
|
||||
function parseSockstat(content) {
|
||||
let socketsUsed = 0, tcpInuse = 0, tcpOrphan = 0, tcpTw = 0, tcpAlloc = 0, tcpMemPages = 0;
|
||||
|
||||
for (const line of content.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.startsWith("sockets:")) {
|
||||
const parts = trimmed.split(/\s+/);
|
||||
const idx = parts.indexOf("used");
|
||||
if (idx !== -1 && idx + 1 < parts.length) {
|
||||
socketsUsed = Number.parseInt(parts[idx + 1], 10) || 0;
|
||||
}
|
||||
} else if (trimmed.startsWith("TCP:")) {
|
||||
const parts = trimmed.split(/\s+/);
|
||||
const map = {};
|
||||
for (let i = 1; i + 1 < parts.length; i += 2) {
|
||||
map[parts[i]] = Number.parseInt(parts[i + 1], 10) || 0;
|
||||
}
|
||||
tcpInuse = map.inuse ?? 0;
|
||||
tcpOrphan = map.orphan ?? 0;
|
||||
tcpTw = map.tw ?? 0;
|
||||
tcpAlloc = map.alloc ?? 0;
|
||||
tcpMemPages = map.mem ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
return { socketsUsed, tcpInuse, tcpOrphan, tcpTw, tcpAlloc, tcpMemPages };
|
||||
}
|
||||
|
||||
export async function compute(db, _peers) {
|
||||
const [load1, load5, load15] = loadavg();
|
||||
|
||||
@ -23,6 +54,13 @@ export async function compute(db, _peers) {
|
||||
diskUsedPct = total > 0 ? Math.round((used / total) * 10000) / 100 : 0;
|
||||
} catch {}
|
||||
|
||||
// TCP socket stats
|
||||
let tcp = { socketsUsed: 0, tcpInuse: 0, tcpOrphan: 0, tcpTw: 0, tcpAlloc: 0, tcpMemPages: 0 };
|
||||
try {
|
||||
const content = await readFile(SOCKSTAT_PATH, "utf8");
|
||||
tcp = parseSockstat(content);
|
||||
} catch {}
|
||||
|
||||
const ts = Date.now();
|
||||
const uptimeSec = Math.round(uptime());
|
||||
|
||||
@ -31,12 +69,19 @@ export async function compute(db, _peers) {
|
||||
memTotalMB, memUsedMB, memUsedPct,
|
||||
diskTotalGB, diskUsedGB, diskUsedPct,
|
||||
uptimeSec,
|
||||
socketsUsed: tcp.socketsUsed,
|
||||
tcpInuse: tcp.tcpInuse,
|
||||
tcpOrphan: tcp.tcpOrphan,
|
||||
tcpTw: tcp.tcpTw,
|
||||
tcpAlloc: tcp.tcpAlloc,
|
||||
tcpMemPages: tcp.tcpMemPages,
|
||||
});
|
||||
|
||||
return {
|
||||
cpu: { load1m: load1, load5m: load5, load15m: load15 },
|
||||
memory: { totalMB: memTotalMB, usedMB: memUsedMB, usedPct: memUsedPct },
|
||||
disk: { totalGB: diskTotalGB, usedGB: diskUsedGB, usedPct: diskUsedPct },
|
||||
tcp: { socketsUsed: tcp.socketsUsed, inuse: tcp.tcpInuse, orphan: tcp.tcpOrphan, tw: tcp.tcpTw, alloc: tcp.tcpAlloc, memPages: tcp.tcpMemPages },
|
||||
uptimeSec,
|
||||
};
|
||||
}
|
||||
|
||||
@ -0,0 +1,6 @@
|
||||
ALTER TABLE snapshots ADD COLUMN sockets_used INTEGER;
|
||||
ALTER TABLE snapshots ADD COLUMN tcp_inuse INTEGER;
|
||||
ALTER TABLE snapshots ADD COLUMN tcp_orphan INTEGER;
|
||||
ALTER TABLE snapshots ADD COLUMN tcp_tw INTEGER;
|
||||
ALTER TABLE snapshots ADD COLUMN tcp_alloc INTEGER;
|
||||
ALTER TABLE snapshots ADD COLUMN tcp_mem_pages INTEGER;
|
||||
@ -1,4 +1,4 @@
|
||||
import { integer, real, sqliteTable } from "drizzle-orm/sqlite-core";
|
||||
import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
||||
|
||||
export const snapshots = sqliteTable("snapshots", {
|
||||
ts: integer("ts").primaryKey(),
|
||||
@ -12,4 +12,11 @@ export const snapshots = sqliteTable("snapshots", {
|
||||
diskUsedGB: real("disk_used_gb").notNull(),
|
||||
diskUsedPct: real("disk_used_pct").notNull(),
|
||||
uptimeSec: integer("uptime_sec").notNull(),
|
||||
// TCP socket stats (merged from linux-tcp-socket-stats)
|
||||
socketsUsed: integer("sockets_used"),
|
||||
tcpInuse: integer("tcp_inuse"),
|
||||
tcpOrphan: integer("tcp_orphan"),
|
||||
tcpTw: integer("tcp_tw"),
|
||||
tcpAlloc: integer("tcp_alloc"),
|
||||
tcpMemPages: integer("tcp_mem_pages"),
|
||||
});
|
||||
|
||||
@ -1,120 +0,0 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { linuxTcpSocketStats } from "./schema.ts";
|
||||
|
||||
const SOCKSTAT_PATH = "/proc/net/sockstat";
|
||||
const RAW_MAX = 4096;
|
||||
|
||||
function parseInt10(s) {
|
||||
const n = Number.parseInt(String(s), 10);
|
||||
return Number.isFinite(n) ? Math.trunc(n) : NaN;
|
||||
}
|
||||
|
||||
function parseSockstat(content) {
|
||||
let socketsUsed = 0;
|
||||
let tcpInuse = 0;
|
||||
let tcpOrphan = 0;
|
||||
let tcpTw = 0;
|
||||
let tcpAlloc = 0;
|
||||
let tcpMemPages = 0;
|
||||
let parseOk = 1;
|
||||
|
||||
let socketsOk = false;
|
||||
let tcpOk = false;
|
||||
|
||||
for (const line of content.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.startsWith("sockets:")) {
|
||||
const parts = trimmed.split(/\s+/);
|
||||
const usedIdx = parts.indexOf("used");
|
||||
if (usedIdx !== -1 && usedIdx + 1 < parts.length) {
|
||||
const v = parseInt10(parts[usedIdx + 1]);
|
||||
if (Number.isFinite(v)) {
|
||||
socketsUsed = v;
|
||||
socketsOk = true;
|
||||
}
|
||||
}
|
||||
} else if (trimmed.startsWith("TCP:")) {
|
||||
const parts = trimmed.split(/\s+/);
|
||||
const map = {};
|
||||
for (let i = 1; i + 1 < parts.length; i += 2) {
|
||||
map[parts[i]] = parseInt10(parts[i + 1]);
|
||||
}
|
||||
const keys = ["inuse", "orphan", "tw", "alloc", "mem"];
|
||||
if (keys.every((k) => Number.isFinite(map[k]))) {
|
||||
tcpInuse = map.inuse;
|
||||
tcpOrphan = map.orphan;
|
||||
tcpTw = map.tw;
|
||||
tcpAlloc = map.alloc;
|
||||
tcpMemPages = map.mem;
|
||||
tcpOk = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!socketsOk || !tcpOk) {
|
||||
parseOk = 0;
|
||||
socketsUsed = 0;
|
||||
tcpInuse = 0;
|
||||
tcpOrphan = 0;
|
||||
tcpTw = 0;
|
||||
tcpAlloc = 0;
|
||||
tcpMemPages = 0;
|
||||
}
|
||||
|
||||
return {
|
||||
socketsUsed,
|
||||
tcpInuse,
|
||||
tcpOrphan,
|
||||
tcpTw,
|
||||
tcpAlloc,
|
||||
tcpMemPages,
|
||||
parseOk,
|
||||
};
|
||||
}
|
||||
|
||||
export async function compute(db, _peers) {
|
||||
const ts = Date.now();
|
||||
let rawSockstat = "";
|
||||
let row = {
|
||||
socketsUsed: 0,
|
||||
tcpInuse: 0,
|
||||
tcpOrphan: 0,
|
||||
tcpTw: 0,
|
||||
tcpAlloc: 0,
|
||||
tcpMemPages: 0,
|
||||
parseOk: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
const content = await readFile(SOCKSTAT_PATH, "utf8");
|
||||
rawSockstat =
|
||||
content.length > RAW_MAX ? content.slice(0, RAW_MAX) : content;
|
||||
row = parseSockstat(content);
|
||||
} catch {
|
||||
rawSockstat = "";
|
||||
}
|
||||
|
||||
await db.insert(linuxTcpSocketStats).values({
|
||||
ts,
|
||||
socketsUsed: row.socketsUsed,
|
||||
tcpInuse: row.tcpInuse,
|
||||
tcpOrphan: row.tcpOrphan,
|
||||
tcpTw: row.tcpTw,
|
||||
tcpAlloc: row.tcpAlloc,
|
||||
tcpMemPages: row.tcpMemPages,
|
||||
parseOk: row.parseOk,
|
||||
rawSockstat,
|
||||
});
|
||||
|
||||
return {
|
||||
ts,
|
||||
socketsUsed: row.socketsUsed,
|
||||
tcpInuse: row.tcpInuse,
|
||||
tcpOrphan: row.tcpOrphan,
|
||||
tcpTw: row.tcpTw,
|
||||
tcpAlloc: row.tcpAlloc,
|
||||
tcpMemPages: row.tcpMemPages,
|
||||
parseOk: row.parseOk,
|
||||
rawSockstat,
|
||||
};
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
-- Migration: 0001_init
|
||||
-- Creates the linux_tcp_socket_stats table for linux-tcp-socket-stats sense.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS linux_tcp_socket_stats (
|
||||
ts INTEGER PRIMARY KEY,
|
||||
sockets_used INTEGER NOT NULL,
|
||||
tcp_inuse INTEGER NOT NULL,
|
||||
tcp_orphan INTEGER NOT NULL,
|
||||
tcp_tw INTEGER NOT NULL,
|
||||
tcp_alloc INTEGER NOT NULL,
|
||||
tcp_mem_pages INTEGER NOT NULL,
|
||||
parse_ok INTEGER NOT NULL,
|
||||
raw_sockstat TEXT NOT NULL
|
||||
);
|
||||
@ -1,13 +0,0 @@
|
||||
import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
||||
|
||||
export const linuxTcpSocketStats = sqliteTable("linux_tcp_socket_stats", {
|
||||
ts: integer("ts").primaryKey(),
|
||||
socketsUsed: integer("sockets_used").notNull(),
|
||||
tcpInuse: integer("tcp_inuse").notNull(),
|
||||
tcpOrphan: integer("tcp_orphan").notNull(),
|
||||
tcpTw: integer("tcp_tw").notNull(),
|
||||
tcpAlloc: integer("tcp_alloc").notNull(),
|
||||
tcpMemPages: integer("tcp_mem_pages").notNull(),
|
||||
parseOk: integer("parse_ok").notNull(),
|
||||
rawSockstat: text("raw_sockstat").notNull(),
|
||||
});
|
||||
@ -3,26 +3,89 @@ import { execSync } from "node:child_process";
|
||||
import { readFileSync, existsSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const NERVE_ROOT = join(process.env.HOME ?? "/home/azureuser", ".uncaged-nerve");
|
||||
const HOME = process.env.HOME ?? "/home/azureuser";
|
||||
const NERVE_ROOT = join(HOME, ".uncaged-nerve");
|
||||
const SENSES_DIR = join(NERVE_ROOT, "senses");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function nerveCommandEnv(): NodeJS.ProcessEnv {
|
||||
const pnpmHome = join(HOME, ".local/share/pnpm");
|
||||
const npmUserBin = join(HOME, ".local/share/npm/bin");
|
||||
return {
|
||||
...process.env,
|
||||
PNPM_HOME: pnpmHome,
|
||||
PATH: `${npmUserBin}:${pnpmHome}:${process.env.PATH ?? ""}`,
|
||||
};
|
||||
}
|
||||
|
||||
function run(cmd: string, cwd?: string): string {
|
||||
return execSync(cmd, {
|
||||
encoding: "utf-8",
|
||||
cwd: cwd ?? NERVE_ROOT,
|
||||
timeout: 300_000,
|
||||
env: {
|
||||
...process.env,
|
||||
PNPM_HOME: join(process.env.HOME ?? "/home/azureuser", ".local/share/pnpm"),
|
||||
PATH: `${join(process.env.HOME ?? "/home/azureuser", ".local/share/pnpm")}:${process.env.PATH}`,
|
||||
},
|
||||
env: nerveCommandEnv(),
|
||||
}).trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the same checks the workflow used to ask Hermes to perform, but locally.
|
||||
* Hermes chat often returns UI prose instead of shell output, which caused false failures.
|
||||
*/
|
||||
function runSenseSmokeTest(senseName: string): { ok: boolean; log: string; reason: string } {
|
||||
const logParts: string[] = [];
|
||||
try {
|
||||
const status = run("nerve status");
|
||||
logParts.push("=== nerve status ===\n" + status);
|
||||
if (!status.includes(senseName)) {
|
||||
return {
|
||||
ok: false,
|
||||
log: logParts.join("\n\n"),
|
||||
reason: `Sense "${senseName}" not listed in \`nerve status\` output`,
|
||||
};
|
||||
}
|
||||
|
||||
const triggerOut = run(`nerve sense trigger ${senseName}`);
|
||||
logParts.push("=== nerve sense trigger ===\n" + triggerOut);
|
||||
|
||||
let lastQuery = "";
|
||||
for (let i = 0; i < 25; i++) {
|
||||
run("sleep 1");
|
||||
try {
|
||||
lastQuery = run(`nerve sense query ${senseName}`);
|
||||
} catch (e) {
|
||||
logParts.push(`=== nerve sense query (attempt ${i + 1}) ===\nERROR: ${String(e)}`);
|
||||
continue;
|
||||
}
|
||||
logParts.push(`=== nerve sense query (attempt ${i + 1}) ===\n${lastQuery}`);
|
||||
if (!lastQuery.includes("(0 rows)")) {
|
||||
return {
|
||||
ok: true,
|
||||
log: logParts.join("\n\n"),
|
||||
reason: "Trigger succeeded and query returned at least one row",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
log: logParts.join("\n\n"),
|
||||
reason: lastQuery.includes("(0 rows)")
|
||||
? "Query still returned 0 rows after trigger (compute error, throttle drop, or DB not written)"
|
||||
: "Timed out waiting for successful sense query",
|
||||
};
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
return {
|
||||
ok: false,
|
||||
log: logParts.join("\n\n"),
|
||||
reason: `Smoke test command failed: ${msg}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Call a cheap LLM with tool_choice to extract structured metadata from text.
|
||||
* Uses DashScope (Alibaba Cloud, OpenAI-compatible) with qwen-plus.
|
||||
@ -61,13 +124,6 @@ function cursorAgent(prompt: string, mode: "plan" | "ask" | "default", cwd: stri
|
||||
);
|
||||
}
|
||||
|
||||
function hermesRun(prompt: string): string {
|
||||
const escaped = prompt.replace(/'/g, "'\\''");
|
||||
return run(
|
||||
`hermes chat -q '${escaped}' --model anthropic/claude-sonnet-4 -t terminal --yolo 2>&1 || true`,
|
||||
);
|
||||
}
|
||||
|
||||
// Build context string with existing sense examples
|
||||
function buildSenseExamples(): string {
|
||||
const examples: string[] = [];
|
||||
@ -263,50 +319,18 @@ Create all files now.`;
|
||||
};
|
||||
}
|
||||
|
||||
// Reload daemon to pick up new sense, then trigger it
|
||||
const testPrompt = `You need to test a newly created Nerve sense called "${senseName}".
|
||||
const smoke = runSenseSmokeTest(senseName);
|
||||
ctx.log(`tester: smoke — ok=${smoke.ok}, reason="${smoke.reason}"`);
|
||||
ctx.log(`tester: log head — ${smoke.log.substring(0, 400)}`);
|
||||
|
||||
Steps:
|
||||
1. Restart the nerve daemon: run "nerve stop && nerve start" (make sure PNPM_HOME and PATH are set)
|
||||
2. Wait 2 seconds
|
||||
3. Check status: run "nerve status" and verify "${senseName}" appears in senses list
|
||||
4. Trigger the sense: run "nerve sense trigger ${senseName}"
|
||||
5. Check the result: run "nerve sense query ${senseName} --limit 1"
|
||||
|
||||
Report whether the sense works. If there are errors, include the full error output.
|
||||
The nerve binary is at ~/.local/share/pnpm/nerve.
|
||||
|
||||
Reply with either:
|
||||
- "PASS: <summary>" if the sense works
|
||||
- "FAIL: <error details>" if it doesn't`;
|
||||
|
||||
const result = hermesRun(testPrompt);
|
||||
ctx.log(`tester: raw result — ${result.substring(0, 300)}`);
|
||||
|
||||
// Use LLM to judge pass/fail instead of fragile string matching
|
||||
const verdict = llmExtract<{ passed: boolean; reason: string }>(
|
||||
`Test output for sense "${senseName}":\n\n${result.substring(0, 4000)}`,
|
||||
"judge_test_result",
|
||||
"Determine whether the test passed or failed based on the output",
|
||||
{
|
||||
type: "object",
|
||||
properties: {
|
||||
passed: { type: "boolean", description: "true if the sense was successfully triggered and returned valid data" },
|
||||
reason: { type: "string", description: "Brief explanation of why it passed or failed" },
|
||||
},
|
||||
required: ["passed", "reason"],
|
||||
},
|
||||
);
|
||||
ctx.log(`tester: verdict — passed=${verdict.passed}, reason="${verdict.reason}"`);
|
||||
|
||||
if (verdict.passed) {
|
||||
return { type: "test_passed", senseName, result: verdict.reason };
|
||||
if (smoke.ok) {
|
||||
return { type: "test_passed", senseName, result: smoke.reason };
|
||||
}
|
||||
|
||||
return {
|
||||
type: "test_failed",
|
||||
senseName,
|
||||
reason: verdict.reason,
|
||||
reason: `${smoke.reason}\n\n--- smoke log ---\n${smoke.log}`,
|
||||
attempt,
|
||||
};
|
||||
},
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user