#!/usr/bin/env python3 """ Export Hermes Agent metrics to Prometheus textfile collector format. Reads ~/.hermes/state.db and writes /var/lib/node_exporter/textfile_collector/hermes.prom. Designed to run as a cron job (every 60s). Metrics produced: hermes_sessions_total{source,model} — session count hermes_messages_total{source} — message count hermes_tokens_input_total{source,model} — input tokens hermes_tokens_output_total{source,model} — output tokens hermes_tokens_cache_read_total{source,model} — cache read tokens hermes_tokens_cache_write_total{source,model} — cache write tokens hermes_tokens_reasoning_total{source,model} — reasoning tokens hermes_cost_usd_total{source,model} — estimated cost in USD hermes_api_calls_total{source,model} — API call count hermes_sessions_active{source} — active sessions (not ended) hermes_sessions_ended{source} — ended sessions in period hermes_errors_total{source} — sessions with errors hermes_db_size_bytes — state.db file size hermes_scrape_duration_seconds — how long the scrape took """ import json import os import sqlite3 import time from collections import defaultdict from pathlib import Path # --- Config ---------------------------------------------------------------- HERMES_HOME = Path(os.environ.get("HERMES_HOME", os.path.expanduser("~/.hermes"))) DB_PATH = HERMES_HOME / "state.db" OUTPUT_PATH = Path("/var/lib/node_exporter/textfile_collector/hermes.prom") # --------------------------------------------------------------------------- # Labels that are safe to expose (no PII) SAFE_SOURCES = {"cli", "webui", "telegram", "discord", "slack", "whatsapp", "signal", "matrix", "teams", "email", "cron", "gateway", "acp"} def _safe_source(source: str) -> str: """Normalise source to a known label or 'other'.""" s = source.lower().strip() return s if s in SAFE_SOURCES else "other" def _fmt_metric(name: str, value, labels: dict = None) -> str: """Format one Prometheus metric line.""" if labels: label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items())) return f"{name}{{{label_str}}} {value}" return f"{name} {value}" def scrape() -> str: """Read state.db and return Prometheus text.""" start = time.monotonic() lines = [] lines.append("# HELP hermes_sessions_total Total Hermes sessions created") lines.append("# TYPE hermes_sessions_total counter") lines.append("# HELP hermes_messages_total Total messages across all sessions") lines.append("# TYPE hermes_messages_total counter") lines.append("# HELP hermes_tokens_input_total Total input tokens consumed") lines.append("# TYPE hermes_tokens_input_total counter") lines.append("# HELP hermes_tokens_output_total Total output tokens generated") lines.append("# TYPE hermes_tokens_output_total counter") lines.append("# HELP hermes_tokens_cache_read_total Total cache read tokens") lines.append("# TYPE hermes_tokens_cache_read_total counter") lines.append("# HELP hermes_tokens_cache_write_total Total cache write tokens") lines.append("# TYPE hermes_tokens_cache_write_total counter") lines.append("# HELP hermes_tokens_reasoning_total Total reasoning tokens") lines.append("# TYPE hermes_tokens_reasoning_total counter") lines.append("# HELP hermes_cost_usd_total Total estimated cost in USD") lines.append("# TYPE hermes_cost_usd_total counter") lines.append("# HELP hermes_api_calls_total Total API calls made") lines.append("# TYPE hermes_api_calls_total counter") lines.append("# HELP hermes_sessions_active Currently active sessions") lines.append("# TYPE hermes_sessions_active gauge") lines.append("# HELP hermes_sessions_ended Sessions ended in the last hour") lines.append("# TYPE hermes_sessions_ended gauge") lines.append("# HELP hermes_errors_total Sessions with non-nil end_reason (errors)") lines.append("# TYPE hermes_errors_total counter") lines.append("# HELP hermes_db_size_bytes Size of state.db on disk") lines.append("# TYPE hermes_db_size_bytes gauge") lines.append("# HELP hermes_scrape_duration_seconds Time spent scraping metrics") lines.append("# TYPE hermes_scrape_duration_seconds gauge") if not DB_PATH.exists(): lines.append(_fmt_metric("hermes_db_size_bytes", 0)) lines.append(_fmt_metric("hermes_scrape_duration_seconds", time.monotonic() - start)) return "\n".join(lines) + "\n" # DB file size lines.append(_fmt_metric("hermes_db_size_bytes", DB_PATH.stat().st_size)) try: conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True) conn.row_factory = sqlite3.Row cur = conn.cursor() now = time.time() one_hour_ago = now - 3600 # --- Per-source, per-model aggregates (all time) --- cur.execute(""" SELECT COALESCE(NULLIF(source, ''), 'unknown') AS source, COALESCE(NULLIF(model, ''), 'unknown') AS model, COUNT(*) AS sessions, SUM(message_count) AS messages, SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens, SUM(cache_read_tokens) AS cache_read_tokens, SUM(cache_write_tokens) AS cache_write_tokens, SUM(reasoning_tokens) AS reasoning_tokens, SUM(estimated_cost_usd) AS cost, SUM(api_call_count) AS api_calls FROM sessions GROUP BY source, model """) for row in cur.fetchall(): src = _safe_source(row["source"]) model = row["model"] labels = {"source": src, "model": model} lines.append(_fmt_metric("hermes_sessions_total", row["sessions"] or 0, labels)) lines.append(_fmt_metric("hermes_messages_total", row["messages"] or 0, labels)) lines.append(_fmt_metric("hermes_tokens_input_total", row["input_tokens"] or 0, labels)) lines.append(_fmt_metric("hermes_tokens_output_total", row["output_tokens"] or 0, labels)) lines.append(_fmt_metric("hermes_tokens_cache_read_total", row["cache_read_tokens"] or 0, labels)) lines.append(_fmt_metric("hermes_tokens_cache_write_total", row["cache_write_tokens"] or 0, labels)) lines.append(_fmt_metric("hermes_tokens_reasoning_total", row["reasoning_tokens"] or 0, labels)) lines.append(_fmt_metric("hermes_cost_usd_total", row["cost"] or 0, labels)) lines.append(_fmt_metric("hermes_api_calls_total", row["api_calls"] or 0, labels)) # --- Active sessions (not ended) --- cur.execute(""" SELECT COALESCE(NULLIF(source, ''), 'unknown') AS source, COUNT(*) AS cnt FROM sessions WHERE ended_at IS NULL GROUP BY source """) for row in cur.fetchall(): lines.append(_fmt_metric("hermes_sessions_active", row["cnt"], {"source": _safe_source(row["source"])})) # --- Sessions ended in last hour --- cur.execute(""" SELECT COALESCE(NULLIF(source, ''), 'unknown') AS source, COUNT(*) AS cnt FROM sessions WHERE ended_at >= ? AND ended_at IS NOT NULL GROUP BY source """, (one_hour_ago,)) for row in cur.fetchall(): lines.append(_fmt_metric("hermes_sessions_ended", row["cnt"], {"source": _safe_source(row["source"])})) # --- Sessions with error end_reason --- cur.execute(""" SELECT COALESCE(NULLIF(source, ''), 'unknown') AS source, COUNT(*) AS cnt FROM sessions WHERE end_reason IS NOT NULL AND end_reason NOT IN ('completed', 'user_exit', 'timeout', 'compression_split', '') GROUP BY source """) for row in cur.fetchall(): lines.append(_fmt_metric("hermes_errors_total", row["cnt"], {"source": _safe_source(row["source"])})) conn.close() except sqlite3.Error as e: lines.append(f"# ERROR scraping state.db: {e}") elapsed = time.monotonic() - start lines.append(_fmt_metric("hermes_scrape_duration_seconds", elapsed)) return "\n".join(lines) + "\n" def main(): try: output = scrape() OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) # Atomic write via temp file + rename tmp = OUTPUT_PATH.with_suffix(".prom.tmp") tmp.write_text(output) tmp.rename(OUTPUT_PATH) except Exception as e: # Write an error metric so Prometheus can alert on it error_output = ( "# HELP hermes_scrape_error 1 if the last scrape failed\n" f"# TYPE hermes_scrape_error gauge\n" f"hermes_scrape_error 1.0\n" ) try: OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) tmp = OUTPUT_PATH.with_suffix(".prom.tmp") tmp.write_text(error_output) tmp.rename(OUTPUT_PATH) except Exception: pass raise if __name__ == "__main__": main()