Initial import: monitoring, power-management, maintenance, web scripts

2026-08-05 12:26:50 +02:00
commit 1936ded08f
10 changed files with 1306 additions and 0 deletions
@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""
+Export Hermes Agent metrics to Prometheus textfile collector format.
+
+Reads ~/.hermes/state.db and writes /var/lib/node_exporter/textfile_collector/hermes.prom.
+Designed to run as a cron job (every 60s).
+
+Metrics produced:
+  hermes_sessions_total{source,model}          — session count
+  hermes_messages_total{source}                 — message count
+  hermes_tokens_input_total{source,model}       — input tokens
+  hermes_tokens_output_total{source,model}      — output tokens
+  hermes_tokens_cache_read_total{source,model}  — cache read tokens
+  hermes_tokens_cache_write_total{source,model}  — cache write tokens
+  hermes_tokens_reasoning_total{source,model}    — reasoning tokens
+  hermes_cost_usd_total{source,model}           — estimated cost in USD
+  hermes_api_calls_total{source,model}          — API call count
+  hermes_sessions_active{source}                — active sessions (not ended)
+  hermes_sessions_ended{source}                 — ended sessions in period
+  hermes_errors_total{source}                   — sessions with errors
+  hermes_db_size_bytes                          — state.db file size
+  hermes_scrape_duration_seconds                — how long the scrape took
+"""
+
+import json
+import os
+import sqlite3
+import time
+from collections import defaultdict
+from pathlib import Path
+
+# --- Config ----------------------------------------------------------------
+HERMES_HOME = Path(os.environ.get("HERMES_HOME", os.path.expanduser("~/.hermes")))
+DB_PATH = HERMES_HOME / "state.db"
+OUTPUT_PATH = Path("/var/lib/node_exporter/textfile_collector/hermes.prom")
+# ---------------------------------------------------------------------------
+
+# Labels that are safe to expose (no PII)
+SAFE_SOURCES = {"cli", "webui", "telegram", "discord", "slack", "whatsapp",
+                "signal", "matrix", "teams", "email", "cron", "gateway", "acp"}
+
+
+def _safe_source(source: str) -> str:
+    """Normalise source to a known label or 'other'."""
+    s = source.lower().strip()
+    return s if s in SAFE_SOURCES else "other"
+
+
+def _fmt_metric(name: str, value, labels: dict = None) -> str:
+    """Format one Prometheus metric line."""
+    if labels:
+        label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
+        return f"{name}{{{label_str}}} {value}"
+    return f"{name} {value}"
+
+
+def scrape() -> str:
+    """Read state.db and return Prometheus text."""
+    start = time.monotonic()
+    lines = []
+    lines.append("# HELP hermes_sessions_total Total Hermes sessions created")
+    lines.append("# TYPE hermes_sessions_total counter")
+    lines.append("# HELP hermes_messages_total Total messages across all sessions")
+    lines.append("# TYPE hermes_messages_total counter")
+    lines.append("# HELP hermes_tokens_input_total Total input tokens consumed")
+    lines.append("# TYPE hermes_tokens_input_total counter")
+    lines.append("# HELP hermes_tokens_output_total Total output tokens generated")
+    lines.append("# TYPE hermes_tokens_output_total counter")
+    lines.append("# HELP hermes_tokens_cache_read_total Total cache read tokens")
+    lines.append("# TYPE hermes_tokens_cache_read_total counter")
+    lines.append("# HELP hermes_tokens_cache_write_total Total cache write tokens")
+    lines.append("# TYPE hermes_tokens_cache_write_total counter")
+    lines.append("# HELP hermes_tokens_reasoning_total Total reasoning tokens")
+    lines.append("# TYPE hermes_tokens_reasoning_total counter")
+    lines.append("# HELP hermes_cost_usd_total Total estimated cost in USD")
+    lines.append("# TYPE hermes_cost_usd_total counter")
+    lines.append("# HELP hermes_api_calls_total Total API calls made")
+    lines.append("# TYPE hermes_api_calls_total counter")
+    lines.append("# HELP hermes_sessions_active Currently active sessions")
+    lines.append("# TYPE hermes_sessions_active gauge")
+    lines.append("# HELP hermes_sessions_ended Sessions ended in the last hour")
+    lines.append("# TYPE hermes_sessions_ended gauge")
+    lines.append("# HELP hermes_errors_total Sessions with non-nil end_reason (errors)")
+    lines.append("# TYPE hermes_errors_total counter")
+    lines.append("# HELP hermes_db_size_bytes Size of state.db on disk")
+    lines.append("# TYPE hermes_db_size_bytes gauge")
+    lines.append("# HELP hermes_scrape_duration_seconds Time spent scraping metrics")
+    lines.append("# TYPE hermes_scrape_duration_seconds gauge")
+
+    if not DB_PATH.exists():
+        lines.append(_fmt_metric("hermes_db_size_bytes", 0))
+        lines.append(_fmt_metric("hermes_scrape_duration_seconds", time.monotonic() - start))
+        return "\n".join(lines) + "\n"
+
+    # DB file size
+    lines.append(_fmt_metric("hermes_db_size_bytes", DB_PATH.stat().st_size))
+
+    try:
+        conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True)
+        conn.row_factory = sqlite3.Row
+        cur = conn.cursor()
+
+        now = time.time()
+        one_hour_ago = now - 3600
+
+        # --- Per-source, per-model aggregates (all time) ---
+        cur.execute("""
+            SELECT
+                COALESCE(NULLIF(source, ''), 'unknown') AS source,
+                COALESCE(NULLIF(model, ''), 'unknown') AS model,
+                COUNT(*) AS sessions,
+                SUM(message_count) AS messages,
+                SUM(input_tokens) AS input_tokens,
+                SUM(output_tokens) AS output_tokens,
+                SUM(cache_read_tokens) AS cache_read_tokens,
+                SUM(cache_write_tokens) AS cache_write_tokens,
+                SUM(reasoning_tokens) AS reasoning_tokens,
+                SUM(estimated_cost_usd) AS cost,
+                SUM(api_call_count) AS api_calls
+            FROM sessions
+            GROUP BY source, model
+        """)
+        for row in cur.fetchall():
+            src = _safe_source(row["source"])
+            model = row["model"]
+            labels = {"source": src, "model": model}
+            lines.append(_fmt_metric("hermes_sessions_total", row["sessions"] or 0, labels))
+            lines.append(_fmt_metric("hermes_messages_total", row["messages"] or 0, labels))
+            lines.append(_fmt_metric("hermes_tokens_input_total", row["input_tokens"] or 0, labels))
+            lines.append(_fmt_metric("hermes_tokens_output_total", row["output_tokens"] or 0, labels))
+            lines.append(_fmt_metric("hermes_tokens_cache_read_total", row["cache_read_tokens"] or 0, labels))
+            lines.append(_fmt_metric("hermes_tokens_cache_write_total", row["cache_write_tokens"] or 0, labels))
+            lines.append(_fmt_metric("hermes_tokens_reasoning_total", row["reasoning_tokens"] or 0, labels))
+            lines.append(_fmt_metric("hermes_cost_usd_total", row["cost"] or 0, labels))
+            lines.append(_fmt_metric("hermes_api_calls_total", row["api_calls"] or 0, labels))
+
+        # --- Active sessions (not ended) ---
+        cur.execute("""
+            SELECT COALESCE(NULLIF(source, ''), 'unknown') AS source, COUNT(*) AS cnt
+            FROM sessions WHERE ended_at IS NULL
+            GROUP BY source
+        """)
+        for row in cur.fetchall():
+            lines.append(_fmt_metric("hermes_sessions_active", row["cnt"],
+                                     {"source": _safe_source(row["source"])}))
+
+        # --- Sessions ended in last hour ---
+        cur.execute("""
+            SELECT COALESCE(NULLIF(source, ''), 'unknown') AS source, COUNT(*) AS cnt
+            FROM sessions WHERE ended_at >= ? AND ended_at IS NOT NULL
+            GROUP BY source
+        """, (one_hour_ago,))
+        for row in cur.fetchall():
+            lines.append(_fmt_metric("hermes_sessions_ended", row["cnt"],
+                                     {"source": _safe_source(row["source"])}))
+
+        # --- Sessions with error end_reason ---
+        cur.execute("""
+            SELECT COALESCE(NULLIF(source, ''), 'unknown') AS source, COUNT(*) AS cnt
+            FROM sessions
+            WHERE end_reason IS NOT NULL
+              AND end_reason NOT IN ('completed', 'user_exit', 'timeout', 'compression_split', '')
+            GROUP BY source
+        """)
+        for row in cur.fetchall():
+            lines.append(_fmt_metric("hermes_errors_total", row["cnt"],
+                                     {"source": _safe_source(row["source"])}))
+
+        conn.close()
+    except sqlite3.Error as e:
+        lines.append(f"# ERROR scraping state.db: {e}")
+
+    elapsed = time.monotonic() - start
+    lines.append(_fmt_metric("hermes_scrape_duration_seconds", elapsed))
+    return "\n".join(lines) + "\n"
+
+
+def main():
+    try:
+        output = scrape()
+        OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+        # Atomic write via temp file + rename
+        tmp = OUTPUT_PATH.with_suffix(".prom.tmp")
+        tmp.write_text(output)
+        tmp.rename(OUTPUT_PATH)
+    except Exception as e:
+        # Write an error metric so Prometheus can alert on it
+        error_output = (
+            "# HELP hermes_scrape_error 1 if the last scrape failed\n"
+            f"# TYPE hermes_scrape_error gauge\n"
+            f"hermes_scrape_error 1.0\n"
+        )
+        try:
+            OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+            tmp = OUTPUT_PATH.with_suffix(".prom.tmp")
+            tmp.write_text(error_output)
+            tmp.rename(OUTPUT_PATH)
+        except Exception:
+            pass
+        raise
+
+
+if __name__ == "__main__":
+    main()