{
 "batch": "deep-24",
 "setup": "Hermes cloned from github.com/NousResearch/hermes-agent (HEAD 8ec22731) and built with uv sync --locked --python 3.14 (uv and CPython kept under test/scratch); tryit.py from the gist ran each plugin against tests/fakes/fake_llm_provider.py. Plugin tools are not in the default offered toolsets, so added_by_plugin was empty for all, but planned calls to plugin tools were dispatched and ran. Extra: a tryit variant that sets the scripted final reply and plugin config (for say-do-gate); plugin CLIs run against each test home; pytest, pytest-asyncio, numpy, websockets, pyyaml added to the Hermes venv for plugin tests. No GPU, docker, PipeWire, whisper.cpp, ffmpeg or real keys on this machine.",
 "plugins": [
  {
   "pr": 150800,
   "name": "hypnos",
   "validate": "passed: hermes plugins validate --json ok=true, all checks green (manifest, loadable, capability probe, declared tools/hooks/middleware, collisions, security scan safe, no core override, desktop surface inside SDK; python_runtime external)",
   "inside": "Loaded (doctor: 0 tools, 9 hooks, llm_execution middleware; Hermes warns provides_middleware is an unknown manifest field but loads it). A chat with terminal ls and a bad read_file ran; hooks captured it (data.db: 1 session, 1 episode, 3 steps, 2 signals, 3 routes). hermes hypnos status/doctor/sleep worked; sleep stopped cleanly with an exact trainer-install hint. With a CPU torch trainer env installed, demo timelapse --tiny ran day 1 (21 synthetic turns) but night 1 training did not finish in 7 min on CPU (first try failed because the worker strips proxy env and could not reach Hugging Face).",
   "tests": "passed: 229 passed, 2 skipped (e2e trainer tests need HYPNOS_E2E_TRAINER)",
   "useful": 3,
   "reliable": 4,
   "polish": 5,
   "creative": 5,
   "beyond": 5,
   "total": 27,
   "rank": 1,
   "best": "Captures approvals, denials and corrections through nine hooks and trains a personal LoRA brain that Hermes routes mastered turns to, a change to Hermes's own model only a plugin can make.",
   "worst": "The core loop needs a separate torch trainer and ideally a GPU; on CPU here a tiny-model night did not finish in 7 minutes, and its headline numbers come from a synthetic GPU run.",
   "blocked": null
  },
  {
   "pr": 149968,
   "name": "lahja",
   "validate": "passed: hermes plugins validate --json ok=true, all checks green (manifest, loadable, capability probe, declared tools/hooks/middleware, collisions, security scan safe, no core override)",
   "inside": "Loaded (doctor: 0 tools, 0 hooks; registers a transcription provider, a /stt-fix command and hermes lahja CLI). hermes lahja models listed 3 checksum-pinned models; lahja status reported whisper-cli, ffmpeg and models MISSING; lahja test on a missing file failed cleanly with 'whisper-cli not found (brew install whisper-cpp)'. No real transcription possible here (no whisper.cpp or 574 MB model).",
   "tests": "passed: 99 passed, 6 skipped (real-engine tests need whisper.cpp)",
   "useful": 3,
   "reliable": 4,
   "polish": 4,
   "creative": 4,
   "beyond": 4,
   "total": 23,
   "rank": 6,
   "best": "Plugs a local dialect-tuned Whisper into Hermes's speech-to-text with VAD, hallucination filtering and language routing, backed by a reproducible WER benchmark.",
   "worst": "Serves a niche of Arabic-dialect voice users, and its setup hints are macOS-centric (brew, Metal); the real engine path could not be exercised here.",
   "blocked": null
  },
  {
   "pr": 150696,
   "name": "privacy-xray",
   "validate": "passed: hermes plugins validate --json ok=true, all checks green (manifest, loadable, capability probe, declared tools/hooks/middleware, collisions, security scan safe, no core override)",
   "inside": "Loaded (doctor: 1 tool, 2 hooks; plus /xray command, skill, dashboard tab). privacy_xray check returned 0 red, 4 amber, 24 green with exact fix commands; map recorded the 2 outgoing requests to 127.0.0.1 with no personal data; find (valid and 2-char query) was refused as not on this machine because the in-process session had no surface (fail-closed); an invalid action was rejected by Hermes's schema validation.",
   "tests": "passed: 79 tests OK with its documented runner (Hermes python from the Hermes dir); under plain pytest all 79 error on a relative-import collection issue",
   "useful": 3,
   "reliable": 4,
   "polish": 4,
   "creative": 4,
   "beyond": 4,
   "total": 23,
   "rank": 5,
   "best": "Hooks every main and auxiliary model request to show, as metadata only, which vendors and countries see your chats, with a canary person to trace and a config audit with fix commands.",
   "worst": "Mostly for privacy-minded or EU users, and find fails closed whenever Hermes does not report a local surface, as in this run.",
   "blocked": null
  },
  {
   "pr": 153853,
   "name": "keryx",
   "validate": "passed: hermes plugins validate --json ok=true, all checks green (manifest, loadable, capability probe, declared tools/hooks/middleware, collisions, security scan safe, no core override; 2 python requirements installed; isolation in_process because of ctx.post_message)",
   "inside": "Loaded (doctor: 1 tool place_call, 2 hooks; /calls and hermes keryx CLI). place_call to 911 was refused as an emergency number; a valid booking brief was refused as inside quiet hours (21:00-09:00); a call missing 'to' was rejected by schema validation. hermes keryx simulate ran a fake call end to end with the approval prompt, disclosure line, ask_principal and outcome; keryx doctor listed missing PipeWire tools with a fix.",
   "tests": "75 passed, 3 failed (test_linux_bluetooth: pw-record/pw-play/pw-dump and a modem not present on this machine)",
   "useful": 3,
   "reliable": 4,
   "polish": 5,
   "creative": 5,
   "beyond": 4,
   "total": 25,
   "rank": 3,
   "best": "Lets Hermes place real phone calls from the user's own phone over Bluetooth, with a per-call approval, emergency and premium-number blocks, quiet hours and a rate limit enforced in code, and the key read through Hermes's secret scope.",
   "worst": "Needs Linux with PipeWire 1.4 HFP, a paired phone and an OpenAI realtime key, so few users can run it, and 3 of its tests assume that hardware stack.",
   "blocked": null
  },
  {
   "pr": 149365,
   "name": "say-do-gate",
   "validate": "passed: hermes plugins validate --json ok=true, all checks green (manifest, loadable, capability probe, declared tools/hooks/middleware, collisions, security scan safe, no core override)",
   "inside": "Loaded (doctor: 0 tools, 3 hooks). In annotate mode inside Hermes, a reply claiming 'all tests pass' after only a failing command got the footer '[say-do] Claimed: tests pass. Ledger: 0 test commands run.'; the unbacked 'wrote report.md' claim in the same reply was not flagged, and inline code `echo 3 passed` was misread as a '3 tests pass' claim.",
   "tests": "passed: 393 passed once SAYDO_UPSTREAM points at the Hermes checkout (1 load test fails without it, as it expects ../upstream)",
   "useful": 4,
   "reliable": 4,
   "polish": 3,
   "creative": 4,
   "beyond": 4,
   "total": 23,
   "rank": 4,
   "best": "Deterministically checks the final reply's claims against a ledger of what actually ran this turn and appends a receipt, at zero token cost.",
   "worst": "The claim extractor misses some claims and misreads inline code, and the README has no screenshots or demo beyond a text example.",
   "blocked": null
  },
  {
   "pr": 149159,
   "name": "understudy",
   "validate": "passed: hermes plugins validate --json ok=true, all checks green (manifest, loadable, capability probe, declared tools/hooks/middleware, collisions, security scan safe, no core override, desktop surface inside SDK; isolation in_process because of ctx.post_message)",
   "inside": "Loaded (doctor: 2 tools; also /understudy, a hermes understudy CLI, a skill, and dashboard and Desktop pages; it asks for llm.model_override and provider_override, which were not granted non-interactively). understudy_report with no history returned 'no audition found yet'; a dry-run audition returned an unpriced estimate with 'No calls made and nothing spent'; empty candidates were rejected by schema; a real audition from the CLI refused with a clear message to grant model-override consent.",
   "tests": "passed: 94 passed",
   "useful": 4,
   "reliable": 5,
   "polish": 4,
   "creative": 4,
   "beyond": 4,
   "total": 25,
   "rank": 2,
   "best": "Replays the user's real Hermes decisions on cheaper candidate models with a blind judge, a baseline noise floor and a hard budget cap, so switching models becomes a measured decision.",
   "worst": "It needs real history and spends real tokens, so nothing beyond the estimate could be shown here, and it is read-only advice rather than a change it makes.",
   "blocked": null
  }
 ]
}