{ "batch": "deep-31", "setup": "Cloned https://github.com/NousResearch/hermes-agent at fdde27e7dae174cb4ba842589e92508852b7dd9f into scratch and ran uv sync --locked --python 3.14. Used the RUN.md tryit.py fake-model runner, enabling each plugin toolset in its isolated Hermes home and selecting Procgen Studio as the image provider; installed matching Playwright Chromium in scratch for the Hallway Test demo. No model key or external account was used.", "plugins": [ { "pr": 152475, "name": "hallway-test", "validate": "Passed catalog validation; no check failures or warnings.", "inside": "Loaded in Hermes with one deferred hallway_test tool, a slash command and Desktop page. After enabling its toolset, Hermes refused a file URL and ran the bundled demo in Chromium for one persona, producing a report with accessibility findings; the scripted model did not finish the task in four steps.", "tests": "35 passed, 0 failed after installing the matching Playwright Chromium in scratch.", "useful": 3, "reliable": 4, "polish": 5, "creative": 5, "beyond": 5, "rank": 4, "best": "Code-enforced accessibility personas drove a real browser and produced an actionable replay and report.", "worst": "The browser needs a separate matching Chromium install and a full five-person run makes many model calls.", "blocked": null, "total": 27 }, { "pr": 150293, "name": "procgen-studio", "validate": "Passed catalog validation; no check failures or warnings.", "inside": "Loaded as an image and video provider in Hermes, selected for image_generate; the scripted model supplied no drawing block, so the valid prompt returned a bounded render_failed result and an empty prompt was rejected. Unit tests exercised local rendering successfully, but the fake-model conversation did not produce an image.", "tests": "177 passed, 1 failed on Python 3.14: the sandbox rejects NamedTuple-generated __replace__; tests ran from a scratch copy to avoid pytest collecting the plugin root entry point as a package.", "useful": 4, "reliable": 3, "polish": 5, "creative": 5, "beyond": 5, "rank": 3, "best": "It plugs a local, reproducible code-rendering backend into Hermes image and video generation and includes editable source, styles and a gallery.", "worst": "The renderer has a Python 3.14 NamedTuple compatibility failure, and the scripted model could not complete a real Hermes render.", "blocked": null, "total": 27 }, { "pr": 149414, "name": "xray", "validate": "Passed catalog validation; no check failures or warnings.", "inside": "Loaded in Hermes with two hooks and CLI commands rather than model tools. Through Hermes terminal calls, detonate marked the clean sample GREEN and the delayed malicious sample RED; verdicts listed both and a nonexistent path returned a clear error.", "tests": "13 passed, 0 failed.", "useful": 4, "reliable": 4, "polish": 5, "creative": 5, "beyond": 5, "rank": 2, "best": "It tests installed plugins with bait and clock shifts, then connects observed behavior to runtime warnings and an optional approval gate.", "worst": "Its child process is a Python audit tripwire running as the user, so native code can bypass its egress and spawn checks.", "blocked": null, "total": 28 }, { "pr": 151135, "name": "wake-me-when", "validate": "Passed catalog validation; no check failures or warnings.", "inside": "Loaded in Hermes with seven deferred tools, a pre-LLM hook and a watch command. A demo live watch started and appeared in the list, a local-file request from this chat was refused, recording search returned no matching moment, and an invalid stop returned usage guidance; an actual alert was not observed before the scripted chat ended.", "tests": "112 passed, 3 failed, 7 skipped in the full suite; each of the three failed end-to-end tests passed when rerun alone.", "useful": 3, "reliable": 4, "polish": 5, "creative": 5, "beyond": 5, "rank": 5, "best": "It adds persistent live audio-video monitoring with evidence, clips, catch-up and chat alerts beyond a one-turn prompt.", "worst": "The full end-to-end suite showed order-sensitive failures, and the short Hermes run did not verify an emitted alert.", "blocked": null, "total": 27 }, { "pr": 152190, "name": "hermes-tuneup", "validate": "Passed catalog validation with a warning for many private Hermes core imports under catalog rule 9.", "inside": "Loaded in Hermes with tuneup_check, a slash command, CLI command and hooks. Active and all-profile checks returned read-only compaction thresholds; a nonexistent profile returned a structured error.", "tests": "320 passed, 0 failed with the local HTTP proxy unset; six public-host assertions failed only when that proxy redirected them to loopback.", "useful": 4, "reliable": 3, "polish": 5, "creative": 4, "beyond": 4, "rank": 7, "best": "It turns Hermes compaction and cost configuration into concrete route-level findings with guarded fix commands and a Desktop pane.", "worst": "Its many private-core imports drew a catalog warning and could break when Hermes internals change.", "blocked": null, "total": 24 }, { "pr": 153144, "name": "signet", "validate": "Passed catalog validation; no check failures or warnings.", "inside": "Loaded in Hermes with two deferred tools and three approval hooks. Outbox listing worked; a valid draft with reserved example addresses was blocked pending human approval and appeared held in the outbox, while missing sender and malformed recipient inputs were refused.", "tests": "108 passed, 0 failed, plus 140 passing subtests.", "useful": 4, "reliable": 5, "polish": 5, "creative": 5, "beyond": 5, "rank": 1, "best": "It binds a person’s approval to the exact outgoing email and blocks direct mailer routes, with a ledger and clear refusal paths.", "worst": "No real person or mail account was present, so delivery after approval could only be checked by its local tests.", "blocked": null, "total": 29 }, { "pr": 0, "name": "done-bell", "validate": "Passed catalog validation; no check failures or warnings.", "inside": "Loaded in Hermes with one deferred status tool and four hooks. Status and mute calls worked; Hermes rejected a non-boolean mute argument before invocation. OS notification and chime were not observable in the scripted session.", "tests": "9 passed, 0 failed.", "useful": 4, "reliable": 3, "polish": 2, "creative": 2, "beyond": 4, "rank": 8, "best": "A small hook and Desktop integration gives completion and approval-wait notifications without credentials.", "worst": "Its README centers on a local Windows developer path, and the scripted run did not verify actual notifications.", "blocked": null, "total": 19 }, { "pr": 0, "name": "homeassistant", "validate": "Passed catalog validation with one private-core import warning for gateway.restart.is_supervised_gateway_launch.", "inside": "Loaded in Hermes as a gateway platform and registered four Home Assistant tools. With no HASS_TOKEN or Home Assistant account, those tools were not offered to the scripted model, and attempted calls reported unavailable; Doctor passed.", "tests": "212 passed, 0 failed, 4 skipped.", "useful": 4, "reliable": 4, "polish": 4, "creative": 2, "beyond": 5, "rank": 6, "best": "It gives Hermes a full Home Assistant event platform and four guarded device tools with broad test coverage.", "worst": "It needs a Home Assistant token and instance to expose its tools, and catalog validation flagged one private core import.", "blocked": null, "total": 24 } ] }