[ { "slug": "claude-code-token-savers", "kind": "shootout", "status": "testing", "tier": "deep", "title": "Claude Code token-saver shootout", "category": "coding-agents/skills-plugins", "repos": [ { "repo": "thedotmack/claude-mem", "stars": 101376, "license": "Apache-2.0" }, { "repo": "mksglu/context-mode", "stars": 30474, "license": "NOASSERTION" }, { "repo": "JuliusBrussee/caveman", "stars": 113243, "license": "Apache-2.0" }, { "repo": "xerj-org/xerj", "stars": 7622, "license": "Apache-2.0" } ], "claims": [ { "claim": "These repos stop Claude Code from burning through tokens and forgetting context", "source": "https://x.com/RoundtableSpace/status/2111483624679969197" }, { "claim": "They actually reduce Claude Code token waste", "source": "https://x.com/RoundtableSpace/status/2110955662544982061" } ], "plan": "Run the same coding task 3 times with each tool and once with no tool, using the same model and budget. Measure tokens used, task success and wall time.", "who_its_for": "People who use Claude Code every day and pay for tokens.", "notes_so_far": [ "gstack and Orca appear in some versions of this list but are parallel-agent tools, so they're tested in the parallel-agents shootout instead.", "One listed repo, anthropics/claude-code-mcp, returned 404 on 2026-10-17.", "Method: smoke test runs in a throwaway Docker container through Rocky's test queue. Claim runs use Claude Code's own usage report (output_tokens, total_cost_usd). Raw data is kept for the final review." ], "recycle_group": "\"Claude Code token-saver\" repo list", "topic": "coding-agents", "explainer": "Four add-ons that try to make AI coding agents like Claude Code waste fewer tokens. claude-mem saves and re-injects memory between sessions, context-mode shrinks tool output, Caveman makes the agent answer in fewer words, and XERJ swaps grep-style searching for an index.", "measure": [ "Tokens used on the same coding task, with each tool and with no tool", "Whether the task still gets done correctly", "Wall-clock time per run", "How hard each one is to install" ], "sneak_peek": [ "Smoke test: claude-mem and caveman installed. context-mode's npm install failed with an npm internal error.", "caveman, 3 questions x 2 runs: answers ran about 20% shorter to read (130 vs 161 median words), but billed output tokens didn't drop (712 vs 631). The sample is small." ], "gallery": [], "charts": [ { "type": "bars", "title": "Caveman: visible answer length (median words)", "unit": "words", "note": "Same 3 questions, 2 runs each, Claude Code on Haiku, tools off. Caveman's own page reports a 3% median output cut; Rocky saw the same ballpark.", "groups": [ { "label": "Setup", "bars": [ { "name": "No instruction", "value": 161, "verdict": "neutral" }, { "name": "\"Answer concisely.\"", "value": 182, "verdict": "neutral" }, { "name": "caveman SKILL.md", "value": 130, "verdict": "half" } ] } ] }, { "type": "bars", "title": "Billed output tokens (median, as Claude Code reports)", "unit": "tokens", "note": "Shorter text did NOT mean fewer billed tokens here (n=6 per setup). This is why there's no verdict yet.", "groups": [ { "label": "Setup", "bars": [ { "name": "No instruction", "value": 631, "verdict": "neutral" }, { "name": "\"Answer concisely.\"", "value": 693, "verdict": "neutral" }, { "name": "caveman SKILL.md", "value": 712, "verdict": "half" } ] } ] } ], "results": { "jobs": [ { "id": 1, "title": "token-savers: smoke test (clone/install/build, 4 repos)", "kind": "smoke", "status": "done", "started": "2026-10-17T23:05:10Z", "finished": "2026-10-17T23:08:16Z", "wall_seconds": 186, "install_friction": null, "artifacts": [] } ], "smoke_only": true, "ingested": "2026-10-18T15:01:47Z" }, "last_test_finished": "2026-10-17T23:08:16Z", "results_in": true }, { "slug": "lyogavin-airllm", "kind": "review", "status": "published", "tier": "claim", "title": "AirLLM", "category": "local-models/runtimes", "repos": [ { "repo": "lyogavin/airllm", "stars": 39145, "license": "Apache-2.0" } ], "claims": [ { "claim": "Kimi K3, 2.8 trillion parameters, in 3.72 GB \u2026 70B models on a single 4 GB card", "source": "https://x.com/RoundtableSpace/status/2111045739430269341", "measured": "Qwen2.5-72B-Instruct (bf16) ran with PyTorch GPU memory capped at 4 GB: peak GPU allocation 2.34 GB, whole-process peak 5.91 GB, 63.9 seconds per token, correct answer. The post's Kimi K3 (2.8T) and DeepSeek-V3 (671B) examples were not tested.", "result": "HALF-REAL" } ], "plan": "Run 9B, 27B and a 70B-class model (Qwen2.5-72B) with PyTorch GPU memory capped at 4 GB. Measure whether each loads and answers, peak memory, and tokens per second, against a normal full load of the same model.", "who_its_for": "Tinkerers who want to run big models on small GPUs.", "notes_so_far": [ "The post doesn't mention speed, so Rocky measured it.", "9B and 27B shards were pre-split before the timed runs, so split time isn't in their load times. The 72B run is different: see the load-time note under Measured results.", "The 4 GB cap is PyTorch's per-process GPU memory limit (see bench.py in the rerun bundle). It does not limit the OS file cache that streams layers in, or the rest of the Python process.", "Only one short prompt (24 new tokens, greedy decoding) per model, so speed for long outputs or long prompts is not measured yet." ], "topic": "ai-models", "explainer": "A Python library for running very large language models on a small GPU. Its README says 70B models fit on a single 4 GB card without quantization, by streaming the model a layer at a time.", "measure": [ "Does a 70B model load with GPU memory capped at 4 GB?", "Peak GPU and system memory used", "Tokens per second (the post doesn't mention speed)" ], "sneak_peek": [ "Setup: AirLLM streams a model one layer at a time from disk. Rocky capped PyTorch's GPU memory at 4 GB, asked each model the same question, and compared with a normal load of the same model. Rocky's other big model (gpt-oss-120b) was paused during runs so nothing else competed for memory.", "Qwen3.5-9B: about 45x slower than a normal load to save about 14 GB of GPU memory.", "Qwen3.6-27B: about 140x slower than a normal load to save about 47 GB.", "Qwen2.5-72B (the 70B-class claim): it ran and answered correctly at a 2.34 GB GPU allocation, but at 63.9 seconds per token. A normal 4-bit load of the same model did 4.53 tokens per second, about 290x faster.", "Caveat: on this box GPU and system memory are the same pool, so the cap limits PyTorch's allocations, not the OS file cache that streams the layers in. The AirLLM process itself peaked at 5.91 GB for 72B." ], "gallery": [], "speed_table": [ { "model": "Qwen3.5-9B", "airllm_tps": 0.176952, "normal_tps": 8.0174 }, { "model": "Qwen3.6-27B", "airllm_tps": 0.022472, "normal_tps": 3.2208 }, { "model": "Qwen2.5-72B (70B-class)", "airllm_label": "AirLLM, 4 GB cap (bf16)", "airllm_tps": 0.015646, "normal_tps": 4.5291, "normal_label": "Normal load, ollama Q4_K_M (4-bit, different precision)" } ], "charts": [ { "type": "bars", "title": "Speed: seconds per token (shorter bar = faster)", "unit": "s", "note": "Measured on Rocky's box, 24-token runs. AirLLM at a 4 GB cap is about 45x (9B), 140x (27B) and 290x (72B) slower than a normal load. The 72B normal load is ollama's 4-bit Q4 build, a different precision.", "groups": [ { "label": "Qwen3.5-9B (bf16 vs bf16)", "bars": [ { "name": "AirLLM 4 GB cap", "value": 5.651, "verdict": "half" }, { "name": "Normal load", "value": 0.125, "verdict": "neutral" } ] }, { "label": "Qwen3.6-27B (bf16 vs bf16)", "bars": [ { "name": "AirLLM 4 GB cap", "value": 44.499, "verdict": "half" }, { "name": "Normal load", "value": 0.31, "verdict": "neutral" } ] }, { "label": "Qwen2.5-72B (bf16 vs 4-bit Q4, different precision)", "bars": [ { "name": "AirLLM 4 GB cap", "value": 63.913, "verdict": "half" }, { "name": "Normal load (Q4)", "value": 0.221, "verdict": "neutral" } ] } ] }, { "type": "bars", "title": "Memory: GPU stays under 4 GB, the whole process doesn't (GB)", "unit": "GB", "note": "Green = PyTorch's GPU allocation, under the 4 GB cap every time. Yellow = the whole AirLLM process (RSS), over 4 GB every time. The yellow bars are the ones that matter for a real 4 GB card. Peak over load plus generation.", "groups": [ { "label": "The claim", "bars": [ { "name": "4 GB card", "value": 4.0, "verdict": "neutral" } ] }, { "label": "Qwen3.5-9B", "bars": [ { "name": "GPU allocation", "value": 2.79, "verdict": "real" }, { "name": "Whole process", "value": 7.52, "verdict": "half" } ] }, { "label": "Qwen3.6-27B", "bars": [ { "name": "GPU allocation", "value": 3.33, "verdict": "real" }, { "name": "Whole process", "value": 6.39, "verdict": "half" } ] }, { "label": "Qwen2.5-72B", "bars": [ { "name": "GPU allocation", "value": 2.34, "verdict": "real" }, { "name": "Whole process", "value": 5.91, "verdict": "half" } ] } ] } ], "verdict": "HALF-REAL", "published_ct": "Oct 18, 2026 (Central Time)", "verdict_why": "The core trick works: Qwen2.5-72B ran and answered correctly with PyTorch's GPU memory capped at 4 GB (it peaked at 2.3 GB). But Rocky's box shares 121 GB between CPU and GPU, it isn't a real 4 GB card, and the AirLLM process itself needed 5.9 GB, more than a 4 GB card has. It is also slow: 64 seconds per token, so a 50-token reply takes about 53 minutes, about 290x slower than a normal 4-bit load. So \"70B on a single 4 GB card\" is stretched.", "results_table": { "intro": "Every number below comes from Rocky's runs on an NVIDIA GB10 box (121.7 GB shared by CPU and GPU, Samsung 4 TB NVMe SSD), AirLLM 4.0.0 at commit fd3bd7e, throwaway Docker containers with no network. Same prompt for every run, 24 new tokens, greedy decoding. 9B and 27B ran Oct 17, 7:06 to 7:32 PM CT; 72B ran Oct 17, 11:30 to 11:59 PM CT.", "headers": [ "Run", "Weights", "Load time", "Generate 24 tokens", "Tokens per second", "Seconds per token", "Peak GPU allocation (PyTorch)", "Peak process memory (RSS)", "Answer" ], "rows": [ [ "9B AirLLM, 4 GB cap", "bf16", "6.4 s", "135.6 s", "0.177 tok/s", "5.7 s", "2.79 GB", "7.52 GB", "Correct, token-for-token same as normal load" ], [ "9B normal full load", "bf16", "95.4 s", "3.0 s", "8.02 tok/s", "0.125 s", "16.73 GB", "17.61 GB", "Correct" ], [ "27B AirLLM, 4 GB cap", "bf16", "5.9 s", "1,068 s (17.8 min)", "0.0225 tok/s", "44.5 s", "3.33 GB", "6.39 GB", "Correct, token-for-token same as normal load" ], [ "27B normal full load", "bf16", "257.9 s", "7.5 s", "3.22 tok/s", "0.31 s", "50.21 GB", "47.69 GB", "Correct" ], [ "72B AirLLM, 4 GB cap", "bf16", "100.4 s (see note)", "1,534 s (25.6 min)", "0.0156 tok/s", "63.9 s", "2.34 GB", "5.91 GB", "Correct: \"The capital of France is Paris, and the Seine River runs through it.\"" ], [ "72B normal load (ollama 4-bit Q4_K_M baseline, different precision)", "4-bit quantized", "not measured", "5.3 s", "4.53 tok/s", "0.22 s", "not measured", "not measured", "Correct, same first sentence" ], [ "72B one-time layer split (AirLLM prep, CPU only)", "bf16", "191.1 s to split", "\u2013", "\u2013", "\u2013", "none (no GPU)", "7.55 GB", "\u2013" ] ], "notes": [ "72B baseline: the full bf16 72B model (about 145 GB) can't fit in this box's 121 GB, so the normal-load baseline is ollama's 4-bit Q4_K_M build. That is a different precision and a different engine, so the 290x speed gap is not a pure like-for-like comparison.", "72B load time: the separate one-time split step was still finishing when the timed run started, so AirLLM wrote the last layer files itself during the 100.4 s load. That load time includes some splitting and isn't a clean load time.", "72B answer: the first sentence matches the Q4 baseline word for word. After about 20 tokens the wording drifts (\"a more complex question\" vs \"a more challenging question\"), which is expected between bf16 and 4-bit weights. 9B and 27B were compared at the same precision and matched token for token.", "What RSS counts: everything the Python process holds in RAM, including Python, the PyTorch and CUDA libraries, and layer data read from disk, not just GPU tensors. That is why it sits above PyTorch's GPU number.", "Peak process memory (RSS) was sampled every 0.2 s for the whole run, load plus generation. Every AirLLM run went over 4 GB here (5.9 to 7.5 GB), even though PyTorch's GPU allocation stayed under the cap.", "Rocky has not tried a discrete GPU that physically has only 4 GB, so this does not prove the claim on a real 4 GB card.", "Where the time goes (estimate from measured numbers): the 72B layer files are 136 GB on disk, and AirLLM reads every layer once per new token, so roughly 136 GB passes through per token. At 63.9 s per token that is about 2.1 GB/s on average. Some of it comes from the OS file cache rather than the drive (a Samsung 4 TB NVMe SSD)." ] }, "speed_note": "Example from the measured 72B rates: a short 50-token reply takes about 53 minutes with AirLLM at 63.9 s per token, versus about 11 seconds with the ollama Q4 load. A one-page summary (about 700 tokens) takes about 12.4 hours versus about 2.6 minutes. These are estimates derived from the measured rates, not separate runs.", "bundle": { "intro": "The exact files Rocky ran, plus the raw result lines every number on this page comes from. To rerun you need a Linux box with an NVIDIA GPU and Docker, and disk for the 72B weights (136 GB) plus AirLLM's layer files (another 136 GB) and the ollama Q4 baseline. Results on a real 4 GB card will differ.", "files": [ { "name": "Dockerfile", "href": "/reviews/lyogavin-airllm/bundle/Dockerfile", "what": "pinned AirLLM image (commit fd3bd7e)" }, { "name": "bench.py", "href": "/reviews/lyogavin-airllm/bundle/bench.py", "what": "the benchmark: split, AirLLM with the 4 GB cap, or normal load; writes one JSON line per run" }, { "name": "prep.sh", "href": "/reviews/lyogavin-airllm/bundle/prep.sh", "what": "builds the image and pre-splits the 9B and 27B models" }, { "name": "run.sh", "href": "/reviews/lyogavin-airllm/bundle/run.sh", "what": "9B and 27B runs, AirLLM and normal load" }, { "name": "dl70.sh", "href": "/reviews/lyogavin-airllm/bundle/dl70.sh", "what": "downloads Qwen2.5-72B-Instruct, splits it, pulls the ollama Q4 baseline" }, { "name": "run70.sh", "href": "/reviews/lyogavin-airllm/bundle/run70.sh", "what": "72B AirLLM run plus the ollama Q4 baseline" }, { "name": "results-latest.jsonl", "href": "/reviews/lyogavin-airllm/bundle/results-latest.jsonl", "what": "raw 9B / 27B results" }, { "name": "results-70b-latest.jsonl", "href": "/reviews/lyogavin-airllm/bundle/results-70b-latest.jsonl", "what": "raw 72B results" }, { "name": "split72.jsonl", "href": "/reviews/lyogavin-airllm/bundle/split72.jsonl", "what": "raw 72B split-step result" } ] }, "runs": [ { "label": "9B + 27B runs", "started": "2026-10-18T00:04:51Z", "finished": "2026-10-18T00:40:18Z" }, { "label": "72B run", "started": "2026-10-18T04:29:36Z", "finished": "2026-10-18T05:06:51Z" } ], "verdict_caveats": [ "Not tested on a real 4 GB graphics card. The 4 GB limit was PyTorch's memory cap on a 121 GB shared-memory box, so the OS could cache layer files in spare RAM.", "The 290x figure compares AirLLM (bf16) with ollama's 4-bit Q4 build, a different precision and engine. The 9B and 27B comparisons are bf16 vs bf16: 45x and 140x slower.", "Practical use: fine for an unattended batch job you can leave for hours. Not usable for chat or coding help.", "One short prompt, 24 new tokens, one run per model. No repeat runs yet." ], "claim_scope_note": "Scope: Rocky tested the \"70B models on a single 4 GB card\" part with Qwen2.5-72B-Instruct, an open 70B-class model. The post's bigger examples, Kimi K3 (2.8 trillion parameters in 3.72 GB) and DeepSeek-V3 (671 billion), were not tested in this run, so this verdict says nothing about them.", "results": { "jobs": [ { "id": 2, "title": "AirLLM: 9B + 27B with 4 GB GPU cap vs normal full load (tokens/sec, peak memory)", "kind": "claim", "status": "done", "started": "2026-10-18T00:04:51Z", "finished": "2026-10-18T00:40:18Z", "wall_seconds": 2127, "install_friction": null, "artifacts": [ "raw/lyogavin-airllm/job-2/env.txt" ] }, { "id": 12, "title": "AirLLM 70B: Qwen2.5-72B bf16 at 4 GB cap + ollama Q4 baseline (pauses 120B)", "kind": "claim", "status": "done", "started": "2026-10-18T04:29:36Z", "finished": "2026-10-18T05:06:51Z", "wall_seconds": 2235, "install_friction": null, "artifacts": [ "raw/lyogavin-airllm/job-12/env.txt" ] } ], "smoke_only": false, "ingested": "2026-10-18T15:02:03Z" }, "last_test_finished": "2026-10-18T05:06:51Z", "score": { "slug": "lyogavin-airllm", "version": 1, "subscores": { "claim": 50, "performance": 0, "install": null, "reliability": 60, "docs": null, "safety": null }, "weights": { "claim": 30, "performance": 30, "install": 15, "reliability": 15, "docs": 5, "safety": 5 }, "score": 32, "grade": "F", "confidence": "medium", "basis": { "claim": "72B ran with torch GPU allocation under 4 GB cap, but process RSS 5.9 GB and not on a real 4 GB card; v2 strict-box retest queued", "performance": "about 64 s/token on 72B vs 11 s for a whole reply on a normal Q4 load: far beyond 100x slower", "reliability": "one run per config (v2 adds repeats)" }, "tags": [ "local-models", "runtime", "slow-but-works" ], "keywords": [ "run 70B on 4GB", "layer streaming", "AirLLM" ] } }, { "slug": "morluto-rea", "kind": "review", "status": "testing", "tier": "deep", "title": "REA (reverse engineer anything)", "category": "coding-agents/skills-plugins", "repos": [ { "repo": "morluto/rea", "stars": 304631, "license": "MIT" } ], "claims": [ { "claim": "REA gives Claude Code or Codex the power to pull any app apart \u2026 25K+ stars on GitHub in one day", "source": "https://x.com/humzaakhalid/status/2110305451565031610" } ], "plan": "Point it at open-source apps only and check whether its answers come with proof. Then try the rebuild claim on a small open-source app with an existing test suite.", "who_its_for": "Developers learning unfamiliar codebases.", "notes_so_far": [ "Rocky only tests on open-source apps whose licenses allow it." ], "topic": "security", "explainer": "An MCP server that lets an AI agent take apart an app it doesn't have the source for (native binaries, Electron and JavaScript apps, .NET, websites) and explain how a feature works, with the evidence behind each answer.", "measure": [ "Do its explanations of an open-source app match the real source code?", "Does every answer point to proof?", "Can it rebuild a small feature that passes the app's existing tests?" ], "sneak_peek": [ "REA ran on Stretchly, an open-source Electron app (shipped build and source). Its source-side map lines up with Rocky's regex ground truth: 8 BrowserWindows (truth 8), 30 main-process IPC handlers (truth 30 literal ipcMain channels), and 10 context-bridge APIs.", "The install took 340 seconds. The rebuild claim hasn't been tested yet." ], "gallery": [], "results": { "jobs": [ { "id": 4, "title": "REA: analyze open-source Electron app (shipped + source), verify evidence", "kind": "claim", "status": "done", "started": "2026-10-18T01:46:08Z", "finished": "2026-10-18T02:45:04Z", "wall_seconds": 3536, "install_friction": null, "artifacts": [ "raw/morluto-rea/job-4/artifact-sha256.txt", "raw/morluto-rea/job-4/install.txt", "raw/morluto-rea/job-4/source-commit.txt" ] } ], "smoke_only": false, "ingested": "2026-10-18T15:02:13Z" }, "last_test_finished": "2026-10-18T02:45:04Z", "results_in": true }, { "slug": "parallel-coding-agents", "kind": "shootout", "status": "testing", "tier": "deep", "title": "Parallel AI coding agents shootout", "category": "agent-frameworks/orchestrators", "repos": [ { "repo": "stablyai/orca", "stars": 95173, "license": "MIT" }, { "repo": "ruvnet/ruflo", "stars": 75950, "license": "MIT" }, { "repo": "OpenHands/OpenHands", "stars": null, "license": null }, { "repo": "NousResearch/hermes-agent", "stars": 254625, "license": "MIT" }, { "repo": "kdlbs/kandev", "stars": 963, "license": "AGPL-3.0" } ], "claims": [ { "claim": "5 GitHub repos that let you run multiple AI coding agents in parallel", "source": "https://x.com/RoundtableSpace/status/2111181634821062671" }, { "claim": "Run many AI coding tasks at once, then review all changes in one place", "source": "https://x.com/GithubProjects/status/2109708440235057612" } ], "plan": "Give every tool the same 3 independent issues on a sample repo, with the same model and budget. Measure correct merged changes, conflicts, cost and wall time.", "who_its_for": "Developers juggling several coding tasks at once.", "notes_so_far": [], "topic": "robotics", "explainer": "Tools that run several AI coding agents at the same time, each on its own task or git worktree, and then let you review all the changes in one place. Orca, Kandev and OpenHands are workspaces for this; Ruflo is a multi-agent harness; Hermes Agent is a self-improving agent.", "measure": [ "Correct, mergeable changes on the same 3 independent issues", "Merge conflicts between agents", "Cost and wall-clock time", "Setup effort" ], "sneak_peek": [ "Install smoke: ruflo, OpenHands, kandev and orca installed. hermes-agent failed to install (about 20 minutes, no binary).", "Three Claude Code CLI agents in parallel fixed 3 of 3 planted bugs in about 340 seconds of wall time. Each fix passed the original tests.", "Merging the 3 branches produced 2 conflicts. That came from compiled .pyc files committed by the agents, not from the code." ], "gallery": [], "results": { "jobs": [ { "id": 5, "title": "Parallel coding agents: smoke 5 tools; 3 issues x 3 parallel agents (Claude Code, Hermes)", "kind": "claim", "status": "done", "started": "2026-10-18T02:45:04Z", "finished": "2026-10-18T04:26:40Z", "wall_seconds": 6096, "install_friction": null, "artifacts": [ "raw/parallel-coding-agents/job-5/hermes-build.txt" ] } ], "smoke_only": false, "ingested": "2026-10-18T15:02:22Z" }, "last_test_finished": "2026-10-18T04:26:40Z", "results_in": true }, { "slug": "bherbruck-solidcraft", "kind": "review", "status": "published", "tier": "claim", "title": "SolidCraft", "category": "cad-3d/cad-apps", "repos": [ { "repo": "bherbruck/solidcraft", "stars": null, "license": "Apache-2.0" } ], "claims": [ { "claim": "A free SolidWorks alternative that runs in your browser, and Claude Code can drive it \u2026 demo builds a full Raspberry Pi case", "source": "https://x.com/RoundtableSpace/status/2111053289127333933" } ], "plan": "Reproduce the Raspberry Pi case demo through its MCP server. Check that the STEP file is valid and the dimensions are right, then add it to the CAD shootout.", "who_its_for": "Hobby and maker CAD users who want agent-driven modeling.", "notes_so_far": [ "The repo URL redirected on 2026-10-17 (moved or renamed). Rocky will test the canonical repo." ], "topic": "cad", "explainer": "An open-source parametric 3D CAD app written in Rust that runs on the desktop or in the browser. It has a built-in MCP server so AI agents can sketch, extrude and edit models.", "measure": [ "Can an agent rebuild the README's Raspberry Pi 4 case through the MCP server?", "Is the exported STEP file valid?", "Are the key dimensions right (58 \u00d7 49 mm hole pattern)?" ], "sneak_peek": [ "Vendor demo over MCP: 15 tools and 22 of 22 steps passed. It exported a valid STEP file (one closed solid) that re-imported cleanly.", "Attempt 2 succeeded in 943 seconds with 34 tool calls. It built a 93 x 64 x 25 mm case with its mounting holes exactly on the 58 x 49 mm Pi pattern (0.0 mm error).", "Attempts 1 and 3 hit the 25-minute limit without a STEP file (58 and 27 tool calls)." ], "gallery": [], "results": { "jobs": [ { "id": 3, "title": "SolidCraft: vendor demo over MCP + agent builds Pi 4 case (3 tries), STEP check", "kind": "claim", "status": "done", "started": "2026-10-18T00:40:18Z", "finished": "2026-10-18T01:46:08Z", "wall_seconds": 3950, "install_friction": null, "artifacts": [ "raw/bherbruck-solidcraft/job-3/claim-2.step", "raw/bherbruck-solidcraft/job-3/sha256.txt", "raw/bherbruck-solidcraft/job-3/smoke-enclosure.step" ] } ], "smoke_only": false, "ingested": "2026-10-18T15:01:45Z" }, "last_test_finished": "2026-10-18T01:46:08Z", "score": { "slug": "bherbruck-solidcraft", "version": 1, "subscores": { "claim": 67, "performance": null, "install": null, "reliability": 33, "docs": null, "safety": null }, "weights": { "claim": 40, "performance": 20, "install": 15, "reliability": 15, "docs": 10, "safety": 0 }, "score": 58, "grade": "C-", "confidence": "medium", "basis": { "claim": "vendor demo 22/22 (100) + agent Pi case 1/3 (33), mean", "reliability": "1 of 3 agent attempts succeeded" }, "tags": [ "cad", "mcp", "browser-cad" ], "keywords": [ "SolidWorks alternative", "CAD MCP", "Raspberry Pi case" ] }, "verdict": "HALF-REAL", "verdict_why": "The vendor demo ran 22 of 22 steps over MCP. An agent produced a correct Raspberry Pi 4 case in 1 of 3 attempts, and the other 2 timed out after 25 minutes.", "published_ct": "Oct 18, 2026", "verdict_caveats": [ "The agent was Claude Code's CLI driving Rocky's local open model, not Anthropic's Claude, so a stronger model may do better." ] }, { "slug": "microsoft-markitdown", "kind": "claim", "status": "published", "tier": "claim", "title": "markitdown: does it really turn any document into clean text?", "category": "data-rag/doc-parsers", "topic": "coding-agents", "repos": [ { "repo": "microsoft/markitdown", "stars": 190787, "license": "MIT" } ], "claims": [ { "claim": "One command turns any document (Office, PDF, HTML, more) into Markdown your agent can read", "source": "https://x.com/victor_explore/status/2111462082499482057" } ], "plan": "Build 9 fixture files (DOCX, PPTX, XLSX, PDF, HTML, CSV, JSON, EPUB, and a scanned image-only PDF), each carrying 6 known canary sentences and a known 3x4 table. Convert each and count how many canaries and table cells survive, and whether a real Markdown table comes out.", "who_its_for": "Anyone feeding documents to an LLM or a RAG pipeline.", "explainer": "A Microsoft tool that converts Office files, PDFs, HTML, CSV, JSON, EPUB and more into Markdown so an agent can read them. Rocky checks what actually survives the conversion, including a scanned page with no text layer.", "measure": [ "How many known sentences survive each format", "Whether table cells come through", "Whether a real Markdown table is produced", "What happens to a scanned, image-only PDF" ], "notes_so_far": [ "Method: throwaway python:3.12 container on Rocky's test box, markitdown installed from the pinned commit 4cc9fa1 (v0.1.8), no keys. Rocky built the fixtures so the right answers are known in advance.", "First run was on the box; the same test is queued in the sandbox to confirm." ], "sneak_peek": [ "Every one of the 6 canary sentences survived in DOCX, PPTX, XLSX, PDF, HTML, JSON and EPUB.", "All 16 table cells survived in CSV, DOCX, PPTX, XLSX, HTML, PDF and JSON, and CSV, DOCX, PPTX, XLSX and HTML produced real Markdown tables.", "The scanned, image-only PDF returned 1 character: 0 of 6 sentences and 0 of 16 cells.", "Each file took 0.5 to 0.7 seconds to convert." ], "charts": [ { "type": "strip", "title": "Did the document convert? (first run)", "note": "Green = converted with its text intact. Red = came back empty. The scanned PDF has no text layer, so a text extractor can't read it.", "cells": [ { "label": "DOCX", "state": "pass", "detail": "6/6 lines, table" }, { "label": "PPTX", "state": "pass", "detail": "6/6 lines, table" }, { "label": "XLSX", "state": "pass", "detail": "6/6 lines, table" }, { "label": "PDF (text)", "state": "pass", "detail": "6/6 lines" }, { "label": "HTML", "state": "pass", "detail": "6/6 lines, table" }, { "label": "CSV", "state": "pass", "detail": "16/16 cells" }, { "label": "JSON", "state": "pass", "detail": "6/6 lines" }, { "label": "EPUB", "state": "pass", "detail": "6/6 lines" }, { "label": "PDF (scanned)", "state": "fail", "detail": "0 chars, no text layer" } ] }, { "type": "bars", "title": "Table cells recovered (of 16)", "unit": "cells", "note": "A known 3x4 table (header + 3 rows = 16 cells) embedded in each format, then counted in the Markdown.", "groups": [ { "label": "Formats with a table", "bars": [ { "name": "CSV", "value": 16, "verdict": "real" }, { "name": "DOCX", "value": 16, "verdict": "real" }, { "name": "PPTX", "value": 16, "verdict": "real" }, { "name": "XLSX", "value": 16, "verdict": "real" }, { "name": "HTML", "value": 16, "verdict": "real" }, { "name": "PDF (text)", "value": 16, "verdict": "real" }, { "name": "PDF (scanned)", "value": 0, "verdict": "vapor" } ] } ] } ], "gallery": [], "results": { "jobs": [ { "id": 13, "title": "markitdown: convert 9 known docs (DOCX/PPTX/XLSX/PDF/HTML/CSV/JSON/EPUB/scanned) and count what survived", "kind": "claim", "status": "done", "started": "2026-10-18T04:26:40Z", "finished": "2026-10-18T04:29:26Z", "wall_seconds": 166, "install_friction": null, "artifacts": [ "raw/microsoft-markitdown/job-13/fixtures/pdf-preview.png", "raw/microsoft-markitdown/job-13/fixtures/scanned-preview.png", "raw/microsoft-markitdown/job-13/md/csv.md", "raw/microsoft-markitdown/job-13/md/docx.md", "raw/microsoft-markitdown/job-13/md/epub.md", "raw/microsoft-markitdown/job-13/md/html.md", "raw/microsoft-markitdown/job-13/md/json.md", "raw/microsoft-markitdown/job-13/md/pdf.md", "raw/microsoft-markitdown/job-13/md/pptx.md", "raw/microsoft-markitdown/job-13/md/scanned-pdf.md", "raw/microsoft-markitdown/job-13/md/xlsx.md" ] } ], "smoke_only": false, "ingested": "2026-10-18T18:34:11Z" }, "last_test_finished": "2026-10-18T04:29:26Z", "score": { "slug": "microsoft-markitdown", "version": 1, "subscores": { "claim": 89, "performance": 100, "install": 100, "reliability": 60, "docs": null, "safety": null }, "weights": { "claim": 35, "performance": 25, "install": 15, "reliability": 15, "docs": 5, "safety": 5 }, "score": 89, "grade": "A-", "confidence": "medium", "basis": { "claim": "8/9 formats intact; scanned PDF 0/6", "performance": "0.5-0.7 s per file", "install": "pip install from pinned commit, 146 s, 0 fixes", "reliability": "single run" }, "tags": [ "doc-parser", "rag", "no-ocr" ], "keywords": [ "markdown conversion", "pdf to markdown", "docx", "rag" ] }, "verdict": "REAL", "verdict_why": "8 of 9 formats came through with every test sentence and table cell. The one miss was a scanned PDF with no text layer, which returned 1 character.", "published_ct": "Oct 18, 2026", "verdict_caveats": [ "\"Any document\" means documents that contain text. Scans and photos come back empty unless you add an OCR or LLM plugin." ] }, { "slug": "panniantong-agent-reach", "kind": "claim", "status": "published", "tier": "claim", "title": "Agent-Reach: do the 'zero config' internet channels really work free, no keys?", "category": "data-rag/scrapers", "topic": "mcp-servers", "repos": [ { "repo": "Panniantong/Agent-Reach", "stars": 98758, "license": "MIT" } ], "claims": [ { "claim": "Give your AI agent eyes to see the entire internet: Twitter, Reddit, YouTube, GitHub, Bilibili and more, chosen and installed for you", "source": "https://x.com/trending_repos/status/2106356614559391800" } ], "plan": "Install Agent-Reach in a clean container with no cookies or accounts, run `agent-reach doctor`, then do one real read on each platform the project calls 'zero config' (Web, GitHub, YouTube, Bilibili, RSS, V2EX). Count how many work with no keys.", "who_its_for": "People wiring an agent up to read social and web content without juggling a dozen tools.", "explainer": "A Python package that picks, installs and health-checks the most reliable way for an agent to read each platform. It advertises six 'zero config' channels that need no login. Rocky tests whether those actually return content on a fresh machine.", "measure": [ "How many zero-config channels return real content with no keys", "What `agent-reach doctor` reports", "Whether install stays safe (no silent system changes) by default" ], "notes_so_far": [ "Method: throwaway python:3.12 container, package pinned to 94f06c1 (v1.5.0), no cookies, no API keys. Only the channels the README calls zero-config are counted.", "The cookie/login channels (Twitter, Reddit, XiaoHongShu) are out of scope: they need the user's own session and Rocky tests with no accounts." ], "sneak_peek": [ "Zero-config reads that worked: Web, YouTube, RSS, V2EX and web search, for 5 of 7.", "Zero-config reads that failed: GitHub and Bilibili.", "The install worked." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 18, "title": "Agent-Reach: do the zero-config platforms (Web/GitHub/YouTube/Bilibili/RSS/V2EX) work free, no keys?", "kind": "claim", "status": "done", "started": "2026-10-18T05:08:37Z", "finished": "2026-10-18T05:10:02Z", "wall_seconds": 85, "install_friction": null, "artifacts": [ "raw/panniantong-agent-reach/job-18/logs/web.md" ] } ], "smoke_only": false, "ingested": "2026-10-18T15:02:20Z" }, "last_test_finished": "2026-10-18T05:10:02Z", "score": { "slug": "panniantong-agent-reach", "version": 1, "subscores": { "claim": 71, "performance": null, "install": null, "reliability": 60, "docs": null, "safety": null }, "weights": { "claim": 35, "performance": 25, "install": 15, "reliability": 15, "docs": 5, "safety": 5 }, "score": 68, "grade": "C+", "confidence": "medium", "basis": { "claim": "5/7 zero-config channels read", "reliability": "single run" }, "tags": [ "scraper", "agent-tools" ], "keywords": [ "web reading", "youtube", "rss", "agent internet access" ] }, "verdict": "HALF-REAL", "verdict_why": "It installed, and 5 of the 7 channels it calls zero-config actually read content with no setup. GitHub and Bilibili failed.", "published_ct": "Oct 18, 2026", "verdict_caveats": [ "Channels that need logins (Twitter, Reddit and others) weren't configured, because Rocky uses no personal accounts." ] }, { "slug": "agent-skill-scanners", "kind": "shootout", "status": "published", "tier": "claim", "title": "Agent-skill security scanners: SkillSpector vs HackMyAgent", "category": "bot-ops/security-scanners", "topic": "security", "repos": [ { "repo": "NVIDIA/SkillSpector", "stars": 20439, "license": "Apache-2.0" }, { "repo": "opena2a-org/hackmyagent", "stars": 84, "license": "Apache-2.0" } ], "claims": [ { "claim": "SkillSpector: a security scanner that detects prompt injection, data exfiltration and tool poisoning in agent skills before you install", "source": "https://x.com/LLMpsycho/status/2110275358683701658" }, { "claim": "HackMyAgent: Metasploit for AI agents, scan a package, repo or skill for prompt injection before you run it", "source": "https://x.com/RoundtableSpace/status/2107202918508269573" } ], "plan": "Build 8 agent skills with known answers: 5 malicious (file exfiltration, env-var theft, a prompt-injection SKILL.md, curl-pipe-bash, a hidden base64 exec) and 3 clean (word count, a Markdown TOC builder, a hex-to-RGB helper). Run each scanner's free, no-LLM mode on all 8 and score how many threats it catches and how many clean skills it wrongly flags.", "who_its_for": "Anyone about to install a third-party Claude/Codex skill or MCP server.", "explainer": "Two open-source scanners that promise to catch dangerous agent skills before you install them. Rocky plants known-bad and known-good skills and measures each scanner's catch rate and false-alarm rate on the same set, using their free offline mode.", "measure": [ "How many of 5 planted threats each scanner catches", "How many of 3 clean skills each wrongly flags", "Speed per scan" ], "notes_so_far": [ "Method: throwaway container, SkillSpector cf5cac7 (v2.13.0) in `--no-llm` mode, HackMyAgent 0.33.2 via its free static `check`. No ANTHROPIC_API_KEY, so neither used a paid LLM. Exfiltration URLs in the fixtures are inert.", "Fair-fight note: both were run in their FREE offline mode. HackMyAgent also has a `secure --deep` mode that calls a paid LLM; Rocky did not use it, so this measures the no-cost tier only." ], "sneak_peek": [ "SkillSpector caught 5 of 5 malicious skills and raised 0 false alarms on 3 clean skills. Each scan took about 1.3 to 1.6 seconds.", "HackMyAgent caught 3 of 5. It missed the webhook exfiltration skill and the hidden base64 exec, and it flagged all 3 clean skills as high risk. Each scan took about 3.4 to 5.0 seconds." ], "charts": [ { "type": "matrix", "title": "Did the scanner get it right? (8 planted skills)", "note": "Green = correct call (caught a threat, or cleared a clean skill). Red = wrong call (missed a threat, or false alarm). Free offline mode for both.", "cols": [ "SkillSpector", "HackMyAgent" ], "rows": [ "exfil-webhook", "env-steal", "prompt-injection", "curl-bash", "hidden-base64", "clean-wordcount", "clean-markdown-toc", "clean-color" ], "row_tags": [ "malicious", "malicious", "malicious", "malicious", "malicious", "clean", "clean", "clean" ], "cells": [ [ "pass", "fail" ], [ "pass", "pass" ], [ "pass", "pass" ], [ "pass", "pass" ], [ "pass", "fail" ], [ "pass", "fail" ], [ "pass", "fail" ], [ "pass", "fail" ] ] }, { "type": "scoreboard", "title": "Score out of 8 (higher is better)", "unit": "/8", "note": "One point for each correct call across the 5 malicious and 3 clean skills.", "rows": [ { "label": "SkillSpector", "value": 8, "max": 8, "verdict": "real", "suffix": "/8" }, { "label": "HackMyAgent (free)", "value": 3, "max": 8, "verdict": "vapor", "suffix": "/8" } ] } ], "gallery": [], "results": { "jobs": [ { "id": 17, "title": "Scanner shootout: SkillSpector vs HackMyAgent on 8 planted skills (5 malicious, 3 clean)", "kind": "claim", "status": "done", "started": "2026-10-18T05:06:55Z", "finished": "2026-10-18T05:08:37Z", "wall_seconds": 102, "install_friction": null, "artifacts": [] } ], "smoke_only": false, "ingested": "2026-10-18T15:01:41Z" }, "last_test_finished": "2026-10-18T05:08:37Z", "score": { "slug": "agent-skill-scanners", "version": 1, "subscores": { "claim": 65, "performance": null, "install": null, "reliability": 60, "docs": null, "safety": null }, "weights": { "claim": 35, "performance": 25, "install": 10, "reliability": 10, "docs": 5, "safety": 15 }, "score": 64, "grade": "C", "confidence": "medium", "basis": { "claim": "SkillSpector 5/5, 0 FP (100); HackMyAgent 3/5 recall, 3/3 FP (30); mean", "reliability": "single run, deterministic" }, "tags": [ "security", "scanner", "shootout" ], "keywords": [ "prompt injection", "skill scanner", "SkillSpector", "HackMyAgent" ] }, "verdict": "HALF-REAL", "verdict_why": "SkillSpector's claim holds: it caught 5 of 5 planted malicious skills with no false alarms. HackMyAgent caught 3 of 5 and flagged all 3 clean skills as high risk.", "published_ct": "Oct 18, 2026", "verdict_caveats": [ "Both tools ran in their free modes with no LLM. The test used 8 planted skills: 5 malicious and 3 clean." ], "per_repo_verdicts": { "NVIDIA/SkillSpector": "REAL", "opena2a-org/hackmyagent": "HALF-REAL" } }, { "slug": "mukul975-cyber-skills", "kind": "claim", "status": "published", "tier": "claim", "title": "Anthropic-Cybersecurity-Skills: are there really 817 skills across 6 frameworks?", "category": "data-rag/security", "topic": "security", "repos": [ { "repo": "mukul975/Anthropic-Cybersecurity-Skills", "stars": 34620, "license": "Apache-2.0" } ], "claims": [ { "claim": "817 structured cybersecurity skills for AI agents, mapped to 6 frameworks (MITRE ATT&CK, OWASP, NIST and more)", "source": "https://x.com/tom_doerr/status/2109535383188799957" } ], "plan": "Count every SKILL.md in the repo, check each has valid YAML frontmatter with a name and description, and count how many actually name a security framework. Pure static read, no code run.", "who_its_for": "Security folks eyeing a big skill bundle for their coding agent.", "explainer": "A large bundle that claims 817 cybersecurity skills for AI agents, each tied to frameworks like MITRE ATT&CK and OWASP. Rocky counts them and checks the framework claim, rather than taking the number on faith.", "measure": [ "How many SKILL.md files exist vs the claimed 817", "How many have valid frontmatter", "How many name a real framework", "Which frameworks appear and how often" ], "notes_so_far": [ "Method: clone at 54a7988 (v1.3.0), static scan only. 'Named framework' means the skill text mentions a known framework (MITRE ATT&CK, OWASP, NIST, CIS, D3FEND, PCI DSS, ISO 27001, and others)." ], "sneak_peek": [ "818 skill folders and 818 SKILL.md at commit 54a7988; all 818 frontmatters parse. 809 agent.py and 282 process.py, every one byte-unique; median agent.py is 182 lines.", "Frontmatter framework coverage: ATT&CK 806, NIST CSF 805, D3FEND 139, AI RMF 97, F3 94, ATLAS 93 -- within one of the README's claimed numbers on every field.", "ID validity: 290 of 292 distinct ATT&CK IDs are valid and current in v20.0 (2 deprecated ICS IDs); 52 of 52 NIST CSF IDs valid; 22 of 22 ATLAS IDs valid.", "Anatomy mismatch: the documented layout (standards.md / workflows.md / process.py / template.md) exists in only 34-43% of skills; the real dominant layout (agent.py + api-reference.md, 99%) is undocumented.", "Prompt-injection scan over all 818 SKILL.md found no live injection aimed at the reading agent; 11 files carry vendor curl|sh install one-liners an agent could run verbatim.", "The '308 runtime pip installs' lead was wrong in the interesting direction: 308 scripts mention pip install in help text, zero invoke it at runtime." ], "charts": [ { "type": "bars", "title": "Claimed vs counted", "unit": "skills", "note": "Rocky counted every SKILL.md and checked its frontmatter.", "groups": [ { "label": "Skills", "bars": [ { "name": "Claimed", "value": 817, "verdict": "neutral" }, { "name": "Found", "value": 818, "verdict": "real" }, { "name": "Valid frontmatter", "value": 818, "verdict": "real" }, { "name": "Name a framework", "value": 817, "verdict": "real" } ] } ] }, { "type": "scoreboard", "title": "Framework mentions across the skills", "unit": "", "note": "How many skills reference each framework (a skill can map to several). 11 distinct frameworks appeared; claim said 6.", "rows": [ { "label": "NIST", "value": 817, "verdict": "teal" }, { "label": "CIS", "value": 270, "verdict": "teal" }, { "label": "MITRE ATT&CK", "value": 255, "verdict": "teal" }, { "label": "D3FEND", "value": 141, "verdict": "teal" }, { "label": "OWASP", "value": 110, "verdict": "teal" }, { "label": "PCI DSS", "value": 64, "verdict": "teal" }, { "label": "ISO 27001", "value": 14, "verdict": "teal" } ] } ], "gallery": [], "results": { "jobs": [ { "id": 14, "title": "Cyber-Skills: verify the 817-skills / 6-frameworks claim by counting + validating every SKILL.md", "kind": "claim", "status": "done", "started": "2026-10-18T04:29:26Z", "finished": "2026-10-18T04:29:32Z", "wall_seconds": 6, "install_friction": null, "artifacts": [] } ], "smoke_only": false, "ingested": "2026-10-18T18:34:11Z" }, "last_test_finished": "2026-10-18T04:29:32Z", "score": { "slug": "mukul975-cyber-skills", "version": 1, "subscores": { "claim": 100, "performance": null, "install": null, "reliability": 60, "docs": null, "safety": null }, "weights": { "claim": 35, "performance": 25, "install": 15, "reliability": 15, "docs": 5, "safety": 5 }, "score": 88, "grade": "A-", "confidence": "medium", "basis": { "claim": "818 SKILL.md vs 817 claimed; 11 frameworks vs 6 claimed", "reliability": "deterministic count, single run (capped at 60 by spec)" }, "tags": [ "skills", "security", "count-verified" ], "keywords": [ "cybersecurity skills", "MITRE ATT&CK", "NIST", "Claude skills" ] }, "verdict": "UNPROVEN", "verdict_why": "The 818-skill count and the six frontmatter framework fields are real and machine-checked, and 364 of 366 framework IDs we validated exist in the official catalogs. But counting files can't tell you whether a skill helps an agent, and the claim ledger logs 8 contradicted README claims -- so usefulness stays unproven until the A/B lab run.", "published_ct": "Oct 18, 2026", "verdict_caveats": [ "Do not read UNPROVEN as fake: the structure claims largely hold (818 skills, six mapping fields, valid IDs). It means nobody -- including us, yet -- has measured whether these skills make an agent better.", "The claim ledger (data/claims/mukul975-cyber-skills.json) has 28 rows: 3 supported, 11 partly, 8 contradicted, 6 unverified. Contradicted items include the documented skill anatomy, the token-cost claims, 'all 22 CSF categories', 'zero deprecated IDs', and 'production-grade' on sampled evidence.", "Both scripts we hand-checked from the README's own examples have confirmed bugs: the showcase memory-forensics agent crashes on every full report run, and the breach-containment agent reports CONTAINED regardless of whether any action succeeded.", "Star-integrity sampling was blocked at the API level from our test box (verified blocked for a control repo too), so no account-age numbers exist either way." ], "v2": { "works": { "verdict": "unproven", "why": "Phase 1 only counted and parsed. The with-vs-without A/B on ~100 skills is queued as Phase 2; until that lands, usefulness is not measured." }, "safe": { "verdict": "caution", "why": "No confirmed malware in the surface scan, but two README-featured agents have confirmed bugs (memory-forensics TypeError; breach-containment clock-derived password + hardcoded CONTAINED). README funnels collect leads (survey with Casky Tokens, casky.ai waitlist). Not a hard Safe fail yet \u2014 no confirmed secret exfil." }, "claims_honest": { "verdict": "mixed", "why": "Claim ledger: 3 supported, 11 partly, 8 contradicted, 6 unverified. Contradictions include skill anatomy, token-cost claims, 'all 22 CSF categories', 'zero deprecated IDs', Navigator release assets, and 'production-grade' on sampled evidence." }, "provenance": { "verdict": "caution", "why": "Independent project with a buried non-affiliation disclaimer that does not travel into plugin.json / CITATION.cff / the GitHub description. Bulk history (611 skills in commit 1). Star-integrity sampling was blocked on our test box (environment limit, not a finding)." } }, "usefulness": "Unproven -- head-to-head A/B test (same task with and without the skill loaded) is queued as Phase 2.", "verdict_draft": true }, { "slug": "dietrichgebert-ponytail", "kind": "smoke", "status": "testing", "tier": "smoke", "title": "ponytail: the 'laziest senior dev' prompt for coding agents", "category": "coding-agents/skills-plugins", "topic": "coding-agents", "repos": [ { "repo": "DietrichGebert/ponytail", "stars": 166274, "license": "MIT" } ], "claims": [ { "claim": "Makes your AI agent think like the laziest senior dev in the room: the best code is the code you never wrote", "source": "https://x.com/trending_repos/status/2106718156463182216" } ], "plan": "Smoke first: confirm the single prompt file (SKILL.md) and AGENTS.md exist and parse, and that the Claude Code plugin marketplace manifest is valid JSON. A real before/after agent run, measuring whether ponytail makes an agent reuse code instead of writing new code, is queued as a claim test.", "who_its_for": "Developers who want their coding agent to stop over-engineering.", "explainer": "A single widely-shared prompt (166k stars) that tells a coding agent to avoid writing unnecessary code and reuse what exists. Rocky first checks it installs cleanly, then will measure whether it actually changes agent behavior.", "measure": [ "Does the prompt and plugin install cleanly", "Does the agent write less new code with it than without (claim run, queued)" ], "notes_so_far": [ "Method: clone at 09c434a (v5.3.2); smoke only for now. The behavioral claim run needs a controlled agent task and is queued." ], "sneak_peek": [ "Smoke test: the repo cloned, and its skill file and AGENTS.md are both present.", "The first run failed because of a quoting bug in Rocky's own test script, not in ponytail. It's fixed, and the real claim test (18 agent runs, with and without the skill) is queued." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 19, "title": "ponytail: SKILL.md/AGENTS.md parse, plugin marketplace manifest valid JSON (smoke)", "kind": "smoke", "status": "failed", "started": "2026-10-18T05:10:02Z", "finished": "2026-10-18T05:10:04Z", "wall_seconds": 2, "install_friction": null, "artifacts": [] }, { "id": 40, "title": "ponytail claim: with vs without skill, coding packet, 3 repeats (18 agent runs, local model) [rerun: /out permissions]", "kind": "claim", "status": "done", "started": "2026-10-18T18:39:18Z", "finished": "2026-10-18T19:54:29Z", "wall_seconds": 4511, "install_friction": null, "artifacts": [] } ], "smoke_only": false, "ingested": "2026-10-18T20:31:30Z" }, "last_test_finished": "2026-10-18T19:54:29Z", "results_in": true }, { "slug": "vibheksoni-stealth-browser-mcp", "kind": "smoke", "status": "testing", "tier": "smoke", "title": "Stealth Browser MCP: the Cloudflare-bypassing browser for agents", "category": "mcp-servers/browser", "topic": "mcp-servers", "repos": [ { "repo": "vibheksoni/stealth-browser-mcp", "stars": 2556, "license": "MIT" } ], "claims": [ { "claim": "Bypass Cloudflare challenges and bot checks by automating real Chrome browser instances over MCP", "source": "https://x.com/tom_doerr/status/2111291280802095294" } ], "plan": "Smoke first: clone, install, import the server and count the MCP tools it registers (claims 101 tools, or a 21-tool core). The live claim, passing a Cloudflare / bot-detection check, needs a Chrome sandbox and is queued separately.", "who_its_for": "Agents that need to read sites behind Cloudflare and bot walls.", "explainer": "An MCP server that drives a real Chrome browser (via nodriver + CDP) to get past Cloudflare and bot checks, with a claimed 101 tools. Rocky first confirms it installs and exposes its tools, then will test the bypass claim against public detectors.", "measure": [ "Does it install and expose its MCP tools", "How many tools are really there vs the claimed 101/21", "Does it pass public bot-detection checks (claim run, queued)" ], "notes_so_far": [ "Method: clone at ba95cca (v0.2.5); smoke only for now. The repo publishes its own weekly detector benchmark; Rocky will run an independent check." ], "sneak_peek": [ "Smoke test: it installed and the server imported, but Rocky's static probe counted 0 tool functions, so that probe was wrong. A real MCP tools/list check against the 101/21 claim, plus web errands, is queued." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 20, "title": "Stealth Browser MCP: install, import server, count the 101/21 MCP tools (smoke)", "kind": "smoke", "status": "done", "started": "2026-10-18T05:10:04Z", "finished": "2026-10-18T05:10:26Z", "wall_seconds": 22, "install_friction": null, "artifacts": [] } ], "smoke_only": true, "ingested": "2026-10-18T15:02:35Z" }, "last_test_finished": "2026-10-18T05:10:26Z", "results_in": true }, { "slug": "trymirai-uzu", "kind": "smoke", "status": "testing", "tier": "smoke", "title": "uzu: the Rust inference engine that 'absolutely rips'", "category": "local-models/runtimes", "topic": "speed-claims", "repos": [ { "repo": "trymirai/uzu", "stars": 2385, "license": "MIT" } ], "claims": [ { "claim": "This local Rust-based inference engine absolutely rips", "source": "https://x.com/Dan_Jeffries1/status/2109639251079950344" } ], "plan": "Smoke first: build the engine from source with cargo at a pinned tag and confirm the CLI runs. A tokens-per-second claim run against a downloaded model is queued.", "who_its_for": "People running local models who care about raw speed.", "explainer": "A high-performance local inference engine written in Rust. Rocky first checks it builds and runs, then will measure real decode speed so 'rips' becomes a number.", "measure": [ "Does it build from source", "Does the CLI run", "Decode tokens/second on a real model (claim run, queued)" ], "notes_so_far": [ "Method: build at tag 0.6.9 with the stock Rust toolchain in a container; smoke only for now." ], "sneak_peek": [ "Smoke test: uzu 0.6.9 built with cargo in about 87 seconds, but the build produced only a library and no uzu command-line tool.", "uzu targets Apple Silicon, so this Linux GPU box can't run its speed claim. A fair test needs a Mac with Apple Silicon." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 21, "title": "uzu: cargo build --release the Rust inference engine, CLI --help (smoke)", "kind": "smoke", "status": "done", "started": "2026-10-18T05:10:26Z", "finished": "2026-10-18T05:12:19Z", "wall_seconds": 113, "install_friction": null, "artifacts": [] } ], "smoke_only": true, "ingested": "2026-10-18T15:02:33Z" }, "last_test_finished": "2026-10-18T05:12:19Z", "results_in": true }, { "slug": "elementalsouls-claude-bughunter", "kind": "claim", "status": "published", "tier": "claim", "title": "Claude-BugHunter: is it really 82 bug-hunting skills?", "category": "coding-agents/skills-plugins", "topic": "security", "repos": [ { "repo": "elementalsouls/Claude-BugHunter", "stars": 5148, "license": "MIT" } ], "claims": [ { "claim": "A curated bundle of 82 skills for Claude Code for vulnerability auditing, CVE tracking and structured red-team work", "source": "https://x.com/tom_doerr/status/2109143185834299899" } ], "plan": "Count every SKILL.md, validate each has frontmatter, and list the slash-commands the bundle registers. Static read only, no red-team run.", "who_its_for": "Bug bounty hunters and security reviewers using Claude Code.", "explainer": "A Claude Code bundle that claims 82 skills for finding vulnerabilities. Rocky counts the skills and commands rather than taking the number on faith; the actual bug-finding ability is a separate, later test.", "measure": [ "Skill count vs the claimed 82", "Valid frontmatter", "How many slash-commands it registers" ], "notes_so_far": [ "Method: clone at 35e7614 (v2.1), static scan only." ], "sneak_peek": [ "83 SKILL.md files against the 82 claimed, and 83 of 83 have valid frontmatter.", "15 slash-commands ship with it, including hunt, recon, triage, report, web3-audit and chain." ], "charts": [ { "type": "bars", "title": "Claimed vs counted", "unit": "", "note": "Static count of the bundle's contents.", "groups": [ { "label": "Contents", "bars": [ { "name": "Claimed skills", "value": 82, "verdict": "neutral" }, { "name": "Found skills", "value": 83, "verdict": "real" }, { "name": "Valid frontmatter", "value": 83, "verdict": "real" }, { "name": "Slash-commands", "value": 15, "verdict": "teal" } ] } ] } ], "gallery": [], "results": { "jobs": [ { "id": 15, "title": "Claude-BugHunter: verify the 82-skills claim, validate frontmatter, list commands", "kind": "claim", "status": "done", "started": "2026-10-18T04:29:32Z", "finished": "2026-10-18T04:29:36Z", "wall_seconds": 4, "install_friction": null, "artifacts": [] } ], "smoke_only": false, "ingested": "2026-10-18T15:01:55Z" }, "last_test_finished": "2026-10-18T04:29:36Z", "score": { "slug": "elementalsouls-claude-bughunter", "version": 1, "subscores": { "claim": 100, "performance": null, "install": null, "reliability": 60, "docs": null, "safety": null }, "weights": { "claim": 35, "performance": 20, "install": 15, "reliability": 15, "docs": 5, "safety": 10 }, "score": 88, "grade": "A-", "confidence": "medium", "basis": { "claim": "83 SKILL.md vs 82 claimed", "reliability": "single run" }, "tags": [ "skills", "security", "count-verified" ], "keywords": [ "bug bounty", "red team", "Claude Code skills" ] }, "verdict": "REAL", "verdict_why": "There are 83 skill files against the 82 claimed, all with valid frontmatter, plus 15 slash-commands.", "published_ct": "Oct 18, 2026", "verdict_caveats": [ "The claim is about the size of the bundle. Whether these skills find real bugs wasn't measured." ] }, { "slug": "plugin87-ux-ui-agent-skills", "kind": "claim", "status": "testing", "tier": "claim", "title": "ux-ui-agent-skills: 52 components and valid design tokens?", "category": "coding-agents/skills-plugins", "topic": "coding-agents", "repos": [ { "repo": "plugin87/ux-ui-agent-skills", "stars": 2037, "license": "MIT" } ], "claims": [ { "claim": "Turns Claude into a senior design architect: production-ready UI components and accessible design tokens", "source": "https://x.com/tom_doerr/status/2109833931914674487" } ], "plan": "Count the skills and component files, and check that every design-token JSON file is valid JSON. Static read only.", "who_its_for": "Front-end developers who want their agent to produce consistent, accessible UI.", "explainer": "A skill pack that claims 52 UI components and WCAG-grade design tokens for Claude. Rocky checks the counts and that the token files actually parse.", "measure": [ "Skill count", "Component files vs the claimed 52", "Whether every design-token JSON parses" ], "notes_so_far": [ "Method: clone at 62092bd (v2.9.8), static scan only. Whether the generated UI is truly accessible is a deeper test, not yet run." ], "sneak_peek": [ "Found 26 skills and 48 component files, against the claim of 52 components. All 30 JSON token files parse (30 of 30)." ], "charts": [ { "type": "bars", "title": "What's in the pack", "unit": "", "note": "Static count. Component files are files whose path names a component; the claim counts 52 components.", "groups": [ { "label": "Contents", "bars": [ { "name": "Skills", "value": 26, "verdict": "teal" }, { "name": "Component files", "value": 48, "verdict": "teal" }, { "name": "Claimed components", "value": 52, "verdict": "neutral" }, { "name": "Valid JSON", "value": 30, "verdict": "real" } ] } ] } ], "gallery": [], "results": { "jobs": [ { "id": 16, "title": "ux-ui-agent-skills: count skills + 52 components, validate DTCG token JSON", "kind": "claim", "status": "done", "started": "2026-10-18T05:06:51Z", "finished": "2026-10-18T05:06:55Z", "wall_seconds": 4, "install_friction": null, "artifacts": [] } ], "smoke_only": false, "ingested": "2026-10-18T15:02:25Z" }, "last_test_finished": "2026-10-18T05:06:55Z", "results_in": true }, { "slug": "justvugg-colibri", "kind": "smoke", "status": "testing", "tier": "smoke", "title": "colibri: frontier MoE models on hardware you already own?", "category": "local-models/runtimes", "topic": "ai-models", "repos": [ { "repo": "JustVugg/colibri", "stars": 42608, "license": "Apache-2.0" } ], "claims": [ { "claim": "Run frontier MoE models (hundreds of billions of parameters) on hardware you already own: pure C, experts streamed from disk", "source": "https://x.com/tom_doerr/status/2109954808618557869" } ], "plan": "Smoke first: build the pure-C engine and run its `--list`, which reports which models fit this machine and why. A real streamed run of a large model, measuring tokens/second, is queued as a heavier claim test.", "who_its_for": "People who want to run very large models without a datacenter GPU.", "explainer": "A pure-C engine that keeps a model's dense parts in RAM and streams its experts from disk, to run huge mixture-of-experts models on ordinary hardware. Rocky first builds it and reads its honesty about what fits; the big streamed run comes later.", "measure": [ "Does the engine build", "Does `--list` report what fits this machine", "Tokens/second on a real streamed model (claim run, queued)" ], "notes_so_far": [ "Method: build at e0db083 (v2.0.0) in a container, CPU-only list; smoke only for now. The repo itself publishes measured tok/s per machine, which Rocky will spot-check." ], "sneak_peek": [ "Smoke test: the pure-C engine built in about 38 seconds, and --list rated these models a good fit for this machine: DeepSeek V4 Flash (167 GB), MiMo-V2.6 Flash, Qwen3.8-Flash-Next, GLM-5.2/5.3, Inkling and MiMo-V2.6 Pro.", "Its list itself says Kimi K3 (1,561 GB) needs more free disk than this box has.", "Speed tests are queued: Qwen3.6-35B-A3B, then DeepSeek V4 Flash (284B) in a 32 GB RAM cap." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 22, "title": "colibri: build the pure-C engine, --list which models fit this machine (smoke)", "kind": "smoke", "status": "done", "started": "2026-10-18T05:12:19Z", "finished": "2026-10-18T05:13:06Z", "wall_seconds": 47, "install_friction": null, "artifacts": [] } ], "smoke_only": true, "ingested": "2026-10-18T15:01:58Z" }, "last_test_finished": "2026-10-18T05:13:06Z", "results_in": true }, { "slug": "tauricresearch-tradingagents", "kind": "smoke", "status": "testing", "tier": "smoke", "title": "TradingAgents: does the multi-agent trading framework stand up?", "category": "trading-bots/paper-trading-agents", "topic": "trading-bots", "repos": [ { "repo": "TauricResearch/TradingAgents", "stars": 111752, "license": "Apache-2.0" } ], "claims": [ { "claim": "A multi-agent LLM financial-trading framework", "source": "https://x.com/Scrizy211/status/2107100015399240107" } ], "plan": "Smoke first: install the package and build a TradingAgentsGraph wired to Rocky's local gpt-oss-120b (no paid API, $0). Confirm the agents construct. A full paper-trade decision run with real market data is queued as a claim test.", "who_its_for": "Quant-curious builders who want an agent team to research trades.", "explainer": "A popular framework (111k stars) where several LLM agents debate and produce a trading decision. Rocky first confirms it installs and the agent graph builds against a local model, then will run an actual paper decision.", "measure": [ "Does it install and import", "Does the multi-agent graph build against a local model", "A paper-trade decision on a fixed date (claim run, queued)" ], "notes_so_far": [ "Method: clone at e8f5503 (v0.7.0); smoke uses the local gpt-oss-120b via its OpenAI-compatible endpoint, so no paid tokens are spent. No real trades are placed, ever." ], "sneak_peek": [ "Smoke test: it installed, and the multi-agent graph built against Rocky's local model.", "A paper-trade replay (3 tickers x 5 dates, against buy-and-hold) is queued." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 23, "title": "TradingAgents: install, build the multi-agent graph on local gpt-oss-120b (smoke)", "kind": "smoke", "status": "done", "started": "2026-10-18T05:13:06Z", "finished": "2026-10-18T05:13:33Z", "wall_seconds": 27, "install_friction": null, "artifacts": [] } ], "smoke_only": true, "ingested": "2026-10-18T15:02:27Z" }, "last_test_finished": "2026-10-18T05:13:33Z", "results_in": true }, { "slug": "earthtojake-text-to-cad", "kind": "smoke", "status": "testing", "tier": "smoke", "title": "text-to-cad: does cadgen actually build a correct part?", "category": "cad-3d/text-to-cad", "topic": "cad", "repos": [ { "repo": "earthtojake/text-to-cad", "stars": 21277, "license": "MIT" } ], "claims": [ { "claim": "Gives your agent CAD superpowers: generate real 3D models as STEP/STL/3MF with design-for-manufacturing checks", "source": "https://x.com/Amit_0717/status/2107213487545938121" } ], "plan": "Smoke first: install the cadgen engine, build a parametric part (a 40x30x10 mm plate with a 5 mm hole) through its runtime, export STEP, then independently re-parse that STEP and check the bounding box and that a 2.5 mm-radius hole exists.", "who_its_for": "Hardware people who want an agent to produce real, manufacturable CAD.", "explainer": "The engine behind the text-to-cad plugin that lets agents generate STEP/STL/3MF parts. Rocky has it build a part with known dimensions, then checks the exported geometry with its own parser, no cadgen code trusted.", "measure": [ "Does cadgen install and export a STEP file", "Does the exported part match the requested dimensions", "Is the hole really where it should be" ], "notes_so_far": [ "Method: install cadgen 0.7.21 (repo 2457312, v0.7.21) via uv; the STEP check is an independent regex parse of the file, not cadgen's own report." ], "sneak_peek": [ "Smoke test: cadgen 0.7.21 installed and exported a STEP plate with the 5 mm hole (radius 2.5 mm). The bounding box read 40 x 30 x 56 mm instead of 40 x 30 x 10, so Rocky's script or the check needs another look.", "The real claim test, an agent with the plugin building 3 packet parts 3 times each, is queued." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 24, "title": "text-to-cad: cadgen builds a parametric 40x30x10 plate w/ 5mm hole, independent STEP check", "kind": "smoke", "status": "done", "started": "2026-10-18T05:13:33Z", "finished": "2026-10-18T05:14:07Z", "wall_seconds": 34, "install_friction": null, "artifacts": [ "raw/earthtojake-text-to-cad/job-24/plate.step" ] } ], "smoke_only": true, "ingested": "2026-10-18T15:01:53Z" }, "last_test_finished": "2026-10-18T05:14:07Z", "results_in": true }, { "slug": "magnitudedev-magnitude", "kind": "claim", "status": "testing", "tier": "claim", "title": "Magnitude: is it really 2x faster than Ollama?", "category": "local-models/runtimes", "topic": "speed-claims", "repos": [ { "repo": "magnitudedev/magnitude", "stars": 7066, "license": "Apache-2.0" } ], "claims": [ { "claim": "An open-source local inference engine that tunes kernels to your hardware, over 2x faster than Ollama (2x faster decode than llama.cpp)", "source": "https://x.com/Shadowfetch/status/2106312008434135485" } ], "plan": "Install the Linux package in a container, start the server, download a recommended small model, and time decode tokens/second on a fixed prompt. Compare against the same task on Ollama on the same box.", "who_its_for": "Anyone running local models who wants more speed for free.", "explainer": "An inference engine that auto-tunes kernels per model and claims to beat Ollama by 2x. Rocky measures real decode speed on Rocky's GPU box. Note: a clean 2x verdict needs the exact same model on both engines, so Rocky is careful about which number means what.", "measure": [ "Does it install and serve on Linux", "Decode tokens/second on a small model", "How that compares to Ollama on the same hardware" ], "notes_so_far": [ "Method: pinned CLI 0.2.9, GPU job on Rocky's box, no paid APIs. Rocky will only call it 2x if the comparison is same-model, same-hardware.", "GPU test queued; results land here when it runs." ], "sneak_peek": [ "Smoke test: the .deb installed and the server started, but no model was picked (the catalog had no recommendations while the box's memory was busy), so the speed test returned HTTP 400.", "Same-prompt baseline: Ollama ran qwen2.5-coder 7B at 45.3 tokens/s.", "A claim rerun with the 120B model paused and 10 packet prompts is queued." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 26, "title": "magnitude: install, serve, pull a small model, measure decode tok/s for the 2x-faster-than-Ollama claim (GPU)", "kind": "smoke", "status": "done", "started": "2026-10-18T05:46:35Z", "finished": "2026-10-18T05:48:26Z", "wall_seconds": 111, "install_friction": null, "artifacts": [ "raw/magnitudedev-magnitude/job-26/logs/model.txt" ] } ], "smoke_only": true, "ingested": "2026-10-18T15:02:07Z" }, "last_test_finished": "2026-10-18T05:48:26Z", "results_in": true }, { "slug": "tiiuae-falcon-perception", "kind": "claim", "status": "testing", "tier": "claim", "title": "Falcon-OCR (270M): accurate full-page OCR from a tiny model?", "category": "data-rag/doc-parsers", "topic": "vision", "repos": [ { "repo": "tiiuae/Falcon-Perception", "stars": 786, "license": "Apache-2.0" } ], "claims": [ { "claim": "A 270M early-fusion OCR model, trained from scratch on OCR with no VLM pretraining, small model with amazing performance", "source": "https://x.com/dahou_yasser/status/2105762692980170889" } ], "plan": "Render 3 images with known text (clean print, small print, a table), run Falcon-OCR's end-to-end mode on the box GPU, and score character accuracy and word recall against the known text.", "who_its_for": "People doing OCR who want a tiny local model instead of a big VLM.", "explainer": "A 270M-parameter OCR model from TII that claims to punch far above its size. Rocky feeds it images whose exact text is known and measures how accurately it reads them.", "measure": [ "Character accuracy vs known text", "Word recall", "Speed per image" ], "notes_so_far": [ "Method: pinned c457916 (v1.0.0), model auto-downloaded from Hugging Face, GPU job. Rocky builds the images so the ground truth is known.", "GPU test queued; results land here when it runs." ], "sneak_peek": [ "First run on 3 images: clean text and small text both came back with 100% character accuracy. The table came back as HTML with all words recalled, but the scorer measured raw characters, so the table score was wrong.", "A rerun with the 5-page OCR packet, scored by table cells and repeated 3 times, is queued." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 25, "title": "Falcon-OCR 270M: OCR 3 known images (print/small/table), score char-accuracy vs ground truth (GPU)", "kind": "smoke", "status": "done", "started": "2026-10-18T05:14:07Z", "finished": "2026-10-18T05:46:35Z", "wall_seconds": 1948, "install_friction": null, "artifacts": [] }, { "id": 33, "title": "Falcon-OCR claim: OCR packet 5 pages x3 repeats, table cell recall, install friction", "kind": "claim", "status": "failed", "started": "2026-10-18T19:58:03Z", "finished": "2026-10-18T19:58:05Z", "wall_seconds": 2, "install_friction": { "method": "README: clone + pip install -e .[torch]", "seconds": 0.0, "ok": false, "manual_fixes": 0, "commands_run": 2 }, "artifacts": [] } ], "smoke_only": true, "ingested": "2026-10-18T20:31:32Z" }, "last_test_finished": "2026-10-18T19:58:05Z", "results_in": true }, { "slug": "openbmb-voxcpm", "kind": "claim", "status": "testing", "tier": "claim", "title": "VoxCPM2: real-time, intelligible text-to-speech?", "category": "media-gen/voice", "topic": "voice-audio", "repos": [ { "repo": "OpenBMB/VoxCPM", "stars": 39490, "license": "Apache-2.0" } ], "claims": [ { "claim": "VoxCPM2: tokenizer-free multilingual TTS with real-time streaming (RTF as low as ~0.3 on an RTX 4090)", "source": "https://x.com/irr/status/2109159825863016475" } ], "plan": "Synthesize 3 fixed sentences on the box GPU, measure the real-time factor (generation time / audio length), and run the audio back through Whisper to check it's intelligible (word recall vs the input).", "who_its_for": "Builders who want local, natural-sounding speech.", "explainer": "A 2B tokenizer-free TTS model that claims real-time streaming. Rocky measures the real-time factor on its own GPU and checks the speech is actually understandable, not just fast.", "measure": [ "Real-time factor (lower is faster)", "Is the speech intelligible (Whisper word recall)", "Load time" ], "notes_so_far": [ "Method: voxcpm at cce58a5, model from Hugging Face, GPU job. Rocky's box is a GB10, not the RTX 4090 the claim cites, so the RTF number is for this hardware.", "GPU test queued; results land here when it runs." ], "sneak_peek": [ "The first run failed before testing because of Rocky's harness bug (wrong container entrypoint). It's fixed, and the voice packet run (5 lines x 2, RTF and intelligibility) is running now." ], "charts": [], "gallery": [], "results": { "jobs": [ { "id": 27, "title": "VoxCPM2 TTS: synth 3 sentences, measure RTF, whisper-check intelligibility (GPU)", "kind": "claim", "status": "failed", "started": "2026-10-18T05:54:03Z", "finished": "2026-10-18T05:54:20Z", "wall_seconds": 17, "install_friction": null, "artifacts": [] }, { "id": 29, "title": "VoxCPM2 claim v2: voice packet 5 lines x2, RTF spread, streaming latency, whisper recall", "kind": "claim", "status": "done", "started": "2026-10-18T15:04:05Z", "finished": "2026-10-18T15:34:06Z", "wall_seconds": 1801, "install_friction": { "method": "pip install voxcpm per README in stock CUDA PyTorch image", "seconds": 1800.0, "ok": false, "manual_fixes": 0 }, "artifacts": [] } ], "smoke_only": false, "ingested": "2026-10-18T15:50:52Z" }, "last_test_finished": "2026-10-18T15:34:06Z", "results_in": true } ]