{"meta":{"preregistration":"docs/PROOF-PREREGISTRATION.md §7-D","axis":"redteam-catch-vs-self-redteam","thesis":"Why not just ask the LLM to red-team itself? Because the model has no ground truth — it cannot catch its own fabricated GitHub citations (Z low). We verify against GitHub (Y high). Measured, not assumed.","generated_at":"2026-06-30T08:59:53.328Z","panel":[{"key":"grok","id":"x-ai/grok-4.3","label":"Grok 4.3"},{"key":"glm","id":"z-ai/glm-5.2","label":"GLM 5.2"}],"plan_set":"in-domain 10 (lib/scenarios 5 + scripts/_proof-plans EXTRA_IN 5)","n_items":20,"corpus_snapshot_version":"v2026-06-15","github_authenticated":true,"isolated_github_cache":"ground-truth X uses an ISOLATED ghCheck cache (prefix 'redteam:') — never the product cache; the product R1 path (STEP3) re-verifies on its own in-process default cache (cannot replay X).","unit_of_analysis":"one true-hallucinated (404) GitHub citation, per (plan × model). Y and Z share this denominator (paired).","success_criteria":{"min_y":0.9,"gate":"Y's Wilson 95% CI lower bound > Z's CI upper bound (non-overlapping) AND Y >= 0.9. K4: if the CIs overlap, there is no structural advantage — reported honestly (claimHolds=false)."}},"per_item":[{"plan_id":"reviewmate","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"coderabbitai/coderabbit","exists":false,"http_status":404},{"repo":"sweepai/sweep","exists":true,"http_status":200},{"repo":"codium-ai/pr-agent","exists":true,"http_status":200},{"repo":"paul-gauthier/aider","exists":true,"http_status":200},{"repo":"opendevin/opendevin","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":["coderabbitai/coderabbit"],"our_caught":["coderabbitai/coderabbit"],"our_missed":[],"model_self_flagged":[],"model_self_caught":[],"model_self_false_positive":[]},{"plan_id":"reviewmate","model":"glm","model_id":"z-ai/glm-5.2","cited_total":5,"cited":[{"repo":"qodo-ai/pr-agent","exists":true,"http_status":200},{"repo":"coderabbitai/ai-pr-reviewer","exists":false,"http_status":404},{"repo":"mattzcarey/code-review-gpt","exists":true,"http_status":200},{"repo":"fluxninja/openai-pr-reviewer","exists":false,"http_status":404},{"repo":"ast-grep/ast-grep","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":["coderabbitai/ai-pr-reviewer","fluxninja/openai-pr-reviewer"],"our_caught":["coderabbitai/ai-pr-reviewer","fluxninja/openai-pr-reviewer"],"our_missed":[],"model_self_flagged":["fluxninja/openai-pr-reviewer"],"model_self_caught":["fluxninja/openai-pr-reviewer"],"model_self_false_positive":[]},{"plan_id":"lawsearch","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"law-gpt-kr/law-gpt-kr","exists":false,"http_status":404},{"repo":"entelecheia/koreanlawbert","exists":false,"http_status":404},{"repo":"klue-org/klue-roberta-law","exists":false,"http_status":404},{"repo":"legal-rag-korea/legal-rag-korea","exists":false,"http_status":404},{"repo":"law-search-korea/law-search-korea","exists":false,"http_status":404}],"unresolved":0,"true_hallucinated":["law-gpt-kr/law-gpt-kr","entelecheia/koreanlawbert","klue-org/klue-roberta-law","legal-rag-korea/legal-rag-korea","law-search-korea/law-search-korea"],"our_caught":["law-gpt-kr/law-gpt-kr","entelecheia/koreanlawbert","klue-org/klue-roberta-law","legal-rag-korea/legal-rag-korea","law-search-korea/law-search-korea"],"our_missed":[],"model_self_flagged":["law-gpt-kr/law-gpt-kr","legal-rag-korea/legal-rag-korea","law-search-korea/law-search-korea"],"model_self_caught":["law-gpt-kr/law-gpt-kr","legal-rag-korea/legal-rag-korea","law-search-korea/law-search-korea"],"model_self_false_positive":[]},{"plan_id":"lawsearch","model":"glm","model_id":"z-ai/glm-5.2","cited_total":5,"cited":[{"repo":"legalai-lab/lawbot","exists":false,"http_status":404},{"repo":"ddolbok/korean-law-chatbot","exists":false,"http_status":404},{"repo":"law-knowledge-assistant/klaw-rag","exists":false,"http_status":404},{"repo":"korea-law-search/klec-rag","exists":false,"http_status":404},{"repo":"ai-lawyer/korean-statutes-qa","exists":false,"http_status":404}],"unresolved":0,"true_hallucinated":["legalai-lab/lawbot","ddolbok/korean-law-chatbot","law-knowledge-assistant/klaw-rag","korea-law-search/klec-rag","ai-lawyer/korean-statutes-qa"],"our_caught":["legalai-lab/lawbot","ddolbok/korean-law-chatbot","law-knowledge-assistant/klaw-rag","korea-law-search/klec-rag","ai-lawyer/korean-statutes-qa"],"our_missed":[],"model_self_flagged":["legalai-lab/lawbot","ddolbok/korean-law-chatbot","law-knowledge-assistant/klaw-rag","korea-law-search/klec-rag","ai-lawyer/korean-statutes-qa"],"model_self_caught":["legalai-lab/lawbot","ddolbok/korean-law-chatbot","law-knowledge-assistant/klaw-rag","korea-law-search/klec-rag","ai-lawyer/korean-statutes-qa"],"model_self_false_positive":[]},{"plan_id":"radnote","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"ohif/viewers","exists":true,"http_status":200},{"repo":"orthanc-server/orthanc","exists":false,"http_status":404},{"repo":"nroduit/weasis","exists":true,"http_status":200},{"repo":"horosproject/horos","exists":true,"http_status":200},{"repo":"fenixmed/meddream-dicom-viewer","exists":false,"http_status":404}],"unresolved":0,"true_hallucinated":["orthanc-server/orthanc","fenixmed/meddream-dicom-viewer"],"our_caught":["orthanc-server/orthanc","fenixmed/meddream-dicom-viewer"],"our_missed":[],"model_self_flagged":["fenixmed/meddream-dicom-viewer"],"model_self_caught":["fenixmed/meddream-dicom-viewer"],"model_self_false_positive":[]},{"plan_id":"radnote","model":"glm","model_id":"z-ai/glm-5.2","cited_total":5,"cited":[{"repo":"ohif/viewers","exists":true,"http_status":200},{"repo":"dcm4che/dcm4chee-arc-light","exists":true,"http_status":200},{"repo":"invesalius/invesalius3","exists":true,"http_status":200},{"repo":"nroduit/weasis","exists":true,"http_status":200},{"repo":"orthanc-team/orthanc","exists":false,"http_status":404}],"unresolved":0,"true_hallucinated":["orthanc-team/orthanc"],"our_caught":["orthanc-team/orthanc"],"our_missed":[],"model_self_flagged":["orthanc-team/orthanc"],"model_self_caught":["orthanc-team/orthanc"],"model_self_false_positive":[]},{"plan_id":"dangolbot","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"chatwoot/chatwoot","exists":true,"http_status":200},{"repo":"rocketchat/rocket.chat","exists":true,"http_status":200},{"repo":"botpress/botpress","exists":true,"http_status":200},{"repo":"rasahq/rasa","exists":true,"http_status":200},{"repo":"jovotech/jovo-framework","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":[],"our_caught":[],"our_missed":[],"model_self_flagged":[],"model_self_caught":[],"model_self_false_positive":[]},{"plan_id":"dangolbot","model":"glm","model_id":"z-ai/glm-5.2","cited_total":0,"cited":[],"unresolved":0,"true_hallucinated":[],"our_caught":[],"our_missed":[],"model_self_flagged":[],"model_self_caught":[],"model_self_false_positive":[]},{"plan_id":"browserqa","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"browserbase/stagehand","exists":true,"http_status":200},{"repo":"multion-org/multion","exists":false,"http_status":404},{"repo":"openai/openai-cua","exists":false,"http_status":404},{"repo":"co-browser/browser-gpt","exists":false,"http_status":404},{"repo":"skyvern-ai/skyvern","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":["multion-org/multion","openai/openai-cua","co-browser/browser-gpt"],"our_caught":["multion-org/multion","openai/openai-cua","co-browser/browser-gpt"],"our_missed":[],"model_self_flagged":["openai/openai-cua","co-browser/browser-gpt"],"model_self_caught":["openai/openai-cua","co-browser/browser-gpt"],"model_self_false_positive":[]},{"plan_id":"browserqa","model":"glm","model_id":"z-ai/glm-5.2","cited_total":5,"cited":[{"repo":"steel-dev/steel-browser","exists":true,"http_status":200},{"repo":"executeautomation/mcp-playwright","exists":true,"http_status":200},{"repo":"browserbase/browser-use","exists":false,"http_status":404},{"repo":"nano-browser/nano-browser","exists":false,"http_status":404},{"repo":"microsoft/playwright-mcp","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":["browserbase/browser-use","nano-browser/nano-browser"],"our_caught":["browserbase/browser-use","nano-browser/nano-browser"],"our_missed":[],"model_self_flagged":["browserbase/browser-use"],"model_self_caught":["browserbase/browser-use"],"model_self_false_positive":[]},{"plan_id":"meetnote","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"tldv-io/tldv","exists":false,"http_status":404},{"repo":"firefliesai/fireflies","exists":false,"http_status":404},{"repo":"fathomlabs/fathom","exists":false,"http_status":404},{"repo":"otterdevs/otter","exists":false,"http_status":404},{"repo":"grainhq/grain","exists":false,"http_status":404}],"unresolved":0,"true_hallucinated":["tldv-io/tldv","firefliesai/fireflies","fathomlabs/fathom","otterdevs/otter","grainhq/grain"],"our_caught":["tldv-io/tldv","firefliesai/fireflies","fathomlabs/fathom","otterdevs/otter","grainhq/grain"],"our_missed":[],"model_self_flagged":["firefliesai/fireflies","fathomlabs/fathom","otterdevs/otter","grainhq/grain"],"model_self_caught":["firefliesai/fireflies","fathomlabs/fathom","otterdevs/otter","grainhq/grain"],"model_self_false_positive":[]},{"plan_id":"meetnote","model":"glm","model_id":"z-ai/glm-5.2","cited_total":5,"cited":[{"repo":"meetnote-ai/meetnote.ios","exists":false,"http_status":404},{"repo":"vansh288/meeting-summarizer","exists":false,"http_status":404},{"repo":"danielbogdoll/meeting-transcripts-summarizer","exists":false,"http_status":404},{"repo":"zolopty/meeting-minutes","exists":false,"http_status":404},{"repo":"aress31/meetingbot","exists":false,"http_status":404}],"unresolved":0,"true_hallucinated":["meetnote-ai/meetnote.ios","vansh288/meeting-summarizer","danielbogdoll/meeting-transcripts-summarizer","zolopty/meeting-minutes","aress31/meetingbot"],"our_caught":["meetnote-ai/meetnote.ios","vansh288/meeting-summarizer","danielbogdoll/meeting-transcripts-summarizer","zolopty/meeting-minutes","aress31/meetingbot"],"our_missed":[],"model_self_flagged":["meetnote-ai/meetnote.ios","vansh288/meeting-summarizer","danielbogdoll/meeting-transcripts-summarizer","zolopty/meeting-minutes","aress31/meetingbot"],"model_self_caught":["meetnote-ai/meetnote.ios","vansh288/meeting-summarizer","danielbogdoll/meeting-transcripts-summarizer","zolopty/meeting-minutes","aress31/meetingbot"],"model_self_false_positive":[]},{"plan_id":"codesearch","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"continuedev/continue","exists":true,"http_status":200},{"repo":"sourcegraph/cody","exists":false,"http_status":404},{"repo":"paul-gauthier/aider","exists":true,"http_status":200},{"repo":"bloopai/bloop","exists":true,"http_status":200},{"repo":"pashpashpash/openclaw","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":["sourcegraph/cody"],"our_caught":["sourcegraph/cody"],"our_missed":[],"model_self_flagged":["pashpashpash/openclaw"],"model_self_caught":[],"model_self_false_positive":["pashpashpash/openclaw"]},{"plan_id":"codesearch","model":"glm","model_id":"z-ai/glm-5.2","cited_total":5,"cited":[{"repo":"bloopai/bloop","exists":true,"http_status":200},{"repo":"continuedev/continue","exists":true,"http_status":200},{"repo":"paul-gauthier/aider","exists":true,"http_status":200},{"repo":"sourcegraph/cody","exists":false,"http_status":404},{"repo":"nus-apr/auto-code-rover","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":["sourcegraph/cody"],"our_caught":["sourcegraph/cody"],"our_missed":[],"model_self_flagged":[],"model_self_caught":[],"model_self_false_positive":[]},{"plan_id":"promptops","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"promptlayer/promptlayer","exists":false,"http_status":404},{"repo":"langchain-ai/langsmith-sdk","exists":true,"http_status":200},{"repo":"helicone/helicone","exists":true,"http_status":200},{"repo":"promptfoo/promptfoo","exists":true,"http_status":200},{"repo":"humanloop/humanloop","exists":false,"http_status":404}],"unresolved":0,"true_hallucinated":["promptlayer/promptlayer","humanloop/humanloop"],"our_caught":["promptlayer/promptlayer","humanloop/humanloop"],"our_missed":[],"model_self_flagged":["promptlayer/promptlayer","humanloop/humanloop"],"model_self_caught":["promptlayer/promptlayer","humanloop/humanloop"],"model_self_false_positive":[]},{"plan_id":"promptops","model":"glm","model_id":"z-ai/glm-5.2","cited_total":5,"cited":[{"repo":"promptfoo/promptfoo","exists":true,"http_status":200},{"repo":"langgenius/dify","exists":true,"http_status":200},{"repo":"agenta-ai/agenta","exists":true,"http_status":200},{"repo":"microsoft/promptflow","exists":true,"http_status":200},{"repo":"pezzolabs/pezzo","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":[],"our_caught":[],"our_missed":[],"model_self_flagged":[],"model_self_caught":[],"model_self_false_positive":[]},{"plan_id":"design2code","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"animaapp/anima","exists":false,"http_status":404},{"repo":"locofyai/locofy-docs","exists":false,"http_status":404},{"repo":"builderio/builder","exists":true,"http_status":200},{"repo":"teleporthq/teleport-code-generators","exists":true,"http_status":200},{"repo":"dhiwise/dhiwise-code-generator","exists":false,"http_status":404}],"unresolved":0,"true_hallucinated":["animaapp/anima","locofyai/locofy-docs","dhiwise/dhiwise-code-generator"],"our_caught":["animaapp/anima","locofyai/locofy-docs","dhiwise/dhiwise-code-generator"],"our_missed":[],"model_self_flagged":["dhiwise/dhiwise-code-generator"],"model_self_caught":["dhiwise/dhiwise-code-generator"],"model_self_false_positive":[]},{"plan_id":"design2code","model":"glm","model_id":"z-ai/glm-5.2","cited_total":5,"cited":[{"repo":"builderio/mitosis","exists":true,"http_status":200},{"repo":"locofy-ai/locofy","exists":false,"http_status":404},{"repo":"teleporthq/teleport-code-generators","exists":true,"http_status":200},{"repo":"bernaferrari/figmatocode","exists":true,"http_status":200},{"repo":"roman01la/html-to-react-components","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":["locofy-ai/locofy"],"our_caught":["locofy-ai/locofy"],"our_missed":[],"model_self_flagged":[],"model_self_caught":[],"model_self_false_positive":[]},{"plan_id":"apidocs","model":"grok","model_id":"x-ai/grok-4.3","cited_total":5,"cited":[{"repo":"stoplightio/spectral","exists":true,"http_status":200},{"repo":"redocly/redocly-cli","exists":true,"http_status":200},{"repo":"openapitools/openapi-diff","exists":true,"http_status":200},{"repo":"tufin/oasdiff","exists":true,"http_status":200},{"repo":"opticdev/optic","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":[],"our_caught":[],"our_missed":[],"model_self_flagged":[],"model_self_caught":[],"model_self_false_positive":[]},{"plan_id":"apidocs","model":"glm","model_id":"z-ai/glm-5.2","cited_total":5,"cited":[{"repo":"redocly/redocly","exists":false,"http_status":404},{"repo":"stoplightio/spectral","exists":true,"http_status":200},{"repo":"oasdiff/oasdiff","exists":true,"http_status":200},{"repo":"openapitools/openapi-diff","exists":true,"http_status":200},{"repo":"swagger-api/swagger-codegen","exists":true,"http_status":200}],"unresolved":0,"true_hallucinated":["redocly/redocly"],"our_caught":["redocly/redocly"],"our_missed":[],"model_self_flagged":[],"model_self_caught":[],"model_self_false_positive":[]}],"summary":{"n_items":20,"cited_total":95,"unresolved":0,"true_hallucinations":40,"Y_catch_rate":1,"Y_caught":40,"Y_ci_wilson":[0.9124,1],"Z_self_catch_rate":0.65,"Z_caught":26,"Z_ci_wilson":[0.4951,0.7787],"Z_self_false_positives":1,"Y_minus_Z":0.35,"ci_non_overlapping":true,"demotions":0,"claimHolds":true}}