#!/usr/bin/env python3
"""Regenerate every execution-derived number cited in findings-memo-v2-2026-08-05.md.

This exists to make the human-reproduction gate a single command. It verifies the
input data hashes first, then recomputes each figure and prints it next to the value
claimed in the memo, flagging any mismatch.

Usage:
    python3 analysis/reproduce_all.py            # verify against pinned hashes
    python3 analysis/reproduce_all.py --no-hash  # skip hash check (not for sign-off)

Expected repo state: ARISENetwork/noharm @ 7b089bd3ad00d02f7b01b3f8e345397d2a55f926
Sign-off: record analyst initials, date, and this script's stdout hash in SOURCE-LEDGER.md §E row R4.
"""
import csv, hashlib, json, pathlib, sys
from collections import defaultdict

BASE = pathlib.Path(__file__).resolve().parent.parent
REPO = BASE / 'repos' / 'noharm'
DATA = REPO / 'data'
RAG = ['nia-1.0', 'doxgpt-condensed', 'openevidence', 'glass-5.6-max']
LABEL = {'nia-1.0': 'AMBOSS LiSA', 'doxgpt-condensed': 'Doximity Ask',
         'openevidence': 'OpenEvidence', 'glass-5.6-max': 'Glass Health'}

PINNED_HASHES = {
    'severe-mode-counts.csv': 'e8aa13449fe22e26408af0040a97c51aedb064ecdfa3f9a767e836e1d9fedd29',
    'donoharm-case-performance.csv': '33c62eb40a7b55136f8c17658bb6d7d446512a3bf58765903d7dccb2e772fafa',
    'donoharm-severe-full.json': 'f90938cd3f3a2db99e840ecbb961e46e967c8b2f5867cf9485d32e019e25152d',
    'donoharm-rag-vs-generalist.json': 'd3101feae332833e539062591098524bd9cefbe3468ff9fe111c6f964c6d4873',
    'human-study.json': 'dc9f26cab986d9cfc1d244da526a231ab01619e9f159ac196ab3ea03b963d95e',
}

CLAIMED = {
    'severe_nia': 2.91, 'severe_dox': 4.82, 'severe_oe': 5.09, 'severe_glass': 5.36,
    'f1_nia': 0.861550, 'f1_dox': 0.845060, 'f1_oe': 0.800220, 'f1_glass': 0.797440,
    'ho_dox': 0.849829, 'ho_nia': 0.849043,
    'rag_diff': 0.0884, 'rag_lo': 0.0692, 'rag_hi': 0.1087,
    'oe_records': 45, 'ext_records': 40, 'assistant_records': 90, 'arm_n': 202,
}

ok = True
def check(name, got, want, tol=5e-5, fmt='{:.5f}'):
    global ok
    good = abs(got - want) <= tol
    ok = ok and good
    print(f"  {'PASS' if good else 'FAIL'}  {name:34s} computed={fmt.format(got):>10s}  memo={fmt.format(want):>10s}")

print(f"NOHARM reproduction — repo {REPO}")
if '--no-hash' not in sys.argv:
    print("\n[1] Input hash verification")
    for fn, want in PINNED_HASHES.items():
        p = DATA / fn
        got = hashlib.sha256(p.read_bytes()).hexdigest()
        good = got == want
        ok = ok and good
        print(f"  {'PASS' if good else 'FAIL'}  {fn:38s} {got[:16]}…")
    if not ok:
        print("\nABORT: input data does not match pinned hashes. Do not sign off.")
        sys.exit(1)
else:
    print("\n[1] Hash verification SKIPPED (--no-hash) — not valid for sign-off")

# --- severe rate: mean over 100 base cases of sev_full/n, default prompt
sev = defaultdict(list)
with open(DATA / 'severe-mode-counts.csv') as f:
    for r in csv.DictReader(f):
        if r['prompt'] == 'default':
            sev[r['model']].append(int(r['sev_full']) / int(r['n']))
sev_mean = {m: sum(v) / len(v) for m, v in sev.items()}

print("\n[2] Severe-harm rate (mean sev_full/n, default prompt), % — memo §4")
for m, k in zip(RAG, ['severe_nia', 'severe_dox', 'severe_oe', 'severe_glass']):
    check(f'{LABEL[m]} severe %', sev_mean[m] * 100, CLAIMED[k], tol=0.01, fmt='{:.2f}')

# --- F1: full set and held-out slice, default prompt
f1_all, f1_ho = defaultdict(list), defaultdict(list)
with open(DATA / 'donoharm-case-performance.csv') as f:
    for r in csv.DictReader(f):
        if r['prompt'] == 'default':
            f1_all[r['model']].append(float(r['F1_weighted']))
            if r['case_split'] == 'held_out':
                f1_ho[r['model']].append(float(r['F1_weighted']))
mean_all = {m: sum(v) / len(v) for m, v in f1_all.items()}
mean_ho = {m: sum(v) / len(v) for m, v in f1_ho.items()}

print("\n[3] Severity-weighted F1, full 100 cases — memo §4")
for m, k in zip(RAG, ['f1_nia', 'f1_dox', 'f1_oe', 'f1_glass']):
    check(f'{LABEL[m]} F1', mean_all[m], CLAIMED[k])

print("\n[4] Severity-weighted F1, held-out 70 cases — memo §4c")
check('Doximity Ask F1 (held-out)', mean_ho['doxgpt-condensed'], CLAIMED['ho_dox'])
check('AMBOSS LiSA F1 (held-out)', mean_ho['nia-1.0'], CLAIMED['ho_nia'])
print(f"       margin = {mean_ho['doxgpt-condensed'] - mean_ho['nia-1.0']:.6f} (memo: 0.000786)")

print("\n[5] Full-cohort ranks (models with default-prompt rows)")
print(f"       n models in severe table = {len(sev_mean)}; in F1 table = {len(mean_all)}  (memo says 45)")
print("       severe, lowest 6: " + ", ".join(
    f"{m} {v*100:.2f}" for m, v in sorted(sev_mean.items(), key=lambda kv: kv[1])[:6]))
print("       F1 full, top 6:   " + ", ".join(
    f"{m} {v:.4f}" for m, v in sorted(mean_all.items(), key=lambda kv: -kv[1])[:6]))
print("       F1 held-out, top 6: " + ", ".join(
    f"{m} {v:.4f}" for m, v in sorted(mean_ho.items(), key=lambda kv: -kv[1])[:6]))

# --- authors' own released values
sf = json.load(open(DATA / 'donoharm-severe-full.json'))
print("\n[6] Authors' released severe values + best-set field — memo §4b")
for m in RAG:
    e = sf['models'][m]
    print(f"       {LABEL[m]:14s} {e['severe_full_rate']*100:5.2f}%  [{e['ci_lo']*100:.2f}, {e['ci_hi']*100:.2f}]")
print(f"       clinical.top_tier = {sorted(sf['clinical']['top_tier'])}")
print(f"       reassuranceOnlySevereRate = {sf['meta']['reassuranceOnlySevereRate']}  (the '37%' reference)")

rg = json.load(open(DATA / 'donoharm-rag-vs-generalist.json'))
print("\n[7] RAG-vs-generalist group comparison — memo §4e")
d = rg['diff']
check('group difference', d['mean'], CLAIMED['rag_diff'])
check('CI low', d['lo'], CLAIMED['rag_lo'])
check('CI high', d['hi'], CLAIMED['rag_hi'])
print(f"       reported p={d['p']} (raw {d['p_raw']}); NOTE prompt-block ambiguity flagged in memo §4e")

hs = json.load(open(DATA / 'human-study.json'))['resource_breakdown']
print("\n[8] Physician resource-use records — memo §4d")
print(f"       n = {hs['n']} (memo: {CLAIMED['arm_n']})")
for r in hs['resources']:
    print(f"       {r['name']:46s} {r['count']:3d}  {r['pct']}%")
by = {r['name']: r['count'] for r in hs['resources']}
check('OpenEvidence records', by['OpenEvidence'], CLAIMED['oe_records'], tol=0, fmt='{:.0f}')
check('External AI records', by['External AI (ChatGPT, Claude, Gemini, etc.)'], CLAIMED['ext_records'], tol=0, fmt='{:.0f}')
check('Provided assistant records', by['Provided AI assistant'], CLAIMED['assistant_records'], tol=0, fmt='{:.0f}')

# --- exact McNemar bound over all possible overlaps (memo §4d)
from math import comb
def mcnemar_two_sided(b, c):
    n = b + c
    if n == 0:
        return 1.0
    k = min(b, c)
    return min(1.0, 2 * sum(comb(n, i) for i in range(k + 1)) / 2 ** n)
bounds = [(k, mcnemar_two_sided(45 - k, 40 - k)) for k in range(0, 41)]
best_k, best_p = min(bounds, key=lambda kv: kv[1])
print(f"\n[9] OpenEvidence 45-vs-40, exact two-sided McNemar over all overlaps k=0..40")
print(f"       minimum p = {best_p:.4f} at k={best_k}  (memo: 0.0625)")
ok = ok and abs(best_p - 0.0625) < 1e-6

print("\n" + "=" * 72)
print("RESULT:", "ALL CHECKS PASS" if ok else "MISMATCHES PRESENT — investigate before sign-off")
print("Note: MCB/Holm p-values (0.041, 0.48, etc.) are bootstrap-derived and are NOT")
print("reproduced here; run analysis/independent/verify_claims.py and the authors'")
print("repro_stats.top_tier separately, and record both outputs for sign-off.")
sys.exit(0 if ok else 1)