"""Score the baseline and four 'accidental regressions' on the golden set, then run the pytest gate on each."""
import json, os, subprocess, sys
from rag import Retriever, Config
from harness import load_golden, evaluate

G = load_golden()
cases = [("baseline", {}),
         ("chunk_size 800->200", {"chunk_size": 200}),
         ("chunk_size 800->3000", {"chunk_size": 3000}),
         ("heading_prefix off", {"heading_prefix": False}),
         ("lowercase off (tokenizer bug)", {"lowercase": False})]
print(f"golden questions: {len(G)}, corpus: FastAPI docs\n")
print(f"{'case':32} {'hit@1':>6} {'hit@5':>6} {'MRR':>6} | pytest gate")
for name, ov in cases:
    m, rows = evaluate(Retriever(Config(**ov)), G)
    env = dict(os.environ, RAG_CONFIG_OVERRIDE=json.dumps(ov))
    p_all = subprocess.run([sys.executable, "-m", "pytest", "-q", "--no-header", "-p", "no:cacheprovider", "test_golden.py"],
                       capture_output=True, text=True, env=env)
    last = p_all.stdout.strip().splitlines()[-1]
    print(f"{name:32} {m['hit@1']:6.3f} {m['hit@5']:6.3f} {m['mrr']:6.3f} | exit={p_all.returncode} {last}")
    if p_all.returncode:
        for l in p_all.stdout.splitlines():
            if l.startswith("E       AssertionError"): print("     ", l[8:200])