"""End-to-end test harness for the M0/M1 LFCS Pricing Agent build.

Per user instruction (2026-05-07 overnight): runs the agent end-to-end against
2602 Munmorah + 2629 Erskine Park reference jobs and reports first-pass error
vs Liam's submitted prices.

What this harness exercises:
  1. M0 schema reachability (Airtable list_records on every new table)
  2. Override capture (offline mode by default — does NOT pollute Airtable;
     pass `--commit-airtable` to write real records into a sandbox Bid)
  3. Sanitisation Layer 1 + Layer 2 against synthetic-leak fixtures + real
     Calc Basis text
  4. Feedback aggregator diversity-criteria evaluation against synthetic data
  5. Acceptance metric: first-pass error pct vs Liam's submitted total per Stage 5 § 1

Outputs a structured report to test-harness/report-{timestamp}.md.

Usage:
    python -m test-harness.run_harness                  # offline mode (default; safe)
    python -m test-harness.run_harness --commit-airtable  # writes to Airtable

Exit codes:
    0 = all sub-tests passed
    1 = one or more sub-tests failed (acceptance criteria report still written)
"""

from __future__ import annotations

import argparse
import datetime as dt
import json
import os
import statistics
import sys
import traceback
import uuid
from pathlib import Path

# Allow `python -m test-harness.run_harness` from the lfcs-pricing-agent root
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
sys.path.insert(0, str(ROOT))

from lib import sanitise, verify  # noqa: E402
from lib.config import FEEDBACK_TRIGGER, REASON_CODE_DOMAINS  # noqa: E402
from lib.feedback_aggregator import _evaluate_diversity  # noqa: E402
from lib.override_capture import OverrideCaptureSession  # noqa: E402

sys.path.insert(0, str(HERE))
from fixtures import (  # type: ignore  # noqa: E402
    JOBS,
    SYNTHETIC_OVERRIDES_2602,
    SYNTHETIC_OVERRIDES_2629,
    SYNTHETIC_OVERRIDES_HCB,
)


REPORT_DIR = HERE / "reports"


def _ts() -> str:
    return dt.datetime.now().strftime("%Y-%m-%d_%H%M%S")


def _pct(numerator: float, denominator: float) -> float:
    if denominator == 0:
        return 0.0
    return (numerator - denominator) / denominator * 100


def test_first_pass_error(report: list[str]) -> dict:
    """Acceptance metric: agent_first_pass within ±15% of Liam-submitted (Stage 5 § 1 pre-gate).

    HONEST: jobs with `agent_first_pass_v1_ex_gst = None` are SKIPPED (not counted toward
    pass/fail). HCB is one such case — its BOQ was used to DERIVE the methodology rate
    templates, so an agent first-pass using those very templates would tautologically
    match. Skipping HCB here is the tautology guard.
    """
    report.append("\n## TEST 1 — First-pass error vs Liam-submitted (Stage 5 acceptance metric)\n")
    results: dict = {}
    pass_count = 0
    eligible_count = 0
    skipped: list[str] = []
    for job_no, job in JOBS.items():
        first_pass = job.get("agent_first_pass_v1_ex_gst")
        submitted = job["liam_submitted_ex_gst"]
        if first_pass is None:
            skipped.append(job_no)
            results[job_no] = {
                "skipped": True,
                "skip_reason": "no agent first-pass; rate templates derived from this job's BOQ (tautology guard)",
                "liam_submitted": submitted,
            }
            report.append(
                f"- **{job_no} {job['name']} ({job['job_category']}):** SKIPPED — "
                f"no agent first-pass on file. {job.get('notes', '')[:200]}..."
            )
            continue
        eligible_count += 1
        delta_pct = _pct(first_pass, submitted)
        passed = abs(delta_pct) <= 15.0
        if passed:
            pass_count += 1
        results[job_no] = {
            "first_pass_v1": first_pass,
            "liam_submitted": submitted,
            "delta_dollars": first_pass - submitted,
            "delta_pct": delta_pct,
            "passed_15pct_band": passed,
        }
        report.append(
            f"- **{job_no} {job['name']} ({job['job_category']}):** "
            f"first_pass=${first_pass:,.2f}, submitted=${submitted:,.2f}, "
            f"delta=${first_pass - submitted:+,.2f} ({delta_pct:+.2f}%) — "
            f"{'PASS' if passed else 'FAIL'} (Stage 5 pre-gate: ±15% band)"
        )
        # Also report the v2-blind comparison if present
        if "agent_first_pass_v2_blind_ex_gst" in job:
            blind = job["agent_first_pass_v2_blind_ex_gst"]
            blind_pct = _pct(blind, submitted)
            results[job_no]["v2_blind"] = blind
            results[job_no]["v2_blind_delta_pct"] = blind_pct
            report.append(
                f"  - v2-blind reprice: ${blind:,.2f} ({blind_pct:+.2f}% vs submitted) "
                "— illustrates agent-without-anchor variance"
            )
    report.append(
        f"\n**Acceptance:** {pass_count}/{eligible_count} eligible jobs within ±15% pre-gate band. "
        f"{len(skipped)} skipped (no first-pass on file): {skipped}"
    )
    results["pass_count"] = pass_count
    results["eligible_count"] = eligible_count
    results["skipped"] = skipped
    results["total"] = len(JOBS)
    return results


def test_schema_reachability(report: list[str]) -> dict:
    """Verify all M0 tables exist + are readable (smoke test)."""
    report.append("\n## TEST 2 — M0 schema reachability\n")
    from lib import airtable_client as at
    from lib.config import BASE_ID, TBL

    if "AIRTABLE_PAT" not in os.environ and "AIRTABLE_API_KEY" not in os.environ:
        msg = "AIRTABLE_PAT not set — skipping reachability test (M0 schema confirmed at build time)."
        report.append(f"- SKIP: {msg}")
        return {"skipped": True, "reason": msg}

    results: dict = {"tables_reachable": [], "tables_unreachable": []}
    for table_name, table_id in TBL.items():
        try:
            # Just fetch one page; we don't care about content
            for _ in at.list_records(BASE_ID, table_id, page_size=1):
                break
            results["tables_reachable"].append(table_name)
        except Exception as e:  # noqa: BLE001
            results["tables_unreachable"].append((table_name, str(e)))
            report.append(f"- UNREACHABLE: `{table_name}` ({e})")
    report.append(
        f"- Reachable: {len(results['tables_reachable'])}/{len(TBL)} tables"
    )
    if results["tables_unreachable"]:
        report.append(f"- **{len(results['tables_unreachable'])} unreachable** — see above")
    return results


def test_sanitisation_layers(report: list[str]) -> dict:
    """Exercise Layer 1 + Layer 2 against known-good + synthetic-leak fixtures."""
    report.append("\n## TEST 3 — Sanitisation Layer 1 + Layer 2 (independence + fail-closed)\n")
    results: dict = {"cases": []}

    # Case A: known-good text (no agent attribution)
    good_text = """
This is a Liam-authored Calc Basis comment. Crew of 4 carpenters, 2 days,
$85/hr ord rate, 64 manhours total = $5,440. Materials supply LFCS, +10% markup.
Defensible-for-variations posture: variation rate matches submission rate.
"""
    layer1 = sanitise.sanitise_markdown(good_text, operator_of_record="Liam Fitzgerald")
    layer2 = verify.verify_text(layer1.payload, location="case-A-known-good")
    case_a_pass = layer1.layer1_pass and layer2.passed and not layer1.regex_strips_applied
    results["cases"].append({"name": "A-known-good", "passed": case_a_pass})
    report.append(
        f"- **Case A — known-good Liam-authored text:** "
        f"L1 strips={layer1.regex_strips_applied}, L2 findings={len(layer2.findings)} "
        f"— {'PASS' if case_a_pass else 'FAIL'}"
    )

    # Case B: synthetic leak — agent attribution token in a Calc Basis comment
    bad_text_1 = """
Calc Basis: agent confidence 0.71. claude-code-opsman first-pass rate.
[source: friend-vault] (PENDING DERIVE) PLACEHOLDER-stormwater-pit-tce-liv.
_internal_first_pass_rate = $4,300.
"""
    layer1_b = sanitise.sanitise_markdown(bad_text_1)
    layer2_b = verify.verify_text(layer1_b.payload, location="case-B-synthetic-leak")
    # Both layers should match patterns; L1 strips, L2 should pass the *cleaned* text
    case_b_pass = (
        len(layer1_b.regex_strips_applied) >= 5  # multiple patterns hit at L1
        and layer2_b.passed  # L2 must pass after L1 cleanup
    )
    results["cases"].append({"name": "B-synthetic-leak-cleaned", "passed": case_b_pass})
    report.append(
        f"- **Case B — synthetic agent-attribution leak (cleaned by L1, verified by L2):** "
        f"L1 strips={len(layer1_b.regex_strips_applied)} patterns, L2 findings={len(layer2_b.findings)} "
        f"on cleaned output — {'PASS' if case_b_pass else 'FAIL'}"
    )

    # Case C: independence test — bypass L1, send raw bad text to L2 (must fail closed)
    layer2_raw = verify.verify_text(bad_text_1, location="case-C-bypass-L1")
    case_c_pass = not layer2_raw.passed and len(layer2_raw.findings) >= 5
    results["cases"].append({"name": "C-bypass-L1-must-fail-at-L2", "passed": case_c_pass})
    report.append(
        f"- **Case C — bypass L1, raw bad text to L2 (must fail-closed):** "
        f"L2 findings={len(layer2_raw.findings)} — "
        f"{'PASS (correctly fails closed)' if case_c_pass else 'FAIL (L2 missed leak)'}"
    )

    # Case D: record-set sanitisation
    record_set_with_leaks = [
        {
            "id": "recX",
            "rate": 100.0,
            "_internal_agent_confidence": 0.71,
            "_internal_first_pass_rate": 100.0,
            "notes": "claude-code-opsman ran this. [source: friend-vault]",
        }
    ]
    sr = sanitise.sanitise_record_set(record_set_with_leaks)
    vr = verify.verify_record_set(sr.payload, location="case-D-records")
    case_d_pass = (
        len(sr.fields_stripped) == 2  # both _internal_* fields stripped
        and vr.passed  # cleaned record set passes L2
    )
    results["cases"].append({"name": "D-record-set-stripped-and-verified", "passed": case_d_pass})
    report.append(
        f"- **Case D — record-set with `_internal_*` fields:** L1 stripped {sr.fields_stripped}, "
        f"L2 findings={len(vr.findings)} — "
        f"{'PASS' if case_d_pass else 'FAIL'}"
    )

    # Case E: independence verification — patterns differ between L1 + L2
    # Look for L1 pattern names + L2 pattern names; should be DIFFERENT lists
    l1_names = {name for name, _ in sanitise._AGENT_DERIVATION_PATTERNS}
    l2_names = {name for name, _ in verify._LAYER2_FORBIDDEN_PATTERNS}
    overlap = l1_names & l2_names
    case_e_pass = len(overlap) == 0  # pattern NAMES must be different (independence proof)
    results["cases"].append({"name": "E-pattern-name-independence", "passed": case_e_pass})
    report.append(
        f"- **Case E — pattern-name independence (L1 + L2 maintain separate lists):** "
        f"L1 names={len(l1_names)}, L2 names={len(l2_names)}, overlap={len(overlap)} — "
        f"{'PASS' if case_e_pass else 'FAIL'}"
    )

    results["pass_count"] = sum(1 for c in results["cases"] if c["passed"])
    results["total"] = len(results["cases"])
    report.append(f"\n**Acceptance:** {results['pass_count']}/{results['total']} cases passed.")
    return results


def test_override_capture_offline(report: list[str]) -> dict:
    """Exercise OverrideCaptureSession in offline mode against synthetic 2629 + 2602 fixtures."""
    report.append("\n## TEST 4 — Override capture (offline mode, synthetic 2602 + 2629 fixtures)\n")
    results: dict = {"sessions": []}
    for job_no, fixtures in [("2629", SYNTHETIC_OVERRIDES_2629), ("2602", SYNTHETIC_OVERRIDES_2602), ("HCB", SYNTHETIC_OVERRIDES_HCB)]:
        try:
            with OverrideCaptureSession(
                bid_record_id=f"recDUMMY{job_no}{uuid.uuid4().hex[:8]}",
                agent_id="test-harness",
                participants=["test-harness"],
                offline=True,
            ) as session:
                for ovr in fixtures:
                    session.capture_override(
                        bid_line_item_id=f"recDUMMYBLI{uuid.uuid4().hex[:8]}",
                        original_rate_ex_gst=ovr["original_rate_ex_gst"],
                        override_rate_ex_gst=ovr["override_rate_ex_gst"],
                        reason_codes=ovr["reason_codes"],
                        reviewer_type=ovr["reviewer_type"],
                        reviewer_name=ovr["reviewer_name"],
                        free_text_rationale=ovr["free_text_rationale"],
                    )
            results["sessions"].append(
                {
                    "job_no": job_no,
                    "session_ref": session.session_ref,
                    "captures": len(session.captures),
                    "passed": len(session.captures) == len(fixtures),
                }
            )
            report.append(
                f"- **{job_no}:** captured {len(session.captures)}/{len(fixtures)} overrides "
                f"(offline; queued to _Internal/audit-trail/pending-overrides-{session.session_ref}.json)"
            )
        except Exception as e:  # noqa: BLE001
            results["sessions"].append({"job_no": job_no, "error": str(e), "passed": False})
            report.append(f"- **{job_no}: ERROR** {e}")

    pass_count = sum(1 for s in results["sessions"] if s.get("passed"))
    results["pass_count"] = pass_count
    results["total"] = len(results["sessions"])
    report.append(f"\n**Acceptance:** {pass_count}/{len(results['sessions'])} session captures passed.")
    return results


def test_feedback_diversity(report: list[str]) -> dict:
    """Exercise diversity criteria evaluation against synthetic signal sets."""
    report.append("\n## TEST 5 — Feedback aggregator diversity criteria\n")
    results: dict = {"cases": []}

    base_date = dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc)

    # Case A — same job, same day: should NOT trigger
    case_a_signals = [
        {
            "rate_template_id": "tplX",
            "job_id": "jobA",
            "override_rate": 100,
            "original_rate": 90,
            "delta_dollars": 10,
            "delta_pct": 11.0,
            "reason_codes": ["RATE-BASIS"],
            "created_at": base_date,
            "session_ref": "sess1",
        }
        for _ in range(5)
    ]
    a_pass, a_evidence = _evaluate_diversity(case_a_signals, FEEDBACK_TRIGGER)
    case_a_correct = not a_pass  # Should NOT trigger
    results["cases"].append({"name": "A-same-job-same-day", "expected_trigger": False, "actual_trigger": a_pass, "passed": case_a_correct})
    report.append(
        f"- **Case A — 5 overrides same job same day same domain:** "
        f"trigger={a_pass} (expected: False) — "
        f"{'PASS' if case_a_correct else 'FAIL'}"
    )

    # Case B — diverse signal: SHOULD trigger
    case_b_signals = [
        {"rate_template_id": "tplX", "job_id": "jobA", "override_rate": 100, "original_rate": 90,
         "delta_dollars": 10, "delta_pct": 11.0, "reason_codes": ["RATE-BASIS"],
         "created_at": base_date, "session_ref": "s1"},
        {"rate_template_id": "tplX", "job_id": "jobB", "override_rate": 105, "original_rate": 90,
         "delta_dollars": 15, "delta_pct": 16.7, "reason_codes": ["PRODUCTIVITY-CALIBRATION"],
         "created_at": base_date + dt.timedelta(days=20), "session_ref": "s2"},
        {"rate_template_id": "tplX", "job_id": "jobC", "override_rate": 110, "original_rate": 90,
         "delta_dollars": 20, "delta_pct": 22.0, "reason_codes": ["SCOPE-SHIFT"],
         "created_at": base_date + dt.timedelta(days=45), "session_ref": "s3"},
    ]
    b_pass, b_evidence = _evaluate_diversity(case_b_signals, FEEDBACK_TRIGGER)
    case_b_correct = b_pass  # Should trigger
    results["cases"].append({"name": "B-3jobs-45d-3domains", "expected_trigger": True, "actual_trigger": b_pass, "passed": case_b_correct})
    report.append(
        f"- **Case B — 3 overrides, 3 jobs, 45-day span, 3 reason domains:** "
        f"trigger={b_pass} (expected: True) — "
        f"{'PASS' if case_b_correct else 'FAIL'}"
    )

    # Case C — same job, 60-day span: should NOT trigger (only 1 distinct job)
    case_c_signals = [
        {"rate_template_id": "tplX", "job_id": "jobA", "override_rate": 100, "original_rate": 90,
         "delta_dollars": 10, "delta_pct": 11.0, "reason_codes": ["RATE-BASIS"],
         "created_at": base_date, "session_ref": "s1"},
        {"rate_template_id": "tplX", "job_id": "jobA", "override_rate": 105, "original_rate": 90,
         "delta_dollars": 15, "delta_pct": 16.7, "reason_codes": ["PRODUCTIVITY-CALIBRATION"],
         "created_at": base_date + dt.timedelta(days=30), "session_ref": "s2"},
        {"rate_template_id": "tplX", "job_id": "jobA", "override_rate": 110, "original_rate": 90,
         "delta_dollars": 20, "delta_pct": 22.0, "reason_codes": ["SCOPE-SHIFT"],
         "created_at": base_date + dt.timedelta(days=60), "session_ref": "s3"},
    ]
    c_pass, c_evidence = _evaluate_diversity(case_c_signals, FEEDBACK_TRIGGER)
    case_c_correct = not c_pass  # Should NOT trigger (only 1 distinct job)
    results["cases"].append({"name": "C-1job-60d-3domains", "expected_trigger": False, "actual_trigger": c_pass, "passed": case_c_correct})
    report.append(
        f"- **Case C — 3 overrides, 1 job, 60-day span, 3 reason domains:** "
        f"trigger={c_pass} (expected: False; min_distinct_jobs=2 not met) — "
        f"{'PASS' if case_c_correct else 'FAIL'}"
    )

    # Case D — only 1 reason domain, 2 jobs, 35 days: should NOT trigger
    case_d_signals = [
        {"rate_template_id": "tplX", "job_id": "jobA", "override_rate": 100, "original_rate": 90,
         "delta_dollars": 10, "delta_pct": 11.0, "reason_codes": ["RATE-BASIS"],
         "created_at": base_date, "session_ref": "s1"},
        {"rate_template_id": "tplX", "job_id": "jobB", "override_rate": 105, "original_rate": 90,
         "delta_dollars": 15, "delta_pct": 16.7, "reason_codes": ["RATE-BASIS"],
         "created_at": base_date + dt.timedelta(days=35), "session_ref": "s2"},
        {"rate_template_id": "tplX", "job_id": "jobB", "override_rate": 102, "original_rate": 90,
         "delta_dollars": 12, "delta_pct": 13.3, "reason_codes": ["RATE-BASIS"],
         "created_at": base_date + dt.timedelta(days=40), "session_ref": "s3"},
    ]
    d_pass, d_evidence = _evaluate_diversity(case_d_signals, FEEDBACK_TRIGGER)
    case_d_correct = not d_pass  # Should NOT trigger (only 1 reason domain)
    results["cases"].append({"name": "D-2jobs-1domain", "expected_trigger": False, "actual_trigger": d_pass, "passed": case_d_correct})
    report.append(
        f"- **Case D — 3 overrides, 2 jobs, 40-day span, 1 reason domain:** "
        f"trigger={d_pass} (expected: False; min_distinct_reason_domains=2 not met) — "
        f"{'PASS' if case_d_correct else 'FAIL'}"
    )

    pass_count = sum(1 for c in results["cases"] if c["passed"])
    results["pass_count"] = pass_count
    results["total"] = len(results["cases"])
    report.append(f"\n**Acceptance:** {pass_count}/{len(results['cases'])} diversity cases correct.")
    return results


def test_harness_independence_diagnosis(report: list[str]) -> dict:
    """Honest self-diagnosis: is the harness's rate-loading path independent from comparison target?

    Per user instruction (Task 3): 'If you can't verify it's independent, flag it as
    a test-harness design issue for me to review.'

    Verdict: NOT INDEPENDENT in current build. Detailed below.
    """
    report.append("\n## TEST 6 — Test Harness Independence Diagnosis (HONEST)\n")
    report.append(
        "User asked: is the test harness rate-loading path genuinely independent from "
        "the comparison target?\n"
    )
    report.append("**Answer: NO. The current harness is NOT independent.** Here's why:\n")
    report.append(
        "1. **No agent rate-loading path exists in M0/M1.** TEST 1 reads `agent_first_pass_v1_ex_gst` "
        "as a hardcoded constant from `fixtures.py` — itself sourced from Stage 2 deep-read recorded "
        "numbers. The harness compares two recorded numbers (`first_pass` vs `submitted`); it does NOT "
        "load rate templates, traverse a BOQ, and compute a fresh first-pass total.\n"
    )
    report.append(
        "2. **2602's +0.00% is tautological in the documented sense.** Stage 2 § 2.2 records that "
        "CC v1 = $407,870 and Liam-final = $407,870 — Liam signed CC's pack as-is. So 2602's perfect "
        "match isn't a forward-looking accuracy result; it's a record of historical agreement. The harness "
        "correctly reports the recorded number; the recorded number happens to be the same on both sides.\n"
    )
    report.append(
        "3. **HCB SKIP is the tautology guard now made explicit.** Methodology Rate Templates "
        "(`formwork-bridge-abutment-hcb-pattern` $207.87/m², `formwork-bridge-deck-hcb-pattern` $624.20/m², "
        "etc.) were DERIVED from HCB's BOQ. If we ran an agent against HCB inputs using those rates, "
        "it would tautologically reproduce HCB's submitted total (or close). Setting HCB's "
        "`agent_first_pass_v1_ex_gst = None` is the deliberate guard — TEST 1 SKIPS rather than "
        "returning a misleading 0%.\n"
    )
    report.append(
        "4. **Forward-looking independence requires `lfcs-pricing-estimator` (Stage 5 milestone 5).** "
        "An honest forward-looking accuracy test loads:\n"
        "   - methodology Rate Templates derived from a strict subset of jobs (training set);\n"
        "   - new bid inputs (RFQ scope email, BOQ, drawings, Pre-Pricing-Grill answers) from a held-out job;\n"
        "   - applies templates × quantities → first-pass total;\n"
        "   - compares to that held-out job's actually-submitted total.\n"
        "   M0/M1 doesn't have any of those pieces. M5 is the prerequisite.\n"
    )
    report.append(
        "5. **What the M0/M1 harness *does* validly test:** sanitisation L1+L2 independence + "
        "fail-closed behaviour, override-capture schema/flow, feedback-aggregator diversity criteria, "
        "schema reachability, structural parity of LoO + BOQ-response output. **These are all forward-"
        "looking.** Only TEST 1 (first-pass error) is the tautology trap, and only because it has no real "
        "first-pass to test against yet.\n"
    )
    report.append(
        "6. **Recommendation:** rename TEST 1 in M5 to 'Recorded baseline' for the M0/M1 era; once "
        "M5 ships `lfcs-pricing-estimator`, add TEST 1b 'Held-out forward-pass' which:\n"
        "   - holds out one bid (e.g. 2629) from rate-template derivation;\n"
        "   - prices that bid using only templates from OTHER jobs (Hasibul friend-vault stormwater-pit "
        "rates + Sol+ Hornsby rates etc.);\n"
        "   - reports first-pass error vs Liam-submitted as a genuine forward-looking accuracy metric.\n"
    )
    report.append(
        "\n**Bottom line:** the +0.00% on 2602 is real (it's a historical record of agreement, not a "
        "test-harness leak), but it's also not informative about agent quality. The harness needs M5 "
        "before TEST 1 becomes a meaningful accuracy gate. Until then, treat TEST 1 as a baseline "
        "snapshot test, not an accuracy test.\n"
    )
    return {
        "verdict": "NOT_INDEPENDENT",
        "reason": "no rate-loading path in M0/M1; TEST 1 compares recorded numbers",
        "passes": True,  # the diagnosis itself ran cleanly; the design issue is logged for Rocky
        "tautology_guard_active": True,  # HCB is correctly skipped
        "fix_owed_to": "Stage 5 milestone 5 (lfcs-pricing-estimator) + held-out forward-pass test",
    }


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--commit-airtable", action="store_true", help="Write real records to Airtable (default: offline)")
    args = parser.parse_args()

    REPORT_DIR.mkdir(parents=True, exist_ok=True)
    report: list[str] = [
        f"# LFCS Pricing Agent — M0/M1 Test Harness Report\n",
        f"**Run:** {_ts()}\n",
        f"**Mode:** {'commit-airtable' if args.commit_airtable else 'offline'}\n",
        f"**Reference jobs:** 2602 Munmorah Bridge, 2629 Erskine Park WWTP Slab\n",
    ]

    all_results: dict = {}
    try:
        all_results["test_1_first_pass_error"] = test_first_pass_error(report)
    except Exception as e:  # noqa: BLE001
        report.append(f"\n**TEST 1 EXCEPTION:** {e}\n```\n{traceback.format_exc()}\n```")
    try:
        all_results["test_2_schema_reachability"] = test_schema_reachability(report)
    except Exception as e:  # noqa: BLE001
        report.append(f"\n**TEST 2 EXCEPTION:** {e}\n```\n{traceback.format_exc()}\n```")
    try:
        all_results["test_3_sanitisation"] = test_sanitisation_layers(report)
    except Exception as e:  # noqa: BLE001
        report.append(f"\n**TEST 3 EXCEPTION:** {e}\n```\n{traceback.format_exc()}\n```")
    try:
        all_results["test_4_override_capture"] = test_override_capture_offline(report)
    except Exception as e:  # noqa: BLE001
        report.append(f"\n**TEST 4 EXCEPTION:** {e}\n```\n{traceback.format_exc()}\n```")
    try:
        all_results["test_5_feedback_diversity"] = test_feedback_diversity(report)
    except Exception as e:  # noqa: BLE001
        report.append(f"\n**TEST 5 EXCEPTION:** {e}\n```\n{traceback.format_exc()}\n```")
    try:
        all_results["test_6_independence_diagnosis"] = test_harness_independence_diagnosis(report)
    except Exception as e:  # noqa: BLE001
        report.append(f"\n**TEST 6 EXCEPTION:** {e}\n```\n{traceback.format_exc()}\n```")

    # Summary
    report.append("\n---\n\n## Headline\n")
    t1 = all_results.get("test_1_first_pass_error", {})
    if t1:
        eligible = t1.get("eligible_count", t1.get("total", 0))
        report.append(
            f"- **Acceptance metric (first-pass within ±15% of Liam-submitted):** "
            f"{t1.get('pass_count', 0)}/{eligible} eligible jobs "
            f"({len(t1.get('skipped', []))} skipped via tautology guard)"
        )
    t3 = all_results.get("test_3_sanitisation", {})
    if t3:
        report.append(
            f"- **Sanitisation layers:** {t3.get('pass_count', 0)}/{t3.get('total', 0)} cases"
        )
    t5 = all_results.get("test_5_feedback_diversity", {})
    if t5:
        report.append(
            f"- **Feedback diversity:** {t5.get('pass_count', 0)}/{t5.get('total', 0)} cases"
        )

    # Write report
    out_md = REPORT_DIR / f"report-{_ts()}.md"
    out_md.write_text("\n".join(report), encoding="utf-8")
    out_json = REPORT_DIR / f"report-{_ts()}.json"
    out_json.write_text(json.dumps(all_results, indent=2, default=str), encoding="utf-8")

    print(f"\nReport: {out_md}")
    print(f"JSON:   {out_json}")

    # Exit code: 1 if any test failed an acceptance bar
    failed = (
        (t1.get("pass_count", 0) < t1.get("total", 0))
        or (t3.get("pass_count", 0) < t3.get("total", 0))
        or (t5.get("pass_count", 0) < t5.get("total", 0))
    )
    return 1 if failed else 0


if __name__ == "__main__":
    sys.exit(main())