import json,glob,os,re
KW=re.compile(r"crack|damag|\bbroke|broken|chipped|\blump|knock|gouge|scrape|scratch|\bdent\b|smash|\bdefect|\bNCR\b|rectif|repair|patch|make good|snag|busted|cut out|fell off|come away",re.I)
KERB=re.compile(r"kerb|curb",re.I)

def rows(pref=None):
    for f in sorted(glob.glob("/root/.hermes/wa-bridge/days/*.jsonl")):
        b=os.path.basename(f)
        if pref and not b.startswith(pref): continue
        for l in open(f,encoding="utf-8"):
            try: r=json.loads(l)
            except: continue
            blob=" ".join(str(r.get(k) or "") for k in ("text","desc","caption"))
            yield b,r,blob

def show(title,pat,pref=None):
    print("=== %s ===" % title)
    n=0
    for b,r,blob in rows(pref):
        if pat.search(blob):
            n+=1
            print("  %-24s %-6s %-10s %s" % (b, r.get("t"), str(r.get("who"))[:10], blob.strip()[:130]))
    print("  TOTAL: %d" % n); print()

show("2636 LIVERPOOL — damage/defect language, EVERY day","",) if False else None
show("2636 LIVERPOOL — damage/defect language, every day", KW, "2636")
show("2636 LIVERPOOL — kerb/curb, every day", KERB, "2636")
show("ALL JOBS — kerb/curb", KERB)
print("=== POSITIVE CONTROLS ===")
for w in ("footpath","concrete","pour"):
    n=sum(1 for b,r,blob in rows() if re.search(w,blob,re.I))
    print("  '%s' -> %d rows (must be >0)" % (w,n))
n36=sum(1 for _ in rows("2636"))
f36=len([f for f in glob.glob("/root/.hermes/wa-bridge/days/2636*.jsonl")])
print("  2636 corpus: %d rows across %d day-logs" % (n36,f36))
print("  2636 day-logs:", sorted(os.path.basename(f) for f in glob.glob("/root/.hermes/wa-bridge/days/2636*.jsonl")))