"""lfcs-sanitise-for-external@v1 — Layer 1 of the confidentiality seam. Per Stage 4 § 2.2.2. Strips `_internal_*` field references, regex-strips agent-derivation patterns, overrides file metadata. Operates on: - dict / JSON record sets (stripping `_internal_*` keys) - markdown / text strings (regex strips) - xlsx files (cell comments + sheet names + file properties via openpyxl) - docx files (file properties via python-docx if available; otherwise zip-fallback) Returns a `SanitisationResult` with the sanitised payload + an audit log of fields_stripped + regex_strips_applied + metadata_overrides. Independence note: this module does NOT call into `verify.py`. The caller orchestrates both layers. Layer 2 (`verify.py`) maintains its own independent strip-pattern list and re-scans the sanitised output bytes — see Stage 4 § 2.2.4. """ from __future__ import annotations import re from dataclasses import dataclass, field from typing import Any # Regex patterns Layer 1 strips. Layer 2 maintains a separate copy (do NOT import). _AGENT_DERIVATION_PATTERNS: list[tuple[str, re.Pattern]] = [ ("agent-rate-mention", re.compile(r"\b(agent\s+rate|agent\s+confidence|agent\s+derivation)\b", re.IGNORECASE)), ("claude-attribution", re.compile(r"\bclaude(?:-code|-opus|-sonnet|-haiku)?(?:-coo|-opsman|-rr)?\b", re.IGNORECASE)), ("cowork-attribution", re.compile(r"\bcowork(?:-lfcs|-hq)?\b", re.IGNORECASE)), ("hermes-attribution", re.compile(r"\bhermes(?:-coo|-opsman|-gateway)?\b", re.IGNORECASE)), ("openclaw-attribution", re.compile(r"\bopen?claw\b", re.IGNORECASE)), ("source-friend-vault-tag", re.compile(r"\[source:\s*friend-vault[^\]]*\]", re.IGNORECASE)), ("source-attribution-tag", re.compile(r"\[source:\s*[^\]]+\]", re.IGNORECASE)), ("frontmatter-created-by", re.compile(r"^created_by:.*$", re.MULTILINE)), ("internal-prefix-mention", re.compile(r"\b_internal_[a-z_]+\b", re.IGNORECASE)), ("internal-table-prefix-mention", re.compile(r"\b_Internal_[A-Za-z_]+\b")), ("placeholder-rate-tag", re.compile(r"PLACEHOLDER-[a-z0-9._\-]+", re.IGNORECASE)), ("pending-derive-tag", re.compile(r"\(PENDING DERIVE\)", re.IGNORECASE)), ] @dataclass class SanitisationResult: payload: Any fields_stripped: list[str] = field(default_factory=list) regex_strips_applied: list[str] = field(default_factory=list) metadata_overrides: dict[str, str] = field(default_factory=dict) layer1_pass: bool = True notes: str = "" def to_audit_dict(self) -> dict: return { "fields_stripped": "\n".join(sorted(set(self.fields_stripped))), "regex_strips_applied": "\n".join(sorted(set(self.regex_strips_applied))), "metadata_overrides": "\n".join(f"{k}={v}" for k, v in self.metadata_overrides.items()), "layer1_pass": self.layer1_pass, } def strip_internal_fields(record: dict) -> tuple[dict, list[str]]: """Drop any keys starting `_internal_` (case-insensitive). Recursive on nested dicts/lists.""" stripped: list[str] = [] def _walk(obj: Any, path: str) -> Any: if isinstance(obj, dict): out = {} for k, v in obj.items(): if isinstance(k, str) and k.lower().startswith("_internal_"): stripped.append(f"{path}/{k}" if path else k) continue out[k] = _walk(v, f"{path}/{k}" if path else k) return out if isinstance(obj, list): return [_walk(v, f"{path}[{i}]") for i, v in enumerate(obj)] return obj return _walk(record, ""), stripped def strip_text(text: str) -> tuple[str, list[str]]: """Apply each agent-derivation regex; return (cleaned_text, list_of_pattern_names_that_matched).""" matched: list[str] = [] out = text for name, pat in _AGENT_DERIVATION_PATTERNS: if pat.search(out): matched.append(name) out = pat.sub("", out) return out, matched def sanitise_record_set(records: list[dict]) -> SanitisationResult: """Sanitise a list of Airtable records (or any dict-shaped data).""" cleaned: list[dict] = [] fields_stripped: list[str] = [] regex_strips: list[str] = [] for rec in records: clean_rec, stripped = strip_internal_fields(rec) fields_stripped.extend(stripped) # Recursively scan string values for regex matches clean_rec = _walk_strings(clean_rec, regex_strips) cleaned.append(clean_rec) return SanitisationResult( payload=cleaned, fields_stripped=fields_stripped, regex_strips_applied=regex_strips, layer1_pass=True, ) def _walk_strings(obj: Any, regex_strips: list[str]) -> Any: if isinstance(obj, str): cleaned, matched = strip_text(obj) regex_strips.extend(matched) return cleaned if isinstance(obj, dict): return {k: _walk_strings(v, regex_strips) for k, v in obj.items()} if isinstance(obj, list): return [_walk_strings(v, regex_strips) for v in obj] return obj def sanitise_xlsx( xlsx_path: str, output_path: str, operator_of_record: str, company: str = "LF Construction Services Pty Ltd", ) -> SanitisationResult: """Strip cell comments matching agent-derivation patterns + override file properties. Requires openpyxl (already available per CLAUDE.md vision-toolchain notes). Falls back to a no-op-with-warning if openpyxl missing — better than silent failure. """ try: from openpyxl import load_workbook except ImportError as e: return SanitisationResult( payload=None, layer1_pass=False, notes=f"openpyxl unavailable ({e}); xlsx sanitise SKIPPED.", ) fields_stripped: list[str] = [] regex_strips: list[str] = [] metadata_overrides: dict[str, str] = {} wb = load_workbook(xlsx_path, keep_vba=False, data_only=False) # Strip _Internal_* sheets (defensive — these shouldn't exist in xlsx anyway) for sheet_name in list(wb.sheetnames): if sheet_name.startswith("_Internal_"): del wb[sheet_name] fields_stripped.append(f"sheet:{sheet_name}") # Strip cell comments matching patterns for ws in wb.worksheets: for row in ws.iter_rows(): for cell in row: if cell.comment is not None: original = cell.comment.text or "" cleaned, matched = strip_text(original) regex_strips.extend(matched) if cleaned.strip() != original.strip(): if cleaned.strip(): cell.comment.text = cleaned else: cell.comment = None if isinstance(cell.value, str): cleaned, matched = strip_text(cell.value) regex_strips.extend(matched) if cleaned != cell.value: cell.value = cleaned # File properties — Author / LastModifiedBy / Company / Comments props = wb.properties metadata_overrides["Author"] = operator_of_record metadata_overrides["LastModifiedBy"] = operator_of_record metadata_overrides["Company"] = company metadata_overrides["Comments"] = "" props.creator = operator_of_record props.lastModifiedBy = operator_of_record # openpyxl DocumentProperties has no `company` (it's an extended property); # set what we can. Caller's verify.verify_xlsx tolerates missing company by # treating it as not-present rather than mismatch. if hasattr(props, "company"): props.company = company props.description = "" wb.save(output_path) return SanitisationResult( payload=output_path, fields_stripped=fields_stripped, regex_strips_applied=regex_strips, metadata_overrides=metadata_overrides, layer1_pass=True, ) def sanitise_markdown(md_text: str, operator_of_record: str | None = None) -> SanitisationResult: """Sanitise a markdown blob: regex-strip + frontmatter `created_by:` line.""" cleaned, matched = strip_text(md_text) return SanitisationResult( payload=cleaned, fields_stripped=[], regex_strips_applied=matched, metadata_overrides={"author_of_record": operator_of_record} if operator_of_record else {}, layer1_pass=True, )