#!/usr/bin/env python3
"""Find long, exact token overlaps in UTF-8 court text files.
This is a review aid, not an assessment of judges or legal compliance.
No network requests; do not use confidential data without authorization.
"""
import argparse
import hashlib
import json
import re
from pathlib import Path
from collections import defaultdict


def tokenize(text):
    return [(m.group(0).casefold(), m.start(), m.end())
            for m in re.finditer(r"\w+", text, flags=re.UNICODE)]


def run(input_dir, output_file, min_words=40):
    if min_words < 15:
        raise ValueError("min_words must be at least 15")
    files = sorted(Path(input_dir).glob("*.txt"))
    if len(files) < 2:
        raise ValueError("At least two UTF-8 .txt files are required")
    if len(files) > 200:
        raise ValueError("Use batches of at most 200 files")
    if sum(p.stat().st_size for p in files) > 8_000_000:
        raise ValueError("Use batches of at most 8 MB total UTF-8 text")
    records = []
    index = defaultdict(list)
    for idx, path in enumerate(files):
        if path.stat().st_size > 8_000_000:
            raise ValueError(f"File too large for this review helper: {path.name}")
        text = path.read_text(encoding="utf-8")
        tokens = tokenize(text)
        records.append((path, text, tokens))
        for i in range(max(0, len(tokens)-min_words+1)):
            key = hashlib.sha256("\0".join(t[0] for t in tokens[i:i+min_words]).encode("utf-8")).digest()
            index[key].append((idx, i))
    candidates = {}
    for occurrences in index.values():
        docs = sorted(set(d for d, _ in occurrences))
        if len(docs) < 2:
            continue
        # Bound repeated boilerplate to prevent excessive duplicate output.
        first = {}
        for d, i in occurrences:
            first.setdefault(d, i)
        for a_pos, a in enumerate(docs):
            for b in docs[a_pos+1:]:
                ia, ib = first[a], first[b]
                ta, tb = records[a][2], records[b][2]
                # Emit only the start of a maximal contiguous overlap.
                if ia > 0 and ib > 0 and ta[ia-1][0] == tb[ib-1][0]:
                    continue
                length = min_words
                while ia+length < len(ta) and ib+length < len(tb) and ta[ia+length][0] == tb[ib+length][0]:
                    length += 1
                key = (a, b, ia, ib)
                candidates[key] = length
    matches = []
    for (a, b, ia, ib), length in sorted(candidates.items()):
        row = {"word_count": length, "interpretation": "manual review required"}
        for label, d, start in [("a", a, ia), ("b", b, ib)]:
            path, text, tokens = records[d]
            char_start, char_end = tokens[start][1], tokens[start+length-1][2]
            row[label] = {"file": path.name,
                          "line_start": text.count("\n", 0, char_start)+1,
                          "line_end": text.count("\n", 0, char_end)+1,
                          "passage": text[char_start:char_end]}
        matches.append(row)
    payload = {
        "method": "exact normalized token overlaps; not semantic or legal judgment",
        "limitations": ["no automatic separation of quotations", "no test of whether evidence was ignored", "repeated occurrences limited to first indexed location", "no conclusion about AI use or bias"],
        "min_words": min_words,
        "files": [{"name": p.name, "sha256": hashlib.sha256(p.read_bytes()).hexdigest()} for p in files],
        "passage_matches": matches
    }
    output_file = Path(output_file)
    output_file.parent.mkdir(parents=True, exist_ok=True)
    output_file.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
    return len(matches)


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--input", required=True, type=Path)
    ap.add_argument("--output", required=True, type=Path)
    ap.add_argument("--min-words", type=int, default=40)
    args = ap.parse_args()
    try:
        n = run(args.input, args.output, args.min_words)
    except (ValueError, OSError, UnicodeError) as exc:
        ap.exit(2, f"Error: {exc}\n")
    print(f"Saved {n} passage candidates to {args.output}")


if __name__ == "__main__":
    main()