"""Session 36: solved-to-unsolved exact word-length fingerprints.
Counts original runes, not Latin transliteration letters. Local sources only.
No decryption hypothesis fitted. Python + NumPy and retained corpus helpers.
"""
from pathlib import Path
from collections import Counter,defaultdict
import hashlib,json
import numpy as np
from lp_core import R,LET,pages,lp,vals
from transition_surrogates import shuffle_pairs
ROOT=Path(__file__).resolve().parent
SEED=3301036

def blocks(source,page_ids):
    result=[];current=[];word=[];start=None;word_end=None;page=0;offset=0;section=0
    def flush_word():
        nonlocal word,start,word_end
        if word:current.append(dict(start=start,end=word_end.copy(),runes=''.join(word),length=len(word)));word=[];start=None
    def flush_block():
        nonlocal current
        flush_word()
        if current:result.append(dict(block=len(result),section=section,words=current));current=[]
    for ch in source:
        if ch in R:
            if not word:start=dict(page=page_ids[page],rune_offset=offset)
            word.append(ch);word_end=dict(page=page_ids[page],rune_offset=offset);offset+=1
        elif ch=='%':page+=1;offset=0
        elif ch in '/\n\r\t ':pass
        elif ch in '&$§' or ch.isascii() and ch.isalnum():
            flush_block()
            if ch=='$':section+=1
        else:flush_word()
    flush_block();return result

def automaton(sequences):
    nxt=[{}];link=[-1];length=[0];last=0
    for seq in sequences:
        for x in list(seq)+[0]:
            x=int(x);cur=len(nxt);nxt.append({});length.append(length[last]+1);link.append(0);p=last
            while p>=0 and x not in nxt[p]:nxt[p][x]=cur;p=link[p]
            if p>=0:
                q=nxt[p][x]
                if length[p]+1==length[q]:link[cur]=q
                else:
                    clone=len(nxt);nxt.append(nxt[q].copy());length.append(length[p]+1);link.append(link[q])
                    while p>=0 and nxt[p].get(x)==q:nxt[p][x]=clone;p=link[p]
                    link[q]=link[cur]=clone
            last=cur
    return nxt,link,length

def longest(sequences,sam):
    nxt,link,length=sam;best=0
    for seq in sequences:
        state=0;n=0
        for x in seq:
            x=int(x)
            while state and x not in nxt[state]:state=link[state];n=min(n,length[state])
            if x in nxt[state]:state=nxt[state][x];n+=1;best=max(best,n)
            else:state=0;n=0
    return best

def lengths(bs):return [np.array([w['length'] for w in b['words']],dtype=np.int64) for b in bs]

def locate(ref,target,n):
    index=defaultdict(list)
    for bi,b in enumerate(ref):
        seq=[w['length'] for w in b['words']]
        for i in range(len(seq)-n+1):index[tuple(seq[i:i+n])].append((bi,i))
    hits=[]
    for bi,b in enumerate(target):
        seq=[w['length'] for w in b['words']]
        for i in range(len(seq)-n+1):
            key=tuple(seq[i:i+n])
            for rb,ri in index.get(key,[]):
                hits.append(dict(length_words=n,length_runes=sum(key),pattern=list(key),
                    reference=dict(block=rb,word_index=ri,words=ref[rb]['words'][ri:ri+n]),
                    target=dict(block=bi,word_index=i,section=b['section'],words=b['words'][i:i+n])))
    return hits

def comparison(samples,observed,seed,method):
    a=np.array(samples);ge=int((a>=observed).sum());p=(ge+1)/(len(a)+1)
    return dict(seed=seed,simulations=len(a),method=method,observed=observed,
        mean=float(a.mean()),central_95_percent=np.quantile(a,[.025,.975]).tolist(),
        distribution={str(k):v for k,v in sorted(Counter(samples).items())},at_least_observed=ge,
        upper_tail_comparison=p,two_comparison_bonferroni=min(1,2*p),samples=samples)

def verification():
    refs=[[1,3,2,4],[2,2,1],[6,1,5]];sam=automaton(refs);rng=np.random.default_rng(SEED+99)
    for _ in range(100):
        qs=[rng.integers(1,7,size=12).tolist(),rng.integers(1,7,size=9).tolist()];brute=0
        for a in refs:
            for b in qs:
                for i in range(len(a)):
                    for j in range(len(b)):
                        n=0
                        while i+n<len(a) and j+n<len(b) and a[i+n]==b[j+n]:n+=1
                        brute=max(brute,n)
        assert brute==longest(qs,sam)
    # Page/line wraps must not split a rune word; ASCII blocks must break matches.
    parsed=blocks(R[0]+R[1]+'/%'+R[2]+'-'+R[3]+'-123-'+R[4]+'-'+R[5],[0,1])
    assert [[w['length'] for w in b['words']] for b in parsed]==[[3,1],[1,1]]
    trailing=blocks(R[0]+'%.'+R[1],['a','b'])
    assert trailing[0]['words'][0]['end']==dict(page='a',rune_offset=0)
    return dict(longest_match_bruteforce_fixtures=100,page_line_and_numeric_barrier_fixture=True,trailing_page_wrap_coordinate_fixture=True)

def main():
    # All 15 parsed LP1 text pages; hard break before the two solved LP2 pages.
    reference_source='%'.join(pages[:15])+'$%'+'%'.join(lp[56:58])
    reference_ids=[f'LP1_transcription_{i}' for i in range(15)]+['LP2_56','LP2_57']
    refs=blocks(reference_source,reference_ids);targets=blocks('%'.join(lp[:56]),list(range(56)))
    rs,ts=lengths(refs),lengths(targets);sam=automaton(rs);observed=longest(ts,sam)
    hits=locate(refs,targets,observed);assert hits
    rng=np.random.default_rng(SEED);independent=[]
    for i in range(9999):independent.append(longest([rng.permutation(x) for x in ts],sam))
    # Reuse the existing adjacent-pair-preserving sampler with length labels
    # mapped to compact integers. No numerical interpretation of labels.
    alphabet=sorted(set(int(x) for seq in ts for x in seq));assert len(alphabet)<=29
    to_index={x:i for i,x in enumerate(alphabet)};inverse=np.array(alphabet)
    encoded=[np.array([to_index[int(x)] for x in seq],dtype=np.int64) for seq in ts]
    rng=np.random.default_rng(SEED+1);paired=[];attempts=[];changed=[]
    for i in range(999):
        sampled=[shuffle_pairs(seq,rng) for seq in encoded]
        simulated=[inverse[a] for a,_ in sampled]
        paired.append(longest(simulated,sam));attempts.append(sum(n for _,n in sampled))
        changed.append(sum(not np.array_equal(a,b) for a,b in zip(ts,simulated)))
    # Independently build rune strings with planted reference length runs and
    # position-varying symbols, then run the complete lexical matching pipeline.
    control_ref=next(b for b in refs if len(b['words'])>=16);controls=[]
    for size in [12,16]:
        selected=control_ref['words'][:size];source='-'.join(''.join(R[(j*7+k*11+5)%29] for k in range(w['length'])) for j,w in enumerate(selected))
        parsed=blocks(source,['constructed']);stat=longest(lengths(parsed),sam)
        assert stat==size and locate(refs,parsed,size)
        controls.append(dict(planted_words=size,recovered_longest=size,reference_block=control_ref['block'],
            pattern=[w['length'] for w in selected],constructed_runes=source,
            note='Detection of planted word lengths with changed rune identities; not plaintext or key recovery.'))
    out=dict(seed=SEED,statistic='Longest exact consecutive word-length match anywhere between fixed solved references and unsolved targets, within eligible blocks.',
        scope=dict(reference_pages=17,reference_page_ids=reference_ids,reference_blocks=len(refs),reference_words=sum(map(len,rs)),reference_runes=sum(int(x.sum()) for x in rs),
            target_pages=56,target_blocks=len(targets),target_words=sum(map(len,ts)),target_runes=sum(int(x.sum()) for x in ts)),
        boundaries='Count runes per word. Word punctuation separates words; clauses may be crossed. Paragraph/segment/chapter markers and ASCII alphanumeric material are hard breaks. Line and page wraps do not split a word. Do not join LP1 to LP2 references.',
        observed_longest_words=observed,maximal_matches=hits,
        independent_order=comparison(independent,observed,SEED,'Shuffle word lengths within every target block; preserve block length and exact word-length inventory.'),
        pair_preserving_order=comparison(paired,observed,SEED+1,'Within each target block preserve ordered adjacent word-length pair counts, endpoints and inventory via last-exit-tree sampling.'),
        pair_sampler_attempts=attempts,pair_sampler_changed_blocks=changed,
        controls=controls,verification=verification(),reference_blocks=refs,target_blocks=targets,
        transcript_sha256=hashlib.sha256((ROOT/'liber-primus__transcription--master.txt').read_bytes()).hexdigest(),
        script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
        limits=['Word lengths are measured directly from the source rune text on solved pages; no new decryption of reference pages is required or claimed.',
            'The self-quotation hypothesis requires retained word order, word boundaries and one-to-one rune lengths. Rewording, inserted nulls and length-changing encodings can evade it.',
            'Reference text is fixed in both comparisons. The maximum statistic scans all positions, not a selected favorite quotation.',
            'Two null comparisons are dependent; Bonferroni over two comparisons is supplied. Tail values are not probabilities of shared wording or cipher hypotheses.',
            'Even pair-preserving shuffles do not reproduce every property of natural language or intentional composition.',
            'No new original-image retranscription or authentication. No novel excerpts or external books are searched.',
            'Previously examined corpus; exploratory analysis, not independently replicated discovery.'])
    (ROOT/'word_length_fingerprint_results.json').write_text(json.dumps(out,ensure_ascii=False,indent=2)+'\n')
    print(json.dumps({k:out[k] for k in ['scope','observed_longest_words','controls']},ensure_ascii=False))
    for k in ['independent_order','pair_preserving_order']:
        print(k,json.dumps({a:b for a,b in out[k].items() if a!='samples'}))
    print('maximal_match_pairs',len(hits),'changed_blocks',min(changed),max(changed),flush=True)
if __name__=='__main__':main()