"""Session57: fixed-window composition changes at additional visual headings.

No cipher/key search. Visual spans are manual annotations. Rune counts, source
hashes, word-start eligibility and the random-placement comparison are computed.
Run beside the retained sources with NumPy, Pillow and matplotlib.
"""
from pathlib import Path
import hashlib,json,re
import numpy as np
from PIL import Image
from lp_core import lp,R,vals
from lexical_words import tokenize_words

ROOT=Path(__file__).resolve().parent
PLAN_HASH='cdacfbfc00f7d918ce184d77594139b73d208228315310396801ffe80d14dab0'
# Page, start and exclusive end. Color transitions on pages8 and40 occur
# at ordinary single-square word separators, not at the first full stop.
SPANS=[(0,0,13),(3,0,16),(3,119,122),(6,71,82),(8,0,12),
       (15,9,20),(23,0,16),(27,0,19),(33,0,10),(33,91,99),
       (39,119,122),(40,0,8),(54,0,9)]

def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
def cumulative(a):return np.r_[0,np.cumsum(a)]
def g_values(v,cuts,h,w):
    counts=np.vstack([np.zeros(29,dtype=int),np.eye(29,dtype=int)[v].cumsum(axis=0)])
    cuts=np.asarray(cuts);left=counts[cuts]-counts[cuts-w]
    right=counts[cuts+h+w]-counts[cuts+h];total=left+right
    out=np.zeros(len(cuts))
    for a in [left,right]:
        nonzero=a>0;terms=np.zeros(a.shape)
        terms[nonzero]=a[nonzero]*np.log(2*a[nonzero]/total[nonzero])
        out+=2*terms.sum(axis=1)
    return out,left,right

def main():
    plan=json.loads((ROOT/'heading_change_plan.json').read_text())
    assert sha(ROOT/'heading_change_plan.json')==PLAN_HASH
    assert sha(ROOT/'liber-primus__transcription--master.txt')=='b0b2e13c3cd35512ee5955a2d9a56c22d7410e283d35dd9c90abbaa7b99279eb'
    assert [[p,a] for p,a,b in SPANS]==plan['heading_page_offsets']
    source='%'.join(lp[:56]);v=vals(source);N=len(v);w=plan['window_runes']
    offsets=np.r_[0,np.cumsum([len(vals(s)) for s in lp[:56]])]
    lengths=[len(vals(s)) for s in source.split('$') if len(vals(s))]
    section_starts=np.r_[0,np.cumsum(lengths)]
    assert list(lengths)==[729,1145,1729,9,1894,1021,1524,1589,3316]
    manifest=json.loads((ROOT/'jpeg_source_manifest.json').read_text())
    hashes={r['file']:r['sha256'] for r in manifest['files']}
    headings=[];heading_mask=np.zeros(N,dtype=int)
    for p,a,b in SPANS:
        image=f'{p+17:02}.jpg';actual=sha(ROOT/image);assert actual==hashes[image]
        start=int(offsets[p]+a);end=int(offsets[p]+b)
        section=int(np.searchsorted(section_starts,start,side='right')-1)
        heading_mask[start:end]=1
        headings.append({'page':p,'image':image,'image_sha256':actual,
           'page_rune_span':[a,b],'global_rune_span':[start,end],
           'old_section':section,'old_section_offset':int(start-section_starts[section]),
           'coincides_with_old_section_start':bool(start==section_starts[section]),
           'visual_end':'single-square word separator at color transition' if p in [8,40] else 'red multi-dot terminator',
           'role_caveat':'Red opening words within a continuing sentence; not established as a separate title' if p in [8,40] else 'Heading role inferred from enlarged initial, red text and terminating mark',
           'alignment':'manual color-span alignment to retained transcription, not OCR'})
    extra=[r for r in headings if not r['coincides_with_old_section_start']]
    assert [[r['page'],r['page_rune_span'][0]] for r in extra]==plan['extra_heading_page_offsets']
    # Internal gaps containing ASCII letters/digits include number labels and
    # the long encoded block. A flank may not bridge one of these gaps.
    gaps=[''];k=0
    for c in source:
        if c in R:k+=1;gaps.append('')
        else:gaps[k]+=c
    inserted=np.array([bool(re.search('[A-Za-z0-9]',s)) for s in gaps],dtype=int)
    H=cumulative(heading_mask);D=cumulative(inserted)
    wordstarts=np.array(sorted(set(t['rune_offset'] for t in tokenize_words(source))),dtype=int)
    def clean(a,b):return H[b]-H[a]==0 and D[b]-D[a+1]==0
    pools=[]
    for r in extra:
        c,e=r['global_rune_span'];h=e-c;s=r['old_section']
        lo=int(section_starts[s]);hi=int(section_starts[s+1])
        def eligible(t):return t-w>=lo and t+h+w<=hi and clean(t-w,t) and clean(t+h,t+h+w)
        assert c in wordstarts and eligible(c),(r,'actual window fails fixed eligibility')
        pool=np.array([int(t) for t in wordstarts if eligible(int(t))],dtype=int)
        assert len(pool)>0 and c in pool
        pools.append(pool)
        r['heading_runes']=h;r['candidate_cuts']=len(pool)
        r['left_window']=[c-w,c];r['right_window']=[e,e+w]
        r['candidate_global_rune_offsets']=pool.tolist()
    # Primary statistic catches the largest of the five changes, with the
    # same maximum taken on every reference trial.
    rng=np.random.default_rng(plan['seed']);draws=[];proposed=0
    while len(draws)<plan['simulations']:
        ix=[int(rng.integers(len(pool))) for pool in pools];proposed+=1
        intervals=[(r['old_section'],int(pool[j])-w,int(pool[j])+r['heading_runes']+w) for r,pool,j in zip(extra,pools,ix)]
        overlap=any(s==t and max(a,c)<min(b,d) for i,(s,a,b) in enumerate(intervals) for t,c,d in intervals[:i])
        if not overlap:draws.append(ix)
    draws=np.asarray(draws,dtype=int)
    def analyze(values):
        observed=[];nulls=[];rows=[]
        for i,(r,pool) in enumerate(zip(extra,pools)):
            gs,ls,rs=g_values(values,pool,r['heading_runes'],w)
            actual=int(np.flatnonzero(pool==r['global_rune_span'][0])[0])
            observed.append(float(gs[actual]));nulls.append(gs[draws[:,i]])
            rows.append({'page':r['page'],'page_rune_span':r['page_rune_span'],
                'G':float(gs[actual]),'left_counts':ls[actual].tolist(),'right_counts':rs[actual].tolist(),
                'candidate_G_median':float(np.median(gs))})
        reference=np.max(np.stack(nulls,axis=1),axis=1);maximum=max(observed)
        ge=int(np.count_nonzero(reference>=maximum-1e-12))
        return {'boundaries':rows,'maximum_G':maximum,'reference_at_least_observed':ge,
                'plus_one_p':(ge+1)/(len(reference)+1),
                'reference_quantiles':{str(q):float(np.quantile(reference,q)) for q in [.025,.5,.95,.975]},
                'reference_maximum_G':reference.tolist()}
    # Two mathematical fixtures for the statistic, independent of the data.
    z=np.tile(np.arange(29),4)[:64];same=np.r_[z,z]
    assert abs(g_values(same,[64],0,64)[0][0])<1e-10
    separate=np.r_[np.zeros(64,dtype=int),np.ones(64,dtype=int)]
    assert abs(g_values(separate,[64],0,64)[0][0]-256*np.log(2))<1e-10
    real=analyze(v)
    crng=np.random.default_rng(plan['seed']+1);control=crng.integers(0,29,N)
    changed=np.zeros(N,dtype=bool)
    for r in extra:
        c,e=r['global_rune_span']
        for start,end,favorite in [(c-w,c,0),(e,e+w,10)]:
            assert not changed[start:end].any();changed[start:end]=True
            probabilities=np.full(29,.7/28);probabilities[favorite]=.3
            control[start:end]=crng.choice(29,end-start,p=probabilities)
    positive=analyze(control)
    out={'plan_sha256':PLAN_HASH,'source_revision':manifest['repository_revision'],
       'transcription_sha256':sha(ROOT/'liber-primus__transcription--master.txt'),
       'runes':N,'alphabet':R,'old_section_lengths':lengths,'visual_openers':headings,
       'extra_boundaries':extra,'nonrunic_alphanumeric_gap_offsets':np.flatnonzero(inserted).tolist(),
       'randomization':{'seed':plan['seed'],'accepted':len(draws),'proposed':proposed,'joint_draw_indices':draws.tolist()},
       'real':real,'positive_control':positive,
       'control_detail':{'seed':plan['seed']+1,'changed_runes':int(changed.sum()),'left_favorite':'F','right_favorite':'I','favorite_probability':.3,'background':'uniform independent29-symbol stream','detected_at_005':positive['plus_one_p']<=.05},
       'fixtures':{'equal_histograms_G_zero':True,'disjoint_histograms_G_256_log2':True},
       'limits':['Comparison is about local rune composition only; key resets can preserve composition.',
         'Five retrospective boundaries, one fixed64-rune flank size, and no semantic matching of pseudo-boundaries.',
         'Monte Carlo tail is relative to the stated observed-text placement model; not a probability that a cipher theory is true.',
         'No individual boundary was separately promoted using an unadjusted test.']}
    (ROOT/'heading_change_results.json').write_text(json.dumps(out,ensure_ascii=False,indent=2)+'\n')
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    fig,axes=plt.subplots(1,2,figsize=(12,4.2),layout='constrained')
    for ax,result,title in zip(axes,[real,positive],['Actual LP2 text','Constructed strong composition change']):
        ax.hist(result['reference_maximum_G'],bins=35,color='#7895b2',edgecolor='white')
        ax.axvline(result['maximum_G'],color='#9c2424',linewidth=2,label='At annotated headings')
        ax.set_title(title);ax.set_xlabel('Largest G across five boundaries');ax.set_ylabel('Random-placement trials')
        ax.text(.97,.95,f"Tail comparison: {result['plus_one_p']:.4f}",transform=ax.transAxes,ha='right',va='top');ax.legend(loc='upper left')
    fig.savefig(ROOT/'heading_change_comparison.png',dpi=150);plt.close(fig)
    print(json.dumps({'visual_openers':len(headings),'old_section_starts':sum(r['coincides_with_old_section_start'] for r in headings),
       'additional_boundaries':len(extra),'candidate_counts':[len(x) for x in pools],
       'real':{k:v for k,v in real.items() if k!='reference_maximum_G'},
       'control':{k:v for k,v in positive.items() if k not in ['reference_maximum_G','boundaries']}},ensure_ascii=False))

if __name__=='__main__':main()