"""Session63: dependency checks and bounded replay of separator-sensitive tests.
Run with Python + NumPy beside retained inputs. Never modifies older artifacts.
The in-memory semicolon view is a compatibility hypothesis, not ground truth.
"""
from pathlib import Path
import ast, hashlib, json
import numpy as np
import lp_core as core
from lp_core import R, vals, positions, render, scores, SEQS, without_first_clauses
from word_core import tokenize, GROUPS
from lexical_words import tokenize_words
from word_length_fingerprint_audit import blocks
from double_context_audit import features, STRONG
import boundary_pass as boundary

ROOT=Path(__file__).resolve().parent
sha=lambda p:hashlib.sha256(p.read_bytes()).hexdigest()
read=lambda f:json.loads((ROOT/f).read_text())
def write(name,obj):
    (ROOT/name).write_text(json.dumps(obj,ensure_ascii=False,indent=2)+'\n')

def inventory():
    out=[]
    for f in sorted(ROOT.glob('*.py')):
        if f.name==Path(__file__).name:continue
        tree=ast.parse(f.read_text());literals=[];imports=[]
        for n in ast.walk(tree):
            if isinstance(n,ast.ImportFrom):imports.append(n.module)
            if isinstance(n,ast.Import):imports.extend(a.name for a in n.names)
            if isinstance(n,ast.Constant) and isinstance(n.value,str) and 0<len(n.value)<40 and (';' in n.value or n.value=='-'):
                literals.append({'line':n.lineno,'literal':n.value})
        out.append({'file':f.name,'sha256':sha(f),'imports':sorted(set(x for x in imports if x)),
                    'short_semicolon_or_hyphen_literals':sorted(literals,key=lambda x:x['line'])})
    return out

def compatibility_pages(book):
    old=core.pages;new=old.copy();changes=[]
    for overlay in book['overlays']:
        p=overlay['parsed_page_index'];gap=overlay['rune_gap']
        page=book['pages'][p];assert page['image']==overlay['image']
        record=next(b for b in page['boundaries'] if b['rune_gap']==gap)
        literal=[e for e in record['literal_events'] if e['character']=='-']
        assert len(literal)==1 and record['observed_black_class']=='three_dot'
        idx=literal[0]['raw_character_index'];s=new[p];assert s[idx]=='-'
        new[p]=s[:idx]+';'+s[idx+1:]
        changes.append({**overlay,'raw_character_index':idx,'compatibility_character':';'})
    assert sum(sum(a!=b for a,b in zip(s,t))for s,t in zip(old,new))==3
    return old,new,changes

def invariants(old,new):
    checks=[]
    def check(name,ok,detail):
        assert ok,name
        checks.append({'name':name,'exactly_equal':True,'detail':detail})
    a='%'.join(old);b='%'.join(new);u='%'.join(old[15:71]);v='%'.join(new[15:71])
    check('all_73_page_rune_streams',all(np.array_equal(vals(s),vals(t)) for s,t in zip(old,new)),'15,933 runes; changes are nonrunic.')
    check('historical_word_records',tokenize(a)==tokenize(b),'Values, word order, offsets, page, section and paragraph metadata.')
    check('current_lexical_word_records',tokenize_words(a)==tokenize_words(b),'Includes apostrophe joining; no word boundary added or removed.')
    check('word_length_fingerprint_blocks',blocks(a,list(range(73)))==blocks(b,list(range(73))),'Every reference/target word-length record and hard break unchanged; this does not repair the older apostrophe convention.')
    for mark in '/%&$§.':
        check('rune_streams_split_on_'+mark,
              [vals(s).tolist() for s in a.split(mark)]==[vals(s).tolist() for s in b.split(mark)],'Same unit ordering and contents.')
    train_indices=[0,3,4,5,6,7,8,9,10,11,14]
    check('training_sources',all(old[i]==new[i] for i in train_indices),'The corrected LP1 parsed page13 is outside lp_core.train and word_core.GROUPS.')
    selective=[('%'.join(old[15:18]),'%'.join(new[15:18])),('%'.join(old[69:71]),'%'.join(new[69:71]))]
    check('selective_F_targets',all(s==t for s,t in selective),'Session2 skip_pass targets LP2 pages0-2 and54-55, outside corrections. Its null targets are unchanged too.')
    sa=u.split('$');sb=v.split('$')
    check('combined_null_targets',all(sa[i]==sb[i] for i in (0,4)),'Session3 combined_null uses sections0 and4, outside corrections.')
    locations=[]
    for p,s in enumerate(old[15:71]):locations.extend((p,i)for i in range(len(vals(s))))
    cursor=0;feature_rows=[]
    for i,(s,t)in enumerate(zip(sa,sb)):
        if len(vals(s))>100:
            x=features(s,i,cursor,locations);y=features(t,i,cursor,locations)
            check('session27_feature_matrix_section_'+str(i),np.array_equal(x[1],y[1]),'Original STRONG='+repr(STRONG)+' excludes both hyphen and semicolon; no claim to test all physical punctuation.')
            feature_rows.append({'section':i,'eligible_edges':len(x[1]),'repeat_events':len(x[2]),'features':len(x[3])})
        cursor+=len(vals(s))
    schedules=[]
    for i,(s,t)in enumerate(zip(sa,sb)):
        if len(vals(s))<60:continue
        for scope,ss,tt in [('all_text',s,t),('first_clause_per_paragraph_removed',without_first_clauses(s),without_first_clauses(t))]:
            check('scope_runes_'+str(i)+'_'+scope,np.array_equal(vals(ss),vals(tt)),'Literal first-period removal unchanged.')
            for name,reset in boundary.reset_modes.items():
                for advance in ('','-.,;'):
                    for skip in (False,True):
                        x,xf=positions(ss,reset,advance,skip);y,yf=positions(tt,reset,advance,skip)
                        diff=np.flatnonzero(x!=y);fdiff=np.flatnonzero(xf!=yf)
                        if len(diff) or len(fdiff):assert name=='clause' and i in (7,8)
                        schedules.append({'section':i,'scope':scope,'reset':name,'advance':bool(advance),'skip_all_F':skip,
                            'runes':len(x),'changed_counter_positions':diff.tolist(),'changed_reset_flags':fdiff.tolist(),
                            'prefix_length':int(.6*len(x)),'changed_prefix_positions':int(np.sum(diff<int(.6*len(x))))})
    return {'checks':checks,'unsolved_runes':len(vals(u)),'historical_unsolved_words':len(tokenize(u)),
            'current_unsolved_words':len(tokenize_words(u)),'session27_sections':feature_rows,'schedules':schedules},sa,sb

def boundary_records(text,label,only_clause=False):
    records=[];count=0
    for scope,source in [('all_text',text),('first_clause_per_paragraph_removed',without_first_clauses(text))]:
        c=vals(source)
        if len(c)<60:continue
        cut=int(len(c)*.6)
        for resetname,reset in boundary.reset_modes.items():
            if only_clause and resetname!='clause':continue
            for advance in ('','-.,;'):
                for skipf in (False,True):
                    ix,_=positions(source,reset,advance,skipf)
                    for seqname,seq in SEQS.items():
                        keys=seq[boundary.offs[:,None]+ix[None,:]]
                        for a in (1,-1):
                            for sign in (-1,1):
                                d=(a*c[None,:]+sign*keys)%29
                                if skipf:d[:,c==0]=0
                                prefix=scores(d[:,:cut]);j=int(np.argmax(prefix))
                                records.append(dict(target=label,scope=scope,reset=resetname,delimiter_advance=bool(advance),
                                    skip_all_cipher_F=skipf,sequence=seqname,a=a,sign=sign,offset=int(boundary.offs[j]),
                                    prefix_score=float(prefix[j]),suffix_score=float(scores(d[j:j+1,cut:])[0]),
                                    rune_count=len(c),split_after_runes=cut,text=render(source,d[j])))
                                count+=len(boundary.offs)
    return records,count

ID=['scope','reset','delimiter_advance','skip_all_cipher_F','sequence','a','sign']
def identity(r):return tuple(r[k] for k in ID)
def assert_records(a,b):
    assert len(a)==len(b)
    for x,y in zip(a,b):
        assert set(x)==set(y),(set(x),set(y))
        for k in x:
            if k.endswith('_score'):assert abs(x[k]-y[k])<1e-11,(k,x[k],y[k])
            else:assert x[k]==y[k],k

def boundary_replay(sa,sb):
    saved=read('boundary_results.json');result=[]
    for i in (7,8):
        old,n=boundary_records(sa[i],f'marked_section_{i}')
        oldtop=sorted(old,key=lambda r:r['prefix_score'],reverse=True)[:10]
        ref=next(s for s in saved['sections'] if s['section_index']==i)['top_prefix_candidates']
        assert_records(oldtop,ref)
        revised,nnew=boundary_records(sb[i],f'marked_section_{i}',True)
        change={identity(r):r for r in revised}
        merged=[change.get(identity(r),r) for r in old]
        newtop=sorted(merged,key=lambda r:r['prefix_score'],reverse=True)[:10]
        diffs=[]
        for x,y in zip(old,merged):
            if x['reset']!='clause':assert x==y;continue
            diffs.append({'family':dict(zip(ID,identity(x))),
                'old_offset':x['offset'],'new_offset':y['offset'],
                'old_prefix_score':x['prefix_score'],'new_prefix_score':y['prefix_score'],
                'old_suffix_score':x['suffix_score'],'new_suffix_score':y['suffix_score'],
                'rendered_output_changed':x['text']!=y['text']})
        best_old=max((r for r in old if r['reset']=='clause'),key=lambda r:r['prefix_score'])
        best_new=max(revised,key=lambda r:r['prefix_score'])
        result.append({'section':i,'baseline_top10_reproduced':True,'baseline_parameter_settings':n,
            'replayed_clause_settings':nnew,'replayed_clause_family_winners':len(revised),
            'changed_selected_offsets':sum(x['old_offset']!=x['new_offset']for x in diffs),
            'changed_family_prefix_scores':sum(abs(x['old_prefix_score']-x['new_prefix_score'])>1e-12 for x in diffs),
            'changed_family_suffix_scores':sum(abs(x['old_suffix_score']-x['new_suffix_score'])>1e-12 for x in diffs),
            'top1_exactly_unchanged':oldtop[0]==newtop[0],
            'top10_parameter_identities_unchanged':[(identity(r),r['offset'])for r in oldtop]==[(identity(r),r['offset'])for r in newtop],
            'old_top10':oldtop,'new_top10':newtop,'best_old_clause':best_old,'best_new_clause':best_new,'clause_deltas':diffs})
        print('Session2 section',i,'clause settings',nnew,'top1 unchanged',result[-1]['top1_exactly_unchanged'],flush=True)
    return result

def combined_replay(sa,sb):
    import combined_pass as combined
    saved=read('combined_results.json');out=[]
    for i in (7,8):
        s=sa[i];t=sb[i];c=vals(s);cut=min(256,int(.6*len(c)));proof=[]
        for skip in (False,True):
            x,_=positions(s,reset=combined.RESETS['clause'],skip_f=skip)
            y,_=positions(t,reset=combined.RESETS['clause'],skip_f=skip)
            assert np.array_equal(x[:cut],y[:cut])
            valid=[]
            for name,seq in combined.STREAMS.items():
                ox=combined.OFFSETS[combined.OFFSETS+int(x.max())<len(seq)]
                oy=combined.OFFSETS[combined.OFFSETS+int(y.max())<len(seq)]
                assert np.array_equal(ox,oy),(i,skip,name,ox,oy)
                valid.append({'sequence':name,'eligible_offsets':ox.tolist()})
            proof.append({'skip_all_F':skip,'selection_counter_equal':True,'eligibility_equal':True,
                'old_max_counter':int(x.max()),'new_max_counter':int(y.max()),'streams':valid})
        target=next(z for z in saved['sections'] if z['target']==f'marked_section_{i}')
        rows=[]
        for rank,r in enumerate(target['top_prefix_candidates'],1):
            ix,_=positions(s,reset=combined.RESETS[r['reset']],skip_f=r['skip_all_cipher_F'])
            iy,_=positions(t,reset=combined.RESETS[r['reset']],skip_f=r['skip_all_cipher_F'])
            def decode(indices):
                d=(r['a']*c+r['sign']*combined.STREAMS[r['sequence']][indices+r['offset']]+r['shift'])%29
                if r['skip_all_cipher_F']:d[c==0]=0
                return d
            old=decode(ix);new=decode(iy)
            assert old.tolist()==r['decoded_rune_values']
            assert abs(float(scores(old[None,:cut])[0])-r['prefix_score'])<1e-11
            assert abs(float(scores(old[None,cut:])[0])-r['suffix_score'])<1e-11
            assert np.array_equal(old[:cut],new[:cut])
            if r['reset']!='clause':assert np.array_equal(old,new);continue
            diff=np.flatnonzero(old!=new)
            rows.append({'rank':rank,'parameters':{k:r[k]for k in ('sequence','reset','skip_all_cipher_F','a','sign','offset','shift')},
                'prefix_score':r['prefix_score'],'old_suffix_score':r['suffix_score'],
                'new_suffix_score':float(scores(new[None,cut:])[0]),'changed_runes':len(diff),
                'changed_rune_offsets':diff.tolist(),'new_text':render(t,new),'new_decoded_rune_values':new.tolist()})
        out.append({'section':i,'all_candidate_ranks_invariant':True,'proof':proof,
            'saved_top20_baselines_exact':True,'unchanged_saved_nonclause_candidates':20-len(rows),
            'fixed_parameter_clause_replays':rows,'prefix_length':cut})
        print('Session3 section',i,'fixed clause continuations',len(rows),'all ranks invariant',flush=True)
    return out

def main():
    before=sha(ROOT/'liber-primus__transcription--master.txt');book=read('boundary_model.json')
    assert before==book['master_sha256']
    scripts=inventory();old,new,changes=compatibility_pages(book)
    inv,sa,sb=invariants(old,new)
    for c in changes:
        if c['lp2_page'] is not None:
            p=c['lp2_page'];prefix='%'.join(new[15:15+p]);prefix+=('%'if prefix else '')+new[15+p][:c['raw_character_index']]
            c['section']=prefix.count('$');c['section_rune_gap']=len(vals(prefix.rsplit('$',1)[-1]))
    first=boundary_replay(sa,sb);second=combined_replay(sa,sb)
    assert sha(ROOT/'liber-primus__transcription--master.txt')==before
    inputs=['boundary_impact_plan.json','boundary_model.json','boundary_results.json','combined_results.json','skip_results.json','combined_null_results.json','double_context_results.json']
    result={'session':63,'master_sha256':before,'master_unchanged':True,'script_sha256':sha(Path(__file__)),
        'input_hashes':{f:sha(ROOT/f)for f in inputs},'compatibility_overlays':changes,'invariants':inv,
        'session2_replay':first,'session3_replay':second,
        'limits':read('boundary_impact_plan.json')['limits'],
        'session27_limit':'Its numerical features are invariant because its predeclared STRONG set excludes both hyphen and semicolon. It does not test all observed three-dot events; adding them would be a new feature definition, not reproduction.',
        'physical_limit':'Semicolon substitution is a historical-code compatibility view only; physical classes and reset semantics remain separate.'}
    write('boundary_impact_inventory.json',{'scripts':scripts,'count':len(scripts),
        'scope':'Static dependency/literal inventory, not formal proof that every program is invariant. Numerical claims use explicit checks in boundary_impact_results.json.'})
    write('boundary_impact_results.json',result)
    print(json.dumps({'scripts_inventoried':len(scripts),'exact_invariant_checks':len(inv['checks']),
        'schedule_comparisons':len(inv['schedules']),'changed_schedules':sum(bool(r['changed_counter_positions']or r['changed_reset_flags'])for r in inv['schedules']),
        'clause_settings_replayed':sum(r['replayed_clause_settings']for r in first)},indent=2),flush=True)

if __name__=='__main__':main()