"""Session73: compare frozen image readings with the unchanged master.
Python, NumPy and Pillow. No OCR, downloads, key search or plaintext fitting.
"""
from pathlib import Path
from collections import Counter
import hashlib, json, re
from PIL import Image

ROOT = Path(__file__).resolve().parent
LET = 'F U TH O R K G W H N I J EO P X S T B E M L ING OE D A AE Y IO EA'.split()
R = 'ᚠᚢᚦᚩᚱᚳᚷᚹᚻᚾᛁᛄᛇᛈᛉᛋᛏᛒᛖᛗᛚᛝᛟᛞᚪᚫᚣᛡᛠ'
FREEZE_SHA = 'f8b3f9e832f8593a6beb980f236c95f1a2f19612f3ae12518037c10c919a012e'
MASTER_SHA = 'b0b2e13c3cd35512ee5955a2d9a56c22d7410e283d35dd9c90abbaa7b99279eb'

def sha(p):
    return hashlib.sha256(p.read_bytes()).hexdigest()

def parse(text):
    values=[]; marks=[]; lines=[]; source=[]; row_starts=[]
    for line_no,line in enumerate(text.splitlines(),1):
        row_starts.append(len(values))
        toks=re.findall(r'\[[br][0-9?]+\]|[A-Z]+',line)
        assert re.sub(r'\[[br][0-9?]+\]|[A-Z]+|\s','',line)=='',line
        for token in toks:
            if token.startswith('['):
                dots=token[2:-1]
                marks.append({'rune_gap':len(values),'line':line_no,'color':token[1],
                              'dots':None if dots=='?' else int(dots)})
                source.append({1:'-',2:';',3:',',4:'.',13:'.'}.get(None if dots=='?' else int(dots),','))
            else:
                assert token in LET,token
                value=LET.index(token);values.append(value);lines.append(line_no);source.append(R[value])
        source.append('/\n')
    return {'values':values,'marks':marks,'lines':lines,'row_starts':row_starts,'source':''.join(source)}

def alignment(a,b):
    n,m=len(a),len(b);dp=[[0]*(m+1) for _ in range(n+1)]
    for i in range(n+1):dp[i][0]=i
    for j in range(m+1):dp[0][j]=j
    for i in range(1,n+1):
        for j in range(1,m+1):
            dp[i][j]=min(dp[i-1][j-1]+(a[i-1]!=b[j-1]),dp[i-1][j]+1,dp[i][j-1]+1)
    i,j=n,m;out=[]
    while i or j:
        if i and j and dp[i][j]==dp[i-1][j-1]+(a[i-1]!=b[j-1]):
            if a[i-1]!=b[j-1]:
                out.append({'kind':'substitution','initial_offset':i-1,'master_offset':j-1,
                            'initial':LET[a[i-1]],'master':LET[b[j-1]]})
            i-=1;j-=1
        elif i and dp[i][j]==dp[i-1][j]+1:
            out.append({'kind':'delete_from_initial','initial_offset':i-1,'master_gap':j,'initial':LET[a[i-1]]});i-=1
        else:
            out.append({'kind':'insert_into_initial','initial_gap':i,'master_offset':j-1,'master':LET[b[j-1]]});j-=1
    return {'minimum_edit_distance':dp[n][m],'edits':out[::-1]}

def summary(readings,tokenize):
    values=[v for r in readings for v in r['values']]
    inventory=Counter(values); words=tokenize('%'.join(r['source'] for r in readings))
    single=[w for w in words if len(w['values'])==1]
    positions=[]
    for page,r in enumerate(readings):
        positions.extend([{'page':page,'offset':i,'line':r['lines'][i],
                           'rune':LET[a],'across_line':r['lines'][i]!=r['lines'][i+1]} for i,(a,b) in enumerate(zip(r['values'],r['values'][1:])) if a==b])
    seams=[]
    off=0
    for page in range(len(readings)-1):
        off+=len(readings[page]['values']);a=values[off-1];b=values[off]
        seams.append({'from_page':page,'to_page':page+1,'global_rune_gap':off,
                      'left':LET[a],'right':LET[b],'is_repeat':a==b})
    repeated=sum(a==b for a,b in zip(values,values[1:]))
    assert repeated==len(positions)+sum(s['is_repeat'] for s in seams)
    crossing=[]
    for s in seams:
        for w in words:
            if w['rune_offset']<s['global_rune_gap']<w['rune_offset']+len(w['values']):
                crossing.append({'from_page':s['from_page'],'to_page':s['to_page'],
                  'word_global_offset':w['rune_offset'],'length':len(w['values']),
                  'cipher_labels':[LET[v] for v in w['values']]})
    return {'runes':len(values),'eligible_neighbor_pairs':len(values)-1,
      'observed_neighbor_repeats':repeated,
      'inventory_matched_expected_repeats':sum(n*(n-1) for n in inventory.values())/len(values),
      'inventory_counts':{LET[v]:inventory[v] for v in sorted(inventory)},
      'lexical_words':len(words),'singletons':len(single),'singleton_types':len({w['values'][0] for w in single}),
      'singleton_locations':[{'page':w['page'],'page_rune_offset':w['page_rune_offset'],
                             'cipher_label':LET[w['values'][0]]} for w in single],
      'repeat_positions':positions,'page_seams':seams,'words_crossing_pages':crossing}

def main():
    freeze=json.loads((ROOT/'opening73_freeze.json').read_text())
    assert sha(ROOT/'opening73_freeze.json')==FREEZE_SHA
    for n,h in freeze['hashes'].items():assert sha(ROOT/n)==h,n
    assert sha(ROOT/'liber-primus__transcription--master.txt')==MASTER_SHA
    crops=json.loads((ROOT/'opening73_crop_boxes.json').read_text())['crops']
    for c in crops:
        actual=Image.open(ROOT/c['source']).crop(c['box']).convert('RGB')
        stored=Image.open(ROOT/c['file']).convert('RGB')
        assert actual.size==stored.size and actual.tobytes()==stored.tobytes(),c['file']
    # Only after checking the saved freeze do these imports load master pages.
    from lp_core import lp,vals,R as canonical_R,LET as canonical_LET
    from lexical_words import tokenize_words
    assert R==canonical_R and LET==canonical_LET
    initial=[parse((ROOT/f'opening73_initial_p{i:02}.txt').read_text()) for i in range(3)]
    reviewed=[parse((ROOT/f'opening73_reviewed_p{i:02}.txt').read_text()) for i in range(3)]
    master=[];comparisons=[];decisions=[];punct=[]
    for page in range(3):
        m=list(map(int,vals(lp[page]))); assert reviewed[page]['values']==m,page
        raw=lp[page];gaps=[''];line_numbers=[];line=1
        for ch in raw:
            if ch in R:gaps.append('');line_numbers.append(line)
            else:gaps[-1]+=ch
            if ch=='/':line+=1
        master.append({'values':m,'lines':line_numbers,'marks':[], 'source':raw})
        c=alignment(initial[page]['values'],m)
        for e in c['edits']:
            i=e.get('initial_offset',e.get('initial_gap',0))
            e['line']=initial[page]['lines'][min(i,len(initial[page]['lines'])-1)]
            decisions.append({'page':page,**e,'source':f'{page+17:02}.jpg',
              'crop':f"opening73_p{page:02}_strip{(e['line']-1)//3}.png",
              'decision':'Retain master rune reading after source comparison; correct this first-pass reading.',
              'confidence_limit':'Same-reader review guided by the master. Minimum-edit alignment may not be unique.'})
        comparisons.append({'page':page,'initial_runes':len(initial[page]['values']),
                 'master_runes':len(m),**c})
        # Punctuation records preserve shapes; master comparison uses only its literal label.
        observed_literals=[]
        for event in reviewed[page]['marks']:
            gap=event['rune_gap'];expected='-' if event['dots']==1 else ';' if event['dots']==2 else ',' if event['dots']==3 else '.'
            literal=''.join(ch for ch in gaps[gap] if ch in '-.,;')
            assert literal==expected,(page,event,literal)
            observed_literals.append((gap,expected))
            punct.append({'page':page,**event,'master_literal':literal,
                          'role':'shape recorded, semantics not established by this audit'})
        master_literals=[(gap,ch) for gap,s in enumerate(gaps) for ch in s if ch in '-.,;']
        assert Counter(observed_literals)==Counter(master_literals),page
        old=Counter((e['color'],e['dots']) for e in initial[page]['marks'])
        new=Counter((e['color'],e['dots']) for e in reviewed[page]['marks'])
        comparisons[-1]['punctuation_counts']={'initial':{str(k):v for k,v in old.items()},
              'reviewed':{str(k):v for k,v in new.items()}}
    saved=json.loads((ROOT/'opening73_initial_comparison.json').read_text())
    assert comparisons[0]['minimum_edit_distance']==saved['records'][0]['minimum_edit_distance']
    assert [(c['minimum_edit_distance'],c['edits']) for c in comparisons]==[(c['minimum_edit_distance'],c['edits']) for c in saved['records']]
    init_sum=summary(initial,tokenize_words);rev_sum=summary(reviewed,tokenize_words);master_sum=summary(master,tokenize_words)
    assert rev_sum==master_sum
    # Exact test fixtures for parser and minimum-edit implementation.
    toy=[0,1,2,3]
    fixtures={}
    for name,a in [('same',toy),('substitution',[0,1,4,3]),('insertion',[0,1,4,2,3]),('deletion',[0,1,3])]:
        found=alignment(a,toy);assert found['minimum_edit_distance']==(0 if name=='same' else 1)
        fixtures[name]=found
    f=parse('F [b1] J\nJ [b4] A [r13]\n')
    assert f['values']==[0,11,11,24] and [e['rune_gap'] for e in f['marks']]==[1,3,4]
    assert f['marks'][-1]['color']=='r' and f['marks'][-1]['dots']==13
    assert len(tokenize_words('%'.join([parse('F U\n')['source'],parse('TH [b1] I\n')['source']])))==2
    assert rev_sum['runes']==729 and rev_sum['observed_neighbor_repeats']==4
    out={'session':73,'freeze_sha256':FREEZE_SHA,'master_sha256':MASTER_SHA,
      'script_sha256':sha(Path(__file__)),'input_hashes':{n:sha(ROOT/n) for n in
        ['opening_section_plan.json','opening73_freeze.json','opening73_crop_boxes.json',
         'opening73_initial_comparison.json',*[f'opening73_reviewed_p{i:02}.txt' for i in range(3)],
         'lexical_words.py','word_core.py','lp_core.py']},
      'source_images':{f'{i+17:02}.jpg':sha(ROOT/f'{i+17:02}.jpg') for i in range(3)},
      'pixel_exact_crops_verified':len(crops),'initial':init_sum,'reviewed':rev_sum,
      'master':master_sum,'comparisons':comparisons,
      'minimum_edit_operations':dict(Counter(e['kind'] for c in comparisons for e in c['edits'])),
      'minimum_edit_distance_total':sum(c['minimum_edit_distance'] for c in comparisons),
      'review_decisions':decisions,'observed_punctuation':punct,
      'punctuation_first_pass_correction':{'page':0,'line':6,'reviewed_rune_gap':124,
         'decision':'Add the visible black single-dot separator between TH P and Y ING EA; it was omitted from the first-pass reading, not from the master.',
         'crop':'opening73_p00_strip1.png'},
      'controls':{'alignment_fixtures':fixtures,'parser_and_seam_fixtures':'passed',
                  'reviewed_equals_master_rune_stream':True,'reviewed_equals_master_metrics':True},
      'source_corrections_established':0,'no_new_plaintext':True,
      'limitations':'Rough first pass: 136 minimum edits, many label confusions and glyph-part overcounting. Final source comparison was master-guided by the same reader; agreement is not independent confirmation or proof of no source error. Dot shape does not establish cipher semantics. Shuffled expectation is a conditional mean, not an English model or a new p-value.'}
    (ROOT/'opening73_results.json').write_text(json.dumps(out,ensure_ascii=False,indent=2)+'\n')
    print(json.dumps({'initial':init_sum,'reviewed':rev_sum,
                     'minimum_edit_operations':out['minimum_edit_operations'],
                     'marks':len(punct),'pixel_exact_crops':len(crops),'controls':'passed'},indent=2))

if __name__=='__main__':
    main()