"""Session 42: source-image observations, exact crops, corrected word grouping.
Python + NumPy + Pillow. Visual judgments are recorded observations, not OCR output.
Recomputes Sessions 40 and 41 under apostrophe-aware lexical grouping.
"""
from pathlib import Path
from io import BytesIO
from collections import Counter
import json,hashlib
import numpy as np
from PIL import Image,ImageDraw
from word_core import R,LET,vals,lp,GROUPS,tokenize
from lexical_words import join_intrarune_apostrophes,tokenize_words
from whole_word_repetition_audit import describe,matched,verify
from word_reset_feedback_audit import bound,decode,hist,recovery_controls
ROOT=Path(__file__).resolve().parent

def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
def tuples(source):return [tuple(w['values']) for w in tokenize_words(source)]
def arrays(ws):return [np.array(w,dtype=np.int64) for w in ws]
def save_png(im,path):
    b=BytesIO();im.save(b,format='PNG');path.write_bytes(b.getvalue())
    with Image.open(path) as check:assert check.size==im.size and np.array_equal(np.asarray(check),np.asarray(im))

def main():
    spec=json.loads((ROOT/'token_boundary_observations.json').read_text())
    manifest={x['file']:x for x in json.loads((ROOT/'jpeg_source_manifest.json').read_text())['files']}
    images={};evidence=[]
    for item in spec['image_regions']:
        filename=item['image'];path=ROOT/filename
        assert sha(path)==manifest[filename]['sha256']
        if filename not in images:images[filename]=Image.open(path).convert('RGB')
        im=images[filename];box=item['box'];crop=im.crop(box);save_png(crop,ROOT/item['crop_file'])
        evidence.append({**item,'source_sha256':sha(path),'source_url':manifest[filename]['url'],
                         'crop_sha256':sha(ROOT/item['crop_file']),'exact_pixel_crop_verified':True})
    # Assemble native-resolution rows; labels are outside unchanged source pixels.
    for group in spec['contact_sheets']:
        parts=[next(x for x in evidence if x['id']==i) for i in group['regions']]
        crops=[Image.open(ROOT/x['crop_file']).convert('RGB') for x in parts]
        width=max(x.width for x in crops);height=sum(x.height+40 for x in crops)
        im=Image.new('RGB',(width,height),'white');draw=ImageDraw.Draw(im);y=0
        for item,crop in zip(parts,crops):
            draw.text((12,y+8),item['caption'],fill='black');im.paste(crop,(0,y+35));y+=crop.height+40
        save_png(im,ROOT/group['file'])
    source='%'.join(lp[:56]);old=tokenize(source);new=tokenize_words(source)
    assert np.array_equal(vals(source),vals(join_intrarune_apostrophes(source)))
    assert len(old)==2899 and len(new)==2896
    changes=[]
    for page,s in enumerate(lp[:56]):
        for at,ch in enumerate(s):
            if ch!="'":continue
            assert s[at-1] in R and s[at+1] in R
            off=len(vals(s[:at]));section=next(w['section'] for w in old if w['page']==page and w['page_rune_offset']==off)
            before=next(w for w in old if w['page']==page and w['page_rune_offset']==off)
            assert len(before['values'])==1
            start=at-1
            while start>=0 and s[start] in R:start-=1
            text=s[start+1:at+2]
            changes.append(dict(page=page,image=f'{page+17:02}.jpg',section=section,suffix_rune_offset=off,
                suffix_label=LET[before['values'][0]],joined_runes=''.join(c for c in text if c in R),
                joined_labels=[LET[R.index(c)] for c in text if c in R],raw_apostrophe_text=text))
    assert [(x['page'],x['suffix_rune_offset']) for x in changes]==[(4,164),(21,36),(41,218)]
    def single_summary(ws):
        singles=[w for w in ws if len(w['values'])==1]
        return dict(tokens=len(singles),distinct_runes=len(set(w['values'][0] for w in singles)),
            labels=sorted(set(LET[w['values'][0]] for w in singles)),
            locations=[{k:w[k] for k in ['page','section','page_rune_offset','rune_offset','word_index','values']} for w in singles])
    ss=single_summary(new);section8=single_summary([w for w in new if w['section']==8])
    assert ss['tokens']==83 and ss['distinct_runes']==28
    assert section8['tokens']==15 and section8['distinct_runes']==13
    sections=[(i,tuples(s)) for i,s in enumerate(source.split('$')) if len(vals(s))>100]
    target=sum([ws for i,ws in sections],[]);reference=sum([tuples(s) for s in GROUPS.values()],[])
    assert len(target)==2894 and sum(map(len,target))==12947
    assert len(reference)==483 and sum(map(len,reference))==1963
    comparison=dict(target=describe(target),reference=describe(reference),pooled=matched(reference,target,3301042),
        per_section=[dict(section=i,inventory=describe(ws),comparison=matched(reference,ws,3301142+i)) for i,ws in sections])
    b=bound(arrays(target),keep=True);per=[dict(section=i,**bound(arrays(ws))) for i,ws in sections]
    independent=np.zeros(29,dtype=np.int64)
    for (i,ws),row in zip(sections,per):
        w=row['witness'];independent+=hist(decode(arrays(ws),w['A'],w['B'],w['C'],-w['target_bin'],w['initial']))
    assert independent[0]==sum(x['maximum_single_rune_count'] for x in per)
    # Known recovery/invariant controls from Session 41, with corrected reference grouping.
    controls=verify(reference)
    controls['word_reset_recovery']=recovery_controls()
    # Lexical behavior and offset invariance fixtures.
    fixture=R[0]+"'"+R[1]+'/%'+R[2]+'-'+R[3]
    got=tokenize_words(fixture);assert [w['values'] for w in got]==[[0,1,2],[3]]
    assert got[1]['page']==1 and got[1]['page_rune_offset']==1 and got[1]['rune_offset']==3
    assert [w['values'] for w in tokenize_words("'"+R[0]+"'-"+R[1])]==[[0],[1]]
    assert [[w['values'] for w in tokenize_words(s)] for s in GROUPS.values()]==[[w['values'] for w in tokenize(s)] for s in GROUPS.values()]
    result=dict(scope='Targeted visual audit of all 16 historical one-rune tokens in section 8, plus the other two apostrophe suffixes; contextual wrap/paragraph crops. Remaining 68 old singleton locations not newly image-audited.',
        source_transcription_unchanged=True,source_sha256=sha(ROOT/'liber-primus__transcription--master.txt'),
        script_sha256=sha(Path(__file__)),lexical_helper_sha256=sha(ROOT/'lexical_words.py'),
        observations_sha256=sha(ROOT/'token_boundary_observations.json'),visual_observations=spec['observations'],image_evidence=evidence,
        apostrophe_joins=changes,full_corpus_words_before=len(old),full_corpus_words_after=len(new),
        single_rune_tokens_after=ss,section8_after=section8,
        section8_excluding_red_initial=dict(tokens=14,distinct_runes=12,note='Sensitivity check excluding the separated red initial on LP2 page 54; not a finding that it is a null.'),
        session41_recalculation=comparison,
        session40_recalculation=dict(pooled=b,per_section=per,independent_sections_maximum_count=int(independent.max()),
            independent_sections_maximum_frequency=float(independent.max()/independent.sum()),independent_witness_histogram=independent.tolist()),
        controls=controls,lexical_parser_fixtures=True,
        limits=['Visible punctuation distinguishes apostrophes from separating dots; joining apostrophe suffixes is the lexical convention adopted here, not a decoded grammatical interpretation.',
            'An apostrophe could still reset a cipher. The older punctuation-split experiments remain reproducible historical tests of that different segmentation.',
            'All 83 revised singletons are transcription-derived; only the scoped 15 surviving section-8 singletons were visually audited here.',
            'Source archives may share ancestry; hashes verify retained copies, not Cicada authorship or cryptographic signatures.',
            'No all-corpus image transcription, null identification or new plaintext. Earlier experiments beyond Sessions 40 and 41 have not all been rerun.'])
    assert result['source_sha256']=='b0b2e13c3cd35512ee5955a2d9a56c22d7410e283d35dd9c90abbaa7b99279eb'
    (ROOT/'token_boundary_results.json').write_text(json.dumps(result,indent=2,ensure_ascii=False)+'\n')
    print(json.dumps(dict(full_words=len(new),primary_words=len(target),singletons=ss['tokens'],distinct_singletons=ss['distinct_runes'],
        section8_singletons=section8['tokens'],section8_distinct=section8['distinct_runes'],
        s41_target_pairs=comparison['target']['equal_word_pairs'],s41_expected=comparison['pooled']['exact_expected_equal_word_pairs'],
        s41_range=comparison['pooled']['central_95_percent_subsample_ranges'],
        s40_peak=b['maximum_single_rune_count'],s40_frequency=b['maximum_single_rune_frequency'],
        s40_independent_peak=int(independent.max()),s40_independent_frequency=float(independent.max()/independent.sum()),
        regions=len(evidence),source_images=len(images)),indent=2))
if __name__=='__main__':main()