"""Session72: bounded placement comparison, with no cipher fitting.
Run beside retained inputs with Python and NumPy: python3 -B singleton_role_audit.py
"""
from pathlib import Path
from collections import Counter
import hashlib, json, re
import numpy as np
from lp_core import R, LET, lp
from lexical_words import tokenize_words, join_intrarune_apostrophes
from word_core import GROUPS

ROOT = Path(__file__).resolve().parent
FEATURES = ['red_span', 'section_edge', 'paragraph_edge', 'page_edge',
            'line_edge', 'literal_stop_adjacent', 'nonrunic_data_adjacent']

def sha(path):
    return hashlib.sha256(path.read_bytes()).hexdigest()

def describe(source, spans=()):
    words = tokenize_words(source)
    gaps = ['']
    for ch in join_intrarune_apostrophes(source):
        if ch in R:
            gaps.append('')
        else:
            gaps[-1] += ch
    assert len(gaps)-1 == sum(len(w['values']) for w in words)
    rows = []
    for i, w in enumerate(words):
        a = w['rune_offset']; b = a + len(w['values'])
        before, after = gaps[a], gaps[b]
        edges = {}
        for key in ['section', 'paragraph']:
            edges[key] = (i == 0 or words[i-1][key] != w[key] or
                          i+1 == len(words) or words[i+1][key] != w[key])
        flags = [any(a < y and b > x for x, y in spans),
                 edges['section'], edges['paragraph'],
                 a == 0 or b == len(gaps)-1 or '%' in before+after,
                 a == 0 or b == len(gaps)-1 or any(ch in before+after for ch in '/%'),
                 any(ch in before+after for ch in '.,;'),
                 bool(re.search('[A-Za-z0-9]', before+after))]
        rows.append({**w, 'length': len(w['values']), 'before_gap': before,
                     'after_gap': after, 'features': dict(zip(FEATURES, map(bool, flags)))})
    return rows

def matrix(rows):
    return np.array([[r['features'][f] for f in FEATURES] for r in rows], dtype=int)

def comparison(rows, selected, draws, seed):
    x = matrix(rows); selected = np.asarray(selected, dtype=bool)
    assert len(selected) == len(rows)
    blocks = []
    mean = np.zeros(len(FEATURES)); variance = mean.copy()
    for section in sorted({r['section'] for r in rows}):
        ix = np.array([i for i,r in enumerate(rows) if r['section'] == section])
        k = int(selected[ix].sum()); n = len(ix); p = x[ix].mean(axis=0)
        mean += k*p
        if n > 1:
            variance += k*p*(1-p)*(n-k)/(n-1)
        blocks.append((ix, k))
    active = variance > 1e-12
    def statistic(counts):
        z = np.zeros_like(counts, dtype=float)
        z[...,active] = (counts[...,active]-mean[active])/np.sqrt(variance[active])
        return np.maximum(0, z[...,active].max(axis=-1)) if active.any() else np.zeros(counts.shape[:-1])
    observed = x[selected].sum(axis=0)
    assert observed.tolist() == [sum(r['features'][f] for r,y in zip(rows,selected) if y) for f in FEATURES]
    rng = np.random.default_rng(seed)
    null = np.zeros((draws, len(FEATURES)), dtype=int)
    for ix,k in blocks:
        if k:
            for j in range(draws):
                null[j] += x[rng.choice(ix,k,replace=False)].sum(axis=0)
    stats = statistic(null); actual = float(statistic(observed))
    ge = int(np.count_nonzero(stats >= actual-1e-12))
    return {'word_pool': len(rows), 'selected_tokens': int(selected.sum()),
            'section_counts': [{'section':rows[int(ix[0])]['section'], 'words':len(ix), 'selected':k} for ix,k in blocks],
            'feature_counts': {f:{'observed':int(observed[j]), 'expected':float(mean[j]),
                'variance':float(variance[j]), 'z_excess':float((observed[j]-mean[j])/np.sqrt(variance[j])) if active[j] else None,
                'pool_count':int(x[:,j].sum())} for j,f in enumerate(FEATURES)},
            'maximum_positive_z':actual, 'reference_at_least_observed':ge,
            'max_corrected_plus_one_p':(ge+1)/(draws+1),
            'draws':draws, 'seed':seed,
            'reference_max_quantiles':{str(q):float(np.quantile(stats,q)) for q in [.5,.95,.99]},
            'monte_carlo_mean_counts':null.mean(axis=0).tolist()}

def main():
    plan=json.loads((ROOT/'singleton_role_plan.json').read_text())
    assert plan['features'] == FEATURES
    assert sha(ROOT/'liber-primus__transcription--master.txt') == 'b0b2e13c3cd35512ee5955a2d9a56c22d7410e283d35dd9c90abbaa7b99279eb'
    headings=json.loads((ROOT/'heading_change_results.json').read_text())
    spans=[h['global_rune_span'] for h in headings['visual_openers']]
    assert len(spans)==13
    rows=describe('%'.join(lp[:56]),spans)
    singles=[r for r in rows if r['length']==1]
    assert len(rows)==2896 and len(singles)==83
    assert len({r['values'][0] for r in singles})==28
    assert sum(r['length'] for r in rows)==12956
    assert {(r['page'],r['page_rune_offset']) for r in singles if r['features']['red_span']} == {(6,75),(54,0)}
    variants={}
    for j,(name,short,body) in enumerate([('primary_all_words',False,False),('short_words',True,False),
                 ('body_all_words',False,True),('body_short_words',True,True)]):
        pool=[r for r in rows if (not short or r['length']<=3) and (not body or not r['features']['red_span'])]
        variants[name]=comparison(pool,[r['length']==1 for r in pool],plan['draws'],plan['seed']+j)
    # Artificial signal planted at word-aligned line edges; preserve section counts.
    rng=np.random.default_rng(plan['seed']+10); planted=np.zeros(len(rows),dtype=bool)
    for section in sorted({r['section'] for r in rows}):
        ix=np.array([i for i,r in enumerate(rows) if r['section']==section])
        k=sum(rows[int(i)]['length']==1 for i in ix)
        order=sorted(rng.permutation(ix).tolist(),key=lambda i:rows[i]['features']['line_edge'],reverse=True)
        planted[order[:k]]=True
    positive=comparison(rows,planted,plan['draws'],plan['seed']+11)
    assert positive['max_corrected_plus_one_p']<=.001
    # Fixtures: wraps inside words are not edges; a page seam does not split a word;
    # apostrophes are joined; data is retained in the gaps.
    fixture=describe('ᚠ/ᚢ-ᚦ/%-ᚩ’ᚱ-123-ᛁ')
    assert [r['length'] for r in fixture]==[2,1,2,1]
    assert fixture[1]['features']['page_edge'] and fixture[1]['features']['line_edge']
    assert fixture[0]['features']['page_edge']  # Document start is a page edge.
    wrap=describe('ᚠ-ᚢ/ᚦ-ᚩ')
    assert not wrap[1]['features']['line_edge']
    seam=describe('ᚠ-ᚢ/%ᚦ-ᚩ')
    assert [r['length'] for r in seam]==[1,2,1] and not seam[1]['features']['page_edge']
    assert fixture[2]['features']['nonrunic_data_adjacent'] and fixture[3]['features']['nonrunic_data_adjacent']
    known=[]
    for name,source in GROUPS.items():
        rr=describe(source); ss=[r for r in rr if r['length']==1]
        known.append({'group':name,'words':len(rr),'runes':sum(r['length'] for r in rr),
            'singleton_count':len(ss),'singleton_labels':dict(Counter(LET[r['values'][0]] for r in ss)),
            'singleton_feature_counts':{f:sum(r['features'][f] for r in ss) for f in FEATURES if f!='red_span'},
            'all_word_feature_counts':{f:sum(r['features'][f] for r in rr) for f in FEATURES if f!='red_span'},
            'red_span':'Not evaluated in the solved controls',
            'singleton_rows':[{**r,'plaintext_label':LET[r['values'][0]]} for r in ss]})
    # Attach earlier source-inspection evidence without making new visual judgments.
    audited={}
    for name in ['singleton43_observations.json','token_boundary_observations.json']:
        for r in json.loads((ROOT/name).read_text())['observations']:
            if r.get('status') in ['singleton_confirmed','red_initial_singleton']:
                audited[(r['page'],r.get('offset',r.get('page_rune_offset')))]= {'file':name,'record':r.get('id',r.get('region'))}
    inventory=[]
    for r in singles:
        k=(r['page'],r['page_rune_offset']); assert k in audited,k
        inventory.append({**r,'cipher_label':LET[r['values'][0]],'source_audit':audited[k]})
    body=[r for r in singles if not r['features']['red_span']]
    noflags=[r for r in body if not any(r['features'].values())]
    inputs=['liber-primus__transcription--master.txt','heading_change_results.json','boundary_model.json',
            'singleton43_observations.json','token_boundary_observations.json','lp_core.py',
            'lexical_words.py','word_core.py','singleton_role_plan.json']
    out={'session':72,'input_hashes':{n:sha(ROOT/n) for n in inputs},'implementation_sha256':sha(Path(__file__)),
         'scope':{'runes':12956,'words':len(rows),'singletons':83,'singleton_types':28,'red_spans':len(spans),
                  'body_singletons':len(body),'body_singleton_types':len({r['values'][0] for r in body}),
                  'body_singletons_without_any_listed_feature':len(noflags),
                  'types_without_any_listed_feature':len({r['values'][0] for r in noflags})},
         'comparisons':variants,'positive_control':positive,'mapping_fixtures':'passed',
         'solved_controls':known,'singleton_inventory':inventory,
         'all_word_positions':[{k:r[k] for k in ['word_index','section','page','page_rune_offset','rune_offset','length','features']} for r in rows],
         'limits':plan['limits'],'no_new_plaintext':True}
    (ROOT/'singleton_role_results.json').write_text(json.dumps(out,ensure_ascii=False,indent=2)+'\n')
    print(json.dumps({'scope':out['scope'],'comparisons':{k:{'p':v['max_corrected_plus_one_p'],'features':v['feature_counts']} for k,v in variants.items()},
        'positive_p':positive['max_corrected_plus_one_p'],'known':[{k:r[k] for k in ['group','words','singleton_count','singleton_labels','singleton_feature_counts']} for r in known]},indent=2))

if __name__=='__main__':
    main()