"""Session 48: word-position implications of the fixed repeat-to-R hypothesis.
No decryption or language identification. Uses corrected lexical grouping.
"""
from pathlib import Path
from collections import Counter
import hashlib,json
import numpy as np
from word_core import R,LET,GROUPS,lp,vals
from lexical_words import tokenize_words
F=Path(__file__).resolve().parent

def scan(words):
 prev=None;sec=None;out=[];offset=0
 for w in words:
  if w['section']!=sec:prev=None;sec=w['section'];offset=0
  mask=[];positions=[]
  for j,v in enumerate(w['values']):
   mask.append('?' if prev is None else 'R' if v==prev else '_')
   if mask[-1]=='R':positions.append(j)
   prev=v
  out.append(dict(section=sec,page=w['page'],page_rune_offset=w['page_rune_offset'],section_offset=offset,word_length=len(mask),ciphertext_labels=[LET[v] for v in w['values']],mask=''.join(mask),repeat_positions=positions))
  offset+=len(mask)
 return out

def pattern(w,r):return ''.join('R' if v==r else '_' for v in w)

def coverage(lex,required):
 out=[]
 for r in range(29):
  matches=[]
  for length,mask in required:
   words=[w for w in lex if len(w)==length and pattern(w,r)==mask]
   matches.append(dict(length=length,mask=mask,examples=[''.join(LET[v] for v in w) for w in sorted(words)]))
  out.append(dict(rune=LET[r],patterns_supported=sum(bool(m['examples']) for m in matches),all_patterns_supported=all(m['examples'] for m in matches),matches=matches))
 return out

def main():
 words=[w for w in tokenize_words('%'.join(lp[:56])) if w['section'] in [0,1,2,4,5,6,7,8]]
 data=scan(words);events=[w for w in data if w['repeat_positions']];short=[w for w in events if w['word_length']<=3]
 assert sum(len(w['repeat_positions']) for w in data)==86
 assert not any(w['word_length']==1 for w in events)
 cnt=Counter((w['word_length'],w['mask']) for w in short);required=sorted(cnt)
 groups={k:tokenize_words(s) for k,s in GROUPS.items()};lex={tuple(w['values']) for ws in groups.values() for w in ws if len(w['values'])<=3}
 cov=coverage(lex,required)
 loo=[]
 for excluded in groups:
  reduced={tuple(w['values']) for k,ws in groups.items() if k!=excluded for w in ws if len(w['values'])<=3}
  loo.append(dict(excluded=excluded,compatible_letters=[x['rune'] for x in coverage(reduced,required) if x['all_patterns_supported']]))
 # Deliberately ordinary counterexamples outside the small solved-text lexicon.
 # These use only single-letter rune labels, avoiding digraph segmentation ambiguity.
 examples={'T':['TO','IT','TOP','ATE','NOT'],'S':['SO','IS','SAD','ASK','HIS'],'N':['NO','ON','NOW','AND','MAN']}
 counterexamples=[]
 for letter,ws in examples.items():
  r=LET.index(letter);vv=[tuple(LET.index(ch if ch!='C' else 'K') for ch in w) for w in ws]
  masks={(len(w),pattern(w,r)) for w in vv};assert set(required)<=masks
  counterexamples.append(dict(letter=letter,words=ws,all_five_patterns=True))
 reference=np.bincount(np.concatenate([vals(s)[1:] for s in GROUPS.values()]),minlength=29)
 # Behavioral control: additive feedback forces repetition exactly at plaintext R.
 controls=[]
 for letter in ['O','T','X']:
  rr=LET.index(letter);total=0
  for source in GROUPS.values():
   ww=tokenize_words(source);prev=7;encoded=[];expected=[];pos=0
   for w in ww:
    cv=[]
    for p in w['values']:
     c=(prev+p-rr)%29;cv.append(c);prev=c
     expected.append(pos>0 and p==rr);pos+=1
    encoded.append({**w,'section':0,'values':cv})
   got=scan(encoded);flat=[x=='R' for w in got for x in w['mask']]
   assert flat==expected;total+=sum(flat)
  assert total==reference[rr];controls.append(dict(letter=letter,repeat_positions_match_plaintext_exactly=True,count=total))
 section=[]
 for s in [0,1,2,4,5,6,7,8]:
  ww=[w for w in data if w['section']==s];section.append(dict(section=s,events=sum(len(w['repeat_positions']) for w in ww),short_words=sum(bool(w['repeat_positions']) and w['word_length']<=3 for w in ww),short_patterns=sorted({w['mask'] for w in ww if w['repeat_positions'] and w['word_length']<=3})))
 out=dict(hypothesis='At every scored position C_i=C_(i-1) iff P_i=one common fixed rune R; rune lengths and lexical boundaries preserved, no resets inside a section.',scope='Eight substantive sections. Section-first rune is unknown, not a non-R assertion. Tokens use corrected apostrophe joining. Conditional constraints from transcription, not a new audit of all source images.',source_sha256=hashlib.sha256((F/'liber-primus__transcription--master.txt').read_bytes()).hexdigest(),script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),events=86,eligible_positions=sum(w['word_length'] for w in data)-8,singleton_events=0,short_word_count=len(short),pattern_counts=[dict(length=n,mask=m,count=v) for (n,m),v in sorted(cnt.items())],short_word_locations=short,all_event_words=events,sections=section,reference_lexicon_sizes={n:sum(len(w)==n for w in lex) for n in [1,2,3]},reference_coverage=cov,leave_one_group_out=loo,ordinary_word_counterexamples=counterexamples,reference_rune_counts=dict(zip(LET,map(int,reference))),reference_scored=int(reference.sum()),controls=controls,limitations=['R is a placeholder, not the rune labelled R. Underscores require a different plaintext rune under the iff hypothesis.','Only one common R across sections is assumed for the pooled five-pattern constraint. Different section values weaken it.','A small reference lexicon cannot identify the letter or exclude ordinary English words outside that lexicon.','All patterns are necessary conditions, not a cipher solution; no word assignments or key were inferred.'])
 (F/'repeat_letter_results.json').write_text(json.dumps(out,ensure_ascii=False,indent=2)+'\n')
 print('patterns',out['pattern_counts'],'words',len(short),'events',out['events'])
 print('reference candidates',[r['rune'] for r in cov if r['all_patterns_supported']],'LOO',loo)
 print('freq O',reference[3]/reference.sum(),'observed',86/out['eligible_positions'],'control',controls)
if __name__=='__main__':main()