"""Historical punctuation-split parser and boundary-aware rune model.
For current lexical words use lexical_words.tokenize_words (Session 42), which
joins intra-rune apostrophes. This parser stays unchanged for reproduction.
"""
from collections import Counter
from lp_core import *
B=29

def tokenize(source):
    words=[];chars=[];page=0;section=0;para=0;wordstart=None;runeoffset=0;pageoffset=0
    def flush():
        nonlocal chars,wordstart
        if chars:
            words.append({**wordstart,'values':chars,'runes':''.join(R[v] for v in chars)})
            chars=[];wordstart=None
    for ch in source:
        if ch in R:
            if not chars:wordstart={'page':page,'section':section,'paragraph':para,'rune_offset':runeoffset,'page_rune_offset':pageoffset}
            chars.append(R.index(ch));runeoffset+=1;pageoffset+=1
        elif ch=='%':page+=1;pageoffset=0
        elif ch in '/\n\r\t ':pass
        else:
            flush()
            if ch=='$':section+=1;para+=1
            elif ch in '&§':para+=1
    flush()
    for i,w in enumerate(words):w['word_index']=i
    assert sum(len(w['values']) for w in words)==len(vals(source))
    return words

def replace_values(source,values):
    it=iter(values)
    return ''.join(R[int(next(it))] if ch in R else ch for ch in source)

def known_groups():
    groups={}
    for name,inds in [('warning',[0]),('wisdom',[3]),('koan',[4,5,6,7]),('divinity',[8,9,10,11]),('instruction',[14])]:
        sources=[]
        for i in inds:
            v=vals(pages[i]);v=(28-v)%29 if i==0 else (2-v)%29 if 4<=i<=7 else v
            sources.append(replace_values(pages[i],v))
        groups[name]='%'.join(sources)
    return groups

class Model:
    def __init__(self,sources):
        uni=np.ones(30)*.5;bc=np.zeros((30,30));tc=np.zeros((30,30,30))
        old_bi=np.full((29,29),.2);old_tri=np.full((29,29,29),.2)
        for source in sources:
            v=vals(source)
            for a,b in zip(v,v[1:]):old_bi[a,b]+=1
            for a,b,c in zip(v,v[1:],v[2:]):old_tri[a,b,c]+=1
            for w in tokenize(source):
                v=[B,B]+w['values']+[B]
                for a,b,c in zip(v,v[1:],v[2:]):uni[c]+=1;bc[b,c]+=1;tc[a,b,c]+=1
        up=uni/uni.sum();bp=(bc+5*up)/(bc.sum(1,keepdims=True)+5)
        tp=(tc+5*bp[None,:,:])/(tc.sum(2,keepdims=True)+5)
        self.log=np.log(tp)
        self.old_bi=np.log(old_bi/old_bi.sum(1,keepdims=True));self.old_tri=np.log(old_tri/old_tri.sum(2,keepdims=True))
    def score(self,v,lengths,old=False):
        if old:return self.old_bi[v[:,:-1],v[:,1:]].mean(1)+self.old_tri[v[:,:-2],v[:,1:-1],v[:,2:]].mean(1)
        lengths=np.array(lengths);start=np.r_[0,np.cumsum(lengths)[:-1]];ends=np.cumsum(lengths)-1
        assert int(lengths.sum())==v.shape[1]
        pos=np.arange(v.shape[1])-np.repeat(start,lengths)
        a=np.full_like(v,B);b=np.full_like(v,B)
        b[:,1:]=v[:,:-1];b[:,pos<1]=B
        a[:,2:]=v[:,:-2];a[:,pos<2]=B
        s=self.log[a,b,v].sum(1)
        prev=v[:,np.maximum(ends-1,0)].copy();prev[:,lengths==1]=B
        s+=self.log[prev,v[:,ends],B].sum(1)
        return s/(v.shape[1]+len(lengths))

GROUPS=known_groups()
FULL_MODEL=Model(list(GROUPS.values()))