"""Session 28. Compare source passages with solved LP; Python standard library.

Usage: python3 -B religious_source_audit.py CACHE_DIR
Inputs: web*.txt / *.web.txt web-extraction captures and dhp01.html..dhp26.html.
Full source text stays in CACHE_DIR. Results contain locators and hashes, not books.
Web captures must retain L<number>: markers and source URLs. Website boilerplate,
duplicate renderings, commentary and footnotes are excluded by recorded ranges.
"""
import hashlib, json, re, sys
from pathlib import Path
from html.parser import HTMLParser

BASE=Path(__file__).resolve().parent
def sha(s): return hashlib.sha256(s.encode()).hexdigest()
def tokens(s): return re.findall(r'[A-Z]+',s.upper())
def windows(ts,n):
    for i in range(len(ts)-n+1): yield i,tuple(ts[i:i+n])

def web_body(cache,url,lo,hi):
    lines={}
    for f in sorted(cache.glob('*web*.txt')):
        for block in f.read_text().split('--------------------------------------------------------------------------------'):
            if url not in block[:400]: continue
            for k,t in re.findall(r'(?:^|\s)L(\d+): ?(.*?)(?=\sL\d+:|\Z)',block,re.S):
                t=re.sub(r'cite.*?','',t).strip()
                k=int(k)
                if k in lines: assert re.sub(r'\s+',' ',lines[k])==re.sub(r'\s+',' ',t), (url,k)
                lines[k]=t
    assert all(i in lines for i in range(lo,hi+1)),url
    text='\n'.join(lines[i] for i in range(lo,hi+1))
    text=re.sub(r'^p\. \d+\s*$','',text,flags=re.M).replace('[paragraph continues]','')
    return text

def numbered(text,pattern,expected,prefix):
    ms=list(re.finditer(pattern,text))
    assert [int(m[1]) for m in ms]==list(range(1,expected+1)),prefix
    return [{'id':prefix+str(m[1]),'text':text[m.end():ms[i+1].start() if i+1<len(ms) else len(text)].strip()} for i,m in enumerate(ms)]

class Plain(HTMLParser):
    def __init__(self):super().__init__();self.parts=[]
    def handle_data(self,t):self.parts.append(t)
def plain(t):
    p=Plain();p.feed(t);return ''.join(p.parts)

def load(cache):
    th=web_body(cache,'chr/thomas.htm',149,234).split('III: Fragments')[0].rstrip('- \n')
    th=numbered(th,r'(?<!\d)(\d+)\)',114,'Thomas:')
    he=numbered(web_body(cache,'gno/th2/th202.htm',200,427),r'(?m)^(\d+)\.\s',32,'Hermeticum-I:')
    cr=web_body(cache,'oto/engccxx.htm',140,571)
    parts=re.split(r'### Chapter (?:I|II|III)\b',cr)[1:]
    assert len(parts)==3
    cw=[]
    for roman,part,n in zip(['I','II','III'],parts,[66,79,75]):
        cw+=numbered(part,r'(?m)^(\d+)\.\s',n,'AL:'+roman+':')
    dh=[];coverage=[]
    for ch in range(1,27):
        s=(cache/f'dhp{ch:02}.html').read_text()
        for p in re.findall(r'<p\b[^>]*>.*?</p>',s,re.S|re.I):
            txt=plain(p).strip(); label=re.match(r'(\d+)(?:[-–](\d+))?\.?\s',txt)
            if not label:continue
            start=int(label[1]);end=int(label[2] or start)
            coverage.extend(range(start,end+1))
            txt=re.sub(r'\[\d+\]','',txt[label.end():]).strip()
            dh.append({'id':f'Dhp:{ch}:{start}'+(f'-{end}' if end!=start else ''),'text':txt})
    assert coverage==list(range(1,424)),coverage
    return {'Thomas-Lambdin':[r for r in th if r['id']!='Thomas:70'],
            'Thomas-70-Pagels':[r for r in th if r['id']=='Thomas:70'],
            'Dhammapada-Buddharakkhita':dh,'Hermeticum-I-Mead':he,'Book-of-the-Law':cw}

def compare(lp,records):
    index={n:{} for n in range(3,9)}
    for r in records:
        ts=tokens(r['text'])
        for n in index:
            for j,w in windows(ts,n):index[n].setdefault(w,[]).append([r['id'],j])
    hits=[]
    for r in lp:
        ts=tokens(r['text'])
        for n in index:
            for j,w in windows(ts,n):
                if w in index[n]: hits.append({'n':n,'lp':r['id'],'lp_token':j,'phrase_sha256':sha(' '.join(w)),'sources':index[n][w]})
    return hits

def main():
    cache=Path(sys.argv[1]); sources=load(cache)
    raw=(BASE/'liber-primus__translation.txt').read_text()
    lp=[{'id':m[1],'text':m[2]} for m in re.finditer(r'^(\d+(?:\.\d+)+)\s+(.+)$',raw,re.M)]
    assert len(lp)==94
    # An artificial copied four-word passage must be found at its exact offsets.
    control=compare([{'id':'control-LP','text':'a amber quiet lantern silver b'}],[{'id':'control-source','text':'c amber quiet lantern silver d'}])
    assert any(h['n']==4 and h['lp_token']==1 and h['sources']==[['control-source',1]] for h in control)
    out={'method':'Exact ASCII alphabetic tokens, uppercase, n=3..8, no passage-boundary crossing; punctuation removed; no spelling correction or stemming. Offsets zero-based.',
         'scope':'Thomas 114 sayings with saying 70 separate; all Dhammapada verses 1..423 (combined verses retained); Corpus Hermeticum I only; AL I..III only, excluding later Comment.',
         'lp_entries':len(lp),'lp_tokens':sum(len(tokens(x['text'])) for x in lp),'lp_file_sha256':sha(raw),'control_pass':True,'sources':{}}
    for name,rs in sources.items():
        hits=compare(lp,rs)
        out['sources'][name]={'passages':len(rs),'tokens':sum(len(tokens(r['text'])) for r in rs),
          'normalized_corpus_sha256':sha('\n'.join(r['id']+'\t'+' '.join(tokens(r['text'])) for r in rs)),
          'passage_fingerprints':[{'id':r['id'],'tokens':len(tokens(r['text'])),'sha256':sha(' '.join(tokens(r['text'])))} for r in rs],
          'matches_by_length':{str(n):len({h['phrase_sha256'] for h in hits if h['n']==n}) for n in range(3,9)},'matches':hits}
    (BASE/'religious_source_results.json').write_text(json.dumps(out,indent=2)+'\n')
    (cache/'extracted_passages.json').write_text(json.dumps(sources,ensure_ascii=False,indent=2))
    for name,r in out['sources'].items():print(name,r['passages'],r['tokens'],r['matches_by_length'])
    print('LP',len(lp),out['lp_tokens'])
if __name__=='__main__':main()