"""Historical rune-name keys and language-independent corpus diagnostics. This is a bounded experiment, not a linguistic identification of ciphertext. """ import json,re,unicodedata,hashlib from collections import Counter from word_core import * NAMES='FEOH UR THORN OS RAD CEN GYFU WYNN HAEGL NYD IS GER EOH PEORTH EOLHX SIGEL TIR BEORC EH MAN LAGU ING ETHEL DAEG AC AESC YR IOR EAR'.split() def norm(s): s=s.lower().replace('ä','æ').replace('æ','ae').replace('œ','oe').replace('þ','th').replace('ð','th').replace('ƿ','w').replace('ȝ','g') return ''.join(c for c in unicodedata.normalize('NFD',s) if not unicodedata.combining(c)) def encode(w,merged=True): w=norm(w).replace('v','u').replace('c','k').replace('q','k').replace('z','s') table={x.lower():i for i,x in enumerate(LET)};table['ng']=21 # NG has historical support; ING is not greedily collapsed here. table.pop('ing');ks=sorted(table,key=len,reverse=True) if merged else [x for x in table if len(x)==1] out=[] while w: for x in ks: if w.startswith(x):out.append(table[x]);w=w[len(x):];break else:raise ValueError(w) return out def sources(): raw=(ROOT/'beowulf_9700.txt').read_text();body=raw.split('\nBEÓWULF.\n',1)[1].split('\nAPPENDIX\n',1)[0] lines=[s for s in body.splitlines() if re.match(r'^(?:\d+ +| {5})',s) and not s.strip().isupper()] text=' '.join(lines).replace('-','');oe=re.findall(r'[^\W\d_]+',text) latin=[];skip_to=0 for s in (ROOT/'latin.conllu').read_text().splitlines(): if not s:skip_to=0;continue if s.startswith('#'):continue row=s.split('\t');idx=row[0] if '-' in idx:skip_to=int(idx.split('-')[1]);word=row[1] elif idx.isdigit() and int(idx)>skip_to:word=row[1] else:continue if word.isalpha():latin.append(word) return {'old_english_beowulf':oe,'latin_perseus':latin} def make_source(words):return '-'.join(''.join(R[x] for x in w) for w in words) def ic(v): c=np.bincount(v,minlength=29);n=len(v) return float(np.sum(c*(c-1))/(n*(n-1))) def diag(words): v=np.array(sum(words,[]));cnt=Counter(tuple(w) for w in words if len(w)>=4) return dict(runes=len(v),words=len(words),index_of_coincidence=ic(v),distinct_runes=len(set(v)),repeated_word_pairs_ge4=sum(n*(n-1)//2 for n in cnt.values())) def bank(): entries=[(n,encode(n)) for n in NAMES] entries += [('ALL_NAMES',sum([encode(n) for n in NAMES],[])),('NAME_LETTER_LENGTHS',[len(n) for n in NAMES]),('NAME_RUNE_LENGTHS',[len(encode(n)) for n in NAMES])] rows=[];meta=[] for name,key in entries: for reverse in [False,True]: k=np.array(key[::-1] if reverse else key) for phase in range(len(k)): rows.append((k,phase));meta.append(dict(name=name,reverse=reverse,phase=phase)) return rows,meta def search(source,models): words=tokenize(source);v=vals(source);lengths=[len(w['values']) for w in words];rows,meta=bank();n=len(v);out={};count=0 for reset in ['continuous','word']: ix=np.arange(n) if reset=='continuous' else np.concatenate([np.arange(k) for k in lengths]) keys=np.array([key[(ix+phase)%len(key)] for key,phase in rows]) for cs,ks in [(1,1),(1,-1),(-1,1),(-1,-1)]: dec=(cs*v+ks*keys)%29;count+=len(dec) for lang,model in models.items(): scores=model.score(dec,lengths);top=np.argsort(scores)[-3:] candidates=[] for j in top: candidates.append(dict(**meta[j],reset=reset,cipher_sign=cs,key_sign=ks,score=float(scores[j]),text=render(source,dec[j]),values=dec[j].tolist())) out[lang]=sorted(out.get(lang,[])+candidates,key=lambda x:x['score'],reverse=True)[:3] return out,count def main(): raw=sources();models={'modern_english':FULL_MODEL};encoded={};report={'rune_names':dict(zip(R,NAMES)),'sources':{},'controls':{},'sections':[],'benchmark':{},'mapping':'Deterministic experimental GP-style transliteration. Not a claim about authentic historical spelling; accents removed, TH/AE/etc optional merges; C,Q=>K,V=>U,Z=>S.','seed':3301008} for lang,words in raw.items(): encoded[lang]=[encode(w) for w in words];cut=int(len(words)*.8) models[lang]=Model([make_source(encoded[lang][:cut])]);report['sources'][lang]=dict(diag(encoded[lang]),training_words=cut,heldout_words=len(words)-cut) # Exact recovery control: withheld continuous 500-rune-or-longer text, key FEOH. for lang in ['old_english_beowulf','latin_perseus']: words=encoded[lang];cut=int(len(words)*.8);plain=make_source(words[cut:cut+140]);v=vals(plain);k=np.array(encode('FEOH'));cipher=(v+k[np.arange(len(v))%len(k)])%29 out,count=search(replace_values(plain,cipher),{lang:models[lang]});best=out[lang][0] exact=best['values']==v.tolist();assert exact report['controls'][lang]=dict(exact_recovery=exact,runes=len(v),best={k:v for k,v in best.items() if k not in ['text','values']}) sections=[(i,s) for i,s in enumerate('%'.join(lp[:56]).split('$')) if len(vals(s))>100] for idx,s in sections: out,count=search(s,models);summary={lang:[{k:v for k,v in row.items() if k!='values'} for row in rows] for lang,rows in out.items()} report['sections'].append(dict(section=idx,diagnostics=diag([w['values'] for w in tokenize(s)]),candidate_settings=count,top=summary)) rng=np.random.default_rng(3301008) # Uniform starting positions in each encoded reference stream; exact matched rune count. for lang,words in raw.items(): report['benchmark'][lang]={} for merged in [False,True]: v=np.array(sum([encode(w,merged) for w in words],[]));bysection=[] for idx,s in sections: n=len(vals(s));starts=rng.integers(0,len(v)-n+1,499);scores=[ic(v[st:st+n]) for st in starts];observed=ic(vals(s)) bysection.append(dict(section=idx,runes=n,cipher_ic=observed,reference_min=min(scores),reference_median=float(np.median(scores)),reference_max=max(scores),reference_at_or_below_cipher=sum(x<=observed for x in scores))) report['benchmark'][lang]['merged' if merged else 'unmerged']=bysection (ROOT/'historical_results.json').write_text(json.dumps(report,indent=2)) print(json.dumps({k:report[k] for k in ['sources','controls','benchmark']},indent=2)) print('settings',sum(s['candidate_settings'] for s in report['sections'])) if __name__=='__main__':main()