"""Session 65: frozen 2012-2014 signed-message extension of Session 64.

Run berkeley_fetch_audit.py first. Requires berkeley_compare_audit.py beside this
file, Python standard library, GPG, and internet for the two pinned new files.
No code or links contained in retrieved messages are executed or followed.
"""
import base64, collections, concurrent.futures, hashlib, json, re, subprocess, tempfile
from pathlib import Path
from urllib.request import urlopen
import berkeley_compare_audit as prior

ROOT=Path(__file__).resolve().parent
COMMIT='10733feb487bb8beb9966d534727ee5897aa54fc'
FP=prior.FP
YEAR_BLOBS={2012:'0d71498db2b85586687af4366559a50bd4116858',2013:'b2709790becb66fe8d8ca00606199bd935d9b410',2014:'22d81c320256dc0d6da030a17b26b6cacfadcea6'}
LP_SHA='adce48cfea746fd5add5071c85a34a998a546ca94c009bcd796cd3e699cc1c47'
def sha(s):return hashlib.sha256(s.encode()).hexdigest()
def blob_sha(b):return hashlib.sha1(b'blob '+str(len(b)).encode()+b'\0'+b).hexdigest()

def get_year(year):
    url=f'https://raw.githubusercontent.com/scream314/cicada3301/{COMMIT}/{year}.md'
    p=ROOT/f'{year}.md'
    if not p.exists():p.write_bytes(urlopen(url,timeout=45).read())
    assert blob_sha(p.read_bytes())==YEAR_BLOBS[year],p
    return year,url,p.read_text()

def get_assets():
    p=ROOT/'repo-tree.json'
    if not p.exists():p.write_bytes(urlopen(f'https://api.github.com/repos/scream314/cicada3301/git/trees/{COMMIT}?recursive=1',timeout=45).read())
    tree=json.loads(p.read_text());assert not tree.get('truncated')
    candidates=[x for x in tree['tree'] if x['type']=='blob' and x['path'].startswith(('assets/2013/','assets/2014/')) and x['path'].endswith(('.asc','.outguess','.html')) and x.get('size',0)<262144]
    assert len(candidates)==23
    directory=ROOT/'assets';directory.mkdir(exist_ok=True)
    def get(x):
        url=f'https://raw.githubusercontent.com/scream314/cicada3301/{COMMIT}/'+x['path']
        p=directory/(sha(x['path'])+'.txt')
        if not p.exists():p.write_bytes(urlopen(url,timeout=45).read())
        data=p.read_bytes();assert blob_sha(data)==x['sha'],x['path']
        return {'path':x['path'],'url':url,'local_file':p.name,'sha256':hashlib.sha256(data).hexdigest(),'git_blob_sha':x['sha'],'signed_starts':data.count(b'-----BEGIN PGP SIGNED MESSAGE-----')}
    with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:return list(pool.map(get,candidates))

def verify_source(year,text,base,tmp,origin=None):
    starts=list(re.finditer(re.escape('-----BEGIN PGP SIGNED MESSAGE-----'),text))
    records,accepted=[],[]
    for i,m in enumerate(starts,1):
        line_start=text.rfind('\n',0,m.start())+1
        wrapper=text[line_start:m.start()]
        indent=re.match(r'[ \t]*',wrapper).group()
        # Remove exactly one Markdown indentation unit; preserve internal tabs.
        indent='\t' if indent.startswith('\t') else ('    ' if indent.startswith('    ') else '')
        tail=text[m.start():starts[i].start() if i<len(starts) else len(text)]
        end=tail.find('-----END PGP SIGNATURE-----')
        rec={'id':f'{origin}#{i}' if origin else f'{year}-{i}','year':year,'source_line':text.count('\n',0,m.start())+1,
             'indent_removed':repr(indent),'opening_wrapper':wrapper.strip()}
        if end<0:
            rec['result']='incomplete block';records.append(rec);continue
        block=tail[:end+len('-----END PGP SIGNATURE-----')]
        if indent:block=re.sub(r'^'+re.escape(indent),'',block,flags=re.M)
        block+='\n'
        p=Path(tmp)/'message.asc';p.write_text(block)
        run=subprocess.run(base+['--allow-weak-digest-algos','--status-fd','1','--verify',str(p)],text=True,capture_output=True)
        status=[l for l in run.stdout.splitlines() if l.startswith('[GNUPG:]')]
        valid=any(l.startswith('[GNUPG:] VALIDSIG '+FP+' ') for l in status)
        rec.update(sha256=sha(block),result='valid' if valid else 'not verified',gpg_status=status)
        sigpart=block.split('-----BEGIN PGP SIGNATURE-----')[-1].split('-----END PGP SIGNATURE-----')[0]
        siglines=[l.strip() for l in sigpart.splitlines() if re.fullmatch(r'[A-Za-z0-9+/]+={0,2}',l.strip())]
        if siglines:
            try:rec['signature_packet_sha256']=hashlib.sha256(base64.b64decode(''.join(siglines),validate=True)).hexdigest()
            except ValueError:pass
        if valid:
            body=re.split(r'\n[ \t]*\n',block,maxsplit=1)[1].split('-----BEGIN PGP SIGNATURE-----')[0]
            body=re.sub(r'^- ','',body,flags=re.M)
            rec['body_sha256']=sha(body)
            rec['prior_method_word_tokens']=sum(map(len,prior.paragraphs(body)))
            accepted.append({'id':rec['id'],'year':year,'body':body,'body_sha256':sha(body)})
        records.append(rec)
    return records,accepted

def prose_view(body):
    # Sensitivity analysis only. Exclude recognizable index/payload lines before
    # the unchanged Session 64 tokenizer, and preserve paragraph breaks.
    lines=[]
    for line in body.splitlines():
        s=line.strip()
        drop=(re.fullmatch(r'(?:[IVXLCDM]+|\d+)(?::\d+)+',s) or
              (re.fullmatch(r'[0-9a-fA-F]+',s) and any(c.isdigit() for c in s)) or
              re.fullmatch(r'[en]\s*=\s*\d+\\?',s) or
              re.fullmatch(r'[A-Za-z]',s))
        lines.append('' if drop else re.sub(r'\b[a-z2-7]{16,56}\.onion\b','',line))
    return '\n'.join(lines)

def main():
    corpus=json.loads((ROOT/'corpus-private.json').read_text())
    baseline=json.loads((ROOT/'berkeley_comparison_results.json').read_text())
    for label in ['target','controls']:
        assert {x['url']:x['raw_sha256'] for x in corpus[label]}=={x['url']:x['raw_sha256'] for x in baseline[label+'_manifest']}
    assert sha((ROOT/'cicada-key.asc').read_text())==baseline['key_sha256']
    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
        sources=list(pool.map(get_year,[2012,2013,2014]))
    assets=get_assets()
    assert sha(sources[0][2])==baseline['archive_sha256']
    records,accepted=[],[]
    with tempfile.TemporaryDirectory(prefix='cicada65-gpg-') as tmp:
        base=['gpg','--homedir',tmp,'--batch','--no-autostart','--no-auto-key-retrieve']
        subprocess.run(base+['--import',str(ROOT/'cicada-key.asc')],text=True,capture_output=True)
        keys=subprocess.run(base+['--with-colons','--fingerprint','--list-keys'],text=True,capture_output=True)
        assert f'fpr:::::::::{FP}:' in keys.stdout
        for year,url,text in sources:
            rs,docs=verify_source(year,text,base,tmp);records+=rs;accepted+=docs
        for asset in assets:
            year=int(asset['path'].split('/')[1])
            text=(ROOT/'assets'/asset['local_file']).read_text(errors='replace')
            rs,docs=verify_source(year,text,base,tmp,origin=asset['path'])
            records+=rs;accepted+=docs
    # A whitespace-normalized body identifies duplicate prose occurrences.
    unique=[];seen={};duplicates=[]
    for d in accepted:
        h=sha(' '.join(d['body'].split()))
        if h in seen:duplicates.append({'id':d['id'],'duplicate_of':seen[h]})
        else:seen[h]=d['id'];unique.append(d)
    old=[x for x in unique if x['year']==2012]
    assert len(old)==10
    assert sum(len(p) for x in old for p in prior.paragraphs(x['body']))==345
    for label in ['target','controls']:
        for mode,strip in [('quoted_lines_removed',True),('quoted_lines_retained',False)]:
            a=prior.compare(corpus[label],old,strip)
            expected=baseline[label+'_vs_3301'][mode]
            # IDs gain year prefixes; phrase sets and left-message provenance must agree.
            for n in a:
                assert [(x['phrase'],x['left']) for x in a[n]]==[(x['phrase'],x['left']) for x in expected[n]]
    replacements=[]
    for r in records:
        if r['result']=='not verified' and 'signature_packet_sha256' in r:
            hits=[a['id'] for a in records if a['result']=='valid' and a.get('signature_packet_sha256')==r['signature_packet_sha256']]
            if hits:replacements.append({'writeup_copy':r['id'],'verified_copies_with_same_signature':hits})
    results={'scope':'Pinned 2012-2014 write-up signed blocks and 23 saved 2013/2014 asset candidates; same 22 targets and 21 controls as Session 64.',
             'fingerprint':FP,'signed_blocks':records,'duplicate_bodies':duplicates,
             'sources':[{'year':y,'url':u,'sha256':sha(s)} for y,u,s in sources],
             'asset_manifest':assets,'bad_writeup_copies_recovered_from_assets':replacements,
             'target_count':len(corpus['target']),'control_count':len(corpus['controls']),
             'baseline_reproduction':'All 2012 phrase sets and left-message provenance match; ten blocks and 345 tokens match.',
             'cohorts':{}}
    for name,docs in [('2012',old),('2013_writeup',[d for d in unique if d['year']==2013 and not d['id'].startswith('assets/')]),('2014_writeup',[d for d in unique if d['year']==2014 and not d['id'].startswith('assets/')]),('combined',unique)]:
        c={'verified_unique_blocks':len(docs),'prior_method_word_tokens':sum(len(p) for d in docs for p in prior.paragraphs(d['body']))}
        for label in ['target','controls']:
            c[label]={mode:prior.compare(corpus[label],docs,strip) for mode,strip in [('quoted_lines_removed',True),('quoted_lines_retained',False)]}
        results['cohorts'][name]=c
    prose=[];prose_seen={};prose_duplicates=[];empty=[]
    for d in unique:
        body=prose_view(d['body']);parts=[p for p in prior.paragraphs(body) if p]
        h=sha(json.dumps(parts))
        if not parts:empty.append(d['id']);continue
        if h in prose_seen:prose_duplicates.append({'id':d['id'],'same_tokenized_prose_as':prose_seen[h]});continue
        prose_seen[h]=d['id'];prose.append({**d,'body':body})
    results['prose_sensitivity']={'nonempty_unique_tokenized_bodies':len(prose),'word_tokens':sum(len(p) for d in prose for p in prior.paragraphs(d['body'])),'duplicates':prose_duplicates,'empty_after_filter':empty}
    for label in ['target','controls']:
        results['prose_sensitivity'][label]={mode:prior.compare(corpus[label],prose,strip) for mode,strip in [('quoted_lines_removed',True),('quoted_lines_retained',False)]}
    # Keep the pre-existing solved-page transcription separate from signed bytes.
    lp=(ROOT/'liber-primus__translation.txt').read_text();assert sha(lp)==LP_SHA
    clauses=[{'id':m.group(1),'body':m.group(2)} for line in lp.splitlines() if (m:=re.match(r'^(\d+(?:\.\d+)+)\s+(.+)$',line))]
    results['solved_lp_auxiliary']={'sha256':LP_SHA,'clause_count':len(clauses),'word_tokens':sum(len(p) for d in clauses for p in prior.paragraphs(d['body'])),'status':'Existing saved translation; no new signature authentication or decryption performed on this derived text.'}
    for label in ['target','controls']:
        results['solved_lp_auxiliary'][label]={mode:prior.compare(corpus[label],clauses,strip) for mode,strip in [('quoted_lines_removed',True),('quoted_lines_retained',False)]}
    (ROOT/'berkeley_extended_results.json').write_text(json.dumps(results,indent=2))
    (ROOT/'extended-verified-private.json').write_text(json.dumps(unique,indent=2))
    print('Signatures:',[(r['id'],r['result'],r.get('prior_method_word_tokens')) for r in records])
    print('Duplicates:',duplicates)
    print('Recovered copies:',replacements)
    print('Prose sensitivity:',results['prose_sensitivity']['nonempty_unique_tokenized_bodies'],results['prose_sensitivity']['word_tokens'])
    print('LP auxiliary:',results['solved_lp_auxiliary']['clause_count'],results['solved_lp_auxiliary']['word_tokens'])
    for name,c in results['cohorts'].items():
        print(name,c['verified_unique_blocks'],c['prior_method_word_tokens'])
        for label in ['target','controls']:
            for mode,res in c[label].items():
                print(label,mode,{n:len(v) for n,v in res.items()})
                print('four-word hits',res['4'])

if __name__=='__main__':main()