import collections, hashlib, json, re, subprocess, tempfile
from pathlib import Path

ROOT = Path(__file__).resolve().parent
FP = '6D854CD7933322A601C3286D181F01E57A35090F'

def sha(s): return hashlib.sha256(s.encode()).hexdigest()

def verify():
    source = (ROOT/'2012.md').read_text()
    # Undo Markdown's four-space code indentation, and nothing else.
    source = re.sub(r'^ {4}', '', source, flags=re.M)
    starts = list(re.finditer(r'^-----BEGIN PGP SIGNED MESSAGE-----', source, re.M))
    records, accepted = [], []
    with tempfile.TemporaryDirectory(prefix='cicada-gpg-') as tmp:
        base = ['gpg', '--homedir', tmp, '--batch', '--no-autostart', '--no-auto-key-retrieve']
        imp = subprocess.run(base+['--import', str(ROOT/'cicada-key.asc')], capture_output=True,text=True)
        # Some sandboxes cannot start gpg-agent. Public-key verification needs
        # no agent; require the exact imported fingerprint below instead.
        keys = subprocess.run(base+['--with-colons','--fingerprint','--list-keys'],capture_output=True,text=True)
        assert f'fpr:::::::::{FP}:' in keys.stdout, keys.stdout
        for i, match in enumerate(starts):
            tail = source[match.start():starts[i+1].start() if i+1<len(starts) else len(source)]
            end = tail.find('-----END PGP SIGNATURE-----')
            rec = {'index': i+1}
            if end < 0:
                rec['result']='incomplete block'; records.append(rec); continue
            block = tail[:end+len('-----END PGP SIGNATURE-----')]+'\n'
            p=Path(tmp)/'message.asc';p.write_text(block)
            # Explicit legacy SHA1 allowance is scoped to this historical check.
            r=subprocess.run(base+['--allow-weak-digest-algos','--status-fd','1','--verify',str(p)],capture_output=True,text=True)
            status=[l for l in r.stdout.splitlines() if l.startswith('[GNUPG:]')]
            valid = any(l.startswith('[GNUPG:] VALIDSIG '+FP+' ') for l in status)
            rec.update(sha256=sha(block),result='valid' if valid else 'not verified',gpg_status=status)
            if valid:
                body=re.split(r'\n[ \t]*\n',block,maxsplit=1)[1].split('-----BEGIN PGP SIGNATURE-----')[0]
                body=re.sub(r'^- ', '', body, flags=re.M)
                accepted.append({'id':i+1,'body':body,'sha256':sha(block)})
            records.append(rec)
    return records, accepted

def clean(body, strip_quotes):
    body = re.sub(r'(?ms)^begin [0-7]{3} [^\n]+\n.*?^end\s*$', '\n', body)
    body = re.sub(r'-----BEGIN PGP SIGNATURE-----.*?-----END PGP SIGNATURE-----', '\n',body,flags=re.S)
    body = re.sub(r'-----BEGIN PGP PUBLIC KEY BLOCK-----.*?-----END PGP PUBLIC KEY BLOCK-----', '\n',body,flags=re.S)
    body = re.sub(r'-----BEGIN COMPRESSED RSA ENCRYPTED MESSAGE-----.*?-----END COMPRESSED RSA ENCRYPTED MESSAGE-----','\n',body,flags=re.S)
    body = re.sub(r'-----BEGIN PGP SIGNED MESSAGE-----\s*Hash:[^\n]*\n','\n',body)
    body = body.replace('-----BEGIN PGP SIGNED MESSAGE-----', '\n')
    body = re.sub(r'(?m)^- ', '', body)
    if strip_quotes:
        # Retain paragraph breaks at removals so unrelated fragments cannot join.
        body = re.sub(r'(?m)^\s*>.*$', '\n', body)
    body = re.sub(r'https?://\S+|\b\S+@\S+','\n',body)
    body = re.sub(r'(?m)^(?:From|Date|Subject|To|Message-ID|Reply-To|Version|Hash):[^\n]*$', '\n',body)
    # Remove binary/base64-only long lines, not ordinary prose.
    body = re.sub(r'(?m)^\s*[A-Za-z0-9+/=]{40,}\s*$', '\n',body)
    return body

def paragraphs(body, strip_quotes=True):
    return [re.findall(r"[a-z]+(?:'[a-z]+)?", p.lower())
            for p in re.split(r'\n\s*\n',clean(body,strip_quotes))]

def ngrams(docs,n,strip_quotes=True):
    out=collections.defaultdict(set)
    for d in docs:
        for p in paragraphs(d['body'],strip_quotes):
            for i in range(len(p)-n+1):out[tuple(p[i:i+n])].add(str(d.get('id',d.get('url'))))
    return out

def compare(left,right,strip_quotes):
    out={}
    for n in [3,4,5,6,7,8]:
        a,b=ngrams(left,n,strip_quotes),ngrams(right,n,strip_quotes)
        shared=sorted(a.keys() & b.keys())
        out[str(n)]=[{'phrase':' '.join(p),'left':sorted(a[p]),'right':sorted(b[p])} for p in shared]
    return out

def main():
    # Recovery and boundary controls for the exact operation being measured.
    a=[{'id':'a','body':'One two three four five.\n\nSix seven.'}]
    b=[{'id':'b','body':'ONE TWO THREE FOUR FIVE.'}]
    assert ('one','two','three','four','five') in ngrams(a,5).keys() & ngrams(b,5).keys()
    assert ('five','six') not in ngrams(a,2)
    assert not ngrams([{'id':'q','body':'> hidden quote words'}],3,True)
    assert ('hidden','quote','words') in ngrams([{'id':'q','body':'> hidden quote words'}],3,False)
    assert not paragraphs('begin 444 X.GIF\nRANDOMWORDS\nend')[0]
    records, accepted = verify()
    print('Verification:',[(r['index'],r['result']) for r in records])
    (ROOT/'verification.json').write_text(json.dumps(records,indent=2))
    if not (ROOT/'corpus-private.json').exists(): return
    corpus=json.loads((ROOT/'corpus-private.json').read_text())
    result={'scope':'22 indexed relay messages and 21 non-domain monthly convenience controls, versus only verified signed blocks in the fixed 2012 archive.',
            'fingerprint':FP,'signed_blocks':records,'constructed_checks_passed':5,
            'archive_sha256':sha((ROOT/'2012.md').read_text()),
            'key_sha256':sha((ROOT/'cicada-key.asc').read_text()),
            'target_count':len(corpus['target']),'control_count':len(corpus['controls']),
            'verified_count':len(accepted),'months':corpus['months'],
            'sources':{k:corpus[k] for k in ['index_url','archive_url','key_url']}}
    for label in ['target','controls']:
        result[label+'_manifest']=[{k:v for k,v in d.items() if k!='body'} for d in corpus[label]]
        result[label+'_vs_3301']={
            'quoted_lines_removed':compare(corpus[label],accepted,True),
            'quoted_lines_retained':compare(corpus[label],accepted,False)}
    # Prespecified privacy/cryptography vocabulary; descriptive counts, not
    # authorship probabilities or a significance test.
    terms=['privacy','anonymous','anonymity','remailer','encryption','encrypted','cryptography','key','keys','pgp','public','private']
    result['vocabulary']={}
    for label,docs in [('target',corpus['target']),('controls',corpus['controls']),('verified_3301',accepted)]:
        bag=collections.Counter(w for d in docs for p in paragraphs(d['body']) for w in p)
        result['vocabulary'][label]={'tokens':sum(bag.values()),'term_counts':{w:bag[w] for w in terms}}
    (ROOT/'berkeley_comparison_results.json').write_text(json.dumps(result,indent=2))
    for label in ['target','controls']:
        for mode,v in result[label+'_vs_3301'].items():
            print(label,mode,{n:len(x) for n,x in v.items()})
            print('4+ matches:',v['4'])
    print('vocabulary',result['vocabulary'])

if __name__=='__main__': main()