import concurrent.futures as cf
import hashlib, html, json, re, time, urllib.request
from pathlib import Path

ROOT = Path(__file__).resolve().parent
CACHE = ROOT / 'cache'
CACHE.mkdir(exist_ok=True)
BASE = 'https://mailing-list-archive.cryptoanarchy.wiki'
INDEX = BASE + '/authors/nobody_at_cicada_berkeley_edu/'

def fetch(url):
    p = CACHE / (hashlib.sha256(url.encode()).hexdigest() + '.txt')
    if not p.exists():
        req = urllib.request.Request(url, headers={'User-Agent': 'Historical-text-research/1.0'})
        for i in range(3):
            try:
                p.write_bytes(urllib.request.urlopen(req, timeout=45).read())
                break
            except Exception:
                if i == 2: raise
                time.sleep(1)
    return p.read_text(errors='replace')

def links(s):
    return sorted(set(BASE + x for x in re.findall(r'href="(/archive/\d{4}/\d{2}/[a-f0-9]{64})/?"', s)))

def mail(url):
    raw_html = fetch(url)
    blocks = re.findall(r'<pre[^>]*><code[^>]*>(.*?)</code></pre>', raw_html, re.S)
    assert len(blocks) == 1, (url, len(blocks))
    raw = html.unescape(re.sub(r'<[^>]+>', '', blocks[0]))
    headers, body = raw.split('\n\n', 1)
    def header(name):
        m = re.search(r'^' + re.escape(name) + r':\s*(.*)', headers, re.M | re.I)
        return m.group(1).strip() if m else None
    return dict(url=url, raw_sha256=hashlib.sha256(raw.encode()).hexdigest(),
                sender=header('From'), date=header('Date'), subject=header('Subject'),
                message_id=header('Message-ID'), body=body)

def main():
    candidates = links(fetch(INDEX))
    with cf.ThreadPoolExecutor(max_workers=6) as pool:
        indexed = list(pool.map(mail, candidates))
    target = [x for x in indexed if x['sender'] == 'nobody@cicada.berkeley.edu']
    assert len(target) == 22, len(target)
    months = sorted(set(re.search(r'/archive/(\d{4}/\d{2})/', x['url']).group(1) for x in target))
    controls = []
    # Fixed, outcome-independent convenience sample: first three URL hashes per
    # target month whose parsed From header is not the target domain.
    for month in months:
        urls = [u for u in links(fetch(BASE + '/archive/' + month + '/'))
                if '/archive/' + month + '/' in u]
        selected = []
        for start in range(0, len(urls), 4):
            with cf.ThreadPoolExecutor(max_workers=4) as pool:
                batch = list(pool.map(mail, urls[start:start+4]))
            for x in batch:
                if 'cicada.berkeley.edu' not in (x['sender'] or '').lower():
                    selected.append(x)
                if len(selected) == 3: break
            if len(selected) == 3: break
        assert len(selected) == 3
        assert all('/archive/' + month + '/' in x['url'] for x in selected)
        controls.extend(selected)
    assert len({x['url'] for x in controls}) == 21
    keyurl = 'https://raw.githubusercontent.com/cicada-solvers/isitcicada/fe59cae1f1a39026d626be32f14bc51a14ec3f81/assets/pgp/cicada_3301_key.txt'
    (ROOT / 'cicada-key.asc').write_text(fetch(keyurl))
    archiveurl = 'https://raw.githubusercontent.com/scream314/cicada3301/10733feb487bb8beb9966d534727ee5897aa54fc/2012.md'
    (ROOT / '2012.md').write_text(fetch(archiveurl))
    data = dict(index_url=INDEX, target=target, controls=controls,
                archive_url=archiveurl, key_url=keyurl, months=months)
    (ROOT / 'corpus-private.json').write_text(json.dumps(data, indent=2))
    print(json.dumps({'targets':len(target),'controls':len(controls),'months':months}))

if __name__ == '__main__': main()