#!/usr/bin/env python3
"""Follow every engineering posting across the four reads of the same boards.

Reads: 15 September (posting level, ../../../public/post/ai-coding-tools-in-job-postings/data/postings.json),
29 September (role level: only the first posting id of each role was kept,
../engineering-by-country/roles.json), 13 October (posting level through the
posting ids of each role, ../visa-sponsorship/roles.json) and 27 October
(OUT_DIR/eng.jsonl from this folder's sweep, plus the raw Greenhouse boards for
updated_at).

A posting is the board's own id: the Greenhouse job id, the Ashby or Lever
uuid. Only boards read on all of 15 September, 13 October and 27 October count,
so a board that failed to answer one day is not mistaken for closed postings.
The 15 September titles go through today's NOT_ENGINEERING filter, the one
added on 29 September, so all reads use the same definition of the job.

Writes postings.json here: one row per posting seen on any of the three
posting-level reads at the common boards. What each read said about the
posting (title, location, mode, seniority, published date) is kept under that
read's key, so a split by mode or seniority can use the 15 September values
for the whole cohort instead of mixing them with today's.
Usage: python3 lifetimes.py RAW_DIR OUT_DIR
"""
import json
import re
import sys
from pathlib import Path

HERE = Path(__file__).resolve().parent
RESEARCH = HERE.parent
SITE = RESEARCH.parents[1]
RAW, OUT = Path(sys.argv[1]), Path(sys.argv[2])

NOT_ENGINEERING = re.compile(
    r'copywriter|curriculum|content developer|creative strategist|land developer|developer support|developer education|'
    r'developer engagement|asset management|operations specialist|customer support|'
    r'applied ai (architect|strategist)|rotational program|product lead|experience advocate|analytics lead',
    re.I)
API = {'jobs.ashbyhq.com': 'ashby', 'jobs.lever.co': 'lever'}


def job_id(url):
    """Board and job id from a posting URL of any of the three systems."""
    m = re.search(r'gh_jid=(\d+)', url) or re.search(r'greenhouse\.io/[^/]+/jobs/(\d+)', url)
    if m:
        return 'greenhouse', m.group(1)
    m = re.search(r'//(jobs\.ashbyhq\.com|jobs\.lever\.co)/[^/]+/([0-9a-f-]{36})', url)
    if m:
        return API[m.group(1)], m.group(2)
    return None, None


def key_of_id(pid):
    """gh-<slug>-<id>, ab-<slug>-<uuid>, lv-<slug>-<uuid> -> (company, job id)."""
    kind, rest = pid.split('-', 1)
    if kind == 'gh':
        slug, job = rest.rsplit('-', 1)
    else:
        slug, job = rest[:-37], rest[-36:]
    return slug, job


def boards(path):
    return set(json.load(open(path))['boards'])


def main():
    common = (boards(RESEARCH / 'ai-tools-postings/boards-used.json')
              & boards(RESEARCH / 'visa-sponsorship/boards-used.json')
              & boards(HERE / 'boards-used.json'))
    slug_api = {b.split(':', 1)[1]: b.split(':', 1)[0] for b in common}

    rows = {}

    def row(slug, job):
        return rows.setdefault((slug, job), {'company': slug, 'api': slug_api[slug], 'job': job})

    # 27 October: full rows, plus Greenhouse updated_at from the raw boards.
    updated = {}
    for slug, api in slug_api.items():
        if api == 'greenhouse':
            for j in json.load(open(RAW / f'greenhouse-{slug}.json'))['jobs']:
                updated[(slug, str(j['id']))] = (j.get('updated_at') or '')[:10]
    for line in open(OUT / 'eng.jsonl'):
        p = json.loads(line)
        if p['company'] not in slug_api:
            continue
        slug, job = key_of_id(p['id'])
        r = row(slug, job)
        r['oct27'] = {'title': p['title'], 'url': p['url'], 'location': p['location'],
                      'more_locations': p.get('more_locations', []), 'seniority': p['seniority'],
                      'mode': p['mode'], 'published': p['published']}
        r['seen_oct27'] = True
        if (slug, job) in updated:
            r['oct27']['updated'] = updated[(slug, job)]

    # 13 October: every posting id of every role.
    for role in json.load(open(RESEARCH / 'visa-sponsorship/roles.json')):
        if role['company'] not in slug_api:
            continue
        for pid in role['posting_ids']:
            slug, job = key_of_id(pid)
            r = row(slug, job)
            r['seen_oct13'] = True
            r['oct13_role'] = {'title': role['title'], 'published': role['published']}

    # 29 September: first posting of each role only; absence there proves nothing.
    first29 = set()
    for role in json.load(open(RESEARCH / 'engineering-by-country/roles.json')):
        first29.add(key_of_id(role['id']))

    # 15 September: posting level.
    unmatched = 0
    for p in json.load(open(SITE / 'public/post/ai-coding-tools-in-job-postings/data/postings.json')):
        if p['company'] not in slug_api or NOT_ENGINEERING.search(p['title']):
            continue
        api, job = job_id(p['url'])
        if job is None:
            unmatched += 1
            continue
        r = row(p['company'], job)
        r['seen_sep15'] = True
        r['sep15'] = {k: p[k] for k in ('title', 'url', 'location', 'seniority', 'mode', 'published')}

    out = []
    for (slug, job), r in sorted(rows.items()):
        for k in ('sep15', 'oct13', 'oct27'):
            r.setdefault('seen_' + k, False)
        r['first_posting_sep29'] = (slug, job) in first29
        out.append(r)
    json.dump(out, open(HERE / 'postings.json', 'w'), ensure_ascii=False, indent=0)
    print(json.dumps({'common_boards': len(common), 'postings': len(out),
                      'sep15': sum(r['seen_sep15'] for r in out), 'oct13': sum(r['seen_oct13'] for r in out),
                      'oct27': sum(r['seen_oct27'] for r in out), 'sep15_url_without_id': unmatched}))


if __name__ == '__main__':
    main()