#!/usr/bin/env python3 """Every number in the article, from postings.json, sentences.json and labels.json. Usage: python3 analyze.py OUT_DIR [RAW_DIR] > results.txt RAW_DIR, the raw 27 October boards, adds one check: how many postings counted as taken down are still on their board under a title the engineering filter no longer keeps. OUT_DIR is the 27 October eng.jsonl (locations for the Ontario and New York counts). Dates are counted in days from the published date the board gives to the day of the read. """ import collections import datetime as dt import json import re import statistics import sys from pathlib import Path HERE = Path(__file__).resolve().parent OUT = Path(sys.argv[1]) RAW = Path(sys.argv[2]) if len(sys.argv) > 2 else None D = dt.date.fromisoformat SEP15, OCT13, OCT27 = D('2026-09-15'), D('2026-10-13'), D('2026-10-27') ONTARIO = re.compile(r'toronto|ontario|ottawa|waterloo|kitchener|mississauga|markham|\bON\b(?!-)', re.I) CANADA = re.compile(r'canada|toronto|ontario|ottawa|waterloo|kitchener|mississauga|markham|vancouver|montr|' r'calgary|edmonton|british columbia|alberta|\bON\b(?!-)|\bBC\b', re.I) NEW_YORK = re.compile(r'new york|\bNYC\b|\bNY\b|brooklyn|manhattan', re.I) AGE_BUCKETS = [(0, 30, 'under 30 days'), (30, 90, '30 to 89 days'), (90, 180, '90 to 179 days'), (180, 365, '180 to 364 days'), (365, 10 ** 6, '365 days or more')] def pct(a, b): return f'{a}/{b} = {100 * a / b:.1f}%' if b else f'{a}/0' def age(published, day): return (day - D(published)).days def head(title): print(f'\n## {title}') def main(): P = json.load(open(HERE / 'postings.json')) sentences = json.load(open(HERE / 'sentences.json')) labels = json.load(open(HERE / 'labels.json'))['labels'] eng = {json.loads(l)['id']: json.loads(l) for l in open(OUT / 'eng.jsonl')} used = json.load(open(HERE / 'boards-used.json')) print('# How long engineering postings stay up: results, read 27 October 2026') head('Sample') print('27 October read: boards used', used['boards_used'], '| postings read', used['postings_read'], '| engineering postings', used['software_postings']) roles = json.load(open(HERE / 'roles.json')) print('27 October roles', len(roles), '| companies', len({r['company'] for r in roles})) comp = {r['company'] for r in P} print('boards read on all of 15 Sep, 13 Oct, 27 Oct:', len(set(json.load(open(HERE.parent / 'ai-tools-postings/boards-used.json'))['boards']) & set(json.load(open(HERE.parent / 'visa-sponsorship/boards-used.json'))['boards']) & set(used['boards']))) print('boards read on all of 15 Sep, 13 Oct, 27 Oct (companies):', len(comp)) for k in ('sep15', 'oct13', 'oct27'): print(f' postings seen {k} at those boards:', sum(r['seen_' + k] for r in P)) # Survival of the 15 September cohort. C = [r for r in P if r['seen_sep15']] head('The 15 September cohort') print('postings', len(C), '| companies', len({r['company'] for r in C})) print('still up 13 Oct (28 days):', pct(sum(r['seen_oct13'] for r in C), len(C))) print('still up 27 Oct (42 days):', pct(sum(r['seen_oct27'] for r in C), len(C))) print('gone 13 Oct, back 27 Oct under the same id:', sum((not r['seen_oct13']) and r['seen_oct27'] for r in C)) print('\nstill up 27 Oct, by age on 15 Sep:') for lo, hi, name in AGE_BUCKETS: g = [r for r in C if lo <= age(r['sep15']['published'], SEP15) < hi] print(f' {name}: {pct(sum(r["seen_oct27"] for r in g), len(g))}') top = {'databricks', 'palantir', 'openai'} print('\nsame, without databricks, palantir and openai:') for lo, hi, name in ((0, 90, 'under 90 days'), (90, 365, '90 to 364 days'), (365, 10 ** 6, '365 days or more')): g = [r for r in C if r['company'] not in top and lo <= age(r['sep15']['published'], SEP15) < hi] print(f' {name}: {pct(sum(r["seen_oct27"] for r in g), len(g))}') within = collections.defaultdict(lambda: {'young': [], 'old': []}) for r in C: a = age(r['sep15']['published'], SEP15) if a < 90: within[r['company']]['young'].append(r['seen_oct27']) elif a >= 365: within[r['company']]['old'].append(r['seen_oct27']) pairs = [(sum(v['old']) / len(v['old']), sum(v['young']) / len(v['young'])) for v in within.values() if len(v['old']) >= 3 and len(v['young']) >= 3] print('companies with 3+ postings under 90 days and 3+ over a year on 15 Sep:', len(pairs), '| year-old ones stayed up more often:', sum(o > y for o, y in pairs), '| less often:', sum(o < y for o, y in pairs), '| same:', sum(o == y for o, y in pairs)) print('\nsame, Greenhouse and Ashby only (Lever dates are createdAt):') for lo, hi, name in AGE_BUCKETS: g = [r for r in C if r['api'] != 'lever' and lo <= age(r['sep15']['published'], SEP15) < hi] print(f' {name}: {pct(sum(r["seen_oct27"] for r in g), len(g))}') for field in ('seniority', 'mode'): print(f'\nstill up 27 Oct, by {field} as read on 15 Sep:') g = collections.defaultdict(list) for r in C: g[r['sep15'][field]].append(r['seen_oct27']) for k, v in sorted(g.items(), key=lambda x: -len(x[1])): print(f' {k}: {pct(sum(v), len(v))}') print('\nstill up 27 Oct, by board system:') g = collections.defaultdict(list) for r in C: g[r['api']].append(r['seen_oct27']) for k, v in sorted(g.items()): print(f' {k}: {pct(sum(v), len(v))}') bycomp = collections.defaultdict(list) for r in C: bycomp[r['company']].append(r['seen_oct27']) shares = sorted(sum(v) / len(v) for v in bycomp.values() if len(v) >= 10) print('\ncompanies with 10+ postings on 15 Sep:', len(shares), '| median company share still up 27 Oct:', f'{100 * statistics.median(shares):.1f}%', '| companies with every one still up:', sum(s == 1 for s in shares), '| with under half still up:', sum(s < 0.5 for s in shares)) gone = [r for r in C if not r['seen_oct27']] last = [age(r['sep15']['published'], OCT13 if r['seen_oct13'] else SEP15) for r in gone] print('taken down by 27 Oct:', len(gone), '| median age when last seen:', statistics.median(last), 'days') # Ages on 27 October. T = [r for r in P if r['seen_oct27']] ages = [age(r['oct27']['published'], OCT27) for r in T] head('Ages of postings up on 27 October (common boards)') print('postings', len(T), '| median age', statistics.median(ages), 'days') for n in (90, 180, 365): print(f' published more than {n} days earlier: {pct(sum(a > n for a in ages), len(ages))}') for api in ('greenhouse', 'ashby', 'lever'): a = [age(r['oct27']['published'], OCT27) for r in T if r['api'] == api] print(f' {api}: {len(a)} postings, median {statistics.median(a)} days, over 180: {pct(sum(x > 180 for x in a), len(a))}') ra = [age(r['published'], OCT27) for r in roles if r.get('published')] print('roles (all 27 Oct boards):', len(ra), '| median', statistics.median(ra), '| over 90:', pct(sum(a > 90 for a in ra), len(ra)), '| over 365:', pct(sum(a > 365 for a in ra), len(ra))) old = [r for r in T if age(r['oct27']['published'], OCT27) > 365] cc = collections.Counter(r['company'] for r in old) allc = collections.Counter(r['company'] for r in T) print('\nover a year old:', len(old), '| at companies:', len(cc), 'of', len(allc)) print(' top companies:', ', '.join(f'{c} {n}' for c, n in cc.most_common(8))) top3 = sum(n for _, n in cc.most_common(3)) print(' share at the top three:', pct(top3, len(old))) for c, n in cc.most_common(5): print(f' {c}: {n} of its {allc[c]} postings are over a year old') nl90 = [r for r in T if r['api'] != 'lever' and age(r['oct27']['published'], OCT27) > 90] nl = [r for r in old if r['api'] != 'lever'] tl = [r for r in T if r['api'] != 'lever'] print(' Greenhouse and Ashby only, over 90 days:', pct(len(nl90), len(tl))) print(' Greenhouse and Ashby only, over a year:', pct(len(nl), len(tl)), '| Lever:', len(old) - len(nl), 'of the', len(old), '| top:', ', '.join(f'{c} {n}' for c, n in collections.Counter(r['company'] for r in nl).most_common(5))) print(' boards of the top companies:', ', '.join(f'{c} {next(r["api"] for r in T if r["company"] == c)}' for c, _ in cc.most_common(5))) print(' companies (5+ postings) with more than half their postings over 180 days:', sum(1 for c in allc if allc[c] >= 5 and sum(age(r['oct27']['published'], OCT27) > 180 for r in T if r['company'] == c) / allc[c] > 0.5), 'of', sum(1 for c in allc if allc[c] >= 5)) # Dates that move. both = [r for r in C if r['seen_oct27']] head('Same posting id on 15 Sep and 27 Oct, published date changed') for api in ('greenhouse', 'ashby', 'lever'): b = [r for r in both if r['api'] == api] moved = [r for r in b if r['sep15']['published'] != r['oct27']['published']] print(f' {api}: {pct(len(moved), len(b))}') if moved: shift = [(D(r['oct27']['published']) - D(r['sep15']['published'])).days for r in moved] print(f' all later: {all(s > 0 for s in shift)} | median shift {statistics.median(shift)} days | ' f'over 90 days: {sum(s > 90 for s in shift)} | companies {len({r["company"] for r in moved})}') print(' by company:', ', '.join(f'{c} {n}' for c, n in collections.Counter(r['company'] for r in moved).most_common(8))) print(' old date over 90 days before 15 Sep:', sum(age(r['sep15']['published'], SEP15) > 90 for r in moved)) dc = json.load(open(HERE / 'date-check.json'))['rows'] print(' Ashby pages opened (verify-dates.py):', len(dc), '| page datePosted equals the later date:', sum(o['page_datePosted'] == o['oct27_api'] for o in dc), '| page not readable:', sum(str(o['page_datePosted']).startswith('error') for o in dc)) # Reposts under a new id. norm = lambda t: re.sub(r'\W+', ' ', t.lower()).strip() new = [r for r in T if not r['seen_sep15']] idx = collections.defaultdict(list) for r in new: idx[(r['company'], norm(r['oct27']['title']))].append(r) rep, rep_loc, rep_after = [], [], [] for r in gone: m = [x for x in idx.get((r['company'], norm(r['sep15']['title'])), []) if x['oct27']['published'] > r['sep15']['published']] if m: rep.append(r) ml = [x for x in m if x['oct27']['location'] == r['sep15']['location']] if ml: rep_loc.append(r) last_seen = '2026-10-13' if r['seen_oct13'] else '2026-09-15' if any(x['oct27']['published'] > last_seen for x in ml): rep_after.append(r) head('Taken down, and a posting with the same company and title appeared under a new id with a later date') print('of', len(gone), 'taken down:', len(rep), '| same location string too:', len(rep_loc), '| companies', len({r['company'] for r in rep_loc})) print(' of those, the new one dated after the old one was last seen:', len(rep_after), '| companies', len({r['company'] for r in rep_after}), '| most common titles:', ', '.join(f'{k} {n}' for k, n in collections.Counter( norm(r['sep15']['title']) for r in rep_after).most_common(3))) if RAW: on_board = set() for f in RAW.glob('*.json'): api, slug = f.stem.split('-', 1) data = json.load(open(f)) jobs = data['jobs'] if isinstance(data, dict) and 'jobs' in data else data for j in jobs: on_board.add((slug, str(j['id']))) still = [r for r in gone if (r['company'], r['job']) in on_board] print('taken down from the engineering list but still on the board under another title (raw boards):', len(still), 'of', len(gone)) print('new on 27 Oct (not seen 15 Sep):', len(new), '| of them dated before 15 Sep:', sum(r['oct27']['published'] < '2026-09-15' for r in new)) # What postings say about the opening (all 27 October engineering postings). by_label = collections.defaultdict(set) comp_label = collections.defaultdict(set) for s in sentences: lab = labels[s['key']] for pid in s['postings']: by_label[lab].add(pid) comp_label[lab].add(s['company']) head('What 27 October postings say about the opening (labeled sentences)') print('engineering postings:', len(eng)) for lab in ('not_current', 'existing_vacancy', 'several_seats', 'team_match', 'no_end_date', 'end_date'): print(f' {lab}: {len(by_label[lab])} postings at {len(comp_label[lab])} companies: ' + ', '.join(sorted(comp_label[lab]))) # Ontario. def places(p): return [p['location']] + p.get('more_locations', []) on = [p for p in eng.values() if any(ONTARIO.search(l) for l in places(p))] on26 = [p for p in on if p['published'] >= '2026-01-01'] head('Ontario: postings naming an Ontario place, first published on or after 1 January 2026') print('all naming Ontario:', len(on), '| published 2026:', len(on26), '| companies', len({p['company'] for p in on26})) for lab, name in (('existing_vacancy', 'say it is an existing vacancy'), ('not_current', 'say it is not')): g = [p for p in on26 if p['id'] in by_label[lab]] print(f' {name}: {pct(len(g), len(on26))} | companies {len({p["company"] for p in g})} of {len({p["company"] for p in on26})}: ' + ', '.join(f'{c} {n}' for c, n in collections.Counter(p['company'] for p in g).most_common())) moved_ids = {f"{ {'greenhouse': 'gh', 'ashby': 'ab', 'lever': 'lv'}[r['api']] }-{r['company']}-{r['job']}" for r in C if r['seen_oct27'] and r['sep15']['published'] != r['oct27']['published']} print(' of them, seen on 15 September with a different date then:', sum(p['id'] in moved_ids for p in on26), '| that date before 2026:', sum(p['id'] in moved_ids and any( r['sep15']['published'] < '2026-01-01' for r in C if r['job'] in p['id'] and r['company'] == p['company']) for p in on26)) only_on = [p for p in on26 if all(ONTARIO.search(l) for l in places(p) if l.strip())] multi = [p for p in on26 if p not in only_on] vac = by_label['existing_vacancy'] | by_label['not_current'] print(f' every listed location in Ontario: {len(only_on)}, with a vacancy statement {pct(sum(p["id"] in vac for p in only_on), len(only_on))}') print(f' Ontario plus places elsewhere: {len(multi)}, with a vacancy statement {pct(sum(p["id"] in vac for p in multi), len(multi))}') print(' AI statements from Okta (New York City notice):', sum(p['company'] == 'okta' and p['id'] in by_label['ai_screening'] for p in on26)) strict = [p for p in on26 if p['id'] in by_label['ai_screening']] broad = [p for p in on26 if p['id'] in by_label['ai_screening'] | by_label['ai_hiring_general'] | by_label['ai_not_deciding']] print(f' say they use AI to screen, assess or select: {pct(len(strict), len(on26))} | companies ' + ', '.join(f'{c} {n}' for c, n in collections.Counter(p['company'] for p in strict).most_common())) print(f' mention AI in their own hiring at all: {pct(len(broad), len(on26))} | companies {len({p["company"] for p in broad})}') print(' per company (postings, vacancy statement, any AI-in-hiring statement):') for c, n in collections.Counter(p['company'] for p in on26).most_common(): ps = [p for p in on26 if p['company'] == c] print(f' {c}: {n}, {sum(p["id"] in by_label["existing_vacancy"] for p in ps)}, {sum(p in broad for p in ps)}') oc = [o for o in json.load(open(HERE / 'ontario-page-check.json')) if not o.get('control')] ctl = [o for o in json.load(open(HERE / 'ontario-page-check.json')) if o.get('control')] rend = {o['company']: o for o in json.load(open(HERE / 'ontario-page-rendered.json'))['rows'] if not o.get('control')} hosted = re.compile(r'//(job-boards|boards)\.greenhouse\.io/|//jobs\.ashbyhq\.com/|//jobs\.lever\.co/') shown = {} for o in oc: if o['company'] in rend: shown[o['company']] = rend[o['company']]['vacancy_on_page'] elif hosted.search(o['url']): shown[o['company']] = o['vacancy_on_page'] else: shown[o['company']] = None print(' companies without the sentence, one live posting page each:', len(oc), '| text seen (hosted board raw, or rendered in a browser):', sum(v is not None for v in shown.values()), '| "vacancy" on it:', sum(v is True for v in shown.values()), '| not seen:', sorted(c for c, v in shown.items() if v is None)) print(' raw-page controls (companies with the sentence):', ', '.join(f'{o["company"]} {o["vacancy_on_page"]}' for o in ctl)) ca = [p for p in eng.values() if p['published'] >= '2026-01-01' and any(CANADA.search(l) for l in places(p))] cav = [p for p in ca if p['id'] in by_label['existing_vacancy'] | by_label['not_current']] print('Canada (any Canadian place or "Canada"), published 2026:', len(ca), '| with a statement either way:', pct(len(cav), len(ca)), '| companies', ', '.join(sorted({p['company'] for p in cav}))) # New York. ny = [p for p in eng.values() if any(NEW_YORK.search(l) for l in places(p))] nya = [age(p['published'], OCT27) for p in ny] head('New York: postings naming New York (all 27 October boards)') print('postings', len(ny), '| companies', len({p['company'] for p in ny}), '| published more than 90 days earlier:', pct(sum(a > 90 for a in nya), len(nya))) nys = [p for p in ny if p['id'] in by_label['existing_vacancy'] | by_label['not_current']] print(' with a vacancy statement either way:', len(nys)) if __name__ == '__main__': main()