#!/usr/bin/env python3
"""Sentences a reader needs to see to judge what a posting says about the opening.

Three families, matched loosely on purpose (the labels, not the patterns, decide):
  vacancy   any sentence with "vacancy"/"vacancies"                (all postings)
  ai_hiring AI or automation near applicants, screening, interviews  (Ontario postings only)
  opening   pipeline, evergreen, talent pool, future openings, several hires,
            team matching, deadlines and "open until"               (all postings)
A sentence repeated across a company's postings is one string; each string
keeps the ids of the postings it appears in.
Writes sentences.json here. Usage: python3 extract.py OUT_DIR
"""
import json
import re
import sys
from pathlib import Path

HERE = Path(__file__).resolve().parent
OUT = Path(sys.argv[1])

ONTARIO = re.compile(r'toronto|ontario|ottawa|waterloo|kitchener|mississauga|markham|\bON\b(?!-)', re.I)
PATTERNS = {
    'vacancy': re.compile(r'\bvacanc', re.I),
    'ai_hiring': re.compile(
        r'(\bAI\b|artificial intelligence|automated|algorithm|machine learning)'
        r'.{0,160}(applica|applicant|candidate|hiring|recruit|screen|interview|resume|résumé|selection)|'
        r'(applica|applicant|candidate|hiring|recruit|screen|interview|resume|résumé|selection)'
        r'.{0,160}(\bAI\b|artificial intelligence|automated|algorithm)', re.I),
    'opening': re.compile(
        r'pipeline (role|position|posting|req)|evergreen|talent (pool|pipeline|community|network)|'
        r'future (opening|opportunit|role|position|need|vacanc)|'
        r'not (a |an )?(current|active|immediate|specific|open) (opening|vacancy|role|position|need|headcount)|'
        r'no (current|immediate) (opening|vacancy|need)|'
        r'multiple (openings|positions|hires|headcount|seats)|several (openings|positions|hires)|'
        r'hiring (for )?(multiple|several|many) |more than one (hire|position|opening)|'
        r'team[- ]match|match(ed)? (you )?(with|to) (a|the right) team|feeding|'
        r'(posting|position|role|requisition|req)s? (will )?(remain|stay)s? open|open until|'
        r'accepting applications|rolling basis|application deadline|deadline to apply|apply by', re.I),
}


def sentences(text):
    return [s.strip() for s in re.split(r'(?<=[.!?])\s+|\n+', text) if 15 <= len(s.strip()) <= 700]


def main():
    found = {}
    for line in open(OUT / 'eng.jsonl'):
        p = json.loads(line)
        ontario = any(ONTARIO.search(l) for l in [p['location']] + p.get('more_locations', []))
        for s in sentences(p['text']):
            for fam, pat in PATTERNS.items():
                if fam == 'ai_hiring' and not ontario:
                    continue
                if pat.search(s):
                    row = found.setdefault((p['company'], s), {'company': p['company'], 'sentence': s,
                                                               'families': [], 'postings': []})
                    if fam not in row['families']:
                        row['families'].append(fam)
                    if p['id'] not in row['postings']:
                        row['postings'].append(p['id'])
    rows = sorted(found.values(), key=lambda r: (r['company'], r['sentence']))
    for i, r in enumerate(rows, 1):
        r['key'] = f's{i:04d}'
    json.dump(rows, open(HERE / 'sentences.json', 'w'), ensure_ascii=False, indent=0)
    from collections import Counter
    print(len(rows), Counter(f for r in rows for f in r['families']))


if __name__ == '__main__':
    main()