#!/usr/bin/env python3
"""Re-verify each vendor's primary CTA from their live homepage.

Fetches the homepage, strips script/style, and classifies the CTA-like text
found in links and buttons. Conservative by design: anything that cannot be
fetched or classified is recorded as UNKNOWN and excluded from percentages —
an error is never counted as a demo gate.

Usage: verify_cta.py <out.json> [limit] [offset]
"""
import csv, json, re, sys, ssl, urllib.request, urllib.error
from concurrent.futures import ThreadPoolExecutor

CSV = "/Users/prasadthammineni/.claude/uploads/8cdf6720-77d6-4b43-9b82-ff1141fdb368/5fdb4098-healthcare_marketmap_enriched.csv"

# Order matters: self-serve wins if BOTH appear, because a site offering a real
# signup is not gated even when it also has a demo button. This biases AGAINST
# our own headline finding, which is the safe direction.
SELF_SERVE = [
    r'\bsign\s?up\s+free\b', r'\bstart\s+free\b', r'\bfree\s+trial\b',
    r'\bstart\s+(your\s+)?free\s+trial\b', r'\btry\s+(it\s+)?free\b',
    r'\bcreate\s+(a\s+|your\s+)?account\b', r'\bsign\s?up\b',
    r'\bget\s+started\s+free\b', r'\bstart\s+for\s+free\b',
]
GATED = [
    r'\b(get|request|book|schedule)\s+(a\s+)?demo\b', r'\bdemo\b',
    r'\bcontact\s+(us|sales)\b', r'\btalk\s+to\s+(us|sales|an\s+expert)\b',
    r'\bget\s+in\s+touch\b', r'\brequest\s+(a\s+)?quote\b',
    r'\bbook\s+a\s+call\b', r'\bschedule\s+a\s+call\b',
]

UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '\
     '(KHTML, like Gecko) Chrome/124.0 Safari/537.36'

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE


def fetch(url):
    if not url.startswith('http'):
        url = 'https://' + url
    req = urllib.request.Request(url, headers={'User-Agent': UA,
                                               'Accept': 'text/html'})
    with urllib.request.urlopen(req, timeout=20, context=ctx) as r:
        raw = r.read(400_000)
    try:
        return raw.decode('utf-8', 'replace')
    except Exception:
        return raw.decode('latin-1', 'replace')


def cta_text(html):
    """Collect CTA-ish text.

    Links and buttons first (where CTAs live), then fall back to the whole
    visible page text. The pilot showed why the fallback is needed: sites
    render CTAs inside divs with click handlers, so link/button extraction
    alone reported UNCLEAR on pages that plainly say "Book a demo".
    """
    h = re.sub(r'<(script|style|noscript)[^>]*>.*?</\1>', ' ', html,
               flags=re.S | re.I)
    chunks = re.findall(r'<(?:a|button)\b[^>]*>(.*?)</(?:a|button)>', h,
                        flags=re.S | re.I)
    out = []
    for c in chunks:
        t = re.sub(r'<[^>]+>', ' ', c)
        t = re.sub(r'&[a-z]+;', ' ', t)
        t = re.sub(r'\s+', ' ', t).strip().lower()
        if 0 < len(t) <= 60:
            out.append(t)
    if not any(re.search(p, ' | '.join(out))
               for p in SELF_SERVE + GATED):
        body = re.sub(r'<[^>]+>', ' ', h)
        body = re.sub(r'&[a-z]+;', ' ', body)
        out.append(re.sub(r'\s+', ' ', body).strip().lower()[:20000])
    return out


def js_rendered(html):
    """A near-empty document means the CTA never reached us."""
    body = re.sub(r'<(script|style|noscript)[^>]*>.*?</\1>', ' ', html,
                  flags=re.S | re.I)
    body = re.sub(r'<[^>]+>', ' ', body)
    return len(re.sub(r'\s+', ' ', body).strip()) < 400


def classify(texts):
    joined = ' | '.join(texts)
    self_hit = next((p for p in SELF_SERVE if re.search(p, joined)), None)
    gate_hit = next((p for p in GATED if re.search(p, joined)), None)
    if self_hit:
        return 'SELF_SERVE', self_hit
    if gate_hit:
        return 'GATED', gate_hit
    return 'UNCLEAR', None


def check(row):
    rec = {'company': row['Company'], 'website': row['Website'],
           'subcategory': row['Subcategory'], 'march_cta': row['Primary CTA']}
    try:
        html = fetch(row['Website'])
        if js_rendered(html):
            rec['verdict'] = 'UNKNOWN'
            rec['error'] = 'JS_RENDERED'
            return rec
        verdict, hit = classify(cta_text(html))
        rec['verdict'] = verdict
        rec['matched'] = hit
    except urllib.error.HTTPError as e:
        rec['verdict'] = 'UNKNOWN'
        rec['error'] = 'HTTP %s' % e.code
    except Exception as e:
        rec['verdict'] = 'UNKNOWN'
        rec['error'] = type(e).__name__
    return rec


def main():
    out = sys.argv[1]
    limit = int(sys.argv[2]) if len(sys.argv) > 2 else 0
    offset = int(sys.argv[3]) if len(sys.argv) > 3 else 0
    rows = list(csv.DictReader(open(CSV, encoding='utf-8-sig')))
    rows = rows[offset:offset + limit] if limit else rows[offset:]
    with ThreadPoolExecutor(max_workers=12) as ex:
        res = list(ex.map(check, rows))
    json.dump(res, open(out, 'w'), indent=1)
    from collections import Counter
    c = Counter(r['verdict'] for r in res)
    print('checked %d ->' % len(res), dict(c))


if __name__ == '__main__':
    main()