#!/usr/bin/env python3
"""Re-fetch every page a vendor claim rests on and fail if a quoted sentence has gone."""
import csv, html, os, re, sys, urllib.request
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/130.0 Safari/537.36"
rows = list(csv.DictReader(open(os.path.join(os.path.dirname(__file__), 'vendor-claims.csv'))))
def text(url):
    req = urllib.request.Request(url, headers={'User-Agent': UA})
    raw = urllib.request.urlopen(req, timeout=30).read().decode('utf-8', 'ignore')
    raw = re.sub(r'(?is)<script.*?</script>|<style.*?</style>', ' ', raw)
    return re.sub(r'\s+', ' ', html.unescape(re.sub(r'(?s)<[^>]+>', ' ', raw)))
cache, failed = {}, 0
for r in rows:
    cache.setdefault(r['url'], text(r['url']))
    ok = r['quote_test'] in cache[r['url']]
    print(('OK   ' if ok else 'GONE '), r['id'], r['url'])
    failed += 0 if ok else 1
sys.exit(1 if failed else 0)
