// Driver for the prescribed method (section 2.4 fetch, 2.7 validation, 2.8
// repetitions, 2.9 discards, 2.11 statistics). Resumable: each row is appended
// to out/rows.jsonl the moment it exists and work already present is skipped.
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { parseHead, parseTail, stripQuery } from './parse-head.mjs';

const HERE = path.dirname(new URL(import.meta.url).pathname);
const OUT = path.join(HERE, 'out');
const ROWS = path.join(OUT, 'rows.jsonl');
const FILES = path.join(OUT, 'files.jsonl');
const META = path.join(OUT, 'run-meta.json');
const DATASET = path.join(OUT, 'dataset.json');
const SALT_FILE = path.join(OUT, 'salt.txt');
const INPUT = path.join(HERE, 'inputs', 'files.jsonl');

const UA = 'article-research/1.0 (viallo metadata survival run; contact info@zava-solutions.com)';
const WINDOW = 131072;
const TIMEOUT = 30000;
const BUDGET_MS = 7 * 60 * 1000;
const MAX_GLOBAL = 8;
const BODY_CAP = 64 * 1024 * 1024;
const IMAGE_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/tiff']);

const EXT_CONTAINER = {
  jpg: 'jpeg', jpeg: 'jpeg', jpe: 'jpeg', png: 'png', webp: 'webp',
  tif: 'tiff', tiff: 'tiff',
};
const MIME_CONTAINER = {
  'image/jpeg': 'jpeg', 'image/png': 'png', 'image/webp': 'webp',
  'image/tiff': 'tiff',
};

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// ------------------------------------------------------------------ plan

function extOf(u) {
  const m = stripQuery(u).toLowerCase().match(/\.([a-z0-9]+)$/);
  return m ? m[1] : null;
}

function pageNumberOf(u) {
  const m = stripQuery(u).match(/page(\d+)/);
  return m ? Number(m[1]) : null;
}

function hostOf(u) {
  try { return new URL(u).host; } catch { return 'invalid'; }
}

function buildPlan() {
  const records = fs.readFileSync(INPUT, 'utf8').trim().split('\n').map((l) => JSON.parse(l));
  const units = [];
  const push = (rec, arm, kind, url, range, extra) => {
    units.push({
      unit_key: `${rec.file_id}|${arm}|${kind}`,
      file_id: rec.file_id,
      frame: rec.frame,
      arm,
      kind,
      url,
      host: hostOf(url),
      range: range ?? null,
      hint: MIME_CONTAINER[rec.mime] || EXT_CONTAINER[extOf(url)] || null,
      accepted_order: rec.accepted_order,
      api_size: rec.api_size,
      subsample: Boolean(rec.subsample),
      title: rec.title ?? null,
      ...extra,
    });
  };

  for (const rec of records) {
    if (rec.frame === 'W') {
      const cOrig = EXT_CONTAINER[extOf(rec.url_original)] || null;
      const cDisp = rec.url_display ? (EXT_CONTAINER[extOf(rec.url_display)] || null) : null;
      const pairBase = {
        container_original: cOrig,
        container_display: cDisp,
        display_is_original: rec.url_display
          ? stripQuery(rec.url_original) === stripQuery(rec.url_display) : null,
        display_page_number: rec.url_display ? pageNumberOf(rec.url_display) : null,
      };
      push(rec, 'original', 'head', rec.url_original, 'bytes=0-131071', pairBase);
      push(rec, 'display', 'head', rec.url_display, 'bytes=0-131071', {
        ...pairBase,
        display_page_number: rec.url_display ? pageNumberOf(rec.url_display) : null,
      });
      if (rec.subsample && rec.url_display_w200) {
        push(rec, 'display_w200', 'head', rec.url_display_w200, 'bytes=0-131071', {
          ...pairBase,
          display_is_original: stripQuery(rec.url_original) === stripQuery(rec.url_display_w200),
          display_page_number: pageNumberOf(rec.url_display_w200),
        });
      }
      if (rec.subsample && rec.url_display_w1200) {
        push(rec, 'display_w1200', 'head', rec.url_display_w1200, 'bytes=0-131071', {
          ...pairBase,
          display_is_original: stripQuery(rec.url_original) === stripQuery(rec.url_display_w1200),
          display_page_number: pageNumberOf(rec.url_display_w1200),
        });
      }
      if (rec.subsample && rec.archived_url) {
        push(rec, 'archive_oldest', 'head', rec.archived_url, 'bytes=0-131071', pairBase);
      }
      if (rec.api_size != null && rec.api_size > WINDOW) {
        push(rec, 'tail', 'head', rec.url_original, 'bytes=-131072', pairBase);
      }
      if (rec.subsample && (rec.api_size == null || rec.api_size <= 20000000)) {
        push(rec, 'original', 'full', rec.url_original, null, pairBase);
      }
    } else {
      push(rec, 's_full', 'head', rec.url_full, 'bytes=0-131071', {});
      push(rec, 's_thumb', 'head', rec.url_thumb, 'bytes=0-131071', {});
      if (rec.url_remote) push(rec, 's_remote', 'head', rec.url_remote, 'bytes=0-131071', {});
      if (rec.subsample) push(rec, 's_full', 'full', rec.url_full, null, {});
    }
  }
  return { units, records };
}

// ------------------------------------------------------------------ salt

function loadSalt() {
  fs.mkdirSync(OUT, { recursive: true });
  if (fs.existsSync(SALT_FILE)) return fs.readFileSync(SALT_FILE, 'utf8').trim();
  const salt = crypto.randomBytes(32).toString('hex');
  fs.writeFileSync(SALT_FILE, salt + '\n', { mode: 0o600 });
  return salt;
}

// ------------------------------------------------------------------ fetch

async function readBody(res) {
  if (!res.body) return { buf: Buffer.alloc(0), total: 0, capped: false };
  const reader = res.body.getReader();
  const parts = [];
  let total = 0;
  let capped = false;
  for (;;) {
    const { done, value } = await reader.read();
    if (done) break;
    if (value && value.length) {
      parts.push(Buffer.from(value));
      total += value.length;
    }
    if (total > BODY_CAP) {
      capped = true;
      try { await reader.cancel(); } catch { /* already closed */ }
      break;
    }
  }
  return { buf: Buffer.concat(parts), total, capped };
}

class TimeoutError extends Error {}

async function request(url, range) {
  const headers = { 'User-Agent': UA, Accept: '*/*' };
  if (range) headers.Range = range;
  let res;
  try {
    res = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT) });
  } catch (e) {
    if (e && (e.name === 'TimeoutError' || e.name === 'AbortError')) throw new TimeoutError('timeout');
    throw e;
  }
  const body = await readBody(res);
  return { res, ...body };
}

function retryAfterMs(res) {
  const h = res.headers.get('retry-after');
  if (!h) return null;
  const n = Number(h);
  if (Number.isFinite(n)) return Math.max(0, n * 1000);
  const d = Date.parse(h);
  if (!Number.isNaN(d)) return Math.max(0, d - Date.now());
  return null;
}

async function fetchUnit(unit) {
  const trace = [];
  let throttled = false;

  const attempt = async () => {
    const r = await request(unit.url, unit.range);
    trace.push(r.res.status);
    return r;
  };

  try {
    let r = await attempt();
    for (let i = 0; i < 3 && (r.res.status === 429 || r.res.status === 503); i++) {
      throttled = true;
      const wait = retryAfterMs(r.res) ?? 1000 * Math.pow(2, i + 1);
      await sleep(Math.min(wait, 120000));
      r = await attempt();
    }
    if (r.res.status === 404 || r.res.status === 403 || r.res.status === 410) {
      await sleep(30000);
      r = await attempt();
    }
    if (r.res.status === 429 || r.res.status === 503) {
      return { outcome: 'throttled', r, throttled, trace };
    }
    if (!(r.res.status >= 200 && r.res.status < 300)) {
      return { outcome: 'fetch_failed', r, throttled, trace };
    }
    return { outcome: 'ok', r, throttled, trace };
  } catch (e) {
    if (e instanceof TimeoutError) {
      await sleep(30000);
      try {
        const r2 = await attempt();
        if (r2.res.status >= 200 && r2.res.status < 300) {
          return { outcome: 'ok', r: r2, throttled, trace };
        }
        return { outcome: 'fetch_failed', r: r2, throttled, trace };
      } catch {
        return { outcome: 'fetch_failed', r: null, throttled, trace };
      }
    }
    return { outcome: 'fetch_failed', r: null, throttled, trace, error: String(e && e.message) };
  }
}

// ------------------------------------------------------------------ row

const BLANK_INSTRUMENT = {
  has_exif: null, gps_ifd_present: null, gps_coord_tag_present: null,
  gps_coord_all_zero: null, gps_tag_count: null, make_present: null,
  make_is_consumer_device: null, model_present: null, serial_tag_present: null,
  datetime_original_present: null, software_present: null, icc_present: null,
  icc_desc_is_stock: null, xmp_present: null, mpf_index_present: null,
  jpeg_dqt_present: null, width: null, height: null, container: null,
  parse_error: null,
};

async function runUnit(unit, salt) {
  const started = Date.now();
  const base = {
    unit_key: unit.unit_key,
    unit_id: unit.unit_key,
    frame: unit.frame,
    file_id: unit.file_id,
    arm: unit.arm,
    kind: unit.kind,
    host: unit.host,
    accepted_order: unit.accepted_order,
    api_size: unit.api_size ?? null,
    subsample: unit.subsample,
    url_ref: 'sha256:' + crypto.createHash('sha256').update(salt + unit.url).digest('hex').slice(0, 16),
    title: unit.frame === 'W' ? unit.title : null,
    fetched_at: new Date().toISOString(),
    range_header: unit.range,
    http_status: null,
    content_type: null,
    range_honoured: null,
    bytes_on_wire: null,
    bytes_read: null,
    body_capped: false,
    sha256_head: null,
    read_complete: null,
    fetch_failed: false,
    throttled: false,
    attempts: 0,
    discarded: false,
    discard_reason: null,
    container_original: unit.container_original ?? null,
    container_display: unit.container_display ?? null,
    display_is_original: unit.display_is_original ?? null,
    display_page_number: unit.display_page_number ?? null,
    ...BLANK_INSTRUMENT,
    tail_disagrees: null,
    full_disagrees: null,
    py_pil_full_gps_ifd_present: null,
  };

  const { outcome, r, throttled, trace, error } = await fetchUnit(unit);
  base.attempts = trace.length;
  base.throttled = throttled;
  base.request_trace = trace.join(',');
  if (error) base.fetch_error = error;

  if (outcome !== 'ok') {
    base.http_status = r ? r.res.status : null;
    base.fetch_failed = outcome === 'fetch_failed';
    base.duration_ms = Date.now() - started;
    base.no_body = true;
    base.discard_reason = outcome === 'throttled' ? 'throttled_no_body' : 'fetch_failed';
    return base;
  }

  base.http_status = r.res.status;
  const ct = (r.res.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
  base.content_type = ct || null;
  base.range_honoured = r.res.status === 206 ? true : (r.res.status === 200 ? false : null);
  base.bytes_on_wire = r.total;
  base.body_capped = r.capped;

  const cr = r.res.headers.get('content-range');
  const m = cr ? /bytes\s+(\d+)-(\d+)\/(\d+|\*)/.exec(cr) : null;
  const totalSize = m && m[3] !== '*' ? Number(m[3]) : null;

  let buf = r.buf;
  if (unit.kind !== 'full' && buf.length > WINDOW) {
    buf = unit.range === 'bytes=-131072' ? buf.subarray(buf.length - WINDOW) : buf.subarray(0, WINDOW);
  }
  base.bytes_read = buf.length;
  base.sha256_head = crypto.createHash('sha256').update(buf).digest('hex');

  if (unit.kind === 'full') base.read_complete = !r.capped;
  else if (totalSize !== null) base.read_complete = totalSize <= WINDOW;
  else if (r.res.status === 200) base.read_complete = !r.capped && r.total <= WINDOW;
  else base.read_complete = null;

  if (!IMAGE_TYPES.has(ct)) {
    base.discarded = true;
    base.discard_reason = 'content_type';
    base.duration_ms = Date.now() - started;
    return base;
  }
  if (!r.capped && (r.res.status === 200 || r.res.status === 206)) {
    Object.assign(base, unit.kind === 'full' || unit.range !== 'bytes=-131072'
      ? parseHead(buf) : parseTail(buf, unit.hint));
  } else {
    base.parse_error = 'body_capped';
  }
  base.duration_ms = Date.now() - started;
  return base;
}

// ------------------------------------------------------------------ runner

function loadDone() {
  const done = new Map();
  if (!fs.existsSync(ROWS)) return done;
  const txt = fs.readFileSync(ROWS, 'utf8');
  for (const line of txt.split('\n')) {
    if (!line.trim()) continue;
    try {
      const o = JSON.parse(line);
      if (o && o.unit_key) done.set(o.unit_key, o);
    } catch { /* partial trailing line */ }
  }
  return done;
}

async function main() {
  fs.mkdirSync(OUT, { recursive: true });
  const salt = loadSalt();
  const { units, records } = buildPlan();
  const done = loadDone();
  const todo = units.filter((u) => !done.has(u.unit_key));

  console.log(`plan ${units.length} units, ${done.size} rows already present, ${todo.length} to do`);
  if (process.argv.includes('--plan')) {
    const byHost = {};
    for (const u of todo) byHost[u.host] = (byHost[u.host] || 0) + 1;
    const byArm = {};
    for (const u of todo) byArm[`${u.frame}|${u.arm}|${u.kind}`] = (byArm[`${u.frame}|${u.arm}|${u.kind}`] || 0) + 1;
    console.log(JSON.stringify({ byArm, byHostTop: Object.entries(byHost).sort((a, b) => b[1] - a[1]).slice(0, 8), hosts: Object.keys(byHost).length }, null, 2));
    return;
  }
  if (todo.length === 0) {
    await summarize();
    return;
  }

  const start = Date.now();
  const queue = todo.slice();
  const hostNext = new Map();
  const inflight = new Map();
  let written = 0;
  const sink = fs.createWriteStream(ROWS, { flags: 'a' });

  const emit = (row) => {
    sink.write(JSON.stringify(row) + '\n');
    written++;
    if (written % 25 === 0) {
      console.log(`  ${written} rows, ${queue.length} queued, ${Math.round((Date.now() - start) / 1000)}s`);
    }
  };

  while (queue.length > 0) {
    if (Date.now() - start > BUDGET_MS) {
      console.log('budget reached, exiting cleanly');
      break;
    }
    let dispatched = false;
    for (let i = 0; i < queue.length; i++) {
      if (inflight.size >= MAX_GLOBAL) break;
      const u = queue[i];
      if (inflight.has(u.host)) continue;
      const now = Date.now();
      if (now < (hostNext.get(u.host) || 0)) continue;
      queue.splice(i, 1);
      i -= 1;
      hostNext.set(u.host, now + 1000);
      dispatched = true;
      const p = runUnit(u, salt)
        .then((row) => emit(row))
        .catch((e) => emit({ unit_key: u.unit_key, unit_id: u.unit_key, frame: u.frame, file_id: u.file_id, arm: u.arm, kind: u.kind, host: u.host, driver_error: String(e && e.message) }))
        .finally(() => inflight.delete(u.host));
      inflight.set(u.host, p);
    }
    if (!dispatched) await sleep(50);
    else await sleep(15);
  }
  if (inflight.size) await Promise.allSettled([...inflight.values()]);
  await new Promise((res) => sink.end(res));
  console.log(`wrote ${written} rows this invocation`);
  if (queue.length === 0) await summarize();
  else console.log(`${queue.length} units still pending; run again to resume`);
}

// ------------------------------------------------------------- statistics

function wilson(k, n) {
  if (n === 0) return null;
  const z = 1.959963984540054;
  const p = k / n;
  const denom = 1 + (z * z) / n;
  const centre = (p + (z * z) / (2 * n)) / denom;
  const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denom;
  return [Math.max(0, centre - half), Math.min(1, centre + half)];
}

function isUsable(row) {
  return !row.fetch_failed && !row.throttled && !row.discarded
    && !row.no_body && row.parse_error == null && row.http_status >= 200 && row.http_status < 300;
}

function armSummary(rows, dupKeys) {
  const all = rows.filter((r) => r.kind === 'head');
  const fetchFailed = all.filter((r) => r.fetch_failed).length;
  const throttled = all.filter((r) => r.throttled).length;
  const discarded = all.filter((r) => r.discarded && !r.fetch_failed).length;
  const parseErrors = all.filter((r) => !r.fetch_failed && !r.throttled && !r.discarded
    && !r.no_body && r.parse_error != null).length;
  const dups = all.filter((r) => dupKeys.has(r.unit_key)).length;
  const usable = all.filter((r) => isUsable(r) && !dupKeys.has(r.unit_key));
  const numerator = usable.filter((r) => r.gps_coord_tag_present === true && r.gps_coord_all_zero === false).length;
  return {
    units: all.length,
    n: usable.length,
    numerator,
    share: usable.length ? numerator / usable.length : null,
    wilson_95: wilson(numerator, usable.length),
    fetch_failed: fetchFailed,
    throttled,
    discarded,
    parse_error: parseErrors,
    duplicate_sha_dropped: dups,
  };
}

async function summarize() {
  const rows = [];
  const txt = fs.readFileSync(ROWS, 'utf8');
  for (const line of txt.split('\n')) {
    if (!line.trim()) continue;
    try { rows.push(JSON.parse(line)); } catch { /* partial */ }
  }
  const byKey = new Map(rows.map((r) => [r.unit_key, r]));
  const records = fs.readFileSync(INPUT, 'utf8').trim().split('\n').map((l) => JSON.parse(l));

  const dupKeys = new Set();
  const seen = new Map();
  for (const r of rows) {
    if (!r.sha256_head || r.kind !== 'head') continue;
    const k = `${r.frame}|${r.arm}|${r.sha256_head}`;
    if (seen.has(k)) dupKeys.add(r.unit_key);
    else seen.set(k, r.unit_key);
  }

  const armsW = ['original', 'display', 'archive_oldest', 'display_w200', 'display_w1200', 'tail'];
  const armsS = ['s_full', 's_thumb', 's_remote'];
  const frameW = { arms: {}, tail_validation: {}, full_validation: {}, paired: {} };
  const frameS = { arms: {}, full_validation: {} };
  for (const a of armsW) frameW.arms[a] = armSummary(rows.filter((r) => r.frame === 'W' && r.arm === a), dupKeys);
  for (const a of armsS) frameS.arms[a] = armSummary(rows.filter((r) => r.frame === 'S' && r.arm === a), dupKeys);
  frameW.display_is_original_pairs = rows.filter((r) => r.frame === 'W' && r.kind === 'head'
    && ['original', 'display', 'display_w200', 'display_w1200'].includes(r.arm)
    && r.display_is_original === true).length;

  // tail / full validation
  const pairs = (headArm, tailArm) => {
    let compared = 0, disagree = 0, unionAdds = 0;
    const superseded = [];
    for (const r of rows) {
      if (r.arm !== headArm || r.kind !== 'head') continue;
      const t = byKey.get(`${r.file_id}|${tailArm}|head`);
      if (!t || t.parse_error != null || t.fetch_failed || t.discarded) continue;
      if (r.fetch_failed || r.discarded) continue;
      compared++;
      if (r.parse_error != null || r.gps_ifd_present !== t.gps_ifd_present) {
        disagree++;
        if (t.gps_ifd_present === true && r.gps_ifd_present !== true) {
          unionAdds++;
          superseded.push({ file_id: r.file_id, head_gps_ifd: r.gps_ifd_present, head_parse_error: r.parse_error });
        }
      }
    }
    return {
      compared,
      disagree,
      rate: compared ? disagree / compared : null,
      union_adds: unionAdds,
      union_rate: compared ? unionAdds / compared : null,
      union_units: superseded,
      note: 'rate follows 2.7 literally: any inequality between head and tail counts, including tail=false where the metadata sits at the front of the file. union_rate counts only the one-directional case the union rule of 2.7 names.',
    };
  };

  const tailW = pairs('original', 'tail');
  frameW.tail_validation = tailW;
  // tail_disagrees / full_disagrees annotation
  const annotate = [];
  for (const r of rows) {
    const copy = { ...r };
    if (r.kind === 'head' && r.arm === 'original') {
      const t = byKey.get(`${r.file_id}|tail|head`);
      if (t) copy.tail_disagrees = (r.gps_ifd_present !== t.gps_ifd_present);
      const f = byKey.get(`${r.file_id}|original|full`);
      if (f) copy.full_disagrees = (r.gps_ifd_present !== f.gps_ifd_present);
    }
    if (r.kind === 'head' && r.arm === 's_full') {
      const f = byKey.get(`${r.file_id}|s_full|full`);
      if (f) copy.full_disagrees = (r.gps_ifd_present !== f.gps_ifd_present);
    }
    annotate.push(copy);
  }
  fs.writeFileSync(path.join(OUT, 'rows-annotated.jsonl'), annotate.map((r) => JSON.stringify(r)).join('\n') + '\n');

  const fullStats = (arm, frame) => {
    let compared = 0, disagree = 0, excluded = 0;
    for (const r of rows) {
      if (r.frame !== frame || r.arm !== arm || r.kind !== 'head') continue;
      const f = byKey.get(`${r.file_id}|${arm}|full`);
      if (!f) { if (r.subsample !== false) excluded++; continue; }
      if (r.fetch_failed || r.discarded || f.fetch_failed || f.discarded) continue;
      compared++;
      if (r.gps_ifd_present !== f.gps_ifd_present || r.parse_error != null) disagree++;
    }
    return { compared, disagree, rate: compared ? disagree / compared : null };
  };
  frameW.full_validation = fullStats('original', 'W');
  frameW.full_validation.excluded_over_20mb = records.filter((r) => r.frame === 'W'
    && r.subsample && r.api_size != null && r.api_size > 20000000).length;
  frameS.full_validation = fullStats('s_full', 'S');

  // paired frame-W difference
  const pairStat = (subset) => {
    const eligible = [];
    for (const r of rows) {
      if (r.frame !== 'W' || r.arm !== 'display' || r.kind !== 'head') continue;
      const o = byKey.get(`${r.file_id}|original|head`);
      if (!o || !isUsable(o) || !isUsable(r)) continue;
      if (o.display_is_original === true || r.display_is_original === true) continue;
      if (subset === 'restricted' && !(o.read_complete === true && r.read_complete === true)) continue;
      if (subset === 'same_container' && !(o.container_original === r.container_display)) continue;
      if (subset === 'cross_container' && !(o.container_original !== r.container_display)) continue;
      const yn = (x) => (x.gps_coord_tag_present === true && x.gps_coord_all_zero === false) ? 1 : 0;
      eligible.push({ o: yn(o), d: yn(r) });
    }
    const n = eligible.length;
    if (n === 0) return { n: 0 };
    const so = eligible.reduce((a, x) => a + x.o, 0) / n;
    const sd = eligible.reduce((a, x) => a + x.d, 0) / n;
    const diffs = eligible.map((x) => x.o - x.d);
    const mean = diffs.reduce((a, b) => a + b, 0) / n;
    let se = null;
    if (n > 1) {
      const varr = diffs.reduce((a, b) => a + (b - mean) * (b - mean), 0) / (n - 1);
      se = Math.sqrt(varr / n);
    }
    return {
      n,
      share_original: so,
      share_display: sd,
      original_minus_display: mean,
      difference_ci95: se === null ? null : [mean - 1.96 * se, mean + 1.96 * se],
      discordant: diffs.filter((d) => d !== 0).length,
    };
  };
  frameW.paired = {
    all_eligible: pairStat('all'),
    restricted_under_131072: pairStat('restricted'),
    same_container: pairStat('same_container'),
    cross_container: pairStat('cross_container'),
  };

  // full-file instrument cross-check for the 10% subsample, per format
  const formatTable = {};
  for (const r of rows) {
    if (r.kind !== 'head' || !r.subsample) continue;
    const k = `${r.frame}|${r.container || 'unparsed'}`;
    formatTable[k] = (formatTable[k] || 0) + 1;
  }

  const prereg = JSON.parse(fs.readFileSync(META, 'utf8'));
  const calibPath = path.join(OUT, 'calibration.json');
  const calibration = fs.existsSync(calibPath) ? JSON.parse(fs.readFileSync(calibPath, 'utf8')) : null;

  const all = frameW.paired.all_eligible;
  const restricted = frameW.paired.restricted_under_131072;
  const opposite = all.n > 0 && restricted.n >= 20
    && Math.sign(all.original_minus_display) !== Math.sign(restricted.original_minus_display);

  const dataset = {
    run: 'viallo-metadata-survival replication',
    generated_at: new Date().toISOString(),
    generator: prereg.commons_generator,
    instance_version: prereg.instance_version,
    instance: prereg.instance,
    instrument_calibration: calibration,
    window_bytes: WINDOW,
    rows_total: rows.length,
    frame_w: frameW,
    frame_s: frameS,
    validation_subsample_format_mix: formatTable,
    paired_difference_published: opposite ? 'restricted_under_131072' : 'all_eligible',
    paired_directions_disagree: opposite,
    head_share_is_lower_bound: (tailW.rate !== null && tailW.rate > 0.05)
      || (frameW.full_validation.rate !== null && frameW.full_validation.rate > 0.05),
    frame_s_note: 'distinct accounts unavailable: the accepted file list carries no account field; the frame-S interval is over files',
    notes: [
      'frame-S rows carry no URL and no account; url_ref is a salted SHA-256 prefix',
      'the instrument reports presence only: no coordinate, serial number, make string or ICC description is written',
    ],
  };
  fs.mkdirSync(OUT, { recursive: true });
  fs.writeFileSync(DATASET, JSON.stringify(dataset, null, 2) + '\n');

  const files = records.map((rec) => {
    const head = byKey.get(`${rec.file_id}|${rec.frame === 'W' ? 'original' : 's_full'}|head`);
    return {
      frame: rec.frame,
      file_id: rec.file_id,
      accepted_order: rec.accepted_order,
      subsample: Boolean(rec.subsample),
      title: rec.title ?? null,
      page_url: rec.frame === 'W' ? 'https://commons.wikimedia.org/wiki/' + encodeURIComponent(rec.title) : null,
      container_original: head ? head.container_original : null,
      container_display: head ? head.container_display : null,
      display_is_original: head ? head.display_is_original : null,
      display_page_number: head ? head.display_page_number : null,
      archived_version_count: null,
      location_on_page: null,
      provenance_class: null,
      make_is_consumer_device: head ? head.make_is_consumer_device : null,
      size_bytes: rec.api_size,
      url_ref: head ? head.url_ref : null,
    };
  });
  fs.writeFileSync(FILES, files.map((f) => JSON.stringify(f)).join('\n') + '\n');
  console.log('dataset written to', DATASET);
  console.log('frame W original:', JSON.stringify(frameW.arms.original));
  console.log('frame W display :', JSON.stringify(frameW.arms.display));
  console.log('paired all      :', JSON.stringify(all));
  console.log('tail rate       :', tailW.rate, 'full rate W:', frameW.full_validation.rate);
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});