#!/usr/bin/env node
// Turns the measured rows into the published dataset. Every number here is derived from
// data/units.jsonl and data/files.jsonl; nothing is typed in by hand.
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

const RUN_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '..');
const DIR = resolve(RUN_DIR, 'data');
const OUT = resolve(RUN_DIR, 'DATASET.json');
const CSV = resolve(RUN_DIR, 'DATASET.csv');

const readJsonl = (p) => existsSync(p)
  ? readFileSync(p, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l))
  : [];
const readJson = (p) => existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null;

export function wilson(k, n, z = 1.959964) {
  if (!n) return [null, null];
  const p = k / n;
  const d = 1 + (z * z) / n;
  const c = p + (z * z) / (2 * n);
  const m = z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n));
  return [Math.max(0, (c - m) / d), Math.min(1, (c + m) / d)];
}

const round = (x) => (x === null || x === undefined ? null : Number(x.toFixed(4)));

// The numerator fixed in the pre-registration: a coordinate tag that is present and not all zero.
const carries = (r) => r.gps_coord_tag_present === true && r.gps_coord_all_zero === false;
const determined = (r) => r.gps_coord_tag_present !== null && r.gps_coord_tag_present !== undefined;

export function share(rows) {
  const d = rows.filter(determined);
  const k = d.filter(carries).length;
  const [lo, hi] = wilson(k, d.length);
  return {
    n_units: rows.length,
    n_determined: d.length,
    n_unmeasured: rows.length - d.length,
    n_carrying: k,
    share: d.length ? round(k / d.length) : null,
    ci95: [round(lo), round(hi)],
  };
}

const units = readJsonl(`${DIR}/units.jsonl`);
const files = readJsonl(`${DIR}/files.jsonl`);
const frames = readJson(`${DIR}/frames.json`);
const calibration = readJson(`${DIR}/calibration.json`);
const validation = readJson(`${DIR}/validation.json`);

const heads = units.filter((u) => u.kind === 'head');
const fileById = new Map(files.map((f) => [f.file_id, f]));
const byKey = new Map(heads.map((u) => [`${u.file_id}|${u.arm}`, u]));

const ARMS = ['original', 'display', 'display_w200', 'display_w1200', 'archive_oldest',
  's_full', 's_thumb', 's_remote'];
const perArm = {};
for (const arm of ARMS) {
  const rows = heads.filter((u) => u.arm === arm);
  if (!rows.length) continue;
  perArm[arm] = {
    ...share(rows),
    frame: rows[0].frame,
    carrying_make: rows.filter((r) => r.make_is_consumer_device === true).length,
    determined_make: rows.filter((r) => r.make_is_consumer_device !== null
      && r.make_is_consumer_device !== undefined).length,
    carrying_make_share: (() => {
      const d = rows.filter((r) => r.make_is_consumer_device !== null
        && r.make_is_consumer_device !== undefined);
      return d.length
        ? round(d.filter((r) => r.make_is_consumer_device === true).length / d.length) : null;
    })(),
    model_present: rows.filter((r) => r.model_present === true).length,
    serial_present: rows.filter((r) => r.serial_tag_present === true).length,
    has_exif: rows.filter((r) => r.has_exif === true).length,
    parse_errors: rows.filter((r) => r.parse_error !== null).length,
    by_container: rows.reduce((a, r) => {
      const c = r.container || 'unknown';
      a[c] = a[c] || { n: 0, carrying: 0 };
      a[c].n++;
      if (determined(r) && carries(r)) a[c].carrying++;
      return a;
    }, {}),
  };
}

// Frame S repeats: one account can contribute many attachments, so the unit-level share and the
// account-level share are both reported and the interval is taken over distinct accounts.
const sRows = heads.filter((u) => u.arm === 's_full');
const accounts = new Map();
for (const r of sRows) {
  const f = fileById.get(r.file_id);
  const a = (f && f.account_hash) || 'unknown';
  if (!accounts.has(a)) accounts.set(a, []);
  accounts.get(a).push(r);
}
const accountShares = [...accounts.values()].filter((rows) => rows.some(determined))
  .map((rows) => (rows.filter((r) => determined(r) && carries(r)).length ? 1 : 0));
const acctMean = accountShares.length
  ? accountShares.reduce((a, b) => a + b, 0) / accountShares.length : null;
const [alo, ahi] = wilson(accountShares.reduce((a, b) => a + b, 0), accountShares.length);

const frameS = {
  ...share(sRows),
  distinct_accounts: accounts.size,
  accounts_with_any_carrying: accountShares.reduce((a, b) => a + b, 0),
  share_over_accounts: round(acctMean),
  ci95_over_accounts: [round(alo), round(ahi)],
};

// The paired comparison: the same file's original against the derivative offered for display.
const pairs = [];
for (const f of files) {
  if (f.frame !== 'W') continue;
  const o = byKey.get(`${f.file_id}|original`);
  const d = byKey.get(`${f.file_id}|display`);
  const f2 = byKey.get(`${f.file_id}|full`);
  if (!o || !d || !determined(o) || !determined(d)) continue;
  pairs.push({
    file_id: f.file_id,
    display_is_original: d.display_is_original === true,
    same_container: o.container === d.container,
    original_carries: carries(o),
    original_full_file_carries: f2 && determined(f2) ? carries(f2) : null,
    display_carries: carries(d),
  });
}
const paired = (rows) => {
  const both = rows.filter((p) => !p.display_is_original);
  const o = both.filter((p) => p.original_carries).length;
  const d = both.filter((p) => p.display_carries).length;
  const diff = both.length ? (o - d) / both.length : null;
  const [lo, hi] = wilson(Math.round(diff * both.length), both.length);
  return {
    n_pairs: rows.length,
    n_excluded_display_is_original: rows.length - both.length,
    n_compared: both.length,
    original_carrying: o,
    display_carrying: d,
    difference: round(diff),
    ci95: [round(lo), round(hi)],
  };
};
const pairedAll = paired(pairs);
const pairedSame = paired(pairs.filter((p) => p.same_container));
const pairedCross = paired(pairs.filter((p) => !p.same_container));

const out = {
  run_id: '2026-09-10-viallo-metadata-survival',
  product: 'viallo',
  tier: 'micro',
  generated_at: new Date().toISOString(),
  frames: {
    W: {
      source: 'Wikimedia Commons, namespace 6, list=random',
      generator: frames.w_generator,
      captured_at: frames.w_captured_at,
      files_accepted: files.filter((f) => f.frame === 'W').length,
      files_with_archived_version: files.filter((f) => f.frame === 'W' && f.archived_url).length,
    },
    S: {
      source: `${frames.s_instance} public timeline, only_media=true, 8 sequential max_id pages`,
      instance: frames.s_instance,
      instance_version: frames.s_version,
      captured_at: frames.s_captured_at,
      instances_refused: frames.instances_refused,
      files_accepted: files.filter((f) => f.frame === 'S').length,
    },
  },
  instrument_calibration: calibration,
  validation_head_tail_full: validation,
  per_arm: perArm,
  frame_S_account_clustered: frameS,
  paired_W_original_vs_display: {
    all: pairedAll,
    same_container: pairedSame,
    cross_container: pairedCross,
  },
  counts: {
    unit_rows: units.length,
    file_rows: files.length,
    head_rows: heads.length,
    tail_rows: units.filter((u) => u.kind === 'tail').length,
    full_rows: units.filter((u) => u.kind === 'full').length,
    fetch_failures: units.filter((u) => u.fetch_failed).length,
    throttled_rows: units.filter((u) => u.throttled).length,
  },
};
writeFileSync(OUT, JSON.stringify(out, null, 2) + '\n');

const cols = ['unit_key', 'file_id', 'frame', 'arm', 'kind', 'status', 'range_honoured',
  'bytes_on_wire', 'content_range_total', 'throttled', 'fetch_failed',
  'has_exif', 'gps_ifd_present', 'gps_coord_tag_present', 'gps_coord_all_zero', 'gps_tag_count',
  'make_present', 'make_is_consumer_device', 'model_present', 'serial_tag_present',
  'datetime_original_present', 'software_present', 'icc_present', 'icc_desc_is_stock',
  'xmp_present', 'mpf_index_present', 'jpeg_dqt_present', 'width', 'height', 'container',
  'parse_error', 'metadata_region_complete', 'tail_gps_ifd_present', 'display_is_original'];
const cell = (v) => (v === undefined || v === null ? '' : String(v).replace(/[",\n]/g, ' '));
writeFileSync(CSV, [cols.join(',')]
  .concat(units.map((u) => cols.map((c) => cell(u[c])).join(','))).join('\n') + '\n');

console.log(JSON.stringify({
  original: perArm.original,
  display: perArm.display,
  s_full: perArm.s_full,
  s_thumb: perArm.s_thumb,
  paired: pairedAll,
  frame_S_accounts: frameS.distinct_accounts,
}, null, 2));