vyndr/tests/unit/publicationLineage.test.js

'use strict';

/**
 * PUBLICATION LINEAGE — the semantic gates.
 *
 * The central finding these lock down: a model_snapshots row is NOT a published
 * Read. `gradeSlateService` fires its retention hook with BOTH sides, graded AND
 * refused, before any filtering, and only the higher-confidence graded side
 * becomes the served Read. Measured on mlb 2026-08-26: 8,443 of 13,012 captured
 * rows (64.9%) describe a state no user was ever shown.
 */

const path = require('path');
const fs = require('fs');
const L = require('../../src/services/read/readLineage');
const retention = require('../../src/services/retentionService');

const ROOT = path.resolve(__dirname, '..', '..');

const CLAIM = {
  sport: 'mlb', game_date: '2026-08-27', player_key: 'aaron judge',
  stat: 'hits', side: 'over', line: 0.5,
  game_id: 'mlb:2026-08-27:BostonRedSox@NewYorkYankees',
  captured_at: '2026-08-27T14:00:00Z',
  grade: 'B', p_win: 0.61, confidence: 61, projection: 1.2,
  over_odds: 120, under_odds: -145, book: 'dk', ev_pct: 3.2, takeable: true,
  published: true,
};

const persisted = (id, claim, res) => ({
  id, read_id: res.read_id, read_natural_key: res.read_natural_key,
  game_id: claim.game_id, claim_digest: res.claim_digest,
  revision_ordinal: res.revision_ordinal, lineage_action: res.action,
  supersedes_id: res.supersedes_id ?? null, captured_at: claim.captured_at,
  claim,
});
const mint = (v) => () => v;

describe('PUBLICATION MAPPING — a capture is not a publication', () => {
  test('an unmarked capture is refused as not_published', () => {
    const r = L.resolveLineage({ candidate: { ...CLAIM, published: false }, existing: [] });
    expect(r.ok).toBe(false);
    expect(r.refused).toBe('not_published');
  });

  test('publication is opt-IN — an absent marker is not a publication', () => {
    const noMarker = { ...CLAIM };
    delete noMarker.published;
    expect(L.resolveLineage({ candidate: noMarker, existing: [] }).refused).toBe('not_published');
  });

  test('the collector marks ONLY the side that became the served Read', () => {
    const c = retention.createCollector({
      snapshotId: 's1', capturedAt: '2026-08-27T14:00:00Z',
      gameDate: '2026-08-27', gameIdFor: () => 'mlb:2026-08-27:BOS@NYY',
    });
    const base = { player: 'Aaron Judge', stat_type: 'hits', line: 0.5, sport: 'mlb', book: 'dk' };
    const over = { ...base, direction: 'over', grade: 'B', confidence: 61, p_win: 0.61 };
    const under = { ...base, direction: 'under', grade: 'C', confidence: 39, p_win: 0.39 };

    c.onGraded(base, [over, under]);
    expect(c.rows).toHaveLength(2);
    expect(c.rows.filter((r) => r.published)).toHaveLength(0);

    c.onPublished(base, over);
    expect(c.rows.filter((r) => r.published).map((r) => r.side)).toEqual(['over']);
    expect(c.rows.filter((r) => !r.published).map((r) => r.side)).toEqual(['under']);
  });

  test('every collected row DECLARES published, defaulting false', () => {
    const c = retention.createCollector({
      snapshotId: 's1', capturedAt: '2026-08-27T14:00:00Z',
      gameDate: '2026-08-27', gameIdFor: () => 'g',
    });
    c.onGraded({ player: 'X', stat_type: 'hits', line: 0.5, sport: 'mlb' },
      [{ player: 'X', stat_type: 'hits', line: 0.5, direction: 'over', grade: 'B' }]);
    for (const r of c.rows) {
      expect('published' in r).toBe(true);
      expect(r.published).toBe(false);
    }
  });

  test('a refusal is never published even if the signal is misfired at it', () => {
    const c = retention.createCollector({
      snapshotId: 's1', capturedAt: '2026-08-27T14:00:00Z',
      gameDate: '2026-08-27', gameIdFor: () => 'g',
    });
    const base = { player: 'X', stat_type: 'hits', line: 0.5, sport: 'mlb' };
    const refused = { ...base, direction: 'over', insufficient_data: true };
    c.onGraded(base, [refused]);
    expect(c.rows[0].refused).toBe(true);
    // Even marked, a refusal cannot form a published claim: the resolver still
    // needs a claim, and the slate never served this.
    c.onPublished(base, refused);
    expect(c.rows[0].published).toBe(true); // the signal is literal…
    // …but the product never fires it for a refusal — the winner is chosen from
    // `cands`, which excludes refusals. Locked by the source assertion below.
    const src = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8');
    expect(src).toMatch(/const cands = sides\.filter\(\(s\) => s && s\.grade && !s\.insufficient_data\)/);
    expect(src.indexOf('onPublished')).toBeGreaterThan(src.indexOf('const winner'));
  });
});

describe('CHANGE SEMANTICS — belief, market and comparison are distinct', () => {
  const base = { line: 0.5, side: 'over', book: 'dk', over_odds: 120, under_odds: -145, p_win: 0.61, grade: 'B', ev_pct: 3.2, takeable: true };
  const t = (patch) => L.classifyChange(base, { ...base, ...patch });

  test.each([
    ['price only', { over_odds: 180, ev_pct: 5.1 }, 'MARKET_REPRICE', false, true],
    ['under price only', { under_odds: -200, ev_pct: 4.0 }, 'MARKET_REPRICE', false, true],
    ['line change', { line: 1.5, ev_pct: 1.0 }, 'MARKET_LINE_CHANGE', false, true],
    ['book change', { book: 'fd' }, 'MARKET_REPRICE', false, true],
    ['probability', { p_win: 0.55 }, 'BELIEF_CHANGE', true, false],
    ['grade', { grade: 'C' }, 'BELIEF_CHANGE', true, false],
    ['projection', { projection: 2.1 }, 'BELIEF_CHANGE', true, false],
    ['comparison only', { ev_pct: 5.0 }, 'COMPARISON_CHANGE', false, false],
    ['takeable only', { takeable: false }, 'COMPARISON_CHANGE', false, false],
    ['belief + market', { p_win: 0.55, over_odds: 180 }, 'MIXED_CHANGE', true, true],
    ['provenance only', { confidence_basis: 'x' }, 'PROVENANCE_CHANGE', false, false],
    ['nothing', {}, 'NO_MATERIAL_PUBLISHED_CHANGE', false, false],
  ])('%s -> %s', (_l, patch, expected, belief, market) => {
    const r = t(patch);
    expect(r.change_type).toBe(expected);
    expect(r.belief_changed).toBe(belief);
    expect(r.market_changed).toBe(market);
  });

  test('a model version change with identical numbers is NOT a belief change', () => {
    // The output is what was published. A rebuild that produces the same number
    // did not change what the user was told.
    expect(t({}).change_type).toBe('NO_MATERIAL_PUBLISHED_CHANGE');
    expect(L.CLAIM_FIELD_CLASSES.model_version).toBeUndefined();
  });

  test('COMPARISON never masquerades as BELIEF', () => {
    // ev_pct moves whenever either belief or market moves; promoting it would
    // report every reprice as a changed opinion.
    for (const f of ['ev_pct', 'edge_pct', 'value', 'kelly', 'takeable']) {
      expect(L.CLAIM_FIELD_CLASSES[f]).toBe(L.FIELD_CLASS.COMPARISON);
    }
  });

  test('the resolver stamps change_type onto a real revision', () => {
    const r1 = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
    expect(r1.change_type).toBe(L.CHANGE_TYPE.INITIAL_PUBLICATION);
    const row1 = persisted(101, CLAIM, r1);
    const repriced = { ...CLAIM, over_odds: 180, captured_at: '2026-08-27T19:00:00Z' };
    const r2 = L.resolveLineage({ candidate: repriced, existing: [row1] });
    expect(r2.change_type).toBe(L.CHANGE_TYPE.MARKET_REPRICE);
    expect(r2.belief_changed).toBe(false);
  });
});

describe('CLAIM VERSIONING', () => {
  test('every resolution stamps the claim and digest versions', () => {
    const r = L.resolveLineage({ candidate: CLAIM, existing: [], mintReadId: mint('R1') });
    expect(r.claim_schema_version).toBe(L.CLAIM_SCHEMA_VERSION);
    expect(r.digest_algorithm_version).toBe(L.DIGEST_ALGORITHM_VERSION);
    expect(L.CLAIM_SCHEMA_VERSION).toMatch(/@\d+$/);
  });

  test('key ORDER cannot change a digest', () => {
    const a = { line: 0.5, side: 'over', p_win: 0.61, grade: 'B', over_odds: 120 };
    const b = { over_odds: 120, grade: 'B', p_win: 0.61, side: 'over', line: 0.5 };
    expect(L.claimDigest(a)).toBe(L.claimDigest(b));
  });

  test('BOTH price spellings are in the claim, so neither store is blind', () => {
    for (const f of ['locked_odds', 'over_odds', 'under_odds']) {
      expect(L.CLAIM_MARKET_FIELDS).toContain(f);
    }
    const ms = { line: 0.5, side: 'over', over_odds: 120, under_odds: -145 };
    expect(L.claimDigest(ms)).not.toBe(L.claimDigest({ ...ms, over_odds: 180 }));
    expect(L.claimDigest(ms)).not.toBe(L.claimDigest({ ...ms, under_odds: -200 }));
    const led = { line: 0.5, side: 'over', locked_odds: 120 };
    expect(L.claimDigest(led)).not.toBe(L.claimDigest({ ...led, locked_odds: 180 }));
  });

  test('a post-hoc settlement field never moves the claim digest', () => {
    const base = L.claimDigest(CLAIM);
    for (const f of L.POST_HOC_FIELDS) {
      expect(L.claimDigest({ ...CLAIM, [f]: 'anything' })).toBe(base);
    }
  });

  test('an internal non-claim field never moves the digest', () => {
    const base = L.claimDigest(CLAIM);
    for (const f of ['snapshot_id', 'captured_at', 'code_sha', 'cycle_hour_utc', 'chain_shadow']) {
      expect(L.claimDigest({ ...CLAIM, [f]: 'zzz' })).toBe(base);
    }
  });

  test('a legacy row with no version stamp stays truthfully unknown', () => {
    const legacy = {
      id: 55, read_id: null, game_id: CLAIM.game_id, claim_digest: null,
      revision_ordinal: null, lineage_action: null, supersedes_id: null,
      captured_at: '2026-08-01T14:00:00Z',
    };
    const r = L.resolveLineage({ candidate: CLAIM, existing: [legacy], mintReadId: mint('R9') });
    expect(r.lineage_state).toBe(L.LINEAGE_STATE.LEGACY_UNVERIFIED);
  });
});

describe('SETTLEMENT SEPARATION', () => {
  test("the REAL settle patch is confined to evaluation fields", () => {
    const real = {
      outcome: 'hit', actual_value: 2, settled_at: '2026-08-28T02:00:00Z',
      settlement_source: 'statsapi_boxscore',
    };
    expect(L.settlementUpdateIsSafe(real)).toMatchObject({ safe: true, violations: [] });
  });

  test('a patch touching ANY claim-owned field is refused and names it', () => {
    for (const f of ['grade', 'p_win', 'line', 'over_odds', 'claim_digest', 'published', 'revision_ordinal']) {
      const r = L.settlementUpdateIsSafe({ outcome: 'hit', [f]: 'x' });
      expect(r.safe).toBe(false);
      expect(r.violations).toContain(f);
    }
  });

  test('an unrecognised key is reported, not assumed safe', () => {
    expect(L.settlementUpdateIsSafe({ outcome: 'hit', mystery: 1 }).unrecognised).toEqual(['mystery']);
  });

  test('the production settle path writes only evaluation fields', () => {
    const src = fs.readFileSync(path.join(ROOT, 'src/services/snapshotSettlementService.js'), 'utf8');
    const m = src.match(/\.update\(\{([\s\S]*?)\}\)/);
    expect(m).toBeTruthy();
    const keys = [...m[1].matchAll(/^\s*([a-z_0-9]+)\s*:/gm)].map((x) => x[1]);
    expect(keys.length).toBeGreaterThan(0);
    expect(L.settlementUpdateIsSafe(Object.fromEntries(keys.map((k) => [k, 1])))).toMatchObject({ safe: true });
  });
});

describe('DUAL-WRITE FAILURE CONTRACT', () => {
  const rows = () => ([{ ...CLAIM }, { ...CLAIM, side: 'under', published: false }]);

  test('legacy OK + lineage FAILS: rows persist, gap is measurable', async () => {
    const r = rows();
    const out = await retention.attachLineage(r, {
      fetchExisting: async () => { throw new Error('lineage db down'); },
    });
    expect(out.error).toBe('lineage db down');
    for (const row of r) expect(row.read_id).toBeNull();
    for (const row of r) for (const k of retention.LINEAGE_KEYS) expect(k in row).toBe(true);
  });

  test('a failed publication cannot produce lineage — the ordering forbids it', () => {
    // SUPERSEDED MECHANISM, SAME INVARIANT. Lineage used to resolve inside
    // `persist()`, which runs BEFORE the authoritative slate write — so it could
    // have recorded a publication for a slate Redis never served. Lineage now
    // resolves only in `commitPublication`, which is called after that write
    // succeeds, and only over rows already marked published.
    const ret = fs.readFileSync(path.join(ROOT, 'src/services/retentionService.js'), 'utf8');
    const persistBody = ret.slice(ret.indexOf('async function persist('));
    // Capture no longer resolves lineage at all.
    expect(persistBody).not.toMatch(/attachLineage\(/);
    // Exactly one table write inside persist, and it is model_snapshots.
    const writes = [...persistBody.matchAll(/\.from\('([a-z_]+)'\)/g)].map((m) => m[1]);
    expect(new Set(writes)).toEqual(new Set(['model_snapshots']));

    // And the commit is downstream of the authoritative write.
    const snap = fs.readFileSync(path.join(ROOT, 'src/services/snapshotService.js'), 'utf8');
    expect(snap.indexOf('cacheSet(`snapshot:${sp}:latest`'))
      .toBeLessThan(snap.indexOf('commitPublication({'));
  });

  test('an unpublished capture is counted separately from a failure', async () => {
    const r = rows();
    const out = await retention.attachLineage(r, { fetchExisting: async () => [], mintReadId: mint('R1') });
    expect(out.not_published).toBe(1);
    expect(out.refused).toBe(0);
    expect(out.origins).toBe(1);
  });

  test('no database configured leaves NULL lineage without throwing', async () => {
    const r = rows();
    const out = await retention.attachLineage(r, { fetchExisting: async () => null });
    expect(out.error).toBeNull();
    for (const row of r) expect(row.read_id).toBeNull();
  });

  test('malformed identity is counted as unresolved, not published', async () => {
    const bad = [{ ...CLAIM, player_key: null }];
    const out = await retention.attachLineage(bad, { fetchExisting: async () => [] });
    expect(out.unresolved).toBe(1);
    expect(out.origins).toBe(0);
  });

  test('change types are counted for telemetry', async () => {
    const r = [
      { ...CLAIM },
      { ...CLAIM, captured_at: '2026-08-27T19:00:00Z', over_odds: 180 },
      { ...CLAIM, captured_at: '2026-08-27T22:00:00Z', over_odds: 180, p_win: 0.55, grade: 'C' },
    ];
    const out = await retention.attachLineage(r, { fetchExisting: async () => [], mintReadId: mint('R1') });
    expect(out.change_types.INITIAL_PUBLICATION).toBe(1);
    expect(out.change_types.MARKET_REPRICE).toBe(1);
    expect(out.change_types.BELIEF_CHANGE).toBe(1);
  });
});

describe('NON-AUTHORITY — lineage cannot serve the product', () => {
  const walk = (dir, out = []) => {
    for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
      if (e.name === 'node_modules' || e.name.startsWith('.')) continue;
      const full = path.join(dir, e.name);
      if (e.isDirectory()) walk(full, out);
      else if (/\.(js|jsx|ts|tsx)$/.test(e.name)) out.push(full);
    }
    return out;
  };

  test('no product path imports the lineage resolver', () => {
    const allowed = new Set([
      path.join(ROOT, 'src/services/read/readLineage.js'),
      path.join(ROOT, 'src/services/retentionService.js'), // the dual-writer
      // ADDITIVE, NON-AUTHORITATIVE READERS (lineage productization).
      // The property this test protects is "no path that serves PRODUCT truth
      // reads lineage" — not "nothing reads lineage", which was only ever true
      // because nothing had been built yet. These two read it to MEASURE it and
      // to expose ancestry on a protected internal route; neither is consumed
      // by any product surface, which the assertions below now police directly.
      path.join(ROOT, 'src/services/lineageCoverage.js'),
      path.join(ROOT, 'src/services/read/readAncestry.js'),
    ]);
    const files = [
      ...walk(path.join(ROOT, 'src')),
      ...(fs.existsSync(path.join(ROOT, 'web/src')) ? walk(path.join(ROOT, 'web/src')) : []),
    ].filter((f) => !allowed.has(f));
    const offenders = files.filter((f) => {
      const src = fs.readFileSync(f, 'utf8')
        .replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:])\/\/.*$/gm, '$1');
      return /(require\(|from\s+)['"`][^'"`]*readLineage['"`]/.test(src);
    }).map((f) => path.relative(ROOT, f));
    expect(offenders).toEqual([]);

    // STRONGER THAN BEFORE AT THE LAYERS THAT MATTER.
    // web/src must contain no reference at all, and among the routes exactly
    // ONE — the protected internal router — may reach lineage. Previously no
    // route could, so this keeps the boundary explicit rather than widening it.
    const webDir = path.join(ROOT, 'web/src');
    if (fs.existsSync(webDir)) {
      const webHits = walk(webDir).filter((f) => /readLineage|readAncestry|lineageCoverage/
        .test(fs.readFileSync(f, 'utf8'))).map((f) => path.relative(ROOT, f));
      expect(webHits).toEqual([]);
    }
    const routeHits = walk(path.join(ROOT, 'src/routes'))
      .filter((f) => /readLineage|readAncestry|lineageCoverage/.test(fs.readFileSync(f, 'utf8')))
      .map((f) => path.basename(f));
    expect(routeHits).toEqual(['internal.js']);
  });

  test('no route or web file reads a publication-semantics column', () => {
    const cols = ['claim_digest', 'revision_ordinal', 'lineage_action', 'change_type',
      'claim_schema_version', 'read_natural_key'];
    const roots = [path.join(ROOT, 'src/routes')];
    if (fs.existsSync(path.join(ROOT, 'web/src'))) roots.push(path.join(ROOT, 'web/src'));
    const hits = [];
    for (const r of roots) {
      for (const f of walk(r)) {
        const src = fs.readFileSync(f, 'utf8');
        for (const c of cols) if (src.includes(c)) hits.push(`${path.relative(ROOT, f)} -> ${c}`);
      }
    }
    expect(hits).toEqual([]);
  });

  test('the served envelope comes from the cache, not from model_snapshots', () => {
    const src = fs.readFileSync(path.join(ROOT, 'src/routes/snapshot.js'), 'utf8');
    expect(src).toMatch(/cacheGet/);
    expect(src).not.toMatch(/model_snapshots/);
  });
});

describe('EVENT IDENTITY — doubleheader and same-day occurrence', () => {
  const g = { ...CLAIM };

  test('TODAY identity is DERIVED_TEAM_PAIR and says so', () => {
    expect(L.identityMethod(g)).toBe(L.IDENTITY_METHOD.DERIVED_TEAM_PAIR);
    expect(L.eventOccurrence(g)).toBeNull();
    const r = L.resolveLineage({ candidate: g, existing: [], mintReadId: mint('R1') });
    expect(r.identity_method).toBe(L.IDENTITY_METHOD.DERIVED_TEAM_PAIR);
    expect(r.identity_version).toBe(L.IDENTITY_VERSION);
  });

  test('a canonical occurrence SEPARATES two games of a doubleheader', () => {
    const g1 = { ...g, game_pk: 776001 };
    const g2 = { ...g, game_pk: 776002 };
    expect(L.readNaturalKey(g1)).not.toBe(L.readNaturalKey(g2));
    const r1 = L.resolveLineage({ candidate: g1, existing: [], mintReadId: mint('RA') });
    const row1 = persisted(101, g1, r1);
    row1.claim = g1;
    const r2 = L.resolveLineage({ candidate: g2, existing: [row1], mintReadId: mint('RB') });
    expect(r2.read_id).toBe('RB');
    expect(r2.action).toBe(L.LINEAGE_ACTION.ORIGIN);
    expect(r2.identity_method).toBe(L.IDENTITY_METHOD.CANONICAL_EVENT_ID);
  });

  test('a canonical occurrence OVERRIDES the team-name heuristic entirely', () => {
    // Same teams, same spelling, different real game: the heuristic would merge
    // these; the occurrence must win.
    const g1 = { ...g, game_pk: 1 };
    const g2 = { ...g, game_pk: 2 };
    expect(L.sameEvent(g1.game_id, g2.game_id)).toBe(true); // heuristic says same
    const r1 = L.resolveLineage({ candidate: g1, existing: [], mintReadId: mint('RA') });
    const row1 = { ...persisted(101, g1, r1), claim: g1 };
    const r2 = L.resolveLineage({ candidate: g2, existing: [row1], mintReadId: mint('RB') });
    expect(r2.read_id).toBe('RB'); // occurrence overrules it
  });

  test('KNOWN LIMIT: without an occurrence a doubleheader is indistinguishable', () => {
    // `ledgerService.gameIdFor` emits sport:date:away@home with NO game number,
    // so both halves produce a byte-identical id and NO rule over that string
    // can separate them. Asserted so the limit is visible, not implied.
    const gameIdFor = require('../../src/services/ledgerService').__internals.gameIdFor;
    const prop = { away_team: 'Detroit Tigers', home_team: 'Kansas City Royals' };
    expect(gameIdFor('mlb', prop, '2026-08-27')).toBe(gameIdFor('mlb', prop, '2026-08-27'));
    expect(L.readNaturalKey(g)).toBe(L.readNaturalKey({ ...g }));
  });

  test('the dedupe key now carries the EVENT — the data-loss defect is closed', () => {
    // PREVIOUSLY: `player::stat_type::line` with no event component, so the
    // second half of a doubleheader was DROPPED before grading and the merge
    // was avoided only by losing the game. Now the event leads the key, so both
    // real games survive. Behavioural proof lives in eventPublication.test.js.
    const src = fs.readFileSync(path.join(ROOT, 'src/services/gradeSlateService.js'), 'utf8');
    expect(src).toMatch(/const key = `\$\{propositionEventKey\(p\)\}::/);
    expect(src).not.toMatch(/const key = `\$\{p\.player\}::\$\{p\.stat_type\}::\$\{p\.line\}`/);
  });

  test('different participants, stats, lines and sides never share a Read', () => {
    const base = L.readNaturalKey(g);
    for (const patch of [{ player_key: 'other' }, { stat: 'runs' }, { line: 1.5 }, { side: 'under' }, { sport: 'wnba' }, { game_date: '2026-08-28' }]) {
      expect(L.readNaturalKey({ ...g, ...patch })).not.toBe(base);
    }
  });
});