vyndr/tests/unit/lowParamCalibrator.test.js

'use strict';

/**
 * The two-parameter correction that replaces isotonic on a thin sample.
 *
 * What these protect: that it CANNOT chase day-structure (two parameters over
 * the whole curve), and that a thin fit is applied at reduced strength rather
 * than at face value.
 */

const lp = require('../../src/services/model/lowParamCalibrator');

/** An over-confident forecaster: predicts p, actually hits closer to the mean. */
function overConfident(n, dates = 10, seed = 3) {
  let s = seed;
  const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648;
  const rows = [];
  for (let i = 0; i < n; i += 1) {
    const p = 0.5 + rnd() * 0.45;
    const truth = 0.5 + (p - 0.5) * 0.4;       // real skill is 40% of claimed
    rows.push({ date: `d${i % dates}`, p, won: rnd() < truth ? 1 : 0 });
  }
  return rows;
}

describe('it fits the favourite-longshot shape', () => {
  it('FLATTENS an over-confident forecaster (a < 1)', () => {
    const m = lp.fitPlatt(overConfident(2000));
    expect(m).not.toBeNull();
    expect(m.flattens).toBe(true);
    expect(m.a).toBeLessThan(1);
  });

  it('pulls high predictions down and leaves the middle nearly alone', () => {
    const m = lp.fitPlatt(overConfident(2000));
    const hi = lp.applyPlatt(m, 0.92);
    const mid = lp.applyPlatt(m, 0.55);
    expect(hi).toBeLessThan(0.92);
    expect(Math.abs(mid - 0.55)).toBeLessThan(Math.abs(hi - 0.92));
  });

  it('stays monotone — ordering is never disturbed', () => {
    const m = lp.fitPlatt(overConfident(2000));
    let prev = -1;
    for (let p = 0.05; p <= 0.95; p += 0.05) {
      const v = lp.applyPlatt(m, p);
      expect(v).toBeGreaterThan(prev);
      prev = v;
    }
  });

  it('leaves an already-honest forecaster essentially alone', () => {
    let s = 11;
    const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648;
    const rows = [];
    for (let i = 0; i < 2000; i += 1) {
      const p = 0.3 + rnd() * 0.6;
      rows.push({ date: `d${i % 12}`, p, won: rnd() < p ? 1 : 0 });
    }
    const m = lp.fitPlatt(rows);
    expect(Math.abs(lp.applyPlatt(m, 0.8) - 0.8)).toBeLessThan(0.06);
  });
});

describe('shrinkage — a thin fit is applied at reduced strength', () => {
  it('scales with the number of fit DATES, not rows', () => {
    const few = lp.fitPlatt(overConfident(2000, 5));
    const many = lp.fitPlatt(overConfident(2000, 40));
    expect(few.fit_rows).toBe(many.fit_rows);          // same rows
    expect(few.shrinkage).toBeLessThan(many.shrinkage); // different dates
    expect(few.shrinkage).toBeCloseTo(5 / 15, 3);
    expect(many.shrinkage).toBeCloseTo(40 / 50, 3);
  });

  it('a 5-date fit corrects less than a 40-date fit on the same input', () => {
    const few = lp.fitPlatt(overConfident(2000, 5));
    const many = lp.fitPlatt(overConfident(2000, 40));
    // Both flatten; the thin one is held closer to the raw number.
    expect(Math.abs(lp.applyPlatt(few, 0.92) - 0.92))
      .toBeLessThan(Math.abs(lp.applyPlatt(many, 0.92) - 0.92));
  });
});

describe('honesty', () => {
  it('refuses to fit below the row floor', () => {
    expect(lp.fitPlatt(overConfident(40))).toBeNull();
    expect(lp.fitPlatt([])).toBeNull();
    expect(lp.fitPlatt(null)).toBeNull();
  });

  it('an unreadable input returns null, never an uncorrected number', () => {
    const m = lp.fitPlatt(overConfident(2000));
    expect(lp.applyPlatt(m, null)).toBeNull();
    expect(lp.applyPlatt(null, 0.7)).toBeNull();
  });

  it('has exactly two parameters — it CANNOT encode a single odd day', () => {
    // This is the whole reason it replaces isotonic here.
    const m = lp.fitPlatt(overConfident(2000));
    const shape = Object.keys(m).filter((k) => k === 'a' || k === 'b');
    expect(shape.sort()).toEqual(['a', 'b']);
  });
});

describe('the slope must CORRECT, not abandon the forecast', () => {
  /** A forecaster whose p_win carries no information at all. */
  function uninformative(n, seed = 7) {
    let s = seed;
    const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648;
    const rows = [];
    for (let i = 0; i < n; i += 1) rows.push({ date: `d${i % 8}`, p: 0.4 + rnd() * 0.5, won: rnd() < 0.55 ? 1 : 0 });
    return rows;
  }

  it('REFUSES a fit whose slope collapses to a constant', () => {
    // Shrinking a miscalibrated forecaster toward its base rate always lowers
    // Brier, so this would score as a win while destroying all resolution.
    const m = lp.fitPlatt(uninformative(3000));
    expect(m.refused).toBe(true);
    expect(m.reason).toMatch(/collapses to a constant|invert/);
  });

  it('a refused fit produces no calibrated number at all', () => {
    const m = lp.fitPlatt(uninformative(3000));
    expect(lp.applyPlatt(m, 0.8)).toBeNull();
  });

  it('an INVERTING slope is refused by name', () => {
    // Real case: runs fitted a = -0.032, which would reverse every ordering.
    let s = 5;
    const rnd = () => (s = (s * 1103515245 + 12345) % 2147483648) / 2147483648;
    const rows = [];
    for (let i = 0; i < 3000; i += 1) {
      const p = 0.4 + rnd() * 0.5;
      rows.push({ date: `d${i % 8}`, p, won: rnd() < (0.9 - p) ? 1 : 0 });   // backwards
    }
    const m = lp.fitPlatt(rows);
    expect(m.refused).toBe(true);
    expect(m.a).toBeLessThanOrEqual(lp.MIN_SLOPE);
  });

  it('still accepts a genuine flattening', () => {
    const m = lp.fitPlatt(overConfident(2000));
    expect(m.refused).toBeUndefined();
    expect(m.a).toBeGreaterThan(lp.MIN_SLOPE);
    expect(m.a).toBeLessThan(1);
  });
});