Sessions 5-7a: 955 tests, deployment ready
This commit is contained in:
@@ -0,0 +1,267 @@
|
||||
/**
|
||||
* Engine 2 — LLM analysis layer on top of Engine 1 grades.
|
||||
*
|
||||
* Engine 2 doesn't REPLACE Engine 1. It runs after Engine 1 produces a
|
||||
* grade for an A/B-tier prop, applies natural-language reasoning over the
|
||||
* full feature vector + trap signals, and either agrees or disagrees.
|
||||
* Disagreement is itself a signal — surface it in the UI so users can
|
||||
* see when our two systems diverge.
|
||||
*
|
||||
* Architecture choices:
|
||||
* - Async + non-blocking. Engine 1 returns immediately; Engine 2 fills
|
||||
* in 5-30 seconds later via the queue.
|
||||
* - Queue is in-memory (Map keyed by gradeId). On process restart we
|
||||
* lose the queue, which is acceptable — n8n can re-queue from
|
||||
* grade_history WHERE engine2_analyzed_at IS NULL.
|
||||
* - Only A/B-tier props qualify. C/D/F grades skip Engine 2 entirely;
|
||||
* they're already flagged as low-confidence and don't need narrative.
|
||||
* - Prompt is GENERIC — no 'VYNDR' brand string. The model has no idea
|
||||
* who we are. That keeps our system prompt out of any provider's
|
||||
* training/QA pipeline.
|
||||
*/
|
||||
|
||||
const openRouter = require('../adapters/openRouterAdapter');
|
||||
const { getSupabaseServiceClient } = require('../../utils/supabase');
|
||||
|
||||
const BATCH_SIZE = Number(process.env.ENGINE2_BATCH_SIZE) || 10;
|
||||
const ENABLED = String(process.env.ENGINE2_ENABLED || 'true').toLowerCase() !== 'false';
|
||||
|
||||
// Grades that qualify for Engine 2 analysis. C/D/F skip.
|
||||
const ELIGIBLE_GRADES = new Set(['A+', 'A', 'A-', 'B+', 'B', 'B-']);
|
||||
const VALID_GRADES = new Set([
|
||||
'A+', 'A', 'A-', 'B+', 'B', 'B-', 'C+', 'C', 'C-', 'D', 'F', null,
|
||||
]);
|
||||
|
||||
// In-process FIFO queue. Map preserves insertion order — values carry the
|
||||
// context needed to build the prompt without re-querying upstream.
|
||||
const queue = new Map();
|
||||
|
||||
const SYSTEM_MESSAGE = (
|
||||
"You are a sports analytics engine analyzing player prop bets. "
|
||||
+ "Respond ONLY with valid JSON. No preamble, no markdown, no explanation "
|
||||
+ "outside the JSON structure. If you cannot analyze this prop, respond "
|
||||
+ 'with { "grade": null, "reason": "insufficient data" }.'
|
||||
);
|
||||
|
||||
function buildPrompt(ctx) {
|
||||
const features = ctx.features || {};
|
||||
const trapSignals = ctx.trap?.signals || {};
|
||||
const recent = ctx.recentGames || [];
|
||||
|
||||
const featureLines = Object.entries(features)
|
||||
.map(([k, v]) => {
|
||||
if (typeof v === 'number') {
|
||||
return `${k}: ${Number.isInteger(v) ? v : v.toFixed(2)}`;
|
||||
}
|
||||
return `${k}: ${v}`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
const activeTraps = Object.entries(trapSignals)
|
||||
.filter(([, s]) => s?.active && s?.score > 0)
|
||||
.map(([name, s]) => `- ${name}: ${s.score.toFixed(2)} (${s.explanation})`)
|
||||
.join('\n') || 'none';
|
||||
|
||||
const recentLines = recent
|
||||
.map((g) => ` ${g.date}: ${g.value} vs ${g.opponent}${g.home ? ' (home)' : ''}`)
|
||||
.join('\n') || ' (no recent games)';
|
||||
|
||||
return [
|
||||
`PLAYER: ${ctx.player_name} (${ctx.team || 'unknown'})`,
|
||||
`SPORT: ${ctx.sport}`,
|
||||
`PROP: ${ctx.direction} ${ctx.line} ${ctx.stat_type}`,
|
||||
`GAME: ${ctx.away_team || '?'} @ ${ctx.home_team || '?'}, ${ctx.game_date || '?'}`,
|
||||
'',
|
||||
'FEATURES:',
|
||||
featureLines || ' (no features computed)',
|
||||
'',
|
||||
`ENGINE 1 GRADE: ${ctx.engine1_grade} (${(ctx.engine1_factors || []).slice(0, 3).join(', ') || 'no factors'})`,
|
||||
'',
|
||||
'TRAP SIGNALS:',
|
||||
activeTraps,
|
||||
`Trap composite: ${(ctx.trap?.composite ?? 0).toFixed(2)} (${ctx.trap?.recommendation || 'unknown'})`,
|
||||
'',
|
||||
`CONSISTENCY: ${ctx.consistency?.consistency || 'unknown'} (cv=${(ctx.consistency?.cv ?? 0).toFixed(2)}, score=${(ctx.consistency?.score ?? 0).toFixed(2)})`,
|
||||
'',
|
||||
...(ctx.probability && Number.isFinite(ctx.probability.p_over) ? [
|
||||
`PROBABILITY: P(Over) = ${ctx.probability.p_over.toFixed(2)} | P(Under) = ${(1 - ctx.probability.p_over).toFixed(2)}`,
|
||||
`Components: ${
|
||||
Object.entries(ctx.probability.components || {})
|
||||
.filter(([, v]) => Number.isFinite(Number(v)))
|
||||
.map(([k, v]) => `${k}=${Number(v).toFixed(2)}`)
|
||||
.join(', ') || 'none'
|
||||
}`,
|
||||
'',
|
||||
] : []),
|
||||
'RECENT PERFORMANCE:',
|
||||
recentLines,
|
||||
'',
|
||||
'Analyze this prop and respond with:',
|
||||
'{',
|
||||
' "grade": "A+/A/A-/B+/B/B-/C+/C/C-/D/F",',
|
||||
' "confidence": 0.0-1.0,',
|
||||
' "agrees_with_engine1": true/false,',
|
||||
' "narrative": "2-3 sentence analysis",',
|
||||
' "trap_concern": "specific trap risk if any, or null",',
|
||||
' "key_factor": "single most important factor"',
|
||||
'}',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
// Four-strategy parser. The model is supposed to return raw JSON, but
|
||||
// "supposed to" is doing a lot of work — we layer fallbacks so a chatty
|
||||
// model doesn't make us drop the whole analysis. Strategy 4 (regex field
|
||||
// extraction) is the last-ditch — at least we capture the grade.
|
||||
function parseResponse(raw) {
|
||||
if (!raw || typeof raw !== 'string') return null;
|
||||
|
||||
// 1. Direct parse.
|
||||
try {
|
||||
const j = JSON.parse(raw.trim());
|
||||
if (j && typeof j === 'object') return j;
|
||||
} catch { /* fall through */ }
|
||||
|
||||
// 2. Markdown fenced block.
|
||||
const fence = raw.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
||||
if (fence?.[1]) {
|
||||
try {
|
||||
const j = JSON.parse(fence[1].trim());
|
||||
if (j && typeof j === 'object') return j;
|
||||
} catch { /* fall through */ }
|
||||
}
|
||||
|
||||
// 3. First {...} block.
|
||||
const obj = raw.match(/\{[\s\S]*\}/);
|
||||
if (obj) {
|
||||
try {
|
||||
const j = JSON.parse(obj[0]);
|
||||
if (j && typeof j === 'object') return j;
|
||||
} catch { /* fall through */ }
|
||||
}
|
||||
|
||||
// 4. Field-level regex extraction — last resort. We at least want the
|
||||
// grade letter; the narrative becomes a flag string so the row is
|
||||
// distinguishable from a model that returned valid JSON.
|
||||
const gradeMatch = raw.match(/["']?grade["']?\s*[:=]\s*["']?([A-F][+-]?)/i);
|
||||
if (gradeMatch) {
|
||||
const confMatch = raw.match(/["']?confidence["']?\s*[:=]\s*([\d.]+)/i);
|
||||
const conf = confMatch ? parseFloat(confMatch[1]) : NaN;
|
||||
return {
|
||||
grade: gradeMatch[1].toUpperCase(),
|
||||
confidence: Number.isFinite(conf) && conf >= 0 && conf <= 1 ? conf : 0.5,
|
||||
narrative: 'Extracted from malformed response',
|
||||
agrees_with_engine1: null,
|
||||
key_factor: null,
|
||||
trap_concern: null,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function validateAnalysis(parsed) {
|
||||
if (!parsed) return null;
|
||||
// Allow the explicit "I can't" response.
|
||||
if (parsed.grade === null) return { grade: null, reason: parsed.reason || 'insufficient data' };
|
||||
|
||||
if (!VALID_GRADES.has(parsed.grade)) return null;
|
||||
const confidence = Number(parsed.confidence);
|
||||
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) return null;
|
||||
const narrative = typeof parsed.narrative === 'string' ? parsed.narrative.slice(0, 500) : null;
|
||||
if (!narrative || narrative.length === 0) return null;
|
||||
return {
|
||||
grade: parsed.grade,
|
||||
confidence,
|
||||
narrative,
|
||||
agrees_with_engine1: !!parsed.agrees_with_engine1,
|
||||
trap_concern: typeof parsed.trap_concern === 'string' ? parsed.trap_concern.slice(0, 300) : null,
|
||||
key_factor: typeof parsed.key_factor === 'string' ? parsed.key_factor.slice(0, 200) : null,
|
||||
};
|
||||
}
|
||||
|
||||
function queueAnalysis(gradeId, propContext) {
|
||||
if (!ENABLED) return;
|
||||
if (!gradeId || !propContext) return;
|
||||
if (!ELIGIBLE_GRADES.has(propContext.engine1_grade)) return;
|
||||
// De-dupe by gradeId — re-queuing on retry is fine; we just overwrite.
|
||||
queue.set(gradeId, propContext);
|
||||
}
|
||||
|
||||
function getQueueSize() {
|
||||
return queue.size;
|
||||
}
|
||||
|
||||
function clearQueue() {
|
||||
queue.clear();
|
||||
}
|
||||
|
||||
async function persistResult(gradeId, analysis, modelUsed, latencyMs) {
|
||||
const supabase = getSupabaseServiceClient();
|
||||
const patch = {
|
||||
engine2_grade: analysis.grade,
|
||||
engine2_confidence: analysis.confidence,
|
||||
engine2_narrative: analysis.narrative,
|
||||
engine2_agrees: analysis.agrees_with_engine1,
|
||||
engine2_key_factor: analysis.key_factor,
|
||||
engine2_trap_concern: analysis.trap_concern,
|
||||
engine2_model: modelUsed,
|
||||
engine2_latency_ms: latencyMs,
|
||||
engine2_analyzed_at: new Date().toISOString(),
|
||||
};
|
||||
const { error } = await supabase.from('grade_history').update(patch).eq('id', gradeId);
|
||||
if (error) {
|
||||
console.warn('[engine2] persist failed for', gradeId, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function analyzeOne(gradeId, propContext) {
|
||||
const userPrompt = buildPrompt(propContext);
|
||||
const result = await openRouter.analyze(SYSTEM_MESSAGE, userPrompt);
|
||||
if (!result) return { ok: false, reason: 'openrouter unavailable' };
|
||||
|
||||
const parsed = parseResponse(result.response);
|
||||
const analysis = validateAnalysis(parsed);
|
||||
if (!analysis) return { ok: false, reason: 'parse/validate failed' };
|
||||
if (analysis.grade === null) return { ok: false, reason: analysis.reason };
|
||||
|
||||
await persistResult(gradeId, analysis, result.modelUsed, result.latencyMs);
|
||||
return { ok: true, analysis, modelUsed: result.modelUsed, latencyMs: result.latencyMs };
|
||||
}
|
||||
|
||||
async function processQueue() {
|
||||
if (!ENABLED) return { processed: 0, succeeded: 0, failed: 0 };
|
||||
let processed = 0;
|
||||
let succeeded = 0;
|
||||
let failed = 0;
|
||||
for (const [gradeId, ctx] of queue.entries()) {
|
||||
if (processed >= BATCH_SIZE) break;
|
||||
queue.delete(gradeId);
|
||||
processed += 1;
|
||||
try {
|
||||
const res = await analyzeOne(gradeId, ctx);
|
||||
if (res.ok) succeeded += 1; else failed += 1;
|
||||
} catch (err) {
|
||||
console.warn('[engine2] analyze threw for', gradeId, err.message);
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
return { processed, succeeded, failed, remaining: queue.size };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
queueAnalysis,
|
||||
processQueue,
|
||||
getQueueSize,
|
||||
clearQueue,
|
||||
__internals: {
|
||||
buildPrompt,
|
||||
parseResponse,
|
||||
validateAnalysis,
|
||||
analyzeOne,
|
||||
persistResult,
|
||||
queue,
|
||||
SYSTEM_MESSAGE,
|
||||
ELIGIBLE_GRADES,
|
||||
VALID_GRADES,
|
||||
BATCH_SIZE,
|
||||
},
|
||||
};
|
||||
Reference in New Issue
Block a user