openvibers/openvibe.live Commits

The AI's clip picks against people's, blind, on Bench (item 151, C20) (#573)

OP OpenVibers committed 10 days ago · ecf541330e · 4 files, +250 −0

* The AI's clip picks against people's, blind, on Bench (item 151, C20)

Daily, when LIVE_CLIP_EVAL=on: a recent recording with both a public AI clip and a public clip a person made becomes
one live.clip round (each clip with a 6-hour signed link), once per recording; protected content stays out. People
vote blind on Bench's battle page, and results() reads each side's accept rate back.

* SSRF review: clip-evaluation.js calls only OpenVibe.Bench at its configured address

Changes

ddiff --git a/server/ai/auto-clip-job.js b/server/ai/auto-clip-job.js
iindex 70f81e03..8868a5e5 100644
---- a/server/ai/auto-clip-job.js
++++ b/server/ai/auto-clip-job.js
@@@ -429,6 +429,8 @@ function start() {
_flagTimer = require('../utils/jobs').every('auto-clip-flag-sync', FLAG_SYNC_INTERVAL_MS, () => syncAutoClipFlags(), { initialDelayMs: 90 * 1000, jitterMs: 60 * 1000 });
// Once a day, the AI's old clips no person ever opened leave the lists (unlisted, never deleted).
require('./auto-clip-sweep').start();
++ // Once a day (LIVE_CLIP_EVAL=on), recordings with both an AI clip and a person's go to Bench for a blind vote.
++ require('./clip-evaluation').start();
console.log('[AutoClip] Live auto-clip job started (selective chat-spike + AI agreement, cuts via OpenVibe.Media) + persistent VOD backfill');
}
function stop() {
@@@ -436,6 +438,7 @@ function stop() {
if (_backfillTimer) { _backfillTimer(); _backfillTimer = null; }
if (_flagTimer) { _flagTimer(); _flagTimer = null; }
require('./auto-clip-sweep').stop();
++ require('./clip-evaluation').stop();
}
module.exports = { start, stop, clipVodMoment, backfillVodClips, syncAutoClipFlags, _tick, _internals: { checkLiveStream: _checkLiveStream, backfillPool: _backfillPool } };
ddiff --git a/server/ai/clip-evaluation.js b/server/ai/clip-evaluation.js
nnew file mode 100644
iindex 00000000..55ec163c
---- /dev/null
++++ b/server/ai/clip-evaluation.js
@@@ -0,0 +1,131 @@
++'use strict';
++/**
++ * clip-evaluation.js — the AI's clip picks against people's, blind, on OpenVibe.Bench (item 151, C20).
++ *
++ * Once a day, for up to MAX_PER_RUN recent recordings that have both a public clip the AI cut and a public clip a
++ * person made, Live submits one live.clip round to Bench (#105): the AI's pick (policy auto-clip@AUTO_CLIP_POLICY)
++ * and the person's (policy people@1), each with a 6-hour signed playback link from Media. Signed-in people vote at
++ * https://bench.openvibe.services/battle?kind=live.clip (Bench #106: two players, blind, no titles), and
++ * results() reads each side's accept rate back from Bench (both_bad counts against both).
++ *
++ * Off unless LIVE_CLIP_EVAL=on, and never in drill mode (auto-clip-job.js starts it).
++ * Recording-separated: one round per recording, ever (state clip_eval_log keeps the last 2000).
++ * Protected content stays out: only public clips of public recordings, nothing the explicit check flags, never a
++ * mature recording, and never clip 483 or VOD 5709.
++ *
++ * run({ now, max }) → { submitted: [{ vod_id, round_id }], skipped, why }
++ * results() → { policies } from Bench, or null
++ * start() / stop()
++ */
++const db = require('../db/database');
++const media = require('../media-client');
++const principal = require('../net/network-principal');
++const { isExplicit } = require('./explicit');
++const { parseTs } = require('./clip-policy');
++
++const BENCH_URL = String(process.env.LIVE_BENCH_URL || 'https://bench.openvibe.services').replace(/\/+$/, '');
++// The picker as it stands: the date of its last change (score bar, calibration, freshness, sweep). Change it with the
++// picker, so Bench's accept rate is per version.
++const AUTO_CLIP_POLICY = process.env.LIVE_AUTO_CLIP_POLICY || '2026-11-02';
++const LOG = 'clip_eval_log';
++const KEEP = 2000;
++const MAX_PER_RUN = 3;
++const WINDOW_DAYS = 14;
++const PLAYBACK_TTL_S = 6 * 3600;
++const DAY_MS = 24 * 3600 * 1000;
++const NEVER_CLIPS = new Set(['483']);
++const NEVER_VODS = new Set(['5709']);
++
++const enabled = () => String(process.env.LIVE_CLIP_EVAL || '').toLowerCase() === 'on';
++
++async function _log() {
++ try { const d = JSON.parse(await db.getState(LOG) || '[]'); return Array.isArray(d) ? d : []; } catch { return []; }
++}
++
++/** A clip that may be judged: public, ready, not flagged, 5 to 180 seconds, with its window known. */
++function eligible(clip) {
++ if (!clip || clip.visibility !== 'public' || (clip.status && clip.status !== 'ready')) return false;
++ if (NEVER_CLIPS.has(String(clip.id)) || NEVER_VODS.has(String(clip.vod_id))) return false;
++ if (clip.mature || clip.is_mature || isExplicit(clip.title || '', clip.description || '')) return false;
++ const start = Number(clip.start_time);
++ const len = Number(clip.duration_seconds ?? clip.duration ?? (Number(clip.end_time) - start));
++ return Number.isFinite(start) && start >= 0 && Number.isFinite(len) && len >= 5 && len <= 180;
++}
++
++async function bench(method, path, body) {
++ const res = await fetch(`${BENCH_URL}${path}`, {
++ method, headers: { Accept: 'application/json', ...(body ? { 'Content-Type': 'application/json' } : {}), ...(await principal.serviceHeaders('openvibe.bench')) },
++ body: body ? JSON.stringify(body) : undefined, signal: AbortSignal.timeout(15_000),
++ });
++ const text = await res.text();
++ let json = null;
++ try { json = text ? JSON.parse(text) : null; } catch { json = null; }
++ if (!res.ok) { const e = new Error(`Bench ${method} ${path}: HTTP ${res.status}`); e.status = res.status; throw e; }
++ return json;
++}
++
++function entryOf(clip, policy, signed) {
++ const start = Math.round(Number(clip.start_time) * 1000);
++ const len = Math.round(Number(clip.duration_seconds ?? clip.duration ?? (Number(clip.end_time) - Number(clip.start_time))) * 1000);
++ return {
++ source: { recording_id: `vod_${clip.vod_id}`, start_ms: start, end_ms: start + len },
++ playback: { url: signed.url, expires_at: signed.expires_at },
++ policy,
++ ...(clip.title ? { title: String(clip.title).slice(0, 200) } : {}),
++ };
++}
++
++async function run({ now = Date.now(), max = MAX_PER_RUN } = {}) {
++ if (!enabled()) return { submitted: [], skipped: 0, why: 'off (LIVE_CLIP_EVAL is not on)' };
++ if (!principal.configured()) return { submitted: [], skipped: 0, why: 'no OAuth client secret' };
++ const since = new Date(now - WINDOW_DAYS * DAY_MS).toISOString();
++ const ai = await media.listClips({ auto_generated: 1, since, status: 'ready', order: 'newest', limit: 100 }, { timeoutMs: 15_000 });
++ if (!ai || !Array.isArray(ai.clips)) return { submitted: [], skipped: 0, why: 'Media did not answer' };
++ const done = new Set((await _log()).map((x) => String(x.vod_id)));
++ const byVod = new Map();
++ for (const c of ai.clips) if (c.auto_generated === true && eligible(c) && !done.has(String(c.vod_id)) && !byVod.has(String(c.vod_id))) byVod.set(String(c.vod_id), c);
++ const submitted = [];
++ let skipped = 0;
++ for (const [vodId, aiClip] of byVod) {
++ if (submitted.length >= max) break;
++ try {
++ const vod = await media.getVod(vodId, { timeoutMs: 10_000 });
++ if (!vod || vod.visibility !== 'public' || vod.mature || vod.is_mature) { skipped++; continue; }
++ const people = await media.listClips({ vod_id: vodId, auto_generated: 0, status: 'ready', limit: 20 }, { timeoutMs: 10_000 });
++ const person = (people && Array.isArray(people.clips) ? people.clips : []).find((c) => c.auto_generated !== true && eligible(c));
++ if (!person) { skipped++; continue; }
++ const [sa, sp] = await Promise.all([media.signedMediaUrl('clip', aiClip.id, PLAYBACK_TTL_S), media.signedMediaUrl('clip', person.id, PLAYBACK_TTL_S)]);
++ if (!sa || !sa.url || !sp || !sp.url) { skipped++; continue; }
++ const task = await bench('POST', '/api/v1/kinds/live.clip/tasks', { prompt: 'Two clips of one stream', public: false });
++ const round = await bench('POST', `/api/v1/tasks/${task.task.id}/rounds`, {});
++ const roundId = round.round.id;
++ await bench('POST', `/api/v1/rounds/${roundId}/entries`, { payload: entryOf(aiClip, { selector: 'auto-clip', version: AUTO_CLIP_POLICY, models: [] }, sa) });
++ await bench('POST', `/api/v1/rounds/${roundId}/entries`, { payload: entryOf(person, { selector: 'people', version: '1', models: [] }, sp) });
++ const log = await _log();
++ log.unshift({ vod_id: vodId, ai_clip: aiClip.id, person_clip: person.id, round_id: roundId, at: new Date(now).toISOString() });
++ await db.setState(LOG, JSON.stringify(log.slice(0, KEEP)));
++ submitted.push({ vod_id: vodId, round_id: roundId });
++ } catch (e) {
++ if (e && (e.status === 404 || e.status === 410)) { skipped++; continue; }
++ console.warn('[clip-eval] stopped for today:', e && e.message);
++ break; // Bench or Media down: the rest wait for tomorrow
++ }
++ }
++ if (submitted.length) console.log(`[clip-eval] ${submitted.length} recording(s) sent to Bench for a blind vote`);
++ return { submitted, skipped, why: submitted.length ? null : 'no recording with both an AI clip and a person\'s clip' };
++}
++
++/** Each side's results, as Bench counts them (shadow metrics): the AI's picker against people's. */
++async function results() {
++ if (!principal.configured()) return null;
++ try { return await bench('GET', '/api/v1/kinds/live.clip/policies'); } catch { return null; }
++}
++
++let _stop = null;
++function start() {
++ if (_stop || !enabled()) return;
++ _stop = require('../utils/jobs').every('clip-evaluation', DAY_MS, () => run(), { initialDelayMs: 40 * 60 * 1000, jitterMs: 20 * 60 * 1000 });
++}
++function stop() { if (_stop) { _stop(); _stop = null; } }
++
++module.exports = { run, results, start, stop, eligible, entryOf, AUTO_CLIP_POLICY, LOG, MAX_PER_RUN };
ddiff --git a/test/clip-evaluation.test.js b/test/clip-evaluation.test.js
nnew file mode 100644
iindex 00000000..c7956e3f
---- /dev/null
++++ b/test/clip-evaluation.test.js
@@@ -0,0 +1,115 @@
++/**
++ * The AI's clip picks against people's, blind, on Bench (server/ai/clip-evaluation.js; item 151, C20): a recent
++ * recording with a public AI clip and a public clip a person made becomes one live.clip round with both entries and
++ * signed playback links; one round per recording, ever; protected content (private, mature, explicit, clip 483,
++ * VOD 5709) stays out; off unless LIVE_CLIP_EVAL=on; Bench down stops the run for the day.
++ *
++ * node test/run.js clip-evaluation
++ */
++'use strict';
++const assert = require('assert');
++
++process.env.LIVE_CLIP_EVAL = 'on';
++const media = require('../server/media-client');
++const db = require('../server/db/database');
++const principal = require('../server/net/network-principal');
++const state = new Map();
++db.getState = async (k) => state.get(k) || null;
++db.setState = async (k, v) => { state.set(k, v); };
++principal.configured = () => true;
++principal.serviceHeaders = async () => ({ Authorization: 'Bearer svc-live' });
++
++const NOW = Date.parse('2026-11-03T02:00:00Z');
++const clip = (id, vod, ai, extra = {}) => ({ id, vod_id: vod, auto_generated: ai, visibility: 'public', status: 'ready', start_time: 120, duration_seconds: 30, title: `clip ${id}`, ...extra });
++let clips;
++let vods;
++let bench;
++let benchDown;
++const reset = () => {
++ clips = [
++ clip(10, 100, true), clip(11, 100, false), // AI and a person on 100: judged
++ clip(20, 200, true), // nobody else on 200: skipped
++ clip(30, 300, true), clip(31, 300, false, { visibility: 'unlisted' }), // the person's clip is not public
++ clip(40, 400, true, { title: 'explicit sex act' }), clip(41, 400, false),
++ clip(483, 500, true), clip(51, 500, false), // clip 483 never
++ clip(60, 5709, true), clip(61, 5709, false), // VOD 5709 never
++ clip(70, 700, true), clip(71, 700, false), // a mature recording
++ clip(80, 800, true), clip(81, 800, false), // a second good one
++ ];
++ vods = { 100: { id: 100, visibility: 'public' }, 300: { id: 300, visibility: 'public' }, 400: { id: 400, visibility: 'public' }, 500: { id: 500, visibility: 'public' }, 5709: { id: 5709, visibility: 'public' }, 700: { id: 700, visibility: 'public', mature: true }, 800: { id: 800, visibility: 'public' } };
++ bench = [];
++ benchDown = false;
++ state.clear();
++};
++media.listClips = async (q) => {
++ let rows = clips.filter((c) => c.status === 'ready');
++ if (q.auto_generated === 1) rows = rows.filter((c) => c.auto_generated);
++ if (q.auto_generated === 0) rows = rows.filter((c) => !c.auto_generated);
++ if (q.vod_id != null) rows = rows.filter((c) => String(c.vod_id) === String(q.vod_id));
++ return { clips: rows.slice(0, q.limit), total: rows.length };
++};
++media.getVod = async (id) => vods[id] || null;
++media.signedMediaUrl = async (kind, id, ttl) => ({ url: `https://openvibe.media/c/${id}?exp=1&sig=x`, expires_at: new Date(NOW + ttl * 1000).toISOString() });
++global.fetch = async (url, opts) => {
++ const u = new URL(url);
++ const body = opts.body ? JSON.parse(opts.body) : null;
++ bench.push({ method: opts.method, path: u.pathname, body, auth: opts.headers.Authorization });
++ if (benchDown) return new Response('down', { status: 503 });
++ if (u.pathname === '/api/v1/kinds/live.clip/tasks') return Response.json({ task: { id: `bnt_${bench.length}` } }, { status: 201 });
++ if (/\/rounds$/.test(u.pathname)) return Response.json({ round: { id: `brn_${bench.length}` } }, { status: 201 });
++ if (/\/entries$/.test(u.pathname)) return Response.json({ entry: { id: `bte_${bench.length}` } }, { status: 201 });
++ if (u.pathname === '/api/v1/kinds/live.clip/policies') return Response.json({ kind_id: 'live.clip', policies: [{ policy: 'auto-clip@x', accept_rate: 0.5 }] });
++ return new Response('?', { status: 404 });
++};
++
++const evaluation = require('../server/ai/clip-evaluation');
++
++(async () => {
++ reset();
++ let out = await evaluation.run({ now: NOW, max: 5 });
++ assert.deepStrictEqual(out.submitted.map((x) => x.vod_id).sort(), ['100', '800'], JSON.stringify(out));
++ const entries = bench.filter((b) => /\/entries$/.test(b.path));
++ assert.strictEqual(entries.length, 4, 'two entries a round');
++ const ai = entries.find((e) => e.body.payload.policy.selector === 'auto-clip' && e.body.payload.source.recording_id === 'vod_100');
++ const person = entries.find((e) => e.body.payload.policy.selector === 'people' && e.body.payload.source.recording_id === 'vod_100');
++ assert.ok(ai && person, 'the AI\'s pick and the person\'s');
++ assert.deepStrictEqual(ai.body.payload.source, { recording_id: 'vod_100', start_ms: 120000, end_ms: 150000 });
++ assert.match(ai.body.payload.playback.url, /^https:\/\/openvibe\.media\/c\/10\?/);
++ assert.strictEqual(ai.body.payload.policy.version, evaluation.AUTO_CLIP_POLICY);
++ assert.ok(Date.parse(ai.body.payload.playback.expires_at) > NOW, 'a link that lasts');
++ assert.ok(bench.every((b) => b.auth === 'Bearer svc-live'), 'Live\'s own service token');
++ const task = bench.find((b) => b.path === '/api/v1/kinds/live.clip/tasks');
++ assert.deepStrictEqual(task.body, { prompt: 'Two clips of one stream', public: false }, 'a private task that names no recording');
++ const touched = new Set(entries.map((e) => e.body.payload.source.recording_id));
++ for (const never of ['vod_200', 'vod_300', 'vod_400', 'vod_500', 'vod_5709', 'vod_700']) assert.ok(!touched.has(never), `${never} stays out`);
++
++ // One round per recording, ever.
++ const before = bench.length;
++ out = await evaluation.run({ now: NOW + 86400_000, max: 5 });
++ assert.strictEqual(out.submitted.length, 0, 'nothing new');
++ assert.strictEqual(bench.length, before, 'no second round for 100 or 800');
++ assert.strictEqual(JSON.parse(state.get(evaluation.LOG)).length, 2);
++
++ // At most `max` a run; Bench down stops the run (the rest wait for tomorrow).
++ reset();
++ out = await evaluation.run({ now: NOW, max: 1 });
++ assert.strictEqual(out.submitted.length, 1);
++ reset();
++ benchDown = true;
++ out = await evaluation.run({ now: NOW, max: 5 });
++ assert.strictEqual(out.submitted.length, 0);
++ assert.strictEqual(bench.length, 1, 'stopped at the first refusal');
++
++ // Off unless switched on.
++ reset();
++ process.env.LIVE_CLIP_EVAL = '';
++ out = await evaluation.run({ now: NOW });
++ assert.match(out.why, /off/);
++ assert.strictEqual(bench.length, 0);
++ process.env.LIVE_CLIP_EVAL = 'on';
++
++ // The shadow metrics come back from Bench.
++ const r = await evaluation.results();
++ assert.strictEqual(r.policies[0].accept_rate, 0.5);
++ console.log('clip evaluation: all checks passed');
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/test/security-ssrf.test.js b/test/security-ssrf.test.js
iindex af14a2c9..50fc6b07 100644
---- a/test/security-ssrf.test.js
++++ b/test/security-ssrf.test.js
@@@ -365,6 +365,7 @@ async function check(name, fn) {
'server/monetization/cosmetics.js': 'legacy quest API (env)',
'server/openre/openre-client.js': 'OpenRestream (configured)',
'server/ai/ai-service.js': 'OpenVibe.AI (configured)',
++ 'server/ai/clip-evaluation.js': 'OpenVibe.Bench (configured LIVE_BENCH_URL): the AI\'s clip picks against people\'s, with Live\'s service token',
'server/ai/llm.js': 'images are Live\'s own frames; a streamer\'s typed provider address (testProvider) goes through egress.postJson',
'server/chat/soundboard-service.js': '101soundboards (host allowlist; the audio download through safeLookup)',
'server/chat/tts-engine.js': 'Google / AWS TTS (fixed hosts)',