openvibers/openvibe.live Commits
The AI's clip picks against people's, blind, on Bench (item 151, C20) (#573)
Changes
@@@ -429,6 +429,8 @@ function start() {_flagTimer = require('../utils/jobs').every('auto-clip-flag-sync', FLAG_SYNC_INTERVAL_MS, () => syncAutoClipFlags(), { initialDelayMs: 90 * 1000, jitterMs: 60 * 1000 });// Once a day, the AI's old clips no person ever opened leave the lists (unlisted, never deleted).require('./auto-clip-sweep').start();++ // Once a day (LIVE_CLIP_EVAL=on), recordings with both an AI clip and a person's go to Bench for a blind vote.++ require('./clip-evaluation').start();console.log('[AutoClip] Live auto-clip job started (selective chat-spike + AI agreement, cuts via OpenVibe.Media) + persistent VOD backfill');}function stop() {@@@ -436,6 +438,7 @@ function stop() {if (_backfillTimer) { _backfillTimer(); _backfillTimer = null; }if (_flagTimer) { _flagTimer(); _flagTimer = null; }require('./auto-clip-sweep').stop();++ require('./clip-evaluation').stop();}module.exports = { start, stop, clipVodMoment, backfillVodClips, syncAutoClipFlags, _tick, _internals: { checkLiveStream: _checkLiveStream, backfillPool: _backfillPool } };nnew file mode 100644@@@ -0,0 +1,131 @@++'use strict';++/**++ * clip-evaluation.js — the AI's clip picks against people's, blind, on OpenVibe.Bench (item 151, C20).++ *++ * Once a day, for up to MAX_PER_RUN recent recordings that have both a public clip the AI cut and a public clip a++ * person made, Live submits one live.clip round to Bench (#105): the AI's pick (policy auto-clip@AUTO_CLIP_POLICY)++ * and the person's (policy people@1), each with a 6-hour signed playback link from Media. Signed-in people vote at++ * https://bench.openvibe.services/battle?kind=live.clip (Bench #106: two players, blind, no titles), and++ * results() reads each side's accept rate back from Bench (both_bad counts against both).++ *++ * Off unless LIVE_CLIP_EVAL=on, and never in drill mode (auto-clip-job.js starts it).++ * Recording-separated: one round per recording, ever (state clip_eval_log keeps the last 2000).++ * Protected content stays out: only public clips of public recordings, nothing the explicit check flags, never a++ * mature recording, and never clip 483 or VOD 5709.++ *++ * run({ now, max }) → { submitted: [{ vod_id, round_id }], skipped, why }++ * results() → { policies } from Bench, or null++ * start() / stop()++ */++const db = require('../db/database');++const media = require('../media-client');++const principal = require('../net/network-principal');++const { isExplicit } = require('./explicit');++const { parseTs } = require('./clip-policy');++++const BENCH_URL = String(process.env.LIVE_BENCH_URL || 'https://bench.openvibe.services').replace(/\/+$/, '');++// The picker as it stands: the date of its last change (score bar, calibration, freshness, sweep). Change it with the++// picker, so Bench's accept rate is per version.++const AUTO_CLIP_POLICY = process.env.LIVE_AUTO_CLIP_POLICY || '2026-11-02';++const LOG = 'clip_eval_log';++const KEEP = 2000;++const MAX_PER_RUN = 3;++const WINDOW_DAYS = 14;++const PLAYBACK_TTL_S = 6 * 3600;++const DAY_MS = 24 * 3600 * 1000;++const NEVER_CLIPS = new Set(['483']);++const NEVER_VODS = new Set(['5709']);++++const enabled = () => String(process.env.LIVE_CLIP_EVAL || '').toLowerCase() === 'on';++++async function _log() {++ try { const d = JSON.parse(await db.getState(LOG) || '[]'); return Array.isArray(d) ? d : []; } catch { return []; }++}++++/** A clip that may be judged: public, ready, not flagged, 5 to 180 seconds, with its window known. */++function eligible(clip) {++ if (!clip || clip.visibility !== 'public' || (clip.status && clip.status !== 'ready')) return false;++ if (NEVER_CLIPS.has(String(clip.id)) || NEVER_VODS.has(String(clip.vod_id))) return false;++ if (clip.mature || clip.is_mature || isExplicit(clip.title || '', clip.description || '')) return false;++ const start = Number(clip.start_time);++ const len = Number(clip.duration_seconds ?? clip.duration ?? (Number(clip.end_time) - start));++ return Number.isFinite(start) && start >= 0 && Number.isFinite(len) && len >= 5 && len <= 180;++}++++async function bench(method, path, body) {++ const res = await fetch(`${BENCH_URL}${path}`, {++ method, headers: { Accept: 'application/json', ...(body ? { 'Content-Type': 'application/json' } : {}), ...(await principal.serviceHeaders('openvibe.bench')) },++ body: body ? JSON.stringify(body) : undefined, signal: AbortSignal.timeout(15_000),++ });++ const text = await res.text();++ let json = null;++ try { json = text ? JSON.parse(text) : null; } catch { json = null; }++ if (!res.ok) { const e = new Error(`Bench ${method} ${path}: HTTP ${res.status}`); e.status = res.status; throw e; }++ return json;++}++++function entryOf(clip, policy, signed) {++ const start = Math.round(Number(clip.start_time) * 1000);++ const len = Math.round(Number(clip.duration_seconds ?? clip.duration ?? (Number(clip.end_time) - Number(clip.start_time))) * 1000);++ return {++ source: { recording_id: `vod_${clip.vod_id}`, start_ms: start, end_ms: start + len },++ playback: { url: signed.url, expires_at: signed.expires_at },++ policy,++ ...(clip.title ? { title: String(clip.title).slice(0, 200) } : {}),++ };++}++++async function run({ now = Date.now(), max = MAX_PER_RUN } = {}) {++ if (!enabled()) return { submitted: [], skipped: 0, why: 'off (LIVE_CLIP_EVAL is not on)' };++ if (!principal.configured()) return { submitted: [], skipped: 0, why: 'no OAuth client secret' };++ const since = new Date(now - WINDOW_DAYS * DAY_MS).toISOString();++ const ai = await media.listClips({ auto_generated: 1, since, status: 'ready', order: 'newest', limit: 100 }, { timeoutMs: 15_000 });++ if (!ai || !Array.isArray(ai.clips)) return { submitted: [], skipped: 0, why: 'Media did not answer' };++ const done = new Set((await _log()).map((x) => String(x.vod_id)));++ const byVod = new Map();++ for (const c of ai.clips) if (c.auto_generated === true && eligible(c) && !done.has(String(c.vod_id)) && !byVod.has(String(c.vod_id))) byVod.set(String(c.vod_id), c);++ const submitted = [];++ let skipped = 0;++ for (const [vodId, aiClip] of byVod) {++ if (submitted.length >= max) break;++ try {++ const vod = await media.getVod(vodId, { timeoutMs: 10_000 });++ if (!vod || vod.visibility !== 'public' || vod.mature || vod.is_mature) { skipped++; continue; }++ const people = await media.listClips({ vod_id: vodId, auto_generated: 0, status: 'ready', limit: 20 }, { timeoutMs: 10_000 });++ const person = (people && Array.isArray(people.clips) ? people.clips : []).find((c) => c.auto_generated !== true && eligible(c));++ if (!person) { skipped++; continue; }++ const [sa, sp] = await Promise.all([media.signedMediaUrl('clip', aiClip.id, PLAYBACK_TTL_S), media.signedMediaUrl('clip', person.id, PLAYBACK_TTL_S)]);++ if (!sa || !sa.url || !sp || !sp.url) { skipped++; continue; }++ const task = await bench('POST', '/api/v1/kinds/live.clip/tasks', { prompt: 'Two clips of one stream', public: false });++ const round = await bench('POST', `/api/v1/tasks/${task.task.id}/rounds`, {});++ const roundId = round.round.id;++ await bench('POST', `/api/v1/rounds/${roundId}/entries`, { payload: entryOf(aiClip, { selector: 'auto-clip', version: AUTO_CLIP_POLICY, models: [] }, sa) });++ await bench('POST', `/api/v1/rounds/${roundId}/entries`, { payload: entryOf(person, { selector: 'people', version: '1', models: [] }, sp) });++ const log = await _log();++ log.unshift({ vod_id: vodId, ai_clip: aiClip.id, person_clip: person.id, round_id: roundId, at: new Date(now).toISOString() });++ await db.setState(LOG, JSON.stringify(log.slice(0, KEEP)));++ submitted.push({ vod_id: vodId, round_id: roundId });++ } catch (e) {++ if (e && (e.status === 404 || e.status === 410)) { skipped++; continue; }++ console.warn('[clip-eval] stopped for today:', e && e.message);++ break; // Bench or Media down: the rest wait for tomorrow++ }++ }++ if (submitted.length) console.log(`[clip-eval] ${submitted.length} recording(s) sent to Bench for a blind vote`);++ return { submitted, skipped, why: submitted.length ? null : 'no recording with both an AI clip and a person\'s clip' };++}++++/** Each side's results, as Bench counts them (shadow metrics): the AI's picker against people's. */++async function results() {++ if (!principal.configured()) return null;++ try { return await bench('GET', '/api/v1/kinds/live.clip/policies'); } catch { return null; }++}++++let _stop = null;++function start() {++ if (_stop || !enabled()) return;++ _stop = require('../utils/jobs').every('clip-evaluation', DAY_MS, () => run(), { initialDelayMs: 40 * 60 * 1000, jitterMs: 20 * 60 * 1000 });++}++function stop() { if (_stop) { _stop(); _stop = null; } }++++module.exports = { run, results, start, stop, eligible, entryOf, AUTO_CLIP_POLICY, LOG, MAX_PER_RUN };nnew file mode 100644@@@ -0,0 +1,115 @@++/**++ * The AI's clip picks against people's, blind, on Bench (server/ai/clip-evaluation.js; item 151, C20): a recent++ * recording with a public AI clip and a public clip a person made becomes one live.clip round with both entries and++ * signed playback links; one round per recording, ever; protected content (private, mature, explicit, clip 483,++ * VOD 5709) stays out; off unless LIVE_CLIP_EVAL=on; Bench down stops the run for the day.++ *++ * node test/run.js clip-evaluation++ */++'use strict';++const assert = require('assert');++++process.env.LIVE_CLIP_EVAL = 'on';++const media = require('../server/media-client');++const db = require('../server/db/database');++const principal = require('../server/net/network-principal');++const state = new Map();++db.getState = async (k) => state.get(k) || null;++db.setState = async (k, v) => { state.set(k, v); };++principal.configured = () => true;++principal.serviceHeaders = async () => ({ Authorization: 'Bearer svc-live' });++++const NOW = Date.parse('2026-11-03T02:00:00Z');++const clip = (id, vod, ai, extra = {}) => ({ id, vod_id: vod, auto_generated: ai, visibility: 'public', status: 'ready', start_time: 120, duration_seconds: 30, title: `clip ${id}`, ...extra });++let clips;++let vods;++let bench;++let benchDown;++const reset = () => {++ clips = [++ clip(10, 100, true), clip(11, 100, false), // AI and a person on 100: judged++ clip(20, 200, true), // nobody else on 200: skipped++ clip(30, 300, true), clip(31, 300, false, { visibility: 'unlisted' }), // the person's clip is not public++ clip(40, 400, true, { title: 'explicit sex act' }), clip(41, 400, false),++ clip(483, 500, true), clip(51, 500, false), // clip 483 never++ clip(60, 5709, true), clip(61, 5709, false), // VOD 5709 never++ clip(70, 700, true), clip(71, 700, false), // a mature recording++ clip(80, 800, true), clip(81, 800, false), // a second good one++ ];++ vods = { 100: { id: 100, visibility: 'public' }, 300: { id: 300, visibility: 'public' }, 400: { id: 400, visibility: 'public' }, 500: { id: 500, visibility: 'public' }, 5709: { id: 5709, visibility: 'public' }, 700: { id: 700, visibility: 'public', mature: true }, 800: { id: 800, visibility: 'public' } };++ bench = [];++ benchDown = false;++ state.clear();++};++media.listClips = async (q) => {++ let rows = clips.filter((c) => c.status === 'ready');++ if (q.auto_generated === 1) rows = rows.filter((c) => c.auto_generated);++ if (q.auto_generated === 0) rows = rows.filter((c) => !c.auto_generated);++ if (q.vod_id != null) rows = rows.filter((c) => String(c.vod_id) === String(q.vod_id));++ return { clips: rows.slice(0, q.limit), total: rows.length };++};++media.getVod = async (id) => vods[id] || null;++media.signedMediaUrl = async (kind, id, ttl) => ({ url: `https://openvibe.media/c/${id}?exp=1&sig=x`, expires_at: new Date(NOW + ttl * 1000).toISOString() });++global.fetch = async (url, opts) => {++ const u = new URL(url);++ const body = opts.body ? JSON.parse(opts.body) : null;++ bench.push({ method: opts.method, path: u.pathname, body, auth: opts.headers.Authorization });++ if (benchDown) return new Response('down', { status: 503 });++ if (u.pathname === '/api/v1/kinds/live.clip/tasks') return Response.json({ task: { id: `bnt_${bench.length}` } }, { status: 201 });++ if (/\/rounds$/.test(u.pathname)) return Response.json({ round: { id: `brn_${bench.length}` } }, { status: 201 });++ if (/\/entries$/.test(u.pathname)) return Response.json({ entry: { id: `bte_${bench.length}` } }, { status: 201 });++ if (u.pathname === '/api/v1/kinds/live.clip/policies') return Response.json({ kind_id: 'live.clip', policies: [{ policy: 'auto-clip@x', accept_rate: 0.5 }] });++ return new Response('?', { status: 404 });++};++++const evaluation = require('../server/ai/clip-evaluation');++++(async () => {++ reset();++ let out = await evaluation.run({ now: NOW, max: 5 });++ assert.deepStrictEqual(out.submitted.map((x) => x.vod_id).sort(), ['100', '800'], JSON.stringify(out));++ const entries = bench.filter((b) => /\/entries$/.test(b.path));++ assert.strictEqual(entries.length, 4, 'two entries a round');++ const ai = entries.find((e) => e.body.payload.policy.selector === 'auto-clip' && e.body.payload.source.recording_id === 'vod_100');++ const person = entries.find((e) => e.body.payload.policy.selector === 'people' && e.body.payload.source.recording_id === 'vod_100');++ assert.ok(ai && person, 'the AI\'s pick and the person\'s');++ assert.deepStrictEqual(ai.body.payload.source, { recording_id: 'vod_100', start_ms: 120000, end_ms: 150000 });++ assert.match(ai.body.payload.playback.url, /^https:\/\/openvibe\.media\/c\/10\?/);++ assert.strictEqual(ai.body.payload.policy.version, evaluation.AUTO_CLIP_POLICY);++ assert.ok(Date.parse(ai.body.payload.playback.expires_at) > NOW, 'a link that lasts');++ assert.ok(bench.every((b) => b.auth === 'Bearer svc-live'), 'Live\'s own service token');++ const task = bench.find((b) => b.path === '/api/v1/kinds/live.clip/tasks');++ assert.deepStrictEqual(task.body, { prompt: 'Two clips of one stream', public: false }, 'a private task that names no recording');++ const touched = new Set(entries.map((e) => e.body.payload.source.recording_id));++ for (const never of ['vod_200', 'vod_300', 'vod_400', 'vod_500', 'vod_5709', 'vod_700']) assert.ok(!touched.has(never), `${never} stays out`);++++ // One round per recording, ever.++ const before = bench.length;++ out = await evaluation.run({ now: NOW + 86400_000, max: 5 });++ assert.strictEqual(out.submitted.length, 0, 'nothing new');++ assert.strictEqual(bench.length, before, 'no second round for 100 or 800');++ assert.strictEqual(JSON.parse(state.get(evaluation.LOG)).length, 2);++++ // At most `max` a run; Bench down stops the run (the rest wait for tomorrow).++ reset();++ out = await evaluation.run({ now: NOW, max: 1 });++ assert.strictEqual(out.submitted.length, 1);++ reset();++ benchDown = true;++ out = await evaluation.run({ now: NOW, max: 5 });++ assert.strictEqual(out.submitted.length, 0);++ assert.strictEqual(bench.length, 1, 'stopped at the first refusal');++++ // Off unless switched on.++ reset();++ process.env.LIVE_CLIP_EVAL = '';++ out = await evaluation.run({ now: NOW });++ assert.match(out.why, /off/);++ assert.strictEqual(bench.length, 0);++ process.env.LIVE_CLIP_EVAL = 'on';++++ // The shadow metrics come back from Bench.++ const r = await evaluation.results();++ assert.strictEqual(r.policies[0].accept_rate, 0.5);++ console.log('clip evaluation: all checks passed');++})().catch((e) => { console.error(e); process.exit(1); });@@@ -365,6 +365,7 @@ async function check(name, fn) {'server/monetization/cosmetics.js': 'legacy quest API (env)','server/openre/openre-client.js': 'OpenRestream (configured)','server/ai/ai-service.js': 'OpenVibe.AI (configured)',++ 'server/ai/clip-evaluation.js': 'OpenVibe.Bench (configured LIVE_BENCH_URL): the AI\'s clip picks against people\'s, with Live\'s service token','server/ai/llm.js': 'images are Live\'s own frames; a streamer\'s typed provider address (testProvider) goes through egress.postJson','server/chat/soundboard-service.js': '101soundboards (host allowlist; the audio download through safeLookup)','server/chat/tts-engine.js': 'Google / AWS TTS (fixed hosts)',
@@ -429,6 +429,8 @@ function start() {
@@ -429,6 +429,8 @@ function start() {
_flagTimer = require('../utils/jobs').every('auto-clip-flag-sync', FLAG_SYNC_INTERVAL_MS, () => syncAutoClipFlags(), { initialDelayMs: 90 * 1000, jitterMs: 60 * 1000 });
_flagTimer = require('../utils/jobs').every('auto-clip-flag-sync', FLAG_SYNC_INTERVAL_MS, () => syncAutoClipFlags(), { initialDelayMs: 90 * 1000, jitterMs: 60 * 1000 });
// Once a day, the AI's old clips no person ever opened leave the lists (unlisted, never deleted).
// Once a day, the AI's old clips no person ever opened leave the lists (unlisted, never deleted).
require('./auto-clip-sweep').start();
require('./auto-clip-sweep').start();
+ // Once a day (LIVE_CLIP_EVAL=on), recordings with both an AI clip and a person's go to Bench for a blind vote.
+ require('./clip-evaluation').start();
console.log('[AutoClip] Live auto-clip job started (selective chat-spike + AI agreement, cuts via OpenVibe.Media) + persistent VOD backfill');
console.log('[AutoClip] Live auto-clip job started (selective chat-spike + AI agreement, cuts via OpenVibe.Media) + persistent VOD backfill');
}
}
function stop() {
function stop() {
@@ -436,6 +438,7 @@ function stop() {
@@ -436,6 +438,7 @@ function stop() {
if (_backfillTimer) { _backfillTimer(); _backfillTimer = null; }
if (_backfillTimer) { _backfillTimer(); _backfillTimer = null; }
if (_flagTimer) { _flagTimer(); _flagTimer = null; }
if (_flagTimer) { _flagTimer(); _flagTimer = null; }
require('./auto-clip-sweep').stop();
require('./auto-clip-sweep').stop();
+ require('./clip-evaluation').stop();
}
}
module.exports = { start, stop, clipVodMoment, backfillVodClips, syncAutoClipFlags, _tick, _internals: { checkLiveStream: _checkLiveStream, backfillPool: _backfillPool } };
module.exports = { start, stop, clipVodMoment, backfillVodClips, syncAutoClipFlags, _tick, _internals: { checkLiveStream: _checkLiveStream, backfillPool: _backfillPool } };
new file mode 100644
new file mode 100644
@@ -0,0 +1,131 @@
@@ -0,0 +1,131 @@
+'use strict';
+/**
+ * clip-evaluation.js — the AI's clip picks against people's, blind, on OpenVibe.Bench (item 151, C20).
+ *
+ * Once a day, for up to MAX_PER_RUN recent recordings that have both a public clip the AI cut and a public clip a
+ * person made, Live submits one live.clip round to Bench (#105): the AI's pick (policy auto-clip@AUTO_CLIP_POLICY)
+ * and the person's (policy people@1), each with a 6-hour signed playback link from Media. Signed-in people vote at
+ * https://bench.openvibe.services/battle?kind=live.clip (Bench #106: two players, blind, no titles), and
+ * results() reads each side's accept rate back from Bench (both_bad counts against both).
+ *
+ * Off unless LIVE_CLIP_EVAL=on, and never in drill mode (auto-clip-job.js starts it).
+ * Recording-separated: one round per recording, ever (state clip_eval_log keeps the last 2000).
+ * Protected content stays out: only public clips of public recordings, nothing the explicit check flags, never a
+ * mature recording, and never clip 483 or VOD 5709.
+ *
+ * run({ now, max }) → { submitted: [{ vod_id, round_id }], skipped, why }
+ * results() → { policies } from Bench, or null
+ * start() / stop()
+ */
+const db = require('../db/database');
+const media = require('../media-client');
+const principal = require('../net/network-principal');
+const { isExplicit } = require('./explicit');
+const { parseTs } = require('./clip-policy');
+
+const BENCH_URL = String(process.env.LIVE_BENCH_URL || 'https://bench.openvibe.services').replace(/\/+$/, '');
+// The picker as it stands: the date of its last change (score bar, calibration, freshness, sweep). Change it with the
+// picker, so Bench's accept rate is per version.
+const AUTO_CLIP_POLICY = process.env.LIVE_AUTO_CLIP_POLICY || '2026-11-02';
+const LOG = 'clip_eval_log';
+const KEEP = 2000;
+const MAX_PER_RUN = 3;
+const WINDOW_DAYS = 14;
+const PLAYBACK_TTL_S = 6 * 3600;
+const DAY_MS = 24 * 3600 * 1000;
+const NEVER_CLIPS = new Set(['483']);
+const NEVER_VODS = new Set(['5709']);
+
+const enabled = () => String(process.env.LIVE_CLIP_EVAL || '').toLowerCase() === 'on';
+
+async function _log() {
+ try { const d = JSON.parse(await db.getState(LOG) || '[]'); return Array.isArray(d) ? d : []; } catch { return []; }
+}
+
+/** A clip that may be judged: public, ready, not flagged, 5 to 180 seconds, with its window known. */
+function eligible(clip) {
+ if (!clip || clip.visibility !== 'public' || (clip.status && clip.status !== 'ready')) return false;
+ if (NEVER_CLIPS.has(String(clip.id)) || NEVER_VODS.has(String(clip.vod_id))) return false;
+ if (clip.mature || clip.is_mature || isExplicit(clip.title || '', clip.description || '')) return false;
+ const start = Number(clip.start_time);
+ const len = Number(clip.duration_seconds ?? clip.duration ?? (Number(clip.end_time) - start));
+ return Number.isFinite(start) && start >= 0 && Number.isFinite(len) && len >= 5 && len <= 180;
+}
+
+async function bench(method, path, body) {
+ const res = await fetch(`${BENCH_URL}${path}`, {
+ method, headers: { Accept: 'application/json', ...(body ? { 'Content-Type': 'application/json' } : {}), ...(await principal.serviceHeaders('openvibe.bench')) },
+ body: body ? JSON.stringify(body) : undefined, signal: AbortSignal.timeout(15_000),
+ });
+ const text = await res.text();
+ let json = null;
+ try { json = text ? JSON.parse(text) : null; } catch { json = null; }
+ if (!res.ok) { const e = new Error(`Bench ${method} ${path}: HTTP ${res.status}`); e.status = res.status; throw e; }
+ return json;
+}
+
+function entryOf(clip, policy, signed) {
+ const start = Math.round(Number(clip.start_time) * 1000);
+ const len = Math.round(Number(clip.duration_seconds ?? clip.duration ?? (Number(clip.end_time) - Number(clip.start_time))) * 1000);
+ return {
+ source: { recording_id: `vod_${clip.vod_id}`, start_ms: start, end_ms: start + len },
+ playback: { url: signed.url, expires_at: signed.expires_at },
+ policy,
+ ...(clip.title ? { title: String(clip.title).slice(0, 200) } : {}),
+ };
+}
+
+async function run({ now = Date.now(), max = MAX_PER_RUN } = {}) {
+ if (!enabled()) return { submitted: [], skipped: 0, why: 'off (LIVE_CLIP_EVAL is not on)' };
+ if (!principal.configured()) return { submitted: [], skipped: 0, why: 'no OAuth client secret' };
+ const since = new Date(now - WINDOW_DAYS * DAY_MS).toISOString();
+ const ai = await media.listClips({ auto_generated: 1, since, status: 'ready', order: 'newest', limit: 100 }, { timeoutMs: 15_000 });
+ if (!ai || !Array.isArray(ai.clips)) return { submitted: [], skipped: 0, why: 'Media did not answer' };
+ const done = new Set((await _log()).map((x) => String(x.vod_id)));
+ const byVod = new Map();
+ for (const c of ai.clips) if (c.auto_generated === true && eligible(c) && !done.has(String(c.vod_id)) && !byVod.has(String(c.vod_id))) byVod.set(String(c.vod_id), c);
+ const submitted = [];
+ let skipped = 0;
+ for (const [vodId, aiClip] of byVod) {
+ if (submitted.length >= max) break;
+ try {
+ const vod = await media.getVod(vodId, { timeoutMs: 10_000 });
+ if (!vod || vod.visibility !== 'public' || vod.mature || vod.is_mature) { skipped++; continue; }
+ const people = await media.listClips({ vod_id: vodId, auto_generated: 0, status: 'ready', limit: 20 }, { timeoutMs: 10_000 });
+ const person = (people && Array.isArray(people.clips) ? people.clips : []).find((c) => c.auto_generated !== true && eligible(c));
+ if (!person) { skipped++; continue; }
+ const [sa, sp] = await Promise.all([media.signedMediaUrl('clip', aiClip.id, PLAYBACK_TTL_S), media.signedMediaUrl('clip', person.id, PLAYBACK_TTL_S)]);
+ if (!sa || !sa.url || !sp || !sp.url) { skipped++; continue; }
+ const task = await bench('POST', '/api/v1/kinds/live.clip/tasks', { prompt: 'Two clips of one stream', public: false });
+ const round = await bench('POST', `/api/v1/tasks/${task.task.id}/rounds`, {});
+ const roundId = round.round.id;
+ await bench('POST', `/api/v1/rounds/${roundId}/entries`, { payload: entryOf(aiClip, { selector: 'auto-clip', version: AUTO_CLIP_POLICY, models: [] }, sa) });
+ await bench('POST', `/api/v1/rounds/${roundId}/entries`, { payload: entryOf(person, { selector: 'people', version: '1', models: [] }, sp) });
+ const log = await _log();
+ log.unshift({ vod_id: vodId, ai_clip: aiClip.id, person_clip: person.id, round_id: roundId, at: new Date(now).toISOString() });
+ await db.setState(LOG, JSON.stringify(log.slice(0, KEEP)));
+ submitted.push({ vod_id: vodId, round_id: roundId });
+ } catch (e) {
+ if (e && (e.status === 404 || e.status === 410)) { skipped++; continue; }
+ console.warn('[clip-eval] stopped for today:', e && e.message);
+ break; // Bench or Media down: the rest wait for tomorrow
+ }
+ }
+ if (submitted.length) console.log(`[clip-eval] ${submitted.length} recording(s) sent to Bench for a blind vote`);
+ return { submitted, skipped, why: submitted.length ? null : 'no recording with both an AI clip and a person\'s clip' };
+}
+
+/** Each side's results, as Bench counts them (shadow metrics): the AI's picker against people's. */
+async function results() {
+ if (!principal.configured()) return null;
+ try { return await bench('GET', '/api/v1/kinds/live.clip/policies'); } catch { return null; }
+}
+
+let _stop = null;
+function start() {
+ if (_stop || !enabled()) return;
+ _stop = require('../utils/jobs').every('clip-evaluation', DAY_MS, () => run(), { initialDelayMs: 40 * 60 * 1000, jitterMs: 20 * 60 * 1000 });
+}
+function stop() { if (_stop) { _stop(); _stop = null; } }
+
+module.exports = { run, results, start, stop, eligible, entryOf, AUTO_CLIP_POLICY, LOG, MAX_PER_RUN };
new file mode 100644
new file mode 100644
@@ -0,0 +1,115 @@
@@ -0,0 +1,115 @@
+/**
+ * The AI's clip picks against people's, blind, on Bench (server/ai/clip-evaluation.js; item 151, C20): a recent
+ * recording with a public AI clip and a public clip a person made becomes one live.clip round with both entries and
+ * signed playback links; one round per recording, ever; protected content (private, mature, explicit, clip 483,
+ * VOD 5709) stays out; off unless LIVE_CLIP_EVAL=on; Bench down stops the run for the day.
+ *
+ * node test/run.js clip-evaluation
+ */
+'use strict';
+const assert = require('assert');
+
+process.env.LIVE_CLIP_EVAL = 'on';
+const media = require('../server/media-client');
+const db = require('../server/db/database');
+const principal = require('../server/net/network-principal');
+const state = new Map();
+db.getState = async (k) => state.get(k) || null;
+db.setState = async (k, v) => { state.set(k, v); };
+principal.configured = () => true;
+principal.serviceHeaders = async () => ({ Authorization: 'Bearer svc-live' });
+
+const NOW = Date.parse('2026-11-03T02:00:00Z');
+const clip = (id, vod, ai, extra = {}) => ({ id, vod_id: vod, auto_generated: ai, visibility: 'public', status: 'ready', start_time: 120, duration_seconds: 30, title: `clip ${id}`, ...extra });
+let clips;
+let vods;
+let bench;
+let benchDown;
+const reset = () => {
+ clips = [
+ clip(10, 100, true), clip(11, 100, false), // AI and a person on 100: judged
+ clip(20, 200, true), // nobody else on 200: skipped
+ clip(30, 300, true), clip(31, 300, false, { visibility: 'unlisted' }), // the person's clip is not public
+ clip(40, 400, true, { title: 'explicit sex act' }), clip(41, 400, false),
+ clip(483, 500, true), clip(51, 500, false), // clip 483 never
+ clip(60, 5709, true), clip(61, 5709, false), // VOD 5709 never
+ clip(70, 700, true), clip(71, 700, false), // a mature recording
+ clip(80, 800, true), clip(81, 800, false), // a second good one
+ ];
+ vods = { 100: { id: 100, visibility: 'public' }, 300: { id: 300, visibility: 'public' }, 400: { id: 400, visibility: 'public' }, 500: { id: 500, visibility: 'public' }, 5709: { id: 5709, visibility: 'public' }, 700: { id: 700, visibility: 'public', mature: true }, 800: { id: 800, visibility: 'public' } };
+ bench = [];
+ benchDown = false;
+ state.clear();
+};
+media.listClips = async (q) => {
+ let rows = clips.filter((c) => c.status === 'ready');
+ if (q.auto_generated === 1) rows = rows.filter((c) => c.auto_generated);
+ if (q.auto_generated === 0) rows = rows.filter((c) => !c.auto_generated);
+ if (q.vod_id != null) rows = rows.filter((c) => String(c.vod_id) === String(q.vod_id));
+ return { clips: rows.slice(0, q.limit), total: rows.length };
+};
+media.getVod = async (id) => vods[id] || null;
+media.signedMediaUrl = async (kind, id, ttl) => ({ url: `https://openvibe.media/c/${id}?exp=1&sig=x`, expires_at: new Date(NOW + ttl * 1000).toISOString() });
+global.fetch = async (url, opts) => {
+ const u = new URL(url);
+ const body = opts.body ? JSON.parse(opts.body) : null;
+ bench.push({ method: opts.method, path: u.pathname, body, auth: opts.headers.Authorization });
+ if (benchDown) return new Response('down', { status: 503 });
+ if (u.pathname === '/api/v1/kinds/live.clip/tasks') return Response.json({ task: { id: `bnt_${bench.length}` } }, { status: 201 });
+ if (/\/rounds$/.test(u.pathname)) return Response.json({ round: { id: `brn_${bench.length}` } }, { status: 201 });
+ if (/\/entries$/.test(u.pathname)) return Response.json({ entry: { id: `bte_${bench.length}` } }, { status: 201 });
+ if (u.pathname === '/api/v1/kinds/live.clip/policies') return Response.json({ kind_id: 'live.clip', policies: [{ policy: 'auto-clip@x', accept_rate: 0.5 }] });
+ return new Response('?', { status: 404 });
+};
+
+const evaluation = require('../server/ai/clip-evaluation');
+
+(async () => {
+ reset();
+ let out = await evaluation.run({ now: NOW, max: 5 });
+ assert.deepStrictEqual(out.submitted.map((x) => x.vod_id).sort(), ['100', '800'], JSON.stringify(out));
+ const entries = bench.filter((b) => /\/entries$/.test(b.path));
+ assert.strictEqual(entries.length, 4, 'two entries a round');
+ const ai = entries.find((e) => e.body.payload.policy.selector === 'auto-clip' && e.body.payload.source.recording_id === 'vod_100');
+ const person = entries.find((e) => e.body.payload.policy.selector === 'people' && e.body.payload.source.recording_id === 'vod_100');
+ assert.ok(ai && person, 'the AI\'s pick and the person\'s');
+ assert.deepStrictEqual(ai.body.payload.source, { recording_id: 'vod_100', start_ms: 120000, end_ms: 150000 });
+ assert.match(ai.body.payload.playback.url, /^https:\/\/openvibe\.media\/c\/10\?/);
+ assert.strictEqual(ai.body.payload.policy.version, evaluation.AUTO_CLIP_POLICY);
+ assert.ok(Date.parse(ai.body.payload.playback.expires_at) > NOW, 'a link that lasts');
+ assert.ok(bench.every((b) => b.auth === 'Bearer svc-live'), 'Live\'s own service token');
+ const task = bench.find((b) => b.path === '/api/v1/kinds/live.clip/tasks');
+ assert.deepStrictEqual(task.body, { prompt: 'Two clips of one stream', public: false }, 'a private task that names no recording');
+ const touched = new Set(entries.map((e) => e.body.payload.source.recording_id));
+ for (const never of ['vod_200', 'vod_300', 'vod_400', 'vod_500', 'vod_5709', 'vod_700']) assert.ok(!touched.has(never), `${never} stays out`);
+
+ // One round per recording, ever.
+ const before = bench.length;
+ out = await evaluation.run({ now: NOW + 86400_000, max: 5 });
+ assert.strictEqual(out.submitted.length, 0, 'nothing new');
+ assert.strictEqual(bench.length, before, 'no second round for 100 or 800');
+ assert.strictEqual(JSON.parse(state.get(evaluation.LOG)).length, 2);
+
+ // At most `max` a run; Bench down stops the run (the rest wait for tomorrow).
+ reset();
+ out = await evaluation.run({ now: NOW, max: 1 });
+ assert.strictEqual(out.submitted.length, 1);
+ reset();
+ benchDown = true;
+ out = await evaluation.run({ now: NOW, max: 5 });
+ assert.strictEqual(out.submitted.length, 0);
+ assert.strictEqual(bench.length, 1, 'stopped at the first refusal');
+
+ // Off unless switched on.
+ reset();
+ process.env.LIVE_CLIP_EVAL = '';
+ out = await evaluation.run({ now: NOW });
+ assert.match(out.why, /off/);
+ assert.strictEqual(bench.length, 0);
+ process.env.LIVE_CLIP_EVAL = 'on';
+
+ // The shadow metrics come back from Bench.
+ const r = await evaluation.results();
+ assert.strictEqual(r.policies[0].accept_rate, 0.5);
+ console.log('clip evaluation: all checks passed');
+})().catch((e) => { console.error(e); process.exit(1); });
@@ -365,6 +365,7 @@ async function check(name, fn) {
@@ -365,6 +365,7 @@ async function check(name, fn) {
'server/monetization/cosmetics.js': 'legacy quest API (env)',
'server/monetization/cosmetics.js': 'legacy quest API (env)',
'server/openre/openre-client.js': 'OpenRestream (configured)',
'server/openre/openre-client.js': 'OpenRestream (configured)',
'server/ai/ai-service.js': 'OpenVibe.AI (configured)',
'server/ai/ai-service.js': 'OpenVibe.AI (configured)',
+ 'server/ai/clip-evaluation.js': 'OpenVibe.Bench (configured LIVE_BENCH_URL): the AI\'s clip picks against people\'s, with Live\'s service token',
'server/ai/llm.js': 'images are Live\'s own frames; a streamer\'s typed provider address (testProvider) goes through egress.postJson',
'server/ai/llm.js': 'images are Live\'s own frames; a streamer\'s typed provider address (testProvider) goes through egress.postJson',
'server/chat/soundboard-service.js': '101soundboards (host allowlist; the audio download through safeLookup)',
'server/chat/soundboard-service.js': '101soundboards (host allowlist; the audio download through safeLookup)',
'server/chat/tts-engine.js': 'Google / AWS TTS (fixed hosts)',
'server/chat/tts-engine.js': 'Google / AWS TTS (fixed hosts)',