openvibers/openvibe.bench Commits
live.clip: Live's AI clip picks judged blind (item 151, C19) (#105)
Changes
nnew file mode 100644@@@ -0,0 +1,79 @@++'use strict';++++/**++ * live.clip entries (item 151, C19 of the clip-quality research): a clip OpenVibe.Live's picker made, or chose not to++ * make, for one held-out stretch of a recording, compared blind against another policy's pick. Bench does not make++ * clips: Live submits them with its service token.++ *++ * payload = {++ * source: { recording_id, start_ms, end_ms, language?, genre? } the stretch the clip was cut from++ * playback: { url, expires_at } a short-lived private URL from Media++ * policy: { selector, version, models: [string] } which picker chose it, pinned on the entry++ * title?: string the title it would be published with++ * abstained: boolean, reason?: string "nothing worth clipping here"++ * }++ *++ * validateEntry(payload, { now }) → { ok: true, payload } | { ok: false, message }++ * forViewer(payload, { now, playback }) → the payload a reader sees: playback only for a voter, never after expiry++ */++const RECORDING_RE = /^[A-Za-z0-9_.:-]{1,80}$/;++const MIN_MS = 5_000;++const MAX_MS = 180_000;++const RUBRIC = Object.freeze([++ 'Interesting on its own, without the stream around it',++ 'Has its setup and its payoff',++ 'Starts and ends at the right moment, not mid-sentence',++ 'The title says what happens',++ 'Not a repeat of a clip the stream already has',++]);++++const str = (v, max) => (typeof v === 'string' && v.trim() ? v.trim().slice(0, max) : null);++++function validateEntry(payload, { now = Date.now() } = {}) {++ const p = payload && typeof payload === 'object' && !Array.isArray(payload) ? payload : null;++ if (!p) return { ok: false, message: 'A live.clip entry is an object.' };++ const s = p.source && typeof p.source === 'object' ? p.source : null;++ if (!s || !RECORDING_RE.test(String(s.recording_id || ''))) return { ok: false, message: 'source.recording_id names the recording the clip comes from.' };++ const start = Number(s.start_ms);++ const end = Number(s.end_ms);++ if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end <= start) return { ok: false, message: 'source.start_ms and end_ms are whole milliseconds, end after start.' };++ const pol = p.policy && typeof p.policy === 'object' ? p.policy : null;++ const version = pol && str(pol.version, 80);++ if (!pol || !str(pol.selector, 80) || !version) return { ok: false, message: 'policy.selector and policy.version name the picker that chose it.' };++ const models = Array.isArray(pol.models) ? pol.models.map((m) => str(m, 120)).filter(Boolean).slice(0, 10) : [];++ const out = {++ source: { recording_id: String(s.recording_id), start_ms: start, end_ms: end, language: str(s.language, 16), genre: str(s.genre, 40) },++ policy: { selector: str(pol.selector, 80), version, models },++ abstained: p.abstained === true,++ };++ if (out.abstained) {++ const reason = str(p.reason, 300);++ if (!reason) return { ok: false, message: 'An abstention says why: reason.' };++ if (p.playback) return { ok: false, message: 'An abstention has no playback.' };++ out.reason = reason;++ return { ok: true, payload: out };++ }++ if (end - start < MIN_MS || end - start > MAX_MS) return { ok: false, message: 'A clip is 5 to 180 seconds long.' };++ const pb = p.playback && typeof p.playback === 'object' ? p.playback : null;++ let url = null;++ try { url = pb && new URL(String(pb.url)); } catch { url = null; }++ if (!url || url.protocol !== 'https:') return { ok: false, message: 'playback.url is an https address.' };++ const exp = pb && Date.parse(pb.expires_at);++ if (!Number.isFinite(exp) || exp <= now) return { ok: false, message: 'playback.expires_at is a time still to come.' };++ out.playback = { url: url.toString(), expires_at: new Date(exp).toISOString() };++ const title = str(p.title, 200);++ if (title) out.title = title;++ return { ok: true, payload: out };++}++++/** What a reader of a live.clip entry sees: the playback only for a voter, and never once it has expired. */++function forViewer(payload, { now = Date.now(), playback = false } = {}) {++ if (!payload || typeof payload !== 'object' || !payload.playback) return payload;++ const { playback: pb, ...rest } = payload;++ if (!playback) return rest;++ const exp = Date.parse(pb.expires_at);++ if (!Number.isFinite(exp) || exp <= now) return { ...rest, playback_expired: true };++ return payload;++}++++module.exports = { validateEntry, forViewer, RUBRIC, MIN_MS, MAX_MS };@@@ -1,4 +1,5 @@'use strict';++const { PRIVATE_KIND_IDS } = require('../kinds');/*** Bench's generation workers. A round (or private eval run) is created as `queued`; this loop claims it, asks@@@ -15,7 +16,8 @@ function createBenchJobs({ s, bench, ai, config, log = console }) {async function claimRound() {return await s.tx(async () => {-- const r = await s.db.maybe("SELECT * FROM bench_rounds WHERE status = 'queued' ORDER BY created_at, id LIMIT 1 FOR UPDATE");++ // A private kind's round (live.clip) is filled by its submitter, never by Bench's AI.++ const r = await s.db.maybe("SELECT * FROM bench_rounds WHERE status = 'queued' AND NOT (kind_id = ANY($1::text[])) ORDER BY created_at, id LIMIT 1 FOR UPDATE", [PRIVATE_KIND_IDS]);if (!r) return null;await s.db.exec("UPDATE bench_rounds SET status = 'running' WHERE id = $1", [r.id]);return { ...r, status: 'running' };@@@ -27,7 +27,7 @@ const express = require('express');const { http, ids } = require('openvibe-contracts');const { asyncRouter } = require('./router');const { sameOrigin } = require('./same-origin');--const { kindById } = require('../kinds');++const { kindById, isPrivateKind } = require('../kinds');const { BenchError } = require('../store');const boardLib = require('../rating/board');@@@ -92,8 +92,26 @@ function createApi(ctx) {});// ── Tasks ───────────────────────────────────────────────++ // A private kind (live.clip) takes tasks, rounds and entries from its submitter only (kinds.js `submitter`).++ const submitterOnly = (req, res, kindId) => {++ if (!isPrivateKind(kindId)) return true;++ const want = (kindById(kindId) || {}).submitter;++ if (req.principal && req.principal.requester === want) return true;++ problem(res, 403, 'bench.private_kind', `Only ${want} submits to ${kindId}.`);++ return false;++ };++++ r.get('/kinds/:kindId/policies', principal.requireCapability('bench.entry.create'), async (req, res) => {++ const kindId = String(req.params.kindId || '');++ if (kindId !== 'live.clip') return problem(res, 404, 'bench.kind_not_found', 'Policy results are kept for live.clip.');++ if (!submitterOnly(req, res, kindId)) return undefined;++ res.set('Cache-Control', 'private, no-store');++ return res.json({ kind_id: kindId, policies: await bench.clipPolicies() });++ });++r.post('/kinds/:kindId/tasks', actor('bench.task.create'), async (req, res) => {const b = req.body && typeof req.body === 'object' ? req.body : {};++ if (!submitterOnly(req, res, String(req.params.kindId || ''))) return undefined;const owner = creatorOf(req);const out = await bench.createTask({kindId: String(req.params.kindId || ''),@@@ -117,6 +135,8 @@ function createApi(ctx) {// ── Rounds ──────────────────────────────────────────────r.post('/tasks/:id/rounds', actor('bench.round.create'), async (req, res) => {const b = req.body && typeof req.body === 'object' ? req.body : {};++ const task = await bench.getTask(String(req.params.id || ''));++ if (task && !submitterOnly(req, res, task.kind_id)) return undefined;const owner = creatorOf(req);try {const out = await bench.createRound({@@@ -135,7 +155,11 @@ function createApi(ctx) {try {const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: subjectOf(req) });if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');-- res.set('Cache-Control', round.blind ? 'private, no-store' : 'public, max-age=60');++ // A private kind's round is for its submitter and for the people who voted on it.++ if (isPrivateKind(round.kind_id) && round.blind && !(req.principal && req.principal.requester === (kindById(round.kind_id) || {}).submitter)) {++ return problem(res, 404, 'bench.round_not_found', 'No such round.');++ }++ res.set('Cache-Control', round.blind || isPrivateKind(round.kind_id) ? 'private, no-store' : 'public, max-age=60');return res.json({ round });} catch (err) { return failed(res, err); }});@@@ -143,6 +167,8 @@ function createApi(ctx) {r.post('/rounds/:id/entries', principal.requireCapability('bench.entry.create'), async (req, res) => {const b = req.body && typeof req.body === 'object' ? req.body : {};try {++ const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: null });++ if (round && !submitterOnly(req, res, round.kind_id)) return undefined;const entry = await bench.addEntry({roundId: String(req.params.id || ''),provider: b.provider,@@@ -185,8 +211,8 @@ function createApi(ctx) {return problem(res, 401, 'bench.sign_in_required', 'Sign in at bench.openvibe.services, or send a service token that may relay anonymous votes.');}const kindRow = kindById(kind);-- if (!kindRow || kindRow.judge !== 'human_pairwise') {-- return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only pairwise kinds can be judged without signing in.');++ if (!kindRow || kindRow.judge !== 'human_pairwise' || kindRow.private) {++ return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only public pairwise kinds can be judged without signing in.');}if (anon.minted) ctx.anon.setCookie(res, anon.opaque);const pair = await bench.nextJudgement({@@@ -217,8 +243,8 @@ function createApi(ctx) {const round = await bench.getRound(roundId, { viewerSubject: null });if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');const row = kindById(round.kind_id);-- if (!row || row.judge !== 'human_pairwise') {-- return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only pairwise kinds can be judged without signing in.');++ if (!row || row.judge !== 'human_pairwise' || row.private) {++ return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only public pairwise kinds can be judged without signing in.');}const out = await bench.castJudgement({roundId, kindId: round.kind_id, judge: 'human_pairwise',@@@ -9,7 +9,7 @@const express = require('express');const crypto = require('crypto');const renderLib = require('../bench/render');--const { kindById } = require('../kinds');++const { kindById, isPrivateKind } = require('../kinds');function createRenderRoutes(ctx) {const r = express.Router();@@@ -18,6 +18,8 @@ function createRenderRoutes(ctx) {const row = await ctx.s.db.maybe('SELECT e.id, e.payload, e.error, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.id = $1',[String(id || '')]);++ // A private kind's entry (a live.clip pick) is never rendered here: its voters see it in the vote itself.++ if (row && isPrivateKind(row.kind_id)) return null;if (!row) {const match = /^((?:brn_)[0-9A-HJKMNP-TV-Z]{26})~([A-Za-z0-9_-]{1,120})$/.exec(String(id || ''));if (!match) return null;@@@ -11,7 +11,7 @@ const { http } = require('openvibe-contracts');const { asyncRouter } = require('./router');const { sameOrigin } = require('./same-origin');const renderLib = require('../bench/render');--const { kindById, SHOWCASE_KIND_IDS } = require('../kinds');++const { kindById, SHOWCASE_KIND_IDS, isPrivateKind } = require('../kinds');const { esc, raw } = require('../render/html');const { send } = require('../render/layout');@@@ -51,7 +51,8 @@ async function voteCount(db, roundId) {async function loadPairs(ctx, opts) {const db = ctx.s.db;const kind = opts && opts.kind ? String(opts.kind) : 'all';-- const kinds = kind && kind !== 'all' ? [kind] : SHOWCASE_KIND_IDS;++ // A private kind (live.clip) is never in the gallery, even when asked for by name.++ const kinds = kind && kind !== 'all' ? (isPrivateKind(kind) ? [] : [kind]) : SHOWCASE_KIND_IDS;const limit = Math.min(100, Math.max(1, num(opts && opts.limit, 24)));const rows = await db.many("SELECT r.id, r.kind_id, r.created_at, r.created_by, t.prompt, t.task_class FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) ORDER BY r.created_at DESC, r.id LIMIT $2",@@@ -44,8 +44,20 @@ const KINDS = Object.freeze([{ id: 'design.interface', room: 'design', entry: 'component_set', judge: 'human_pairwise', name: 'Component systems arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric'], entry: 'component_set', payload: { html: 'string' } } },]);--const byId = new Map(KINDS.map((k) => [k.id, k]));++// Private kinds: their rounds compare things that are not public (a channel's held-out recordings), so they never++// appear in the public kind list, the boards, the open datasets or the vote dump; their submitter reads its own results.++// live.clip (item 151, C19): OpenVibe.Live's AI clip picks, blind against another policy's (server/bench/clip.js).++const PRIVATE_KINDS = Object.freeze([++ { id: 'live.clip', room: 'live', entry: 'video_clip', judge: 'human_pairwise', name: 'AI clip picks', metrics: ['elo', 'cost_usd', 'accept_rate'], private: true, submitter: 'service:live',++ schema: { renderer: 'video', judges: ['human_pairwise'], entry: 'video_clip',++ payload: { source: 'object', playback: 'object', policy: 'object', title: 'string', abstained: 'boolean', reason: 'string' },++ rubric: require('./bench/clip').RUBRIC } },++]);++const PRIVATE_KIND_IDS = Object.freeze(PRIVATE_KINDS.map((k) => k.id));++const ALL_KINDS = Object.freeze(KINDS.concat(PRIVATE_KINDS));++++const byId = new Map(ALL_KINDS.map((k) => [k.id, k]));const SHOWCASE_KIND_IDS = Object.freeze(KINDS.filter((k) => k.schema && k.schema.renderer).map((k) => k.id));--module.exports = { KINDS, JUDGES, WINNERS, SHOWCASE_KIND_IDS, kindById: (id) => byId.get(id) || null };++module.exports = { KINDS, PRIVATE_KINDS, PRIVATE_KIND_IDS, ALL_KINDS, JUDGES, WINNERS, SHOWCASE_KIND_IDS, kindById: (id) => byId.get(id) || null, isPrivateKind: (id) => PRIVATE_KIND_IDS.includes(id) };@@@ -10,7 +10,7 @@const { frontier } = require('./frontier');const { parsePick } = require('./parser');--const { KINDS, kindById, SHOWCASE_KIND_IDS } = require('../kinds');++const { KINDS, kindById, SHOWCASE_KIND_IDS, PRIVATE_KIND_IDS } = require('../kinds');const renderLib = require('../bench/render');const { screenshotOf } = require('../http/gallery');@@@ -52,7 +52,7 @@ async function busiestKind(ctx) {const rows = await ctx.s.db.many(`SELECT r.kind_id AS id, COUNT(e.id)::int AS nFROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id-- WHERE e.error IS NULL GROUP BY r.kind_id ORDER BY n DESC, r.kind_id LIMIT 1`);++ WHERE e.error IS NULL AND NOT (r.kind_id = ANY($1::text[])) GROUP BY r.kind_id ORDER BY n DESC, r.kind_id LIMIT 1`, [PRIVATE_KIND_IDS]);if (rows && rows[0] && rows[0].id) return rows[0].id;return SHOWCASE_KIND_IDS[0] || 'games.play';}@@@ -14,6 +14,8 @@*/const style = require('./style');++// Private kinds' votes (live.clip) are never in the open vote dump or the public ratings.++const { PRIVATE_KIND_IDS } = require('../kinds');const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));@@@ -26,8 +28,8 @@ async function loadVotes(db) {FROM bench_judgements jJOIN bench_entries l ON l.id = (j.pair->>'left')JOIN bench_entries r ON r.id = (j.pair->>'right')-- WHERE j.judge = 'human_pairwise' AND j.outcome IS NOT NULL-- ORDER BY j.at, j.id`);++ WHERE j.judge = 'human_pairwise' AND j.outcome IS NOT NULL AND NOT (j.kind_id = ANY($1::text[]))++ ORDER BY j.at, j.id`, [PRIVATE_KIND_IDS]);const out = [];for (const row of rows) {const left = styleOf(row.left_payload, row.left_length, row.left_markup);@@@ -8,7 +8,8 @@*/const crypto = require('crypto');const { ids } = require('openvibe-contracts');--const { KINDS, JUDGES, WINNERS, kindById } = require('./kinds');++const { KINDS, ALL_KINDS, PRIVATE_KIND_IDS, JUDGES, WINNERS, kindById, isPrivateKind } = require('./kinds');++const clipLib = require('./bench/clip');const ratingLib = require('./rating');const styleLib = require('./rating/style');@@@ -197,7 +198,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =// ── Kinds ───────────────────────────────────────────────async function seedKinds() {-- for (const k of KINDS) {++ for (const k of ALL_KINDS) {await db.exec(`INSERT INTO bench_kinds (id, room, entry, judge, name, schema, metrics, created_at)VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7::jsonb, now())@@@ -207,8 +208,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =);}}++ /** The public kinds; a private one (live.clip) is never listed. */async function listKinds() {-- return (await db.many('SELECT * FROM bench_kinds ORDER BY id')).map(wireKind);++ return (await db.many('SELECT * FROM bench_kinds WHERE NOT (id = ANY($1::text[])) ORDER BY id', [PRIVATE_KIND_IDS])).map(wireKind);}async function getKind(id) {return wireKind(await db.maybe('SELECT * FROM bench_kinds WHERE id = $1', [String(id || '')]));@@@ -218,6 +220,8 @@ function createStore(s, { log = console, random = Math.random, config = null } =async function createTask({ kindId, prompt, taskClass = null, public: isPublic = false, subject }) {const kind = await getKind(kindId);if (!kind) return refusal(422, 'bench.kind_not_found', `No such kind: ${kindId}`);++ // A private kind's tasks name what is not public (a held-out recording): never public.++ if (isPrivateKind(kind.id)) isPublic = false;const text = String(prompt == null ? '' : prompt).replace(/\s+/g, ' ').trim();if (!text) return refusal(422, 'bench.prompt_required', 'A task needs a prompt or spec.');if (text.length > 4000) return refusal(422, 'bench.prompt_too_long', 'The prompt must be at most 4000 characters.');@@@ -302,7 +306,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =const revealed = viewerSubject != null && (r.created_by === viewerSubject|| await db.value('SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND subject = $2', [r.id, viewerSubject]) > 0);const votes = num(await db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND method <> 'game_outcome'", [r.id]));-- const entries = entryRows.map((e) => wireEntry(e, { blind: !revealed }));++ // A live.clip playback is only for a voter (nextJudgement); a round read never carries one.++ const entries = entryRows.map((e) => wireEntry(e, { blind: !revealed }))++ .map((e) => (isPrivateKind(r.kind_id) ? { ...e, payload: clipLib.forViewer(e.payload, { now: s.now(), playback: false }) } : e));const out = { ...wireRound(r), entries, votes };if (!revealed) out.blind = true;return out;@@@ -313,6 +319,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =return await s.tx(async () => {const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1 FOR UPDATE', [String(roundId || '')]);if (!r) throw new BenchError(404, 'bench.round_not_found', 'No such Bench round.');++ if (r.kind_id === 'live.clip') {++ // A clip pick: its shape checked, and the policy that chose it pinned on the entry (route), never rewritten.++ const v = clipLib.validateEntry(payload, { now: s.now() });++ if (!v.ok) throw new BenchError(422, 'bench.invalid_clip', v.message);++ payload = v.payload;++ route = { policy: v.payload.policy };++ model = model || v.payload.policy.models[0] || `${v.payload.policy.selector}@${v.payload.policy.version}`;++ provider = provider || 'live';++ }const id = s.newId('bte');// The style of the output (how long it is, how much markup it wears) is measured once, here, so the// ratings can control for style later without opening the payload again.@@@ -386,7 +401,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =[String(kind || ''), subject, creators],);if (!row) return null;-- const entries = (await db.many('SELECT * FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id', [row.id])).map(wireEntry);++ const entries = (await db.many('SELECT * FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id', [row.id])).map(wireEntry)++ // A voter of a private kind sees the playback, while it lasts (clip.forViewer).++ .map((e) => (isPrivateKind(row.kind_id) ? { ...e, payload: clipLib.forViewer(e.payload, { now: s.now(), playback: true }) } : e));if (entries.length < 2) return null;const [left, right] = shuffle(entries, rng).slice(0, 2);return {@@@ -609,7 +626,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =async function board(kindId, { type = 'elo', category = CATEGORY_DEFAULT, period = 'all', voters = null } = {}) {const kind = await getKind(kindId);-- if (!kind) return null;++ if (!kind || isPrivateKind(kind.id)) return null;const cat = categoryOf(voters, category);const cost = await costStats(kindId);const pass = await passRate(kindId);@@@ -700,7 +717,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =// ── Dataset (open, no voter ids) ─────────────────────────async function dataset(kindId) {const kind = await getKind(kindId);-- if (!kind) return null;++ if (!kind || isPrivateKind(kind.id)) return null;const tasks = (await db.many('SELECT * FROM bench_tasks WHERE kind_id = $1 AND public = true ORDER BY created_at, id', [kindId])).map(wireTask).map(({ created_by, ...rest }) => rest);const rounds = (await db.many("SELECT * FROM bench_rounds WHERE kind_id = $1 AND status IN ('ready', 'finished') ORDER BY created_at, id", [kindId]))@@@ -728,6 +745,43 @@ function createStore(s, { log = console, random = Math.random, config = null } =return lines.length ? `${lines.join('\n')}\n` : '';}++ /**++ * live.clip by policy (selector@version), for its submitter: entries, abstentions, cost, and how its picks fared++ * in blind votes. accept_rate = (wins + ties) / votes it was in; a both_bad vote counts against both sides.++ */++ async function clipPolicies() {++ const entries = await db.many(`SELECT e.id, e.cost_usd, e.payload FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id++ WHERE r.kind_id = 'live.clip' AND e.error IS NULL`);++ const byEntry = new Map();++ const stats = new Map();++ for (const e of entries) {++ const p = typeof e.payload === 'string' ? JSON.parse(e.payload) : (e.payload || {});++ const key = `${(p.policy && p.policy.selector) || '?'}@${(p.policy && p.policy.version) || '?'}`;++ byEntry.set(e.id, key);++ const st = stats.get(key) || { policy: key, entries: 0, abstained: 0, cost_usd: 0, votes: 0, wins: 0, ties: 0, losses: 0, both_bad: 0 };++ st.entries += 1;++ if (p.abstained) st.abstained += 1;++ st.cost_usd += num(e.cost_usd);++ stats.set(key, st);++ }++ const votes = await db.many(`SELECT j.pair, j.outcome FROM bench_judgements j WHERE j.kind_id = 'live.clip' AND j.judge = 'human_pairwise' AND j.pair IS NOT NULL AND j.outcome IS NOT NULL`);++ for (const v of votes) {++ const pair = typeof v.pair === 'string' ? JSON.parse(v.pair) : v.pair;++ const sides = [['left', pair && pair.left], ['right', pair && pair.right]];++ for (const [side, id] of sides) {++ const st = stats.get(byEntry.get(id));++ if (!st) continue;++ st.votes += 1;++ if (v.outcome === 'tie') st.ties += 1;++ else if (v.outcome === 'both_bad') st.both_bad += 1;++ else if (v.outcome === side) st.wins += 1;++ else st.losses += 1;++ }++ }++ return [...stats.values()].map((st) => ({ ...st, cost_usd: Math.round(st.cost_usd * 1e6) / 1e6, accept_rate: st.votes ? Math.round(((st.wins + st.ties) / st.votes) * 1000) / 1000 : null }))++ .sort((a, b) => a.policy.localeCompare(b.policy));++ }++// ── Private evals ────────────────────────────────────────async function createEval({ owner, kindId, name, testSuite }) {const kind = await getKind(kindId);@@@ -786,7 +840,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,insertOutbox, seedKinds, listKinds, getKind,createTask, getTask,-- createRound, getRound, addEntry, maybeReady, finishRound,++ createRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,nextJudgement, castJudgement, pairwiseGames,board, valueRows, modelSummary, dataset, datasetJsonl,createEval, getEval, runEval, evalResults, evalDailyCount,nnew file mode 100644@@@ -0,0 +1,124 @@++'use strict';++/**++ * live.clip (item 151, C19): OpenVibe.Live's AI clip picks, judged blind against another policy's. Typed clip and++ * abstention entries, the policy pinned on each entry, only Live submits, the playback only for a voter and never++ * after it expires, never on a public board, list, gallery, dataset or vote dump, never filled by Bench's AI, and the++ * cost and accept rate per policy (both_bad counts against both).++ * node test/live-clip.test.js++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++++(async () => {++ const t = await boot({ env: { BENCH_ROUNDS_PER_DAY: '100', BENCH_ROUNDS_DAILY: '1000' } });++ const voter = t.network.addUser('voter');++ const voter2 = t.network.addUser('voter2');++ const live = t.network.serviceToken({ sub: 'svc:live', audience: 'openvibe.bench', cap: ['bench.task.create', 'bench.round.create', 'bench.entry.create'] });++ const other = t.network.serviceToken({ sub: 'svc:room', audience: 'openvibe.bench', cap: ['bench.task.create', 'bench.round.create', 'bench.entry.create'] });++ const soon = () => new Date(Date.now() + 3600_000).toISOString();++ const clip = (version, extra = {}) => ({++ source: { recording_id: 'vod_5021', start_ms: 60_000, end_ms: 90_000, language: 'ja', genre: 'irl' },++ playback: { url: `https://media.openvibe.media/p/${version}.mp4?sig=x`, expires_at: soon() },++ policy: { selector: 'moment-picker', version, models: ['qwen3-vl-4b'] },++ title: 'The cat walks onto the keyboard',++ ...extra,++ });++ let roundId;++++ async function newRound(token = live) {++ const task = await t.get('/api/v1/kinds/live.clip/tasks', { bearer: token, json: { prompt: 'vod_5021 30:00-45:00', public: true } });++ if (task.status !== 201) return { status: task.status, text: task.text };++ assert.strictEqual(task.json().task.public, false, 'a private kind\'s task is never public');++ const rr = await t.get(`/api/v1/tasks/${task.json().task.id}/rounds`, { bearer: token, json: {} });++ assert.strictEqual(rr.status, 201, rr.text);++ return { status: 201, id: rr.json().round.id };++ }++++ await check('only Live makes live.clip tasks, rounds and entries', async () => {++ assert.strictEqual((await newRound(other)).status, 403);++ const r = await newRound();++ assert.strictEqual(r.status, 201, r.text);++ roundId = r.id;++ const e = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: other, json: { payload: clip('v1'), cost_usd: 0.002 } });++ assert.strictEqual(e.status, 403);++ });++++ await check('typed entries: a clip and an abstention; the bad ones refused; the policy pinned on the entry', async () => {++ const bad = [++ clip('v1', { source: { recording_id: 'vod_5021', start_ms: 90_000, end_ms: 60_000 } }),++ clip('v1', { source: { recording_id: 'vod_5021', start_ms: 0, end_ms: 2_000 } }),++ clip('v1', { playback: { url: 'http://insecure.example/x.mp4', expires_at: soon() } }),++ clip('v1', { playback: { url: 'https://media.openvibe.media/p.mp4', expires_at: new Date(Date.now() - 1000).toISOString() } }),++ clip('v1', { policy: { selector: 'moment-picker' } }),++ { source: { recording_id: 'vod_5021', start_ms: 0, end_ms: 1 }, policy: { selector: 's', version: 'v2' }, abstained: true },++ ];++ for (const payload of bad) {++ const r = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: live, json: { payload } });++ assert.strictEqual(r.status, 422, JSON.stringify(payload).slice(0, 120));++ }++ const a = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: live, json: { payload: clip('v1', { extra: 'dropped' }), cost_usd: 0.002 } });++ assert.strictEqual(a.status, 201, a.text);++ assert.strictEqual(a.json().entry.payload.extra, undefined, 'only the typed fields are kept');++ const b = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: live, json: { payload: { source: { recording_id: 'vod_5021', start_ms: 0, end_ms: 1000 }, policy: { selector: 'moment-picker', version: 'v2', models: ['smolvlm2-500m'] }, abstained: true, reason: 'Nothing happened worth a clip.' }, cost_usd: 0.0005 } });++ assert.strictEqual(b.status, 201, b.text);++ const row = await t.ctx.s.db.one('SELECT route, model FROM bench_entries WHERE id = $1', [a.json().entry.id]);++ assert.deepStrictEqual(row.route.policy, { selector: 'moment-picker', version: 'v1', models: ['qwen3-vl-4b'] });++ assert.strictEqual(row.model, 'qwen3-vl-4b');++ });++++ await check('the playback is for a voter only: not in a round read, not after it expires', async () => {++ const anon = await t.get(`/api/v1/rounds/${roundId}`);++ assert.strictEqual(anon.status, 404, 'a private round is not readable by just anyone');++ const asLive = await t.get(`/api/v1/rounds/${roundId}`, { bearer: live });++ assert.strictEqual(asLive.status, 200, asLive.text);++ assert.ok(asLive.json().round.entries.length === 2 && asLive.json().round.entries.every((e) => !e.payload.playback), 'no playback in a round read');++ const pair = await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter });++ assert.strictEqual(pair.status, 200, pair.text);++ const sides = [pair.json().left, pair.json().right];++ assert.ok(sides.some((sd) => sd.payload.playback && sd.payload.playback.url.startsWith('https://')), 'the voter sees the clip');++ await t.ctx.s.db.exec("UPDATE bench_entries SET payload = jsonb_set(payload, '{playback,expires_at}', to_jsonb('2020-01-01T00:00:00.000Z'::text)) WHERE round_id = $1 AND payload ? 'playback'", [roundId]);++ const later = await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter2 });++ const ls = [later.json().left, later.json().right];++ assert.ok(ls.every((sd) => !sd.payload.playback), 'never after it expires');++ assert.ok(ls.some((sd) => sd.payload.playback_expired === true));++ const anonNext = await t.get('/api/v1/judgements/next?kind=live.clip');++ assert.ok([401, 403].includes(anonNext.status), `anonymous voting on a private kind is refused (${anonNext.status})`);++ });++++ await check('cost and accept rate per policy; both_bad counts against both', async () => {++ const pair = await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter });++ const p = pair.json();++ const entries = await t.ctx.s.db.many('SELECT id, payload FROM bench_entries WHERE round_id = $1', [roundId]);++ const v1 = entries.find((e) => e.payload.policy.version === 'v1').id;++ const winnerSide = p.left.id === v1 ? 'left' : 'right';++ const vote = await t.get(`/api/v1/rounds/${roundId}/judgements`, { as: voter, json: { judge: 'human_pairwise', pair: { left: p.left.id, right: p.right.id }, outcome: winnerSide } });++ assert.strictEqual(vote.status, 201, vote.text);++ const p2 = (await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter2 })).json();++ const vote2 = await t.get(`/api/v1/rounds/${roundId}/judgements`, { as: voter2, json: { judge: 'human_pairwise', pair: { left: p2.left.id, right: p2.right.id }, outcome: 'both_bad' } });++ assert.strictEqual(vote2.status, 201, vote2.text);++ const r = await t.get('/api/v1/kinds/live.clip/policies', { bearer: live });++ assert.strictEqual(r.status, 200, r.text);++ const by = Object.fromEntries(r.json().policies.map((x) => [x.policy, x]));++ assert.deepStrictEqual([by['moment-picker@v1'].votes, by['moment-picker@v1'].wins, by['moment-picker@v1'].both_bad, by['moment-picker@v1'].accept_rate], [2, 1, 1, 0.5]);++ assert.deepStrictEqual([by['moment-picker@v2'].votes, by['moment-picker@v2'].losses, by['moment-picker@v2'].both_bad, by['moment-picker@v2'].abstained, by['moment-picker@v2'].accept_rate], [2, 1, 1, 1, 0]);++ assert.strictEqual(by['moment-picker@v1'].cost_usd, 0.002);++ assert.strictEqual((await t.get('/api/v1/kinds/live.clip/policies', { bearer: other })).status, 403, 'Live\'s results are Live\'s');++ });++++ await check('never public: the kind list, the board, the dataset, the vote dump, the gallery; never filled by Bench\'s AI', async () => {++ const kinds = (await t.get('/api/v1/kinds')).json().kinds.map((k) => k.id);++ assert.ok(!kinds.includes('live.clip'));++ assert.strictEqual((await t.get('/api/v1/boards/live.clip')).status, 404);++ assert.strictEqual((await t.get('/api/v1/datasets/live.clip.jsonl')).status, 404);++ const votes = await t.get('/api/v1/datasets/votes.jsonl');++ assert.ok(!votes.text.includes('live.clip'), 'no live.clip vote in the dump');++ const gallery = await t.get('/api/v1/showcase/rounds?kind=live.clip');++ if (gallery.status === 200) assert.ok(!gallery.text.includes(roundId), 'not in the gallery');++ const queued = await newRound();++ const claimed = await t.ctx.s.db.maybe("SELECT * FROM bench_rounds WHERE status = 'queued' AND NOT (kind_id = ANY($1::text[])) AND id = $2", [['live.clip'], queued.id]);++ assert.strictEqual(claimed, null, 'the job runner never claims it');++ });++++ await t.close();++ done();++})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,79 @@
@@ -0,0 +1,79 @@
+'use strict';
+
+/**
+ * live.clip entries (item 151, C19 of the clip-quality research): a clip OpenVibe.Live's picker made, or chose not to
+ * make, for one held-out stretch of a recording, compared blind against another policy's pick. Bench does not make
+ * clips: Live submits them with its service token.
+ *
+ * payload = {
+ * source: { recording_id, start_ms, end_ms, language?, genre? } the stretch the clip was cut from
+ * playback: { url, expires_at } a short-lived private URL from Media
+ * policy: { selector, version, models: [string] } which picker chose it, pinned on the entry
+ * title?: string the title it would be published with
+ * abstained: boolean, reason?: string "nothing worth clipping here"
+ * }
+ *
+ * validateEntry(payload, { now }) → { ok: true, payload } | { ok: false, message }
+ * forViewer(payload, { now, playback }) → the payload a reader sees: playback only for a voter, never after expiry
+ */
+const RECORDING_RE = /^[A-Za-z0-9_.:-]{1,80}$/;
+const MIN_MS = 5_000;
+const MAX_MS = 180_000;
+const RUBRIC = Object.freeze([
+ 'Interesting on its own, without the stream around it',
+ 'Has its setup and its payoff',
+ 'Starts and ends at the right moment, not mid-sentence',
+ 'The title says what happens',
+ 'Not a repeat of a clip the stream already has',
+]);
+
+const str = (v, max) => (typeof v === 'string' && v.trim() ? v.trim().slice(0, max) : null);
+
+function validateEntry(payload, { now = Date.now() } = {}) {
+ const p = payload && typeof payload === 'object' && !Array.isArray(payload) ? payload : null;
+ if (!p) return { ok: false, message: 'A live.clip entry is an object.' };
+ const s = p.source && typeof p.source === 'object' ? p.source : null;
+ if (!s || !RECORDING_RE.test(String(s.recording_id || ''))) return { ok: false, message: 'source.recording_id names the recording the clip comes from.' };
+ const start = Number(s.start_ms);
+ const end = Number(s.end_ms);
+ if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end <= start) return { ok: false, message: 'source.start_ms and end_ms are whole milliseconds, end after start.' };
+ const pol = p.policy && typeof p.policy === 'object' ? p.policy : null;
+ const version = pol && str(pol.version, 80);
+ if (!pol || !str(pol.selector, 80) || !version) return { ok: false, message: 'policy.selector and policy.version name the picker that chose it.' };
+ const models = Array.isArray(pol.models) ? pol.models.map((m) => str(m, 120)).filter(Boolean).slice(0, 10) : [];
+ const out = {
+ source: { recording_id: String(s.recording_id), start_ms: start, end_ms: end, language: str(s.language, 16), genre: str(s.genre, 40) },
+ policy: { selector: str(pol.selector, 80), version, models },
+ abstained: p.abstained === true,
+ };
+ if (out.abstained) {
+ const reason = str(p.reason, 300);
+ if (!reason) return { ok: false, message: 'An abstention says why: reason.' };
+ if (p.playback) return { ok: false, message: 'An abstention has no playback.' };
+ out.reason = reason;
+ return { ok: true, payload: out };
+ }
+ if (end - start < MIN_MS || end - start > MAX_MS) return { ok: false, message: 'A clip is 5 to 180 seconds long.' };
+ const pb = p.playback && typeof p.playback === 'object' ? p.playback : null;
+ let url = null;
+ try { url = pb && new URL(String(pb.url)); } catch { url = null; }
+ if (!url || url.protocol !== 'https:') return { ok: false, message: 'playback.url is an https address.' };
+ const exp = pb && Date.parse(pb.expires_at);
+ if (!Number.isFinite(exp) || exp <= now) return { ok: false, message: 'playback.expires_at is a time still to come.' };
+ out.playback = { url: url.toString(), expires_at: new Date(exp).toISOString() };
+ const title = str(p.title, 200);
+ if (title) out.title = title;
+ return { ok: true, payload: out };
+}
+
+/** What a reader of a live.clip entry sees: the playback only for a voter, and never once it has expired. */
+function forViewer(payload, { now = Date.now(), playback = false } = {}) {
+ if (!payload || typeof payload !== 'object' || !payload.playback) return payload;
+ const { playback: pb, ...rest } = payload;
+ if (!playback) return rest;
+ const exp = Date.parse(pb.expires_at);
+ if (!Number.isFinite(exp) || exp <= now) return { ...rest, playback_expired: true };
+ return payload;
+}
+
+module.exports = { validateEntry, forViewer, RUBRIC, MIN_MS, MAX_MS };
@@ -1,4 +1,5 @@
@@ -1,4 +1,5 @@
'use strict';
'use strict';
+const { PRIVATE_KIND_IDS } = require('../kinds');
/**
/**
* Bench's generation workers. A round (or private eval run) is created as `queued`; this loop claims it, asks
* Bench's generation workers. A round (or private eval run) is created as `queued`; this loop claims it, asks
@@ -15,7 +16,8 @@ function createBenchJobs({ s, bench, ai, config, log = console }) {
@@ -15,7 +16,8 @@ function createBenchJobs({ s, bench, ai, config, log = console }) {
async function claimRound() {
async function claimRound() {
return await s.tx(async () => {
return await s.tx(async () => {
- const r = await s.db.maybe("SELECT * FROM bench_rounds WHERE status = 'queued' ORDER BY created_at, id LIMIT 1 FOR UPDATE");
+ // A private kind's round (live.clip) is filled by its submitter, never by Bench's AI.
+ const r = await s.db.maybe("SELECT * FROM bench_rounds WHERE status = 'queued' AND NOT (kind_id = ANY($1::text[])) ORDER BY created_at, id LIMIT 1 FOR UPDATE", [PRIVATE_KIND_IDS]);
if (!r) return null;
if (!r) return null;
await s.db.exec("UPDATE bench_rounds SET status = 'running' WHERE id = $1", [r.id]);
await s.db.exec("UPDATE bench_rounds SET status = 'running' WHERE id = $1", [r.id]);
return { ...r, status: 'running' };
return { ...r, status: 'running' };
@@ -27,7 +27,7 @@ const express = require('express');
@@ -27,7 +27,7 @@ const express = require('express');
const { http, ids } = require('openvibe-contracts');
const { http, ids } = require('openvibe-contracts');
const { asyncRouter } = require('./router');
const { asyncRouter } = require('./router');
const { sameOrigin } = require('./same-origin');
const { sameOrigin } = require('./same-origin');
-const { kindById } = require('../kinds');
+const { kindById, isPrivateKind } = require('../kinds');
const { BenchError } = require('../store');
const { BenchError } = require('../store');
const boardLib = require('../rating/board');
const boardLib = require('../rating/board');
@@ -92,8 +92,26 @@ function createApi(ctx) {
@@ -92,8 +92,26 @@ function createApi(ctx) {
});
});
// ── Tasks ───────────────────────────────────────────────
// ── Tasks ───────────────────────────────────────────────
+ // A private kind (live.clip) takes tasks, rounds and entries from its submitter only (kinds.js `submitter`).
+ const submitterOnly = (req, res, kindId) => {
+ if (!isPrivateKind(kindId)) return true;
+ const want = (kindById(kindId) || {}).submitter;
+ if (req.principal && req.principal.requester === want) return true;
+ problem(res, 403, 'bench.private_kind', `Only ${want} submits to ${kindId}.`);
+ return false;
+ };
+
+ r.get('/kinds/:kindId/policies', principal.requireCapability('bench.entry.create'), async (req, res) => {
+ const kindId = String(req.params.kindId || '');
+ if (kindId !== 'live.clip') return problem(res, 404, 'bench.kind_not_found', 'Policy results are kept for live.clip.');
+ if (!submitterOnly(req, res, kindId)) return undefined;
+ res.set('Cache-Control', 'private, no-store');
+ return res.json({ kind_id: kindId, policies: await bench.clipPolicies() });
+ });
+
r.post('/kinds/:kindId/tasks', actor('bench.task.create'), async (req, res) => {
r.post('/kinds/:kindId/tasks', actor('bench.task.create'), async (req, res) => {
const b = req.body && typeof req.body === 'object' ? req.body : {};
const b = req.body && typeof req.body === 'object' ? req.body : {};
+ if (!submitterOnly(req, res, String(req.params.kindId || ''))) return undefined;
const owner = creatorOf(req);
const owner = creatorOf(req);
const out = await bench.createTask({
const out = await bench.createTask({
kindId: String(req.params.kindId || ''),
kindId: String(req.params.kindId || ''),
@@ -117,6 +135,8 @@ function createApi(ctx) {
@@ -117,6 +135,8 @@ function createApi(ctx) {
// ── Rounds ──────────────────────────────────────────────
// ── Rounds ──────────────────────────────────────────────
r.post('/tasks/:id/rounds', actor('bench.round.create'), async (req, res) => {
r.post('/tasks/:id/rounds', actor('bench.round.create'), async (req, res) => {
const b = req.body && typeof req.body === 'object' ? req.body : {};
const b = req.body && typeof req.body === 'object' ? req.body : {};
+ const task = await bench.getTask(String(req.params.id || ''));
+ if (task && !submitterOnly(req, res, task.kind_id)) return undefined;
const owner = creatorOf(req);
const owner = creatorOf(req);
try {
try {
const out = await bench.createRound({
const out = await bench.createRound({
@@ -135,7 +155,11 @@ function createApi(ctx) {
@@ -135,7 +155,11 @@ function createApi(ctx) {
try {
try {
const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: subjectOf(req) });
const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: subjectOf(req) });
if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');
if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');
- res.set('Cache-Control', round.blind ? 'private, no-store' : 'public, max-age=60');
+ // A private kind's round is for its submitter and for the people who voted on it.
+ if (isPrivateKind(round.kind_id) && round.blind && !(req.principal && req.principal.requester === (kindById(round.kind_id) || {}).submitter)) {
+ return problem(res, 404, 'bench.round_not_found', 'No such round.');
+ }
+ res.set('Cache-Control', round.blind || isPrivateKind(round.kind_id) ? 'private, no-store' : 'public, max-age=60');
return res.json({ round });
return res.json({ round });
} catch (err) { return failed(res, err); }
} catch (err) { return failed(res, err); }
});
});
@@ -143,6 +167,8 @@ function createApi(ctx) {
@@ -143,6 +167,8 @@ function createApi(ctx) {
r.post('/rounds/:id/entries', principal.requireCapability('bench.entry.create'), async (req, res) => {
r.post('/rounds/:id/entries', principal.requireCapability('bench.entry.create'), async (req, res) => {
const b = req.body && typeof req.body === 'object' ? req.body : {};
const b = req.body && typeof req.body === 'object' ? req.body : {};
try {
try {
+ const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: null });
+ if (round && !submitterOnly(req, res, round.kind_id)) return undefined;
const entry = await bench.addEntry({
const entry = await bench.addEntry({
roundId: String(req.params.id || ''),
roundId: String(req.params.id || ''),
provider: b.provider,
provider: b.provider,
@@ -185,8 +211,8 @@ function createApi(ctx) {
@@ -185,8 +211,8 @@ function createApi(ctx) {
return problem(res, 401, 'bench.sign_in_required', 'Sign in at bench.openvibe.services, or send a service token that may relay anonymous votes.');
return problem(res, 401, 'bench.sign_in_required', 'Sign in at bench.openvibe.services, or send a service token that may relay anonymous votes.');
}
}
const kindRow = kindById(kind);
const kindRow = kindById(kind);
- if (!kindRow || kindRow.judge !== 'human_pairwise') {
- return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only pairwise kinds can be judged without signing in.');
+ if (!kindRow || kindRow.judge !== 'human_pairwise' || kindRow.private) {
+ return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only public pairwise kinds can be judged without signing in.');
}
}
if (anon.minted) ctx.anon.setCookie(res, anon.opaque);
if (anon.minted) ctx.anon.setCookie(res, anon.opaque);
const pair = await bench.nextJudgement({
const pair = await bench.nextJudgement({
@@ -217,8 +243,8 @@ function createApi(ctx) {
@@ -217,8 +243,8 @@ function createApi(ctx) {
const round = await bench.getRound(roundId, { viewerSubject: null });
const round = await bench.getRound(roundId, { viewerSubject: null });
if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');
if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');
const row = kindById(round.kind_id);
const row = kindById(round.kind_id);
- if (!row || row.judge !== 'human_pairwise') {
- return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only pairwise kinds can be judged without signing in.');
+ if (!row || row.judge !== 'human_pairwise' || row.private) {
+ return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only public pairwise kinds can be judged without signing in.');
}
}
const out = await bench.castJudgement({
const out = await bench.castJudgement({
roundId, kindId: round.kind_id, judge: 'human_pairwise',
roundId, kindId: round.kind_id, judge: 'human_pairwise',
@@ -9,7 +9,7 @@
@@ -9,7 +9,7 @@
const express = require('express');
const express = require('express');
const crypto = require('crypto');
const crypto = require('crypto');
const renderLib = require('../bench/render');
const renderLib = require('../bench/render');
-const { kindById } = require('../kinds');
+const { kindById, isPrivateKind } = require('../kinds');
function createRenderRoutes(ctx) {
function createRenderRoutes(ctx) {
const r = express.Router();
const r = express.Router();
@@ -18,6 +18,8 @@ function createRenderRoutes(ctx) {
@@ -18,6 +18,8 @@ function createRenderRoutes(ctx) {
const row = await ctx.s.db.maybe(
const row = await ctx.s.db.maybe(
'SELECT e.id, e.payload, e.error, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.id = $1',
'SELECT e.id, e.payload, e.error, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.id = $1',
[String(id || '')]);
[String(id || '')]);
+ // A private kind's entry (a live.clip pick) is never rendered here: its voters see it in the vote itself.
+ if (row && isPrivateKind(row.kind_id)) return null;
if (!row) {
if (!row) {
const match = /^((?:brn_)[0-9A-HJKMNP-TV-Z]{26})~([A-Za-z0-9_-]{1,120})$/.exec(String(id || ''));
const match = /^((?:brn_)[0-9A-HJKMNP-TV-Z]{26})~([A-Za-z0-9_-]{1,120})$/.exec(String(id || ''));
if (!match) return null;
if (!match) return null;
@@ -11,7 +11,7 @@ const { http } = require('openvibe-contracts');
@@ -11,7 +11,7 @@ const { http } = require('openvibe-contracts');
const { asyncRouter } = require('./router');
const { asyncRouter } = require('./router');
const { sameOrigin } = require('./same-origin');
const { sameOrigin } = require('./same-origin');
const renderLib = require('../bench/render');
const renderLib = require('../bench/render');
-const { kindById, SHOWCASE_KIND_IDS } = require('../kinds');
+const { kindById, SHOWCASE_KIND_IDS, isPrivateKind } = require('../kinds');
const { esc, raw } = require('../render/html');
const { esc, raw } = require('../render/html');
const { send } = require('../render/layout');
const { send } = require('../render/layout');
@@ -51,7 +51,8 @@ async function voteCount(db, roundId) {
@@ -51,7 +51,8 @@ async function voteCount(db, roundId) {
async function loadPairs(ctx, opts) {
async function loadPairs(ctx, opts) {
const db = ctx.s.db;
const db = ctx.s.db;
const kind = opts && opts.kind ? String(opts.kind) : 'all';
const kind = opts && opts.kind ? String(opts.kind) : 'all';
- const kinds = kind && kind !== 'all' ? [kind] : SHOWCASE_KIND_IDS;
+ // A private kind (live.clip) is never in the gallery, even when asked for by name.
+ const kinds = kind && kind !== 'all' ? (isPrivateKind(kind) ? [] : [kind]) : SHOWCASE_KIND_IDS;
const limit = Math.min(100, Math.max(1, num(opts && opts.limit, 24)));
const limit = Math.min(100, Math.max(1, num(opts && opts.limit, 24)));
const rows = await db.many(
const rows = await db.many(
"SELECT r.id, r.kind_id, r.created_at, r.created_by, t.prompt, t.task_class FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) ORDER BY r.created_at DESC, r.id LIMIT $2",
"SELECT r.id, r.kind_id, r.created_at, r.created_by, t.prompt, t.task_class FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) ORDER BY r.created_at DESC, r.id LIMIT $2",
@@ -44,8 +44,20 @@ const KINDS = Object.freeze([
@@ -44,8 +44,20 @@ const KINDS = Object.freeze([
{ id: 'design.interface', room: 'design', entry: 'component_set', judge: 'human_pairwise', name: 'Component systems arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric'], entry: 'component_set', payload: { html: 'string' } } },
{ id: 'design.interface', room: 'design', entry: 'component_set', judge: 'human_pairwise', name: 'Component systems arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric'], entry: 'component_set', payload: { html: 'string' } } },
]);
]);
-const byId = new Map(KINDS.map((k) => [k.id, k]));
+// Private kinds: their rounds compare things that are not public (a channel's held-out recordings), so they never
+// appear in the public kind list, the boards, the open datasets or the vote dump; their submitter reads its own results.
+// live.clip (item 151, C19): OpenVibe.Live's AI clip picks, blind against another policy's (server/bench/clip.js).
+const PRIVATE_KINDS = Object.freeze([
+ { id: 'live.clip', room: 'live', entry: 'video_clip', judge: 'human_pairwise', name: 'AI clip picks', metrics: ['elo', 'cost_usd', 'accept_rate'], private: true, submitter: 'service:live',
+ schema: { renderer: 'video', judges: ['human_pairwise'], entry: 'video_clip',
+ payload: { source: 'object', playback: 'object', policy: 'object', title: 'string', abstained: 'boolean', reason: 'string' },
+ rubric: require('./bench/clip').RUBRIC } },
+]);
+const PRIVATE_KIND_IDS = Object.freeze(PRIVATE_KINDS.map((k) => k.id));
+const ALL_KINDS = Object.freeze(KINDS.concat(PRIVATE_KINDS));
+
+const byId = new Map(ALL_KINDS.map((k) => [k.id, k]));
const SHOWCASE_KIND_IDS = Object.freeze(KINDS.filter((k) => k.schema && k.schema.renderer).map((k) => k.id));
const SHOWCASE_KIND_IDS = Object.freeze(KINDS.filter((k) => k.schema && k.schema.renderer).map((k) => k.id));
-module.exports = { KINDS, JUDGES, WINNERS, SHOWCASE_KIND_IDS, kindById: (id) => byId.get(id) || null };
+module.exports = { KINDS, PRIVATE_KINDS, PRIVATE_KIND_IDS, ALL_KINDS, JUDGES, WINNERS, SHOWCASE_KIND_IDS, kindById: (id) => byId.get(id) || null, isPrivateKind: (id) => PRIVATE_KIND_IDS.includes(id) };
@@ -10,7 +10,7 @@
@@ -10,7 +10,7 @@
const { frontier } = require('./frontier');
const { frontier } = require('./frontier');
const { parsePick } = require('./parser');
const { parsePick } = require('./parser');
-const { KINDS, kindById, SHOWCASE_KIND_IDS } = require('../kinds');
+const { KINDS, kindById, SHOWCASE_KIND_IDS, PRIVATE_KIND_IDS } = require('../kinds');
const renderLib = require('../bench/render');
const renderLib = require('../bench/render');
const { screenshotOf } = require('../http/gallery');
const { screenshotOf } = require('../http/gallery');
@@ -52,7 +52,7 @@ async function busiestKind(ctx) {
@@ -52,7 +52,7 @@ async function busiestKind(ctx) {
const rows = await ctx.s.db.many(
const rows = await ctx.s.db.many(
`SELECT r.kind_id AS id, COUNT(e.id)::int AS n
`SELECT r.kind_id AS id, COUNT(e.id)::int AS n
FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id
FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id
- WHERE e.error IS NULL GROUP BY r.kind_id ORDER BY n DESC, r.kind_id LIMIT 1`);
+ WHERE e.error IS NULL AND NOT (r.kind_id = ANY($1::text[])) GROUP BY r.kind_id ORDER BY n DESC, r.kind_id LIMIT 1`, [PRIVATE_KIND_IDS]);
if (rows && rows[0] && rows[0].id) return rows[0].id;
if (rows && rows[0] && rows[0].id) return rows[0].id;
return SHOWCASE_KIND_IDS[0] || 'games.play';
return SHOWCASE_KIND_IDS[0] || 'games.play';
}
}
@@ -14,6 +14,8 @@
@@ -14,6 +14,8 @@
*/
*/
const style = require('./style');
const style = require('./style');
+// Private kinds' votes (live.clip) are never in the open vote dump or the public ratings.
+const { PRIVATE_KIND_IDS } = require('../kinds');
const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
@@ -26,8 +28,8 @@ async function loadVotes(db) {
@@ -26,8 +28,8 @@ async function loadVotes(db) {
FROM bench_judgements j
FROM bench_judgements j
JOIN bench_entries l ON l.id = (j.pair->>'left')
JOIN bench_entries l ON l.id = (j.pair->>'left')
JOIN bench_entries r ON r.id = (j.pair->>'right')
JOIN bench_entries r ON r.id = (j.pair->>'right')
- WHERE j.judge = 'human_pairwise' AND j.outcome IS NOT NULL
- ORDER BY j.at, j.id`);
+ WHERE j.judge = 'human_pairwise' AND j.outcome IS NOT NULL AND NOT (j.kind_id = ANY($1::text[]))
+ ORDER BY j.at, j.id`, [PRIVATE_KIND_IDS]);
const out = [];
const out = [];
for (const row of rows) {
for (const row of rows) {
const left = styleOf(row.left_payload, row.left_length, row.left_markup);
const left = styleOf(row.left_payload, row.left_length, row.left_markup);
@@ -8,7 +8,8 @@
@@ -8,7 +8,8 @@
*/
*/
const crypto = require('crypto');
const crypto = require('crypto');
const { ids } = require('openvibe-contracts');
const { ids } = require('openvibe-contracts');
-const { KINDS, JUDGES, WINNERS, kindById } = require('./kinds');
+const { KINDS, ALL_KINDS, PRIVATE_KIND_IDS, JUDGES, WINNERS, kindById, isPrivateKind } = require('./kinds');
+const clipLib = require('./bench/clip');
const ratingLib = require('./rating');
const ratingLib = require('./rating');
const styleLib = require('./rating/style');
const styleLib = require('./rating/style');
@@ -197,7 +198,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -197,7 +198,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
// ── Kinds ───────────────────────────────────────────────
// ── Kinds ───────────────────────────────────────────────
async function seedKinds() {
async function seedKinds() {
- for (const k of KINDS) {
+ for (const k of ALL_KINDS) {
await db.exec(
await db.exec(
`INSERT INTO bench_kinds (id, room, entry, judge, name, schema, metrics, created_at)
`INSERT INTO bench_kinds (id, room, entry, judge, name, schema, metrics, created_at)
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7::jsonb, now())
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7::jsonb, now())
@@ -207,8 +208,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -207,8 +208,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
);
);
}
}
}
}
+ /** The public kinds; a private one (live.clip) is never listed. */
async function listKinds() {
async function listKinds() {
- return (await db.many('SELECT * FROM bench_kinds ORDER BY id')).map(wireKind);
+ return (await db.many('SELECT * FROM bench_kinds WHERE NOT (id = ANY($1::text[])) ORDER BY id', [PRIVATE_KIND_IDS])).map(wireKind);
}
}
async function getKind(id) {
async function getKind(id) {
return wireKind(await db.maybe('SELECT * FROM bench_kinds WHERE id = $1', [String(id || '')]));
return wireKind(await db.maybe('SELECT * FROM bench_kinds WHERE id = $1', [String(id || '')]));
@@ -218,6 +220,8 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -218,6 +220,8 @@ function createStore(s, { log = console, random = Math.random, config = null } =
async function createTask({ kindId, prompt, taskClass = null, public: isPublic = false, subject }) {
async function createTask({ kindId, prompt, taskClass = null, public: isPublic = false, subject }) {
const kind = await getKind(kindId);
const kind = await getKind(kindId);
if (!kind) return refusal(422, 'bench.kind_not_found', `No such kind: ${kindId}`);
if (!kind) return refusal(422, 'bench.kind_not_found', `No such kind: ${kindId}`);
+ // A private kind's tasks name what is not public (a held-out recording): never public.
+ if (isPrivateKind(kind.id)) isPublic = false;
const text = String(prompt == null ? '' : prompt).replace(/\s+/g, ' ').trim();
const text = String(prompt == null ? '' : prompt).replace(/\s+/g, ' ').trim();
if (!text) return refusal(422, 'bench.prompt_required', 'A task needs a prompt or spec.');
if (!text) return refusal(422, 'bench.prompt_required', 'A task needs a prompt or spec.');
if (text.length > 4000) return refusal(422, 'bench.prompt_too_long', 'The prompt must be at most 4000 characters.');
if (text.length > 4000) return refusal(422, 'bench.prompt_too_long', 'The prompt must be at most 4000 characters.');
@@ -302,7 +306,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -302,7 +306,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const revealed = viewerSubject != null && (r.created_by === viewerSubject
const revealed = viewerSubject != null && (r.created_by === viewerSubject
|| await db.value('SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND subject = $2', [r.id, viewerSubject]) > 0);
|| await db.value('SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND subject = $2', [r.id, viewerSubject]) > 0);
const votes = num(await db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND method <> 'game_outcome'", [r.id]));
const votes = num(await db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND method <> 'game_outcome'", [r.id]));
- const entries = entryRows.map((e) => wireEntry(e, { blind: !revealed }));
+ // A live.clip playback is only for a voter (nextJudgement); a round read never carries one.
+ const entries = entryRows.map((e) => wireEntry(e, { blind: !revealed }))
+ .map((e) => (isPrivateKind(r.kind_id) ? { ...e, payload: clipLib.forViewer(e.payload, { now: s.now(), playback: false }) } : e));
const out = { ...wireRound(r), entries, votes };
const out = { ...wireRound(r), entries, votes };
if (!revealed) out.blind = true;
if (!revealed) out.blind = true;
return out;
return out;
@@ -313,6 +319,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -313,6 +319,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =
return await s.tx(async () => {
return await s.tx(async () => {
const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1 FOR UPDATE', [String(roundId || '')]);
const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1 FOR UPDATE', [String(roundId || '')]);
if (!r) throw new BenchError(404, 'bench.round_not_found', 'No such Bench round.');
if (!r) throw new BenchError(404, 'bench.round_not_found', 'No such Bench round.');
+ if (r.kind_id === 'live.clip') {
+ // A clip pick: its shape checked, and the policy that chose it pinned on the entry (route), never rewritten.
+ const v = clipLib.validateEntry(payload, { now: s.now() });
+ if (!v.ok) throw new BenchError(422, 'bench.invalid_clip', v.message);
+ payload = v.payload;
+ route = { policy: v.payload.policy };
+ model = model || v.payload.policy.models[0] || `${v.payload.policy.selector}@${v.payload.policy.version}`;
+ provider = provider || 'live';
+ }
const id = s.newId('bte');
const id = s.newId('bte');
// The style of the output (how long it is, how much markup it wears) is measured once, here, so the
// The style of the output (how long it is, how much markup it wears) is measured once, here, so the
// ratings can control for style later without opening the payload again.
// ratings can control for style later without opening the payload again.
@@ -386,7 +401,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -386,7 +401,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
[String(kind || ''), subject, creators],
[String(kind || ''), subject, creators],
);
);
if (!row) return null;
if (!row) return null;
- const entries = (await db.many('SELECT * FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id', [row.id])).map(wireEntry);
+ const entries = (await db.many('SELECT * FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id', [row.id])).map(wireEntry)
+ // A voter of a private kind sees the playback, while it lasts (clip.forViewer).
+ .map((e) => (isPrivateKind(row.kind_id) ? { ...e, payload: clipLib.forViewer(e.payload, { now: s.now(), playback: true }) } : e));
if (entries.length < 2) return null;
if (entries.length < 2) return null;
const [left, right] = shuffle(entries, rng).slice(0, 2);
const [left, right] = shuffle(entries, rng).slice(0, 2);
return {
return {
@@ -609,7 +626,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -609,7 +626,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
async function board(kindId, { type = 'elo', category = CATEGORY_DEFAULT, period = 'all', voters = null } = {}) {
async function board(kindId, { type = 'elo', category = CATEGORY_DEFAULT, period = 'all', voters = null } = {}) {
const kind = await getKind(kindId);
const kind = await getKind(kindId);
- if (!kind) return null;
+ if (!kind || isPrivateKind(kind.id)) return null;
const cat = categoryOf(voters, category);
const cat = categoryOf(voters, category);
const cost = await costStats(kindId);
const cost = await costStats(kindId);
const pass = await passRate(kindId);
const pass = await passRate(kindId);
@@ -700,7 +717,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -700,7 +717,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
// ── Dataset (open, no voter ids) ─────────────────────────
// ── Dataset (open, no voter ids) ─────────────────────────
async function dataset(kindId) {
async function dataset(kindId) {
const kind = await getKind(kindId);
const kind = await getKind(kindId);
- if (!kind) return null;
+ if (!kind || isPrivateKind(kind.id)) return null;
const tasks = (await db.many('SELECT * FROM bench_tasks WHERE kind_id = $1 AND public = true ORDER BY created_at, id', [kindId]))
const tasks = (await db.many('SELECT * FROM bench_tasks WHERE kind_id = $1 AND public = true ORDER BY created_at, id', [kindId]))
.map(wireTask).map(({ created_by, ...rest }) => rest);
.map(wireTask).map(({ created_by, ...rest }) => rest);
const rounds = (await db.many("SELECT * FROM bench_rounds WHERE kind_id = $1 AND status IN ('ready', 'finished') ORDER BY created_at, id", [kindId]))
const rounds = (await db.many("SELECT * FROM bench_rounds WHERE kind_id = $1 AND status IN ('ready', 'finished') ORDER BY created_at, id", [kindId]))
@@ -728,6 +745,43 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -728,6 +745,43 @@ function createStore(s, { log = console, random = Math.random, config = null } =
return lines.length ? `${lines.join('\n')}\n` : '';
return lines.length ? `${lines.join('\n')}\n` : '';
}
}
+ /**
+ * live.clip by policy (selector@version), for its submitter: entries, abstentions, cost, and how its picks fared
+ * in blind votes. accept_rate = (wins + ties) / votes it was in; a both_bad vote counts against both sides.
+ */
+ async function clipPolicies() {
+ const entries = await db.many(`SELECT e.id, e.cost_usd, e.payload FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id
+ WHERE r.kind_id = 'live.clip' AND e.error IS NULL`);
+ const byEntry = new Map();
+ const stats = new Map();
+ for (const e of entries) {
+ const p = typeof e.payload === 'string' ? JSON.parse(e.payload) : (e.payload || {});
+ const key = `${(p.policy && p.policy.selector) || '?'}@${(p.policy && p.policy.version) || '?'}`;
+ byEntry.set(e.id, key);
+ const st = stats.get(key) || { policy: key, entries: 0, abstained: 0, cost_usd: 0, votes: 0, wins: 0, ties: 0, losses: 0, both_bad: 0 };
+ st.entries += 1;
+ if (p.abstained) st.abstained += 1;
+ st.cost_usd += num(e.cost_usd);
+ stats.set(key, st);
+ }
+ const votes = await db.many(`SELECT j.pair, j.outcome FROM bench_judgements j WHERE j.kind_id = 'live.clip' AND j.judge = 'human_pairwise' AND j.pair IS NOT NULL AND j.outcome IS NOT NULL`);
+ for (const v of votes) {
+ const pair = typeof v.pair === 'string' ? JSON.parse(v.pair) : v.pair;
+ const sides = [['left', pair && pair.left], ['right', pair && pair.right]];
+ for (const [side, id] of sides) {
+ const st = stats.get(byEntry.get(id));
+ if (!st) continue;
+ st.votes += 1;
+ if (v.outcome === 'tie') st.ties += 1;
+ else if (v.outcome === 'both_bad') st.both_bad += 1;
+ else if (v.outcome === side) st.wins += 1;
+ else st.losses += 1;
+ }
+ }
+ return [...stats.values()].map((st) => ({ ...st, cost_usd: Math.round(st.cost_usd * 1e6) / 1e6, accept_rate: st.votes ? Math.round(((st.wins + st.ties) / st.votes) * 1000) / 1000 : null }))
+ .sort((a, b) => a.policy.localeCompare(b.policy));
+ }
+
// ── Private evals ────────────────────────────────────────
// ── Private evals ────────────────────────────────────────
async function createEval({ owner, kindId, name, testSuite }) {
async function createEval({ owner, kindId, name, testSuite }) {
const kind = await getKind(kindId);
const kind = await getKind(kindId);
@@ -786,7 +840,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
@@ -786,7 +840,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,
wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,
insertOutbox, seedKinds, listKinds, getKind,
insertOutbox, seedKinds, listKinds, getKind,
createTask, getTask,
createTask, getTask,
- createRound, getRound, addEntry, maybeReady, finishRound,
+ createRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
nextJudgement, castJudgement, pairwiseGames,
nextJudgement, castJudgement, pairwiseGames,
board, valueRows, modelSummary, dataset, datasetJsonl,
board, valueRows, modelSummary, dataset, datasetJsonl,
createEval, getEval, runEval, evalResults, evalDailyCount,
createEval, getEval, runEval, evalResults, evalDailyCount,
new file mode 100644
new file mode 100644
@@ -0,0 +1,124 @@
@@ -0,0 +1,124 @@
+'use strict';
+/**
+ * live.clip (item 151, C19): OpenVibe.Live's AI clip picks, judged blind against another policy's. Typed clip and
+ * abstention entries, the policy pinned on each entry, only Live submits, the playback only for a voter and never
+ * after it expires, never on a public board, list, gallery, dataset or vote dump, never filled by Bench's AI, and the
+ * cost and accept rate per policy (both_bad counts against both).
+ * node test/live-clip.test.js
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+
+(async () => {
+ const t = await boot({ env: { BENCH_ROUNDS_PER_DAY: '100', BENCH_ROUNDS_DAILY: '1000' } });
+ const voter = t.network.addUser('voter');
+ const voter2 = t.network.addUser('voter2');
+ const live = t.network.serviceToken({ sub: 'svc:live', audience: 'openvibe.bench', cap: ['bench.task.create', 'bench.round.create', 'bench.entry.create'] });
+ const other = t.network.serviceToken({ sub: 'svc:room', audience: 'openvibe.bench', cap: ['bench.task.create', 'bench.round.create', 'bench.entry.create'] });
+ const soon = () => new Date(Date.now() + 3600_000).toISOString();
+ const clip = (version, extra = {}) => ({
+ source: { recording_id: 'vod_5021', start_ms: 60_000, end_ms: 90_000, language: 'ja', genre: 'irl' },
+ playback: { url: `https://media.openvibe.media/p/${version}.mp4?sig=x`, expires_at: soon() },
+ policy: { selector: 'moment-picker', version, models: ['qwen3-vl-4b'] },
+ title: 'The cat walks onto the keyboard',
+ ...extra,
+ });
+ let roundId;
+
+ async function newRound(token = live) {
+ const task = await t.get('/api/v1/kinds/live.clip/tasks', { bearer: token, json: { prompt: 'vod_5021 30:00-45:00', public: true } });
+ if (task.status !== 201) return { status: task.status, text: task.text };
+ assert.strictEqual(task.json().task.public, false, 'a private kind\'s task is never public');
+ const rr = await t.get(`/api/v1/tasks/${task.json().task.id}/rounds`, { bearer: token, json: {} });
+ assert.strictEqual(rr.status, 201, rr.text);
+ return { status: 201, id: rr.json().round.id };
+ }
+
+ await check('only Live makes live.clip tasks, rounds and entries', async () => {
+ assert.strictEqual((await newRound(other)).status, 403);
+ const r = await newRound();
+ assert.strictEqual(r.status, 201, r.text);
+ roundId = r.id;
+ const e = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: other, json: { payload: clip('v1'), cost_usd: 0.002 } });
+ assert.strictEqual(e.status, 403);
+ });
+
+ await check('typed entries: a clip and an abstention; the bad ones refused; the policy pinned on the entry', async () => {
+ const bad = [
+ clip('v1', { source: { recording_id: 'vod_5021', start_ms: 90_000, end_ms: 60_000 } }),
+ clip('v1', { source: { recording_id: 'vod_5021', start_ms: 0, end_ms: 2_000 } }),
+ clip('v1', { playback: { url: 'http://insecure.example/x.mp4', expires_at: soon() } }),
+ clip('v1', { playback: { url: 'https://media.openvibe.media/p.mp4', expires_at: new Date(Date.now() - 1000).toISOString() } }),
+ clip('v1', { policy: { selector: 'moment-picker' } }),
+ { source: { recording_id: 'vod_5021', start_ms: 0, end_ms: 1 }, policy: { selector: 's', version: 'v2' }, abstained: true },
+ ];
+ for (const payload of bad) {
+ const r = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: live, json: { payload } });
+ assert.strictEqual(r.status, 422, JSON.stringify(payload).slice(0, 120));
+ }
+ const a = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: live, json: { payload: clip('v1', { extra: 'dropped' }), cost_usd: 0.002 } });
+ assert.strictEqual(a.status, 201, a.text);
+ assert.strictEqual(a.json().entry.payload.extra, undefined, 'only the typed fields are kept');
+ const b = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: live, json: { payload: { source: { recording_id: 'vod_5021', start_ms: 0, end_ms: 1000 }, policy: { selector: 'moment-picker', version: 'v2', models: ['smolvlm2-500m'] }, abstained: true, reason: 'Nothing happened worth a clip.' }, cost_usd: 0.0005 } });
+ assert.strictEqual(b.status, 201, b.text);
+ const row = await t.ctx.s.db.one('SELECT route, model FROM bench_entries WHERE id = $1', [a.json().entry.id]);
+ assert.deepStrictEqual(row.route.policy, { selector: 'moment-picker', version: 'v1', models: ['qwen3-vl-4b'] });
+ assert.strictEqual(row.model, 'qwen3-vl-4b');
+ });
+
+ await check('the playback is for a voter only: not in a round read, not after it expires', async () => {
+ const anon = await t.get(`/api/v1/rounds/${roundId}`);
+ assert.strictEqual(anon.status, 404, 'a private round is not readable by just anyone');
+ const asLive = await t.get(`/api/v1/rounds/${roundId}`, { bearer: live });
+ assert.strictEqual(asLive.status, 200, asLive.text);
+ assert.ok(asLive.json().round.entries.length === 2 && asLive.json().round.entries.every((e) => !e.payload.playback), 'no playback in a round read');
+ const pair = await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter });
+ assert.strictEqual(pair.status, 200, pair.text);
+ const sides = [pair.json().left, pair.json().right];
+ assert.ok(sides.some((sd) => sd.payload.playback && sd.payload.playback.url.startsWith('https://')), 'the voter sees the clip');
+ await t.ctx.s.db.exec("UPDATE bench_entries SET payload = jsonb_set(payload, '{playback,expires_at}', to_jsonb('2020-01-01T00:00:00.000Z'::text)) WHERE round_id = $1 AND payload ? 'playback'", [roundId]);
+ const later = await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter2 });
+ const ls = [later.json().left, later.json().right];
+ assert.ok(ls.every((sd) => !sd.payload.playback), 'never after it expires');
+ assert.ok(ls.some((sd) => sd.payload.playback_expired === true));
+ const anonNext = await t.get('/api/v1/judgements/next?kind=live.clip');
+ assert.ok([401, 403].includes(anonNext.status), `anonymous voting on a private kind is refused (${anonNext.status})`);
+ });
+
+ await check('cost and accept rate per policy; both_bad counts against both', async () => {
+ const pair = await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter });
+ const p = pair.json();
+ const entries = await t.ctx.s.db.many('SELECT id, payload FROM bench_entries WHERE round_id = $1', [roundId]);
+ const v1 = entries.find((e) => e.payload.policy.version === 'v1').id;
+ const winnerSide = p.left.id === v1 ? 'left' : 'right';
+ const vote = await t.get(`/api/v1/rounds/${roundId}/judgements`, { as: voter, json: { judge: 'human_pairwise', pair: { left: p.left.id, right: p.right.id }, outcome: winnerSide } });
+ assert.strictEqual(vote.status, 201, vote.text);
+ const p2 = (await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter2 })).json();
+ const vote2 = await t.get(`/api/v1/rounds/${roundId}/judgements`, { as: voter2, json: { judge: 'human_pairwise', pair: { left: p2.left.id, right: p2.right.id }, outcome: 'both_bad' } });
+ assert.strictEqual(vote2.status, 201, vote2.text);
+ const r = await t.get('/api/v1/kinds/live.clip/policies', { bearer: live });
+ assert.strictEqual(r.status, 200, r.text);
+ const by = Object.fromEntries(r.json().policies.map((x) => [x.policy, x]));
+ assert.deepStrictEqual([by['moment-picker@v1'].votes, by['moment-picker@v1'].wins, by['moment-picker@v1'].both_bad, by['moment-picker@v1'].accept_rate], [2, 1, 1, 0.5]);
+ assert.deepStrictEqual([by['moment-picker@v2'].votes, by['moment-picker@v2'].losses, by['moment-picker@v2'].both_bad, by['moment-picker@v2'].abstained, by['moment-picker@v2'].accept_rate], [2, 1, 1, 1, 0]);
+ assert.strictEqual(by['moment-picker@v1'].cost_usd, 0.002);
+ assert.strictEqual((await t.get('/api/v1/kinds/live.clip/policies', { bearer: other })).status, 403, 'Live\'s results are Live\'s');
+ });
+
+ await check('never public: the kind list, the board, the dataset, the vote dump, the gallery; never filled by Bench\'s AI', async () => {
+ const kinds = (await t.get('/api/v1/kinds')).json().kinds.map((k) => k.id);
+ assert.ok(!kinds.includes('live.clip'));
+ assert.strictEqual((await t.get('/api/v1/boards/live.clip')).status, 404);
+ assert.strictEqual((await t.get('/api/v1/datasets/live.clip.jsonl')).status, 404);
+ const votes = await t.get('/api/v1/datasets/votes.jsonl');
+ assert.ok(!votes.text.includes('live.clip'), 'no live.clip vote in the dump');
+ const gallery = await t.get('/api/v1/showcase/rounds?kind=live.clip');
+ if (gallery.status === 200) assert.ok(!gallery.text.includes(roundId), 'not in the gallery');
+ const queued = await newRound();
+ const claimed = await t.ctx.s.db.maybe("SELECT * FROM bench_rounds WHERE status = 'queued' AND NOT (kind_id = ANY($1::text[])) AND id = $2", [['live.clip'], queued.id]);
+ assert.strictEqual(claimed, null, 'the job runner never claims it');
+ });
+
+ await t.close();
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });