openvibers/openvibe.bench Commits

live.clip: Live's AI clip picks judged blind (item 151, C19) (#105)

OP OpenVibers committed 10 days ago · 06f9b4269d · 10 files, +326 −24

A private kind: Live submits typed clip and abstention entries with their policy pinned; voters see the playback
while it lasts; nothing of it is on a public board, list, gallery, dataset or vote dump, and Bench's AI never fills
its rounds. Live reads each policy's cost and accept rate (both_bad counts against both).

Changes

ddiff --git a/server/bench/clip.js b/server/bench/clip.js
nnew file mode 100644
iindex 0000000..a996f86
---- /dev/null
++++ b/server/bench/clip.js
@@@ -0,0 +1,79 @@
++'use strict';
++
++/**
++ * live.clip entries (item 151, C19 of the clip-quality research): a clip OpenVibe.Live's picker made, or chose not to
++ * make, for one held-out stretch of a recording, compared blind against another policy's pick. Bench does not make
++ * clips: Live submits them with its service token.
++ *
++ * payload = {
++ * source: { recording_id, start_ms, end_ms, language?, genre? } the stretch the clip was cut from
++ * playback: { url, expires_at } a short-lived private URL from Media
++ * policy: { selector, version, models: [string] } which picker chose it, pinned on the entry
++ * title?: string the title it would be published with
++ * abstained: boolean, reason?: string "nothing worth clipping here"
++ * }
++ *
++ * validateEntry(payload, { now }) → { ok: true, payload } | { ok: false, message }
++ * forViewer(payload, { now, playback }) → the payload a reader sees: playback only for a voter, never after expiry
++ */
++const RECORDING_RE = /^[A-Za-z0-9_.:-]{1,80}$/;
++const MIN_MS = 5_000;
++const MAX_MS = 180_000;
++const RUBRIC = Object.freeze([
++ 'Interesting on its own, without the stream around it',
++ 'Has its setup and its payoff',
++ 'Starts and ends at the right moment, not mid-sentence',
++ 'The title says what happens',
++ 'Not a repeat of a clip the stream already has',
++]);
++
++const str = (v, max) => (typeof v === 'string' && v.trim() ? v.trim().slice(0, max) : null);
++
++function validateEntry(payload, { now = Date.now() } = {}) {
++ const p = payload && typeof payload === 'object' && !Array.isArray(payload) ? payload : null;
++ if (!p) return { ok: false, message: 'A live.clip entry is an object.' };
++ const s = p.source && typeof p.source === 'object' ? p.source : null;
++ if (!s || !RECORDING_RE.test(String(s.recording_id || ''))) return { ok: false, message: 'source.recording_id names the recording the clip comes from.' };
++ const start = Number(s.start_ms);
++ const end = Number(s.end_ms);
++ if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end <= start) return { ok: false, message: 'source.start_ms and end_ms are whole milliseconds, end after start.' };
++ const pol = p.policy && typeof p.policy === 'object' ? p.policy : null;
++ const version = pol && str(pol.version, 80);
++ if (!pol || !str(pol.selector, 80) || !version) return { ok: false, message: 'policy.selector and policy.version name the picker that chose it.' };
++ const models = Array.isArray(pol.models) ? pol.models.map((m) => str(m, 120)).filter(Boolean).slice(0, 10) : [];
++ const out = {
++ source: { recording_id: String(s.recording_id), start_ms: start, end_ms: end, language: str(s.language, 16), genre: str(s.genre, 40) },
++ policy: { selector: str(pol.selector, 80), version, models },
++ abstained: p.abstained === true,
++ };
++ if (out.abstained) {
++ const reason = str(p.reason, 300);
++ if (!reason) return { ok: false, message: 'An abstention says why: reason.' };
++ if (p.playback) return { ok: false, message: 'An abstention has no playback.' };
++ out.reason = reason;
++ return { ok: true, payload: out };
++ }
++ if (end - start < MIN_MS || end - start > MAX_MS) return { ok: false, message: 'A clip is 5 to 180 seconds long.' };
++ const pb = p.playback && typeof p.playback === 'object' ? p.playback : null;
++ let url = null;
++ try { url = pb && new URL(String(pb.url)); } catch { url = null; }
++ if (!url || url.protocol !== 'https:') return { ok: false, message: 'playback.url is an https address.' };
++ const exp = pb && Date.parse(pb.expires_at);
++ if (!Number.isFinite(exp) || exp <= now) return { ok: false, message: 'playback.expires_at is a time still to come.' };
++ out.playback = { url: url.toString(), expires_at: new Date(exp).toISOString() };
++ const title = str(p.title, 200);
++ if (title) out.title = title;
++ return { ok: true, payload: out };
++}
++
++/** What a reader of a live.clip entry sees: the playback only for a voter, and never once it has expired. */
++function forViewer(payload, { now = Date.now(), playback = false } = {}) {
++ if (!payload || typeof payload !== 'object' || !payload.playback) return payload;
++ const { playback: pb, ...rest } = payload;
++ if (!playback) return rest;
++ const exp = Date.parse(pb.expires_at);
++ if (!Number.isFinite(exp) || exp <= now) return { ...rest, playback_expired: true };
++ return payload;
++}
++
++module.exports = { validateEntry, forViewer, RUBRIC, MIN_MS, MAX_MS };
ddiff --git a/server/bench/jobs.js b/server/bench/jobs.js
iindex b065fa6..ad69664 100644
---- a/server/bench/jobs.js
++++ b/server/bench/jobs.js
@@@ -1,4 +1,5 @@
'use strict';
++const { PRIVATE_KIND_IDS } = require('../kinds');
/**
* Bench's generation workers. A round (or private eval run) is created as `queued`; this loop claims it, asks
@@@ -15,7 +16,8 @@ function createBenchJobs({ s, bench, ai, config, log = console }) {
async function claimRound() {
return await s.tx(async () => {
-- const r = await s.db.maybe("SELECT * FROM bench_rounds WHERE status = 'queued' ORDER BY created_at, id LIMIT 1 FOR UPDATE");
++ // A private kind's round (live.clip) is filled by its submitter, never by Bench's AI.
++ const r = await s.db.maybe("SELECT * FROM bench_rounds WHERE status = 'queued' AND NOT (kind_id = ANY($1::text[])) ORDER BY created_at, id LIMIT 1 FOR UPDATE", [PRIVATE_KIND_IDS]);
if (!r) return null;
await s.db.exec("UPDATE bench_rounds SET status = 'running' WHERE id = $1", [r.id]);
return { ...r, status: 'running' };
ddiff --git a/server/http/api.js b/server/http/api.js
iindex 0de956a..1fd6ee1 100644
---- a/server/http/api.js
++++ b/server/http/api.js
@@@ -27,7 +27,7 @@ const express = require('express');
const { http, ids } = require('openvibe-contracts');
const { asyncRouter } = require('./router');
const { sameOrigin } = require('./same-origin');
--const { kindById } = require('../kinds');
++const { kindById, isPrivateKind } = require('../kinds');
const { BenchError } = require('../store');
const boardLib = require('../rating/board');
@@@ -92,8 +92,26 @@ function createApi(ctx) {
});
// ── Tasks ───────────────────────────────────────────────
++ // A private kind (live.clip) takes tasks, rounds and entries from its submitter only (kinds.js `submitter`).
++ const submitterOnly = (req, res, kindId) => {
++ if (!isPrivateKind(kindId)) return true;
++ const want = (kindById(kindId) || {}).submitter;
++ if (req.principal && req.principal.requester === want) return true;
++ problem(res, 403, 'bench.private_kind', `Only ${want} submits to ${kindId}.`);
++ return false;
++ };
++
++ r.get('/kinds/:kindId/policies', principal.requireCapability('bench.entry.create'), async (req, res) => {
++ const kindId = String(req.params.kindId || '');
++ if (kindId !== 'live.clip') return problem(res, 404, 'bench.kind_not_found', 'Policy results are kept for live.clip.');
++ if (!submitterOnly(req, res, kindId)) return undefined;
++ res.set('Cache-Control', 'private, no-store');
++ return res.json({ kind_id: kindId, policies: await bench.clipPolicies() });
++ });
++
r.post('/kinds/:kindId/tasks', actor('bench.task.create'), async (req, res) => {
const b = req.body && typeof req.body === 'object' ? req.body : {};
++ if (!submitterOnly(req, res, String(req.params.kindId || ''))) return undefined;
const owner = creatorOf(req);
const out = await bench.createTask({
kindId: String(req.params.kindId || ''),
@@@ -117,6 +135,8 @@ function createApi(ctx) {
// ── Rounds ──────────────────────────────────────────────
r.post('/tasks/:id/rounds', actor('bench.round.create'), async (req, res) => {
const b = req.body && typeof req.body === 'object' ? req.body : {};
++ const task = await bench.getTask(String(req.params.id || ''));
++ if (task && !submitterOnly(req, res, task.kind_id)) return undefined;
const owner = creatorOf(req);
try {
const out = await bench.createRound({
@@@ -135,7 +155,11 @@ function createApi(ctx) {
try {
const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: subjectOf(req) });
if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');
-- res.set('Cache-Control', round.blind ? 'private, no-store' : 'public, max-age=60');
++ // A private kind's round is for its submitter and for the people who voted on it.
++ if (isPrivateKind(round.kind_id) && round.blind && !(req.principal && req.principal.requester === (kindById(round.kind_id) || {}).submitter)) {
++ return problem(res, 404, 'bench.round_not_found', 'No such round.');
++ }
++ res.set('Cache-Control', round.blind || isPrivateKind(round.kind_id) ? 'private, no-store' : 'public, max-age=60');
return res.json({ round });
} catch (err) { return failed(res, err); }
});
@@@ -143,6 +167,8 @@ function createApi(ctx) {
r.post('/rounds/:id/entries', principal.requireCapability('bench.entry.create'), async (req, res) => {
const b = req.body && typeof req.body === 'object' ? req.body : {};
try {
++ const round = await bench.getRound(String(req.params.id || ''), { viewerSubject: null });
++ if (round && !submitterOnly(req, res, round.kind_id)) return undefined;
const entry = await bench.addEntry({
roundId: String(req.params.id || ''),
provider: b.provider,
@@@ -185,8 +211,8 @@ function createApi(ctx) {
return problem(res, 401, 'bench.sign_in_required', 'Sign in at bench.openvibe.services, or send a service token that may relay anonymous votes.');
}
const kindRow = kindById(kind);
-- if (!kindRow || kindRow.judge !== 'human_pairwise') {
-- return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only pairwise kinds can be judged without signing in.');
++ if (!kindRow || kindRow.judge !== 'human_pairwise' || kindRow.private) {
++ return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only public pairwise kinds can be judged without signing in.');
}
if (anon.minted) ctx.anon.setCookie(res, anon.opaque);
const pair = await bench.nextJudgement({
@@@ -217,8 +243,8 @@ function createApi(ctx) {
const round = await bench.getRound(roundId, { viewerSubject: null });
if (!round) return problem(res, 404, 'bench.round_not_found', 'No such round.');
const row = kindById(round.kind_id);
-- if (!row || row.judge !== 'human_pairwise') {
-- return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only pairwise kinds can be judged without signing in.');
++ if (!row || row.judge !== 'human_pairwise' || row.private) {
++ return problem(res, 403, 'bench.anon_kind_not_allowed', 'Only public pairwise kinds can be judged without signing in.');
}
const out = await bench.castJudgement({
roundId, kindId: round.kind_id, judge: 'human_pairwise',
ddiff --git a/server/http/render.js b/server/http/render.js
iindex ba5cb86..f35f693 100644
---- a/server/http/render.js
++++ b/server/http/render.js
@@@ -9,7 +9,7 @@
const express = require('express');
const crypto = require('crypto');
const renderLib = require('../bench/render');
--const { kindById } = require('../kinds');
++const { kindById, isPrivateKind } = require('../kinds');
function createRenderRoutes(ctx) {
const r = express.Router();
@@@ -18,6 +18,8 @@ function createRenderRoutes(ctx) {
const row = await ctx.s.db.maybe(
'SELECT e.id, e.payload, e.error, r.kind_id FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id WHERE e.id = $1',
[String(id || '')]);
++ // A private kind's entry (a live.clip pick) is never rendered here: its voters see it in the vote itself.
++ if (row && isPrivateKind(row.kind_id)) return null;
if (!row) {
const match = /^((?:brn_)[0-9A-HJKMNP-TV-Z]{26})~([A-Za-z0-9_-]{1,120})$/.exec(String(id || ''));
if (!match) return null;
ddiff --git a/server/http/showcase.js b/server/http/showcase.js
iindex b686912..a3d2150 100644
---- a/server/http/showcase.js
++++ b/server/http/showcase.js
@@@ -11,7 +11,7 @@ const { http } = require('openvibe-contracts');
const { asyncRouter } = require('./router');
const { sameOrigin } = require('./same-origin');
const renderLib = require('../bench/render');
--const { kindById, SHOWCASE_KIND_IDS } = require('../kinds');
++const { kindById, SHOWCASE_KIND_IDS, isPrivateKind } = require('../kinds');
const { esc, raw } = require('../render/html');
const { send } = require('../render/layout');
@@@ -51,7 +51,8 @@ async function voteCount(db, roundId) {
async function loadPairs(ctx, opts) {
const db = ctx.s.db;
const kind = opts && opts.kind ? String(opts.kind) : 'all';
-- const kinds = kind && kind !== 'all' ? [kind] : SHOWCASE_KIND_IDS;
++ // A private kind (live.clip) is never in the gallery, even when asked for by name.
++ const kinds = kind && kind !== 'all' ? (isPrivateKind(kind) ? [] : [kind]) : SHOWCASE_KIND_IDS;
const limit = Math.min(100, Math.max(1, num(opts && opts.limit, 24)));
const rows = await db.many(
"SELECT r.id, r.kind_id, r.created_at, r.created_by, t.prompt, t.task_class FROM bench_rounds r JOIN bench_tasks t ON t.id = r.task_id WHERE r.status = 'finished' AND r.kind_id = ANY($1::text[]) ORDER BY r.created_at DESC, r.id LIMIT $2",
ddiff --git a/server/kinds.js b/server/kinds.js
iindex da01fb7..8a5837d 100644
---- a/server/kinds.js
++++ b/server/kinds.js
@@@ -44,8 +44,20 @@ const KINDS = Object.freeze([
{ id: 'design.interface', room: 'design', entry: 'component_set', judge: 'human_pairwise', name: 'Component systems arena', metrics: ['elo', 'cost_usd', 'ms'], schema: { renderer: 'script', limit_ms: 20000, judges: ['human_pairwise', 'rubric'], entry: 'component_set', payload: { html: 'string' } } },
]);
--const byId = new Map(KINDS.map((k) => [k.id, k]));
++// Private kinds: their rounds compare things that are not public (a channel's held-out recordings), so they never
++// appear in the public kind list, the boards, the open datasets or the vote dump; their submitter reads its own results.
++// live.clip (item 151, C19): OpenVibe.Live's AI clip picks, blind against another policy's (server/bench/clip.js).
++const PRIVATE_KINDS = Object.freeze([
++ { id: 'live.clip', room: 'live', entry: 'video_clip', judge: 'human_pairwise', name: 'AI clip picks', metrics: ['elo', 'cost_usd', 'accept_rate'], private: true, submitter: 'service:live',
++ schema: { renderer: 'video', judges: ['human_pairwise'], entry: 'video_clip',
++ payload: { source: 'object', playback: 'object', policy: 'object', title: 'string', abstained: 'boolean', reason: 'string' },
++ rubric: require('./bench/clip').RUBRIC } },
++]);
++const PRIVATE_KIND_IDS = Object.freeze(PRIVATE_KINDS.map((k) => k.id));
++const ALL_KINDS = Object.freeze(KINDS.concat(PRIVATE_KINDS));
++
++const byId = new Map(ALL_KINDS.map((k) => [k.id, k]));
const SHOWCASE_KIND_IDS = Object.freeze(KINDS.filter((k) => k.schema && k.schema.renderer).map((k) => k.id));
--module.exports = { KINDS, JUDGES, WINNERS, SHOWCASE_KIND_IDS, kindById: (id) => byId.get(id) || null };
++module.exports = { KINDS, PRIVATE_KINDS, PRIVATE_KIND_IDS, ALL_KINDS, JUDGES, WINNERS, SHOWCASE_KIND_IDS, kindById: (id) => byId.get(id) || null, isPrivateKind: (id) => PRIVATE_KIND_IDS.includes(id) };
ddiff --git a/server/pick/pick.js b/server/pick/pick.js
iindex 12e26a7..27cbf35 100644
---- a/server/pick/pick.js
++++ b/server/pick/pick.js
@@@ -10,7 +10,7 @@
const { frontier } = require('./frontier');
const { parsePick } = require('./parser');
--const { KINDS, kindById, SHOWCASE_KIND_IDS } = require('../kinds');
++const { KINDS, kindById, SHOWCASE_KIND_IDS, PRIVATE_KIND_IDS } = require('../kinds');
const renderLib = require('../bench/render');
const { screenshotOf } = require('../http/gallery');
@@@ -52,7 +52,7 @@ async function busiestKind(ctx) {
const rows = await ctx.s.db.many(
`SELECT r.kind_id AS id, COUNT(e.id)::int AS n
FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id
-- WHERE e.error IS NULL GROUP BY r.kind_id ORDER BY n DESC, r.kind_id LIMIT 1`);
++ WHERE e.error IS NULL AND NOT (r.kind_id = ANY($1::text[])) GROUP BY r.kind_id ORDER BY n DESC, r.kind_id LIMIT 1`, [PRIVATE_KIND_IDS]);
if (rows && rows[0] && rows[0].id) return rows[0].id;
return SHOWCASE_KIND_IDS[0] || 'games.play';
}
ddiff --git a/server/rating/board.js b/server/rating/board.js
iindex 210c8a8..3452ad1 100644
---- a/server/rating/board.js
++++ b/server/rating/board.js
@@@ -14,6 +14,8 @@
*/
const style = require('./style');
++// Private kinds' votes (live.clip) are never in the open vote dump or the public ratings.
++const { PRIVATE_KIND_IDS } = require('../kinds');
const num = (v, d = 0) => (v == null || Number.isNaN(Number(v)) ? d : Number(v));
@@@ -26,8 +28,8 @@ async function loadVotes(db) {
FROM bench_judgements j
JOIN bench_entries l ON l.id = (j.pair->>'left')
JOIN bench_entries r ON r.id = (j.pair->>'right')
-- WHERE j.judge = 'human_pairwise' AND j.outcome IS NOT NULL
-- ORDER BY j.at, j.id`);
++ WHERE j.judge = 'human_pairwise' AND j.outcome IS NOT NULL AND NOT (j.kind_id = ANY($1::text[]))
++ ORDER BY j.at, j.id`, [PRIVATE_KIND_IDS]);
const out = [];
for (const row of rows) {
const left = styleOf(row.left_payload, row.left_length, row.left_markup);
ddiff --git a/server/store.js b/server/store.js
iindex c5269ca..d3cdf34 100644
---- a/server/store.js
++++ b/server/store.js
@@@ -8,7 +8,8 @@
*/
const crypto = require('crypto');
const { ids } = require('openvibe-contracts');
--const { KINDS, JUDGES, WINNERS, kindById } = require('./kinds');
++const { KINDS, ALL_KINDS, PRIVATE_KIND_IDS, JUDGES, WINNERS, kindById, isPrivateKind } = require('./kinds');
++const clipLib = require('./bench/clip');
const ratingLib = require('./rating');
const styleLib = require('./rating/style');
@@@ -197,7 +198,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
// ── Kinds ───────────────────────────────────────────────
async function seedKinds() {
-- for (const k of KINDS) {
++ for (const k of ALL_KINDS) {
await db.exec(
`INSERT INTO bench_kinds (id, room, entry, judge, name, schema, metrics, created_at)
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7::jsonb, now())
@@@ -207,8 +208,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
);
}
}
++ /** The public kinds; a private one (live.clip) is never listed. */
async function listKinds() {
-- return (await db.many('SELECT * FROM bench_kinds ORDER BY id')).map(wireKind);
++ return (await db.many('SELECT * FROM bench_kinds WHERE NOT (id = ANY($1::text[])) ORDER BY id', [PRIVATE_KIND_IDS])).map(wireKind);
}
async function getKind(id) {
return wireKind(await db.maybe('SELECT * FROM bench_kinds WHERE id = $1', [String(id || '')]));
@@@ -218,6 +220,8 @@ function createStore(s, { log = console, random = Math.random, config = null } =
async function createTask({ kindId, prompt, taskClass = null, public: isPublic = false, subject }) {
const kind = await getKind(kindId);
if (!kind) return refusal(422, 'bench.kind_not_found', `No such kind: ${kindId}`);
++ // A private kind's tasks name what is not public (a held-out recording): never public.
++ if (isPrivateKind(kind.id)) isPublic = false;
const text = String(prompt == null ? '' : prompt).replace(/\s+/g, ' ').trim();
if (!text) return refusal(422, 'bench.prompt_required', 'A task needs a prompt or spec.');
if (text.length > 4000) return refusal(422, 'bench.prompt_too_long', 'The prompt must be at most 4000 characters.');
@@@ -302,7 +306,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
const revealed = viewerSubject != null && (r.created_by === viewerSubject
|| await db.value('SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND subject = $2', [r.id, viewerSubject]) > 0);
const votes = num(await db.value("SELECT COUNT(*)::int AS n FROM bench_judgements WHERE round_id = $1 AND method <> 'game_outcome'", [r.id]));
-- const entries = entryRows.map((e) => wireEntry(e, { blind: !revealed }));
++ // A live.clip playback is only for a voter (nextJudgement); a round read never carries one.
++ const entries = entryRows.map((e) => wireEntry(e, { blind: !revealed }))
++ .map((e) => (isPrivateKind(r.kind_id) ? { ...e, payload: clipLib.forViewer(e.payload, { now: s.now(), playback: false }) } : e));
const out = { ...wireRound(r), entries, votes };
if (!revealed) out.blind = true;
return out;
@@@ -313,6 +319,15 @@ function createStore(s, { log = console, random = Math.random, config = null } =
return await s.tx(async () => {
const r = await db.maybe('SELECT * FROM bench_rounds WHERE id = $1 FOR UPDATE', [String(roundId || '')]);
if (!r) throw new BenchError(404, 'bench.round_not_found', 'No such Bench round.');
++ if (r.kind_id === 'live.clip') {
++ // A clip pick: its shape checked, and the policy that chose it pinned on the entry (route), never rewritten.
++ const v = clipLib.validateEntry(payload, { now: s.now() });
++ if (!v.ok) throw new BenchError(422, 'bench.invalid_clip', v.message);
++ payload = v.payload;
++ route = { policy: v.payload.policy };
++ model = model || v.payload.policy.models[0] || `${v.payload.policy.selector}@${v.payload.policy.version}`;
++ provider = provider || 'live';
++ }
const id = s.newId('bte');
// The style of the output (how long it is, how much markup it wears) is measured once, here, so the
// ratings can control for style later without opening the payload again.
@@@ -386,7 +401,9 @@ function createStore(s, { log = console, random = Math.random, config = null } =
[String(kind || ''), subject, creators],
);
if (!row) return null;
-- const entries = (await db.many('SELECT * FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id', [row.id])).map(wireEntry);
++ const entries = (await db.many('SELECT * FROM bench_entries WHERE round_id = $1 AND error IS NULL ORDER BY id', [row.id])).map(wireEntry)
++ // A voter of a private kind sees the playback, while it lasts (clip.forViewer).
++ .map((e) => (isPrivateKind(row.kind_id) ? { ...e, payload: clipLib.forViewer(e.payload, { now: s.now(), playback: true }) } : e));
if (entries.length < 2) return null;
const [left, right] = shuffle(entries, rng).slice(0, 2);
return {
@@@ -609,7 +626,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
async function board(kindId, { type = 'elo', category = CATEGORY_DEFAULT, period = 'all', voters = null } = {}) {
const kind = await getKind(kindId);
-- if (!kind) return null;
++ if (!kind || isPrivateKind(kind.id)) return null;
const cat = categoryOf(voters, category);
const cost = await costStats(kindId);
const pass = await passRate(kindId);
@@@ -700,7 +717,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
// ── Dataset (open, no voter ids) ─────────────────────────
async function dataset(kindId) {
const kind = await getKind(kindId);
-- if (!kind) return null;
++ if (!kind || isPrivateKind(kind.id)) return null;
const tasks = (await db.many('SELECT * FROM bench_tasks WHERE kind_id = $1 AND public = true ORDER BY created_at, id', [kindId]))
.map(wireTask).map(({ created_by, ...rest }) => rest);
const rounds = (await db.many("SELECT * FROM bench_rounds WHERE kind_id = $1 AND status IN ('ready', 'finished') ORDER BY created_at, id", [kindId]))
@@@ -728,6 +745,43 @@ function createStore(s, { log = console, random = Math.random, config = null } =
return lines.length ? `${lines.join('\n')}\n` : '';
}
++ /**
++ * live.clip by policy (selector@version), for its submitter: entries, abstentions, cost, and how its picks fared
++ * in blind votes. accept_rate = (wins + ties) / votes it was in; a both_bad vote counts against both sides.
++ */
++ async function clipPolicies() {
++ const entries = await db.many(`SELECT e.id, e.cost_usd, e.payload FROM bench_entries e JOIN bench_rounds r ON r.id = e.round_id
++ WHERE r.kind_id = 'live.clip' AND e.error IS NULL`);
++ const byEntry = new Map();
++ const stats = new Map();
++ for (const e of entries) {
++ const p = typeof e.payload === 'string' ? JSON.parse(e.payload) : (e.payload || {});
++ const key = `${(p.policy && p.policy.selector) || '?'}@${(p.policy && p.policy.version) || '?'}`;
++ byEntry.set(e.id, key);
++ const st = stats.get(key) || { policy: key, entries: 0, abstained: 0, cost_usd: 0, votes: 0, wins: 0, ties: 0, losses: 0, both_bad: 0 };
++ st.entries += 1;
++ if (p.abstained) st.abstained += 1;
++ st.cost_usd += num(e.cost_usd);
++ stats.set(key, st);
++ }
++ const votes = await db.many(`SELECT j.pair, j.outcome FROM bench_judgements j WHERE j.kind_id = 'live.clip' AND j.judge = 'human_pairwise' AND j.pair IS NOT NULL AND j.outcome IS NOT NULL`);
++ for (const v of votes) {
++ const pair = typeof v.pair === 'string' ? JSON.parse(v.pair) : v.pair;
++ const sides = [['left', pair && pair.left], ['right', pair && pair.right]];
++ for (const [side, id] of sides) {
++ const st = stats.get(byEntry.get(id));
++ if (!st) continue;
++ st.votes += 1;
++ if (v.outcome === 'tie') st.ties += 1;
++ else if (v.outcome === 'both_bad') st.both_bad += 1;
++ else if (v.outcome === side) st.wins += 1;
++ else st.losses += 1;
++ }
++ }
++ return [...stats.values()].map((st) => ({ ...st, cost_usd: Math.round(st.cost_usd * 1e6) / 1e6, accept_rate: st.votes ? Math.round(((st.wins + st.ties) / st.votes) * 1000) / 1000 : null }))
++ .sort((a, b) => a.policy.localeCompare(b.policy));
++ }
++
// ── Private evals ────────────────────────────────────────
async function createEval({ owner, kindId, name, testSuite }) {
const kind = await getKind(kindId);
@@@ -786,7 +840,7 @@ function createStore(s, { log = console, random = Math.random, config = null } =
wireKind, wireTask, wireRound, wireEntry, wireJudgement, wireRating, wireEval, wireEvalRun,
insertOutbox, seedKinds, listKinds, getKind,
createTask, getTask,
-- createRound, getRound, addEntry, maybeReady, finishRound,
++ createRound, getRound, addEntry, maybeReady, finishRound, clipPolicies,
nextJudgement, castJudgement, pairwiseGames,
board, valueRows, modelSummary, dataset, datasetJsonl,
createEval, getEval, runEval, evalResults, evalDailyCount,
ddiff --git a/test/live-clip.test.js b/test/live-clip.test.js
nnew file mode 100644
iindex 0000000..4e2f232
---- /dev/null
++++ b/test/live-clip.test.js
@@@ -0,0 +1,124 @@
++'use strict';
++/**
++ * live.clip (item 151, C19): OpenVibe.Live's AI clip picks, judged blind against another policy's. Typed clip and
++ * abstention entries, the policy pinned on each entry, only Live submits, the playback only for a voter and never
++ * after it expires, never on a public board, list, gallery, dataset or vote dump, never filled by Bench's AI, and the
++ * cost and accept rate per policy (both_bad counts against both).
++ * node test/live-clip.test.js
++ */
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++
++(async () => {
++ const t = await boot({ env: { BENCH_ROUNDS_PER_DAY: '100', BENCH_ROUNDS_DAILY: '1000' } });
++ const voter = t.network.addUser('voter');
++ const voter2 = t.network.addUser('voter2');
++ const live = t.network.serviceToken({ sub: 'svc:live', audience: 'openvibe.bench', cap: ['bench.task.create', 'bench.round.create', 'bench.entry.create'] });
++ const other = t.network.serviceToken({ sub: 'svc:room', audience: 'openvibe.bench', cap: ['bench.task.create', 'bench.round.create', 'bench.entry.create'] });
++ const soon = () => new Date(Date.now() + 3600_000).toISOString();
++ const clip = (version, extra = {}) => ({
++ source: { recording_id: 'vod_5021', start_ms: 60_000, end_ms: 90_000, language: 'ja', genre: 'irl' },
++ playback: { url: `https://media.openvibe.media/p/${version}.mp4?sig=x`, expires_at: soon() },
++ policy: { selector: 'moment-picker', version, models: ['qwen3-vl-4b'] },
++ title: 'The cat walks onto the keyboard',
++ ...extra,
++ });
++ let roundId;
++
++ async function newRound(token = live) {
++ const task = await t.get('/api/v1/kinds/live.clip/tasks', { bearer: token, json: { prompt: 'vod_5021 30:00-45:00', public: true } });
++ if (task.status !== 201) return { status: task.status, text: task.text };
++ assert.strictEqual(task.json().task.public, false, 'a private kind\'s task is never public');
++ const rr = await t.get(`/api/v1/tasks/${task.json().task.id}/rounds`, { bearer: token, json: {} });
++ assert.strictEqual(rr.status, 201, rr.text);
++ return { status: 201, id: rr.json().round.id };
++ }
++
++ await check('only Live makes live.clip tasks, rounds and entries', async () => {
++ assert.strictEqual((await newRound(other)).status, 403);
++ const r = await newRound();
++ assert.strictEqual(r.status, 201, r.text);
++ roundId = r.id;
++ const e = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: other, json: { payload: clip('v1'), cost_usd: 0.002 } });
++ assert.strictEqual(e.status, 403);
++ });
++
++ await check('typed entries: a clip and an abstention; the bad ones refused; the policy pinned on the entry', async () => {
++ const bad = [
++ clip('v1', { source: { recording_id: 'vod_5021', start_ms: 90_000, end_ms: 60_000 } }),
++ clip('v1', { source: { recording_id: 'vod_5021', start_ms: 0, end_ms: 2_000 } }),
++ clip('v1', { playback: { url: 'http://insecure.example/x.mp4', expires_at: soon() } }),
++ clip('v1', { playback: { url: 'https://media.openvibe.media/p.mp4', expires_at: new Date(Date.now() - 1000).toISOString() } }),
++ clip('v1', { policy: { selector: 'moment-picker' } }),
++ { source: { recording_id: 'vod_5021', start_ms: 0, end_ms: 1 }, policy: { selector: 's', version: 'v2' }, abstained: true },
++ ];
++ for (const payload of bad) {
++ const r = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: live, json: { payload } });
++ assert.strictEqual(r.status, 422, JSON.stringify(payload).slice(0, 120));
++ }
++ const a = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: live, json: { payload: clip('v1', { extra: 'dropped' }), cost_usd: 0.002 } });
++ assert.strictEqual(a.status, 201, a.text);
++ assert.strictEqual(a.json().entry.payload.extra, undefined, 'only the typed fields are kept');
++ const b = await t.get(`/api/v1/rounds/${roundId}/entries`, { bearer: live, json: { payload: { source: { recording_id: 'vod_5021', start_ms: 0, end_ms: 1000 }, policy: { selector: 'moment-picker', version: 'v2', models: ['smolvlm2-500m'] }, abstained: true, reason: 'Nothing happened worth a clip.' }, cost_usd: 0.0005 } });
++ assert.strictEqual(b.status, 201, b.text);
++ const row = await t.ctx.s.db.one('SELECT route, model FROM bench_entries WHERE id = $1', [a.json().entry.id]);
++ assert.deepStrictEqual(row.route.policy, { selector: 'moment-picker', version: 'v1', models: ['qwen3-vl-4b'] });
++ assert.strictEqual(row.model, 'qwen3-vl-4b');
++ });
++
++ await check('the playback is for a voter only: not in a round read, not after it expires', async () => {
++ const anon = await t.get(`/api/v1/rounds/${roundId}`);
++ assert.strictEqual(anon.status, 404, 'a private round is not readable by just anyone');
++ const asLive = await t.get(`/api/v1/rounds/${roundId}`, { bearer: live });
++ assert.strictEqual(asLive.status, 200, asLive.text);
++ assert.ok(asLive.json().round.entries.length === 2 && asLive.json().round.entries.every((e) => !e.payload.playback), 'no playback in a round read');
++ const pair = await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter });
++ assert.strictEqual(pair.status, 200, pair.text);
++ const sides = [pair.json().left, pair.json().right];
++ assert.ok(sides.some((sd) => sd.payload.playback && sd.payload.playback.url.startsWith('https://')), 'the voter sees the clip');
++ await t.ctx.s.db.exec("UPDATE bench_entries SET payload = jsonb_set(payload, '{playback,expires_at}', to_jsonb('2020-01-01T00:00:00.000Z'::text)) WHERE round_id = $1 AND payload ? 'playback'", [roundId]);
++ const later = await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter2 });
++ const ls = [later.json().left, later.json().right];
++ assert.ok(ls.every((sd) => !sd.payload.playback), 'never after it expires');
++ assert.ok(ls.some((sd) => sd.payload.playback_expired === true));
++ const anonNext = await t.get('/api/v1/judgements/next?kind=live.clip');
++ assert.ok([401, 403].includes(anonNext.status), `anonymous voting on a private kind is refused (${anonNext.status})`);
++ });
++
++ await check('cost and accept rate per policy; both_bad counts against both', async () => {
++ const pair = await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter });
++ const p = pair.json();
++ const entries = await t.ctx.s.db.many('SELECT id, payload FROM bench_entries WHERE round_id = $1', [roundId]);
++ const v1 = entries.find((e) => e.payload.policy.version === 'v1').id;
++ const winnerSide = p.left.id === v1 ? 'left' : 'right';
++ const vote = await t.get(`/api/v1/rounds/${roundId}/judgements`, { as: voter, json: { judge: 'human_pairwise', pair: { left: p.left.id, right: p.right.id }, outcome: winnerSide } });
++ assert.strictEqual(vote.status, 201, vote.text);
++ const p2 = (await t.get('/api/v1/judgements/next?kind=live.clip', { as: voter2 })).json();
++ const vote2 = await t.get(`/api/v1/rounds/${roundId}/judgements`, { as: voter2, json: { judge: 'human_pairwise', pair: { left: p2.left.id, right: p2.right.id }, outcome: 'both_bad' } });
++ assert.strictEqual(vote2.status, 201, vote2.text);
++ const r = await t.get('/api/v1/kinds/live.clip/policies', { bearer: live });
++ assert.strictEqual(r.status, 200, r.text);
++ const by = Object.fromEntries(r.json().policies.map((x) => [x.policy, x]));
++ assert.deepStrictEqual([by['moment-picker@v1'].votes, by['moment-picker@v1'].wins, by['moment-picker@v1'].both_bad, by['moment-picker@v1'].accept_rate], [2, 1, 1, 0.5]);
++ assert.deepStrictEqual([by['moment-picker@v2'].votes, by['moment-picker@v2'].losses, by['moment-picker@v2'].both_bad, by['moment-picker@v2'].abstained, by['moment-picker@v2'].accept_rate], [2, 1, 1, 1, 0]);
++ assert.strictEqual(by['moment-picker@v1'].cost_usd, 0.002);
++ assert.strictEqual((await t.get('/api/v1/kinds/live.clip/policies', { bearer: other })).status, 403, 'Live\'s results are Live\'s');
++ });
++
++ await check('never public: the kind list, the board, the dataset, the vote dump, the gallery; never filled by Bench\'s AI', async () => {
++ const kinds = (await t.get('/api/v1/kinds')).json().kinds.map((k) => k.id);
++ assert.ok(!kinds.includes('live.clip'));
++ assert.strictEqual((await t.get('/api/v1/boards/live.clip')).status, 404);
++ assert.strictEqual((await t.get('/api/v1/datasets/live.clip.jsonl')).status, 404);
++ const votes = await t.get('/api/v1/datasets/votes.jsonl');
++ assert.ok(!votes.text.includes('live.clip'), 'no live.clip vote in the dump');
++ const gallery = await t.get('/api/v1/showcase/rounds?kind=live.clip');
++ if (gallery.status === 200) assert.ok(!gallery.text.includes(roundId), 'not in the gallery');
++ const queued = await newRound();
++ const claimed = await t.ctx.s.db.maybe("SELECT * FROM bench_rounds WHERE status = 'queued' AND NOT (kind_id = ANY($1::text[])) AND id = $2", [['live.clip'], queued.id]);
++ assert.strictEqual(claimed, null, 'the job runner never claims it');
++ });
++
++ await t.close();
++ done();
++})().catch((e) => { console.error(e); process.exit(1); });