openvibers/openvibe.ai Commits

live.moments.verify (item 151, C09): a clip's title and description checked against the moment (#280)

OP OpenVibers committed 10 days ago · fda0ed43f0 · 4 files, +79 −1

Before Live makes an AI clip public, a second call checks what its title and
description claim against the minute around the moment: scene notes, speech and
sounds as [m:ss] lines, and the chat reaction. It answers supported, unsupported
(naming the claims the lines do not back) or unsure; nothing near the moment is
"unsure" with no model run; a verdict outside the set is no answer, and "supported"
with unbacked claims is read as unsure. Live (next) makes a clip public only on
supported.

Changes

ddiff --git a/server/workflows/hooks.js b/server/workflows/hooks.js
iindex 6988617..9ba6bb3 100644
---- a/server/workflows/hooks.js
++++ b/server/workflows/hooks.js
@@@ -215,6 +215,24 @@ const PREPARE = {
},
};
},
++ // C09: the lines within a minute of the moment, as [m:ss]; nothing to check against is "unsure" with no model run.
++ 'live.moments.verify'(input) {
++ const t = Number(input.t) || 0;
++ const near = (list, key) => (list || []).filter((x) => Math.abs((Number(x.t) || 0) - t) <= 60).map((x) => `[${mmss(Number(x.t) || 0)}] ${oneLine(x[key], 200)}`);
++ const timeline = near(input.timeline, 'text');
++ const transcript = near(input.transcript, 'text');
++ const sounds = near(input.sounds, 'label');
++ if (!timeline.length && !transcript.length && !sounds.length) return { output: { verdict: 'unsure', unsupported: [], why: 'nothing recorded near the moment to check against' } };
++ const c = input.chat || {};
++ const chat = Number(c.peak) > 0 ? `${Math.round(Number(c.peak))} messages in the busiest 30 s (the stream averages ${Math.round(Number(c.avg) || 0)})${Number(c.people) > 0 ? ` from ${Math.round(Number(c.people))} people` : ''}` : 'no chat reaction recorded';
++ return {
++ vars: {
++ title: oneLine(input.title, 120) || 'Untitled stream', at: mmss(t),
++ clip_title: oneLine(input.clip_title, 120), clip_desc: oneLine(input.clip_desc, 400) || '(none)',
++ timeline: timeline.join('\n') || '(none)', transcript: transcript.join('\n') || '(none)', sounds: sounds.join('\n') || '(none)', chat,
++ },
++ };
++ },
'live.translate'(input) {
const text = String(input.text || '').trim();
const from = input.from || 'auto';
@@@ -467,6 +485,14 @@ const POSTPROCESS = {
if (shown.length && t > Math.max(...shown) + 120) return null;
return { t: Math.floor(t), title: oneLine(j.title, 80).replace(/^["']+|["']+$/g, ''), desc: oneLine(j.desc, 400) };
},
++ // C09: only a verdict from the set is an answer; "supported" with claims it calls unbacked is not a yes.
++ 'live.moments.verify'(r) {
++ const j = r.json || parseJsonLoose(r.text);
++ if (!j || typeof j !== 'object' || !['supported', 'unsupported', 'unsure'].includes(j.verdict)) return null;
++ const unsupported = (Array.isArray(j.unsupported) ? j.unsupported : []).map((x) => oneLine(x, 200)).filter(Boolean).slice(0, 8);
++ const verdict = j.verdict === 'supported' && unsupported.length ? 'unsure' : j.verdict;
++ return { verdict, unsupported, why: oneLine(j.why, 200) };
++ },
'ai.classify'(r, input) {
const j = r.json || parseJsonLoose(r.text);
if (!j || !input.labels.includes(j.label)) return null; // a label outside the given set is not an answer
ddiff --git a/server/workflows/live.js b/server/workflows/live.js
iindex c844330..666494c 100644
---- a/server/workflows/live.js
++++ b/server/workflows/live.js
@@@ -241,6 +241,21 @@ const templates = [
user_prompt: 'Below is a single livestream VOD titled "{{title}}", described by its on-screen TIMELINE (visual scene notes) and its AUDIO TRANSCRIPT, each line timestamped [m:ss]. Titles and descriptions are about what happens.\n\nTIMELINE:\n{{timeline}}\n\nTRANSCRIPT:\n{{transcript}}\n\nSOUNDS HEARD (non-speech audio events):\n{{sounds}}\n\nViewers CLIPPED these timestamps (very strong "this was a highlight" signal): {{clipped}}\nChat activity SPIKED around: {{spikes}}\n\n{{flavor_hint}}\n{{#avoid}}ALREADY USED (a paste or clip exists there) — do NOT pick these or anything within 2 minutes of them: {{avoid}}. Find a DIFFERENT moment.{{/avoid}}\n\nFind the SINGLE most interesting/funny/dramatic/surprising/striking moment in this VOD with something clearly VISIBLE happening. Prefer moments backed by the clip/chat signals when they line up with something notable. NEVER pick a black/dark/loading/blank screen, an intro/BRB card, or a moment with no visible content or activity. {{abstain_hint}}Return STRICT JSON only, nothing else: {"at": "<the m:ss timestamp of the moment, exactly as written in the lines above>", "title": "<specific punchy 3-8 word title, not the stream name, funny, never generic>", "desc": "<one vivid sentence describing the moment>"}',
default_route: 'live.legacy', owner: 'live', metadata: { json_name: 'moment_pick' },
},
++ {
++ key: 'live.moments.verify', name: 'Live: is a clip\'s title and description backed by what happened',
++ description: 'Item 151, C09: before an AI clip is made public, its title and description are checked against the lines around the moment (scene notes, speech, sounds, chat). Unbacked claims keep it from being public.',
++ input_schema: { type: 'object' },
++ output_schema: {
++ type: 'object', additionalProperties: false, required: ['verdict', 'unsupported', 'why'], properties: {
++ verdict: { type: 'string', enum: ['supported', 'unsupported', 'unsure'], description: 'supported: the lines back every claim; unsupported: a claim the lines contradict or never mention; unsure: too little to tell' },
++ unsupported: { type: 'array', items: { type: 'string' }, description: 'each claim of the title or description that the lines do not back (empty when supported)' },
++ why: { type: 'string', description: 'a few words' },
++ },
++ },
++ system_prompt: '',
++ user_prompt: 'A clip is about to be published from the livestream "{{title}}". It is about the moment at [{{at}}].\n\nPROPOSED TITLE: {{clip_title}}\nPROPOSED DESCRIPTION: {{clip_desc}}\n\nEverything known about the minute around it, each line timestamped [m:ss]:\nSCENE NOTES:\n{{timeline}}\n\nSPEECH:\n{{transcript}}\n\nSOUNDS:\n{{sounds}}\n\nCHAT: {{chat}}\n\nCheck the title and the description against these lines only. Answer "supported" when everything they say happens in the lines; "unsupported" when they claim something the lines contradict or never mention (an event, an object, a reaction, a person); "unsure" when the lines are too thin to tell. A funny or punchy wording is fine; an invented event is not. Never guess. Return STRICT JSON only: {"verdict": "supported" | "unsupported" | "unsure", "unsupported": ["<each claim the lines do not back>"], "why": "<a few words>"}',
++ default_route: 'live.legacy', owner: 'live', metadata: { json_name: 'moment_verify' },
++ },
{
key: 'live.stream.recap', name: 'Live: after-show report',
description: 'From recap/recap.js aiWriteup (stream_recap).',
@@@ -490,6 +505,20 @@ const structured = [
steps: [{ kind: 'llm', template: 'live.moments.pick', operation: 'generate', output: 'json', prepare: 'live.moments.pick', postprocess: 'live.moments.pick', params: { max_tokens: 300 } }],
cache_mode: 'none',
},
++ {
++ key: 'live.moments.verify', name: 'Check a clip\'s title and description against the moment', namespace: 'live',
++ description: 'Item 151, C09: a candidate clip (its second, title and description) and the lines around it in; supported, unsupported (with the claims) or unsure out. Live makes a clip public only on supported; a failed check is not a yes.',
++ input_schema: TEXT_OUT({
++ title: STR(200), clip_title: STR(200), clip_desc: STR(800), t: { type: 'number', minimum: 0 },
++ timeline: { type: 'array', maxItems: 60, items: { type: 'object', required: ['t', 'text'], additionalProperties: false, properties: { t: { type: 'number', minimum: 0 }, text: STR(2000) } } },
++ transcript: { type: 'array', maxItems: 120, items: { type: 'object', required: ['t', 'text'], additionalProperties: false, properties: { t: { type: 'number', minimum: 0 }, text: STR(2000) } } },
++ sounds: { type: 'array', maxItems: 60, items: { type: 'object', required: ['t', 'label'], additionalProperties: false, properties: { t: { type: 'number', minimum: 0 }, label: STR(120) } } },
++ chat: { type: 'object', additionalProperties: false, properties: { peak: { type: 'number', minimum: 0 }, avg: { type: 'number', minimum: 0 }, people: { type: 'number', minimum: 0 } } },
++ }, ['clip_title', 't']),
++ output_schema: TEXT_OUT({ verdict: { enum: ['supported', 'unsupported', 'unsure'] }, unsupported: { type: 'array', maxItems: 8, items: STR(200) }, why: STR(200) }, ['verdict', 'unsupported', 'why']),
++ steps: [{ kind: 'llm', template: 'live.moments.verify', operation: 'generate', output: 'json', prepare: 'live.moments.verify', postprocess: 'live.moments.verify', params: { max_tokens: 220 } }],
++ cache_mode: 'none',
++ },
{
key: 'live.stream.recap', name: 'After-show report for a finished stream', namespace: 'live',
input_schema: TEXT_OUT({ facts: { type: 'object' }, sources: SOURCES(0, 50) }, ['facts']),
@@@ -560,7 +589,7 @@ const passthrough = PASSTHROUGH.map(([key, name, description]) => ({
const KIND_TO_WORKFLOW = {
ai_viewers_director: 'live.viewers.plan', ai_viewers_reply: 'live.viewers.reply', ai_viewers: 'live.viewers.line', ai_viewers_fold: 'live.viewers.fold',
chat_global: 'live.chat.insight', chat_user: 'live.chat.insight', chat_relay: 'live.chat.insight', chat_anon: 'live.chat.insight', combined_overview: 'live.chat.insight', session_titles: 'live.chat.insight',
-- moment_vod_rank: 'live.moments.rank', moment_pick: 'live.moments.pick', auto_clip_confirm: 'live.clips.confirm', hero_slogans: 'live.hero.slogans', easter_egg: 'live.easter_egg', home_star: 'live.home.star',
++ moment_vod_rank: 'live.moments.rank', moment_pick: 'live.moments.pick', moment_verify: 'live.moments.verify', auto_clip_confirm: 'live.clips.confirm', hero_slogans: 'live.hero.slogans', easter_egg: 'live.easter_egg', home_star: 'live.home.star',
status_check: 'live.status_check',
};
ddiff --git a/test/live-templates.test.js b/test/live-templates.test.js
iindex b8d0c91..2b1df31 100644
---- a/test/live-templates.test.js
++++ b/test/live-templates.test.js
@@@ -71,6 +71,28 @@ t.test('the moment comes back as the lines show it (11-03: "t": 2 for 2:14)', ()
assert.deepStrictEqual(post({ text: '{"at": "1:35", "title": "\\"Up\\"", "desc": "He jumps."}' }, input), { t: 95, title: 'Up', desc: 'He jumps.' });
});
++t.test('the verifier (C09) reads the minute around the moment and only answers from its set', () => {
++ const vt = (live.templates || []).find((x) => x.key === 'live.moments.verify');
++ assert.ok(vt, 'the template exists');
++ const input = { title: 'Rust night', clip_title: 'It Compiles At Last', clip_desc: 'He jumps up cheering.', t: 95,
++ timeline: [{ t: 95, text: 'streamer jumps up' }, { t: 400, text: 'far away' }], transcript: [{ t: 94, text: 'IT COMPILES' }], sounds: [{ t: 96, label: 'Cheering' }], chat: { peak: 22, avg: 3.2, people: 7 } };
++ const out = render(vt.user_prompt, PREPARE['live.moments.verify'](input).vars);
++ assert.match(out, /about the moment at \[1:35\]/);
++ assert.match(out, /PROPOSED TITLE: It Compiles At Last\nPROPOSED DESCRIPTION: He jumps up cheering\./);
++ assert.match(out, /SCENE NOTES:\n\[1:35\] streamer jumps up\n\nSPEECH/);
++ assert.ok(!/far away/.test(out), 'only the minute around the moment');
++ assert.match(out, /CHAT: 22 messages in the busiest 30 s \(the stream averages 3\) from 7 people/);
++ assert.ok(!/\{\{|\$\{/.test(out), 'nothing left unrendered');
++ assert.deepStrictEqual(PREPARE['live.moments.verify']({ clip_title: 'x', t: 10, timeline: [{ t: 500, text: 'later' }] }), { output: { verdict: 'unsure', unsupported: [], why: 'nothing recorded near the moment to check against' } }, 'nothing to check: unsure, no model');
++ const post = POSTPROCESS['live.moments.verify'];
++ assert.deepStrictEqual(post({ json: { verdict: 'supported', unsupported: [], why: 'he cheers' } }), { verdict: 'supported', unsupported: [], why: 'he cheers' });
++ assert.deepStrictEqual(post({ text: '{"verdict":"unsupported","unsupported":["a dragon explodes"],"why":"no dragon"}' }), { verdict: 'unsupported', unsupported: ['a dragon explodes'], why: 'no dragon' });
++ assert.strictEqual(post({ json: { verdict: 'supported', unsupported: ['a cat'], why: '' } }).verdict, 'unsure', 'a yes with unbacked claims is not a yes');
++ assert.strictEqual(post({ json: { verdict: 'probably', unsupported: [] } }), null, 'a verdict outside the set is no answer');
++ assert.strictEqual(post({ text: 'Looks fine to me!' }), null);
++ assert.strictEqual(live.workflows.find((w) => w.key === 'live.moments.verify').cache_mode, 'none');
++});
++
t.test('slogans, the daily secret and the star render as Live rendered them', () => {
const tp = (k) => live.templates.find((x) => x.key === k);
const sl = render(tp('live.hero.slogans').user_prompt, PREPARE['live.hero.slogans']({ users: [{ name: 'goosely', text: 'loves croutons' }], usernames: ['goosely'], count: 20 }).vars);
ddiff --git a/test/workflows.test.js b/test/workflows.test.js
iindex 88474be..e4d117f 100644
---- a/test/workflows.test.js
++++ b/test/workflows.test.js
@@@ -83,6 +83,7 @@ EXAMPLES['live.easter_egg'] = { vibe: 'croutons' };
EXAMPLES['space.app.build'] = { prompt: 'A guestbook where people sign in and leave a one-line note', name: 'Guestbook', current: [{ path: 'index.html', content: '<!doctype html><title>Guestbook</title><script src="/_openvibe/sdk.js"></script>' }] };
EXAMPLES['space.app.ask'] = { space: 'Pirate helper', instructions: 'Talk like a pirate.', transcript: 'visitor: hi\nai: Ahoy!', message: 'What is a good name for a parrot?' };
EXAMPLES['live.home.star'] = { recent_stars: ['bob'], previous_star: 'bob', candidates: [{ username: 'ann', name: 'Ann', language: 'en', last_14d: { sessions: 5, hours: 12 } }] };
++EXAMPLES['live.moments.verify'] = { title: 'Rust night', clip_title: 'It Compiles At Last', clip_desc: 'The streamer jumps up cheering when the build passes.', t: 95, timeline: [{ t: 95, text: 'streamer jumps up cheering' }], transcript: [{ t: 94, text: 'IT COMPILES' }], sounds: [{ t: 96, label: 'Cheering' }], chat: { peak: 22, avg: 3, people: 7 } };
EXAMPLES['live.moments.pick'] = { title: 'Rust night', flavor: 'clip', timeline: [{ t: 12, text: 'the build fails' }, { t: 95, text: 'streamer jumps up cheering' }], transcript: [{ t: 94, text: 'IT COMPILES' }], sounds: [{ t: 96, label: 'Cheering' }], clipped: [95], spikes: [96], avoid: [300] };
const t = suite('workflows');