openvibers/openvibe.ai Commits

live.moments.pick answers with the moment's [m:ss], not seconds (the model wrote minutes) (#278)

OP OpenVibers committed 10 days ago · 8734b8e86e · 4 files, +56 −14

The prompt showed every line as [m:ss] and asked for "t": <seconds>; gpt-5-nano wrote
the minutes (t: 2 for 2:14). In AI's own run log for 11-03 the picks were 0, 1, 2, 17
for VODs whose lines run to 3,964 s, so Live rejected nearly every pick as next to a
used moment and fell back to the objective signals.

The model now copies the timestamp ("at": "12:34", with a pattern for strict
providers); the hook turns it into t, refuses a bare number (minutes or seconds?)
and a time past everything it was shown. Live's contract (t in seconds) is unchanged,
and an older answer's t still reads as seconds. The stub answers a patterned string
with a sample that fits.

Changes

ddiff --git a/server/providers/stub.js b/server/providers/stub.js
iindex 90fcb7a..30d16f5 100644
---- a/server/providers/stub.js
++++ b/server/providers/stub.js
@@@ -45,6 +45,7 @@ function phrase(rnd, n) {
      return out.join(' ');
  }
  
++const PATTERN_SAMPLES = ['0:01', 'en', 'auto', '1024x1024'];
  /** A value shaped like `schema`, drawn from rnd. Strings are marked synthetic. */
  function synthesize(schema, rnd, depth = 0, name = 'value') {
      if (!schema || typeof schema !== 'object' || depth > 6) return null;
@@@ -81,6 +82,10 @@ function synthesize(schema, rnd, depth = 0, name = 'value') {
          case 'boolean': return rnd() < 0.5;
          case 'null': return null;
          default: {
++            // A string with a pattern takes the first common shape that fits it (a clock time, a language code, a size).
++            if (typeof schema.pattern === 'string') {
++                try { const re = new RegExp(schema.pattern); const hit = PATTERN_SAMPLES.find((x) => re.test(x)); if (hit) return hit; } catch { /* not a pattern this engine reads */ }
++            }
              let s = `(synthetic) ${name.replace(/_/g, ' ')}: ${phrase(rnd, 4 + Math.floor(rnd() * 5))}`;
              if (Number.isInteger(schema.maxLength)) s = s.slice(0, Math.max(0, schema.maxLength));
              if (Number.isInteger(schema.minLength) && s.length < schema.minLength) s = s.padEnd(schema.minLength, '.');
ddiff --git a/server/workflows/hooks.js b/server/workflows/hooks.js
iindex ddd90f5..7d12a78 100644
---- a/server/workflows/hooks.js
++++ b/server/workflows/hooks.js
@@@ -73,8 +73,14 @@ const MOMENT_FLAVOR = {
      paste: 'This moment becomes a SCREENSHOT paste: pick something visually striking in a single frame — a face or reaction, a visual gag, something odd on screen, a scene change. What was said matters less than what is SEEN.',
      clip: 'This moment becomes a 25-SECOND VIDEO CLIP: pick a beat that plays out over time — a line that lands, a reaction, a sound, chat exploding, something HAPPENING — not a static pretty frame.',
  };
--const ABSTAIN_CLIP = 'If NOTHING here is clearly worth a clip (only routine talk with no reaction, a static screen, nothing happening, or only moments already used), say so instead of forcing a pick: a missed clip costs less than a bad one. Then answer {"t": null, "title": "", "desc": "", "why": "<a few words>"}. Otherwise: ';
++const ABSTAIN_CLIP = 'If NOTHING here is clearly worth a clip (only routine talk with no reaction, a static screen, nothing happening, or only moments already used), say so instead of forcing a pick: a missed clip costs less than a bad one. Then answer {"at": null, "title": "", "desc": "", "why": "<a few words>"}. Otherwise: ';
  const oneLine = (t, max) => String(t || '').replace(/\s+/g, ' ').trim().slice(0, max);
++/** "12:34", "[1:02:03]" or "75:12" → seconds; anything else (a bare number, words) → NaN. */
++function clockSeconds(v) {
++    const m = /^\s*\[?\s*(?:(\d{1,2}):)?(\d{1,4}):(\d{2})(?:\.\d+)?\s*\]?\s*$/.exec(String(v == null ? '' : v));
++    if (!m || Number(m[3]) > 59 || (m[1] !== undefined && Number(m[2]) > 59)) return NaN;
++    return (Number(m[1] || 0) * 3600) + (Number(m[2]) * 60) + Number(m[3]);
++}
  // Text shown to the public as a description: one paragraph of prose. A model that answers with a list, headings or
  // bold labels ("- Streamer handle: x (@x).") has its markup and labels dropped, line by line, and headings left out,
  // before the lines are joined.
@@@ -443,14 +449,22 @@ const POSTPROCESS = {
          if (!j || !j.username) return null;
          return { username: String(j.username).slice(0, 64), headline: String(j.headline || '').trim().slice(0, 60), reason: String(j.reason || '').trim().slice(0, 170) };
      },
--    'live.moments.pick'(r) {
++    'live.moments.pick'(r, input) {
          const j = r.json || parseJsonLoose(r.text);
--        // An explicit "nothing worth it" (t: null, or abstain: true) is an answer, not a failure (C09).
--        if (j && (j.abstain === true || (Object.prototype.hasOwnProperty.call(j, 't') && j.t === null))) {
++        const has = (k) => j && Object.prototype.hasOwnProperty.call(j, k);
++        // An explicit "nothing worth it" (at or t null, or abstain: true) is an answer, not a failure (C09).
++        if (j && (j.abstain === true || (has('at') && j.at === null) || (!has('at') && has('t') && j.t === null))) {
              return { t: null, title: '', desc: '', abstain: true, why: oneLine(j.why, 200) };
          }
--        const t = j && Number(j.t);
--        if (!j || !Number.isFinite(t) || t < 0) return null;
++        if (!j) return null;
++        // The moment comes back as the lines show it ("12:34", "1:02:03"); a bare number there could be minutes or
++        // seconds, so it is no answer. An older answer's t is seconds.
++        const t = has('at') ? clockSeconds(j.at) : Number(j.t);
++        if (!Number.isFinite(t) || t < 0) return null;
++        // Never after everything the model was shown: a time past the last line (plus two minutes) is made up.
++        const shown = ['timeline', 'transcript', 'sounds'].flatMap((k) => ((input && input[k]) || []).map((x) => Number(x && x.t)))
++            .concat(((input && input.clipped) || []).map(Number), ((input && input.spikes) || []).map(Number)).filter(Number.isFinite);
++        if (shown.length && t > Math.max(...shown) + 120) return null;
          return { t: Math.floor(t), title: oneLine(j.title, 80).replace(/^["']+|["']+$/g, ''), desc: oneLine(j.desc, 400) };
      },
      'ai.classify'(r, input) {
ddiff --git a/server/workflows/live.js b/server/workflows/live.js
iindex db07283..2dde701 100644
---- a/server/workflows/live.js
++++ b/server/workflows/live.js
@@@ -228,15 +228,17 @@ const templates = [
          description: 'From ai-moments-job.js _findBestMoment (moment_pick): the most interesting visible moment for a screenshot paste or a clip, away from moments already used.',
          input_schema: { type: 'object' },
          output_schema: {
--            type: 'object', additionalProperties: false, required: ['t', 'title', 'desc'], properties: {
--                t: { type: ['integer', 'null'], description: 'seconds into the VOD, or null when nothing is worth it' },
++            type: 'object', additionalProperties: false, required: ['at', 'title', 'desc'], properties: {
++                // The moment as the lines show it ("12:34"), not seconds: asked for seconds next to [m:ss] lines, the
++                // model wrote the minutes (t: 2 for 2:14) on nearly every VOD until 11-03. The hook turns it into t.
++                at: { type: ['string', 'null'], pattern: '^\\[?([0-9]{1,2}:)?[0-9]{1,4}:[0-9]{2}\\]?$', description: 'the [m:ss] timestamp of the moment, copied from the lines above (e.g. "12:34"), or null when nothing is worth it' },
                  title: { type: 'string', description: 'specific punchy 3-8 word title, not the stream name, funny, never generic' },
                  desc: { type: 'string', description: 'one vivid sentence describing the moment' },
--                why: { type: 'string', description: 'when t is null: why nothing is worth it, in a few words' },
++                why: { type: 'string', description: 'when at is null: why nothing is worth it, in a few words' },
              },
          },
          system_prompt: '',
--        user_prompt: 'Below is a single livestream VOD titled "{{title}}", described by its on-screen TIMELINE (visual scene notes) and its AUDIO TRANSCRIPT, each line timestamped [m:ss]. Titles and descriptions are about what happens.\n\nTIMELINE:\n{{timeline}}\n\nTRANSCRIPT:\n{{transcript}}\n\nSOUNDS HEARD (non-speech audio events):\n{{sounds}}\n\nViewers CLIPPED these timestamps (very strong "this was a highlight" signal): {{clipped}}\nChat activity SPIKED around: {{spikes}}\n\n{{flavor_hint}}\n{{#avoid}}ALREADY USED (a paste or clip exists there) — do NOT pick these or anything within 2 minutes of them: {{avoid}}. Find a DIFFERENT moment.{{/avoid}}\n\nFind the SINGLE most interesting/funny/dramatic/surprising/striking moment in this VOD with something clearly VISIBLE happening. Prefer moments backed by the clip/chat signals when they line up with something notable. NEVER pick a black/dark/loading/blank screen, an intro/BRB card, or a moment with no visible content or activity. {{abstain_hint}}Return STRICT JSON only, nothing else: {"t": <seconds into the vod>, "title": "<specific punchy 3-8 word title, not the stream name, funny, never generic>", "desc": "<one vivid sentence describing the moment>"}',
++        user_prompt: 'Below is a single livestream VOD titled "{{title}}", described by its on-screen TIMELINE (visual scene notes) and its AUDIO TRANSCRIPT, each line timestamped [m:ss]. Titles and descriptions are about what happens.\n\nTIMELINE:\n{{timeline}}\n\nTRANSCRIPT:\n{{transcript}}\n\nSOUNDS HEARD (non-speech audio events):\n{{sounds}}\n\nViewers CLIPPED these timestamps (very strong "this was a highlight" signal): {{clipped}}\nChat activity SPIKED around: {{spikes}}\n\n{{flavor_hint}}\n{{#avoid}}ALREADY USED (a paste or clip exists there) — do NOT pick these or anything within 2 minutes of them: {{avoid}}. Find a DIFFERENT moment.{{/avoid}}\n\nFind the SINGLE most interesting/funny/dramatic/surprising/striking moment in this VOD with something clearly VISIBLE happening. Prefer moments backed by the clip/chat signals when they line up with something notable. NEVER pick a black/dark/loading/blank screen, an intro/BRB card, or a moment with no visible content or activity. {{abstain_hint}}Return STRICT JSON only, nothing else: {"at": "<the m:ss timestamp of the moment, exactly as written in the lines above>", "title": "<specific punchy 3-8 word title, not the stream name, funny, never generic>", "desc": "<one vivid sentence describing the moment>"}',
          default_route: 'live.legacy', owner: 'live', metadata: { json_name: 'moment_pick' },
      },
      {
ddiff --git a/test/live-templates.test.js b/test/live-templates.test.js
iindex 96c956c..b8d0c91 100644
---- a/test/live-templates.test.js
++++ b/test/live-templates.test.js
@@@ -2,7 +2,8 @@
  // Live prompts moved to versioned templates (roadmap WS-O task 2); each renders as Live rendered it.
  //   live.moments.pick: the prepare hook formats the timeline, transcript and sounds as [m:ss] lines, names clip and
  //     chat signals, adds the flavor hint, and writes the "already used" line only when there is something to avoid;
--//     nothing to work from is no run; the postprocess refuses an answer without a time.
++//     nothing to work from is no run; the model answers with the moment's [m:ss] as written ("at"), since asked for
++//     seconds beside [m:ss] lines it wrote the minutes (11-03); the postprocess refuses an answer without a time.
  const assert = require('assert');
  const { suite } = require('./helpers');
  const { PREPARE, POSTPROCESS } = require('../server/workflows/hooks');
@@@ -33,9 +34,11 @@ t.test('the prompt renders as Live rendered it', () => {
  t.test('only a clip is offered the way out (C09): a paste always names its frame', () => {
      const clip = render(tpl.user_prompt, PREPARE['live.moments.pick']({ title: 'x', flavor: 'clip', timeline: [{ t: 1, text: 'a' }] }).vars);
      const paste = render(tpl.user_prompt, PREPARE['live.moments.pick']({ title: 'x', flavor: 'paste', timeline: [{ t: 1, text: 'a' }] }).vars);
--    assert.match(clip, /If NOTHING here is clearly worth a clip[\s\S]*"t": null[\s\S]*Otherwise: Return STRICT JSON only/);
--    assert.ok(!/"t": null/.test(paste) && !/NOTHING here/.test(paste), 'no way out for a paste');
--    assert.match(paste, /Return STRICT JSON only, nothing else: \{"t": <seconds into the vod>/);
++    assert.match(clip, /If NOTHING here is clearly worth a clip[\s\S]*"at": null[\s\S]*Otherwise: Return STRICT JSON only/);
++    assert.ok(!/"at": null/.test(paste) && !/NOTHING here/.test(paste), 'no way out for a paste');
++    assert.match(paste, /Return STRICT JSON only, nothing else: \{"at": "<the m:ss timestamp of the moment, exactly as written in the lines above>"/);
++    assert.ok(!/seconds into the vod/i.test(tpl.user_prompt), 'never asked for seconds beside [m:ss] lines');
++    assert.deepStrictEqual(tpl.output_schema.required, ['at', 'title', 'desc'], 'the provider is held to the timestamp');
      assert.ok(!/\{\{|\$\{/.test(clip + paste), 'nothing left unrendered');
  });
  
@@@ -50,6 +53,24 @@ t.test('the answer is read like Live read it', () => {
      assert.strictEqual(post({ text: 'no json at all' }), null);
  });
  
++t.test('the moment comes back as the lines show it (11-03: "t": 2 for 2:14)', () => {
++    const post = POSTPROCESS['live.moments.pick'];
++    const input = { title: 'x', timeline: [{ t: 95, text: 'a' }, { t: 3600, text: 'b' }], transcript: [{ t: 3700, text: 'c' }] };
++    const at = (v) => { const r = post({ json: { at: v, title: 'T', desc: 'D' } }, input); return r && r.t; };
++    assert.strictEqual(at('1:35'), 95);
++    assert.strictEqual(at('[1:35]'), 95, 'copied with its brackets');
++    assert.strictEqual(at('60:05'), 3605, 'minutes past the hour, as the hook writes them');
++    assert.strictEqual(at('1:00:05'), 3605, 'or with hours');
++    assert.strictEqual(at('2:14.5'), 134);
++    assert.strictEqual(at('12'), null, 'a bare number could be minutes or seconds: no answer');
++    assert.strictEqual(at('1:75'), null);
++    assert.strictEqual(at('soon'), null);
++    assert.strictEqual(at('70:00'), null, 'past everything it was shown (plus two minutes): made up');
++    assert.strictEqual(at('63:40'), 3820, 'within two minutes of the last line');
++    assert.deepStrictEqual(post({ json: { at: null, title: '', desc: '', why: 'only  talk' } }, input), { t: null, title: '', desc: '', abstain: true, why: 'only talk' });
++    assert.deepStrictEqual(post({ text: '{"at": "1:35", "title": "\\"Up\\"", "desc": "He  jumps."}' }, input), { t: 95, title: 'Up', desc: 'He jumps.' });
++});
++
  t.test('slogans, the daily secret and the star render as Live rendered them', () => {
      const tp = (k) => live.templates.find((x) => x.key === k);
      const sl = render(tp('live.hero.slogans').user_prompt, PREPARE['live.hero.slogans']({ users: [{ name: 'goosely', text: 'loves   croutons' }], usernames: ['goosely'], count: 20 }).vars);