openvibers/openvibe.live Commits

VOD clips start and end on sentences, as live clips do (#592)

OP OpenVibers committed 10 days ago · 90a16f59a7 · 3 files, +43 −8

* VOD clips start and end on sentences, as live clips do

clipVodMoment cut a fixed 16 s before and 9 s after the moment, so the AI's VOD clips
began and ended mid-sentence; only live clips went through clip-trim.js. A moment now
carries the transcript rows within a minute of it (in the VOD's own time, from the
context the picker already read), and the cutter trims with clip-trim.js, never past
the recording's end. Without speech near it the fixed window stands, as before.

* Merge branch 'main' into feat/vod-clip-sentences

Changes

ddiff --git a/server/ai/ai-moments-job.js b/server/ai/ai-moments-job.js
iindex 2dfbac2d..a4611232 100644
---- a/server/ai/ai-moments-job.js
++++ b/server/ai/ai-moments-job.js
@@@ -228,6 +228,15 @@ function _evidenceAt(ctx, t, pickedBy) {
};
}
/** Transcript text around a moment, so dedupe can tell a new conversation from a repeated scene. */
++/** The transcript rows within `span` seconds of a moment (VOD time), for the cutter to start and end on sentences. */
++function _rowsNear(transcript, offset, span = 60) {
++ const t = Number(offset) || 0;
++ return (transcript || [])
++ .filter((x) => x && Number.isFinite(Number(x.start)) && Math.abs(Number(x.start) - t) <= span && String(x.text || '').trim())
++ .slice(0, 80)
++ .map((x) => ({ start: Number(x.start), end: x.end == null ? null : Number(x.end), text: String(x.text) }));
++}
++
function _speechNear(transcript, offset, windowSec = 45) {
try {
return (transcript || [])
@@@ -277,7 +286,7 @@ async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {
t = clamp(t);
if (!farFromUsed(t)) { console.log(`[AI-Moments] VOD ${vod.vod_id}: pick at ${_mmss(t)} is next to a used moment — trying the objective signals instead`); throw new Error('near used'); }
const near = _nearestMemory(ctx.memories, t);
-- const result = { offset: t, title: _publicTitle(j.title, vod, t, j.desc), desc: _cleanText(_deJson(j.desc), 400) || (near && _deJson(near.description)) || '', speech: _speechNear(ctx.said || ctx.transcript, t), tags: _memTags(near), evidence: _evidenceAt(ctx, t, 'ai') };
++ const result = { offset: t, title: _publicTitle(j.title, vod, t, j.desc), desc: _cleanText(_deJson(j.desc), 400) || (near && _deJson(near.description)) || '', speech: _speechNear(ctx.said || ctx.transcript, t), rows: _rowsNear(ctx.said || ctx.transcript, t), tags: _memTags(near), evidence: _evidenceAt(ctx, t, 'ai') };
if (explicit.isExplicit(j.desc, j.title, near && near.description, result.desc)) { console.log(`[AI-Moments] VOD ${vod.vod_id}: the pick at ${_mmss(t)} shows sexual content — no moment from it`); return null; }
return _isEmptyScene(result.desc) ? null : result;
}
@@@ -300,7 +309,7 @@ async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {
const near = _nearestMemory(ctx.memories, offset);
const desc = near ? _deJson(near.description) : '';
if (near && explicit.isExplicit(near.description)) return null; // never a moment of sexual content (server/ai/explicit.js)
-- return _isEmptyScene(desc) ? null : { offset, title: _fallbackTitle(vod, offset, desc), desc, speech: _speechNear(ctx.said || ctx.transcript, offset), tags: _memTags(near), evidence: _evidenceAt(ctx, offset, candidates[0].by) };
++ return _isEmptyScene(desc) ? null : { offset, title: _fallbackTitle(vod, offset, desc), desc, speech: _speechNear(ctx.said || ctx.transcript, offset), rows: _rowsNear(ctx.said || ctx.transcript, offset), tags: _memTags(near), evidence: _evidenceAt(ctx, offset, candidates[0].by) };
}
// True when a description says the frame is essentially empty — black/dark/loading/blank — so we
// never turn "nothing" into a clip or paste.
@@@ -605,7 +614,7 @@ async function tick(opts = {}) {
try {
const second = await _findBestMoment(v, { flavor: 'clip', avoid: [offset] });
if (second && !await registry.isUsed({ vod_id: v.vod_id, stream_id: v.stream_id, offset: second.offset, desc: second.desc, title: second.title, speech: second.speech })) {
-- const clip = await require('./auto-clip-job').clipVodMoment({ vod: v, offset: second.offset, title: second.title, desc: second.desc, speech: second.speech, evidence: second.evidence, archival: v.archival, source: momentSource });
++ const clip = await require('./auto-clip-job').clipVodMoment({ vod: v, offset: second.offset, title: second.title, desc: second.desc, speech: second.speech, segments: second.rows, evidence: second.evidence, archival: v.archival, source: momentSource });
if (clip) console.log(`[AI-Moments] Auto-clip cut for VOD ${v.vod_id} at ${_mmss(second.offset)} ("${String(second.title || '').slice(0, 60)}") — paste was at ${_mmss(offset)}`);
} else console.log(`[AI-Moments] VOD ${v.vod_id}: no second, distinct beat for a clip — skipped`);
} catch { /* clipping is best-effort */ }
@@@ -636,7 +645,7 @@ function start() {
setInterval(() => { tick().catch(() => {}); }, 5 * 60 * 1000);
}
--module.exports = { start, tick, findBestMoment: _findBestMoment, frameTooDark: _frameTooDark, makeSlug: _slug, publicTitle: _publicTitle, _internals: { momentPool: _momentPool, fallbackTitle: _fallbackTitle, captionTitle: _captionTitle, evidenceAt: _evidenceAt, keepHeroFull } };
++module.exports = { start, tick, findBestMoment: _findBestMoment, frameTooDark: _frameTooDark, makeSlug: _slug, publicTitle: _publicTitle, _internals: { rowsNear: _rowsNear, momentPool: _momentPool, fallbackTitle: _fallbackTitle, captionTitle: _captionTitle, evidenceAt: _evidenceAt, keepHeroFull } };
// CLI: force a one-off regeneration, e.g. a whole-dataset "best of all-time" test run:
// node server/ai/ai-moments-job.js --fresh --target=6 --perUser=2
ddiff --git a/server/ai/auto-clip-job.js b/server/ai/auto-clip-job.js
iindex c31e9013..59b59f75 100644
---- a/server/ai/auto-clip-job.js
++++ b/server/ai/auto-clip-job.js
@@@ -318,7 +318,7 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}
if (dark) { console.log(`[AutoClip] backfill skip VOD ${v.vod_id} — dark/empty frame`); newSkip.push(v.vod_id); continue; }
}
} catch { /* probe optional */ }
-- const clip = await clipVodMoment({ vod: v, offset: moment.offset, title: moment.title, desc: moment.desc, speech: moment.speech, evidence: moment.evidence, archival: v.archival, source });
++ const clip = await clipVodMoment({ vod: v, offset: moment.offset, title: moment.title, desc: moment.desc, speech: moment.speech, segments: moment.rows, evidence: moment.evidence, archival: v.archival, source });
if (clip) { made++; if (v.archival) archiveMade++; console.log(`[AutoClip] Backfilled VOD ${v.vod_id}${v.archival ? ' (archive)' : ''} ("${String(moment.title || '').slice(0, 60)}")`); }
else if (v.archival && !clipPolicy.archiveAllowed(await _clipLog(), Date.now())) continue; // another cutter took today's archive slot: not this VOD's fault, don't skip it forever
else newSkip.push(v.vod_id); // cut failed (e.g. unseekable/short) — don't retry forever
@@@ -394,7 +394,13 @@ async function clipVodMoment(o) {
const publication = require('./clip-publication');
if (!publication.mayCut(vodId)) { console.log(`[AutoClip] VOD ${vodId}: protected, not cut`); return null; }
const visibility = publication.visibilityFor({ vodVisibility: await publication.vodVisibility(vod), wanted: rated.verdict === 'public' ? 'public' : 'unlisted' });
-- const start = Math.max(0, Math.floor(offset) - CLIP_PRE);
++ // Start and end on sentences (server/ai/clip-trim.js), as a live clip does: the rows near the moment are in the
++ // VOD's own time. Without speech near it the fixed window stands (11-03: VOD clips always used it, so they
++ // began and ended mid-sentence).
++ const duration = Number(vod.duration_seconds ?? vod.duration) || 0;
++ const cut = clipTrim.trim({ moment: offset, segments: Array.isArray(o.segments) ? o.segments : [], pre: CLIP_PRE, post: CLIP_POST, limit: duration > 1 ? duration - 0.5 : Infinity });
++ const start = Math.max(0, Math.floor(cut.start * 10) / 10);
++ const end = Math.max(start + 1, Math.round(cut.end * 10) / 10);
// One archive clip per UTC day is shared with the AI moments job. Both may pass the pool
// gate before either writes auto_clip_log, so claim the slot atomically before cutting and
// give it back if no clip results.
@@@ -406,7 +412,7 @@ async function clipVodMoment(o) {
archiveHeld = true;
}
const clip = await media.createClip({
-- vod_id: vodId, start_s: start, end_s: start + CLIP_PRE + CLIP_POST,
++ vod_id: vodId, start_s: start, end_s: end,
title: title || 'Standout Moment', user_id: vod.user_id,
stream_id: vod.stream_id || undefined, auto_generated: true, description: desc || '',
visibility,
@@@ -417,7 +423,7 @@ async function clipVodMoment(o) {
await _logClip({ stream_id: vod.stream_id || null, vod_id: vodId, start_time: start, title: title || 'Standout Moment', clip_id: clip.id, dedup: !!clip.deduplicated || undefined, desc: desc || '', speech, archival: archival || undefined, sig: _sceneSig(desc || title),
score: rated.score, visibility: rated.verdict, bucket: rated.bucket, peak: Number(sp.peak) || 0, chatters: evidence.chatters == null ? undefined : evidence.chatters,
picked_by: evidence.picked_by || undefined, policy: VOD_POLICY });
-- console.log(`[AutoClip] VOD ${vodId} @${Math.round(offset)}s cut (${evidence.picked_by || 'signals'}) — score ${rated.score} → ${rated.verdict}, bucket ${rated.bucket}`);
++ console.log(`[AutoClip] VOD ${vodId} @${Math.round(offset)}s cut (${evidence.picked_by || 'signals'}) — score ${rated.score} → ${rated.verdict}, bucket ${rated.bucket}${cut.snapped ? ', on sentences' : ''}`);
}
return clip || null;
} catch { return null; }
ddiff --git a/test/vod-moment-evidence.test.js b/test/vod-moment-evidence.test.js
iindex f92719ee..23ea9178 100644
---- a/test/vod-moment-evidence.test.js
++++ b/test/vod-moment-evidence.test.js
@@@ -61,6 +61,26 @@ console.warn = () => {};
const c2 = await autoClip.clipVodMoment({ vod: vod(503), offset: 300, title: 'A laugh', desc: 'A dog barks and the streamer laughs at the window', evidence: { picked_by: 'ai', spike: { peak: 0, avg: 0 }, chatters: null, sounds: [{ label: 'Laughter', confidence: 0.8 }], speechWords: 30 } });
assert.ok(c2 && c2.id);
assert.deepStrictEqual(created.map((b) => b.visibility), ['public', 'unlisted'], 'public only for the reacted room');
++ assert.deepStrictEqual([created[0].start_s, created[0].end_s], [284, 309], 'no speech near it: the fixed 16 s before and 9 s after');
++
++ // ── A VOD clip starts and ends on sentences (clip-trim.js), as a live clip does (11-03: it always used the window) ──
++ const rows = [
++ { start: 120, end: 126, text: 'Long before the moment.' },
++ { start: 279.2, end: 286.0, text: 'So I was telling chat about the cable.' },
++ { start: 286.4, end: 296.1, text: 'And then I just walked straight into it, look at this.' },
++ { start: 296.5, end: 305.8, text: 'Oh no, oh no, the whole desk is going.' },
++ { start: 306.2, end: 311.0, text: 'Chat, did you see that?' },
++ ];
++ const near = moments._internals.rowsNear(rows, 300);
++ assert.deepStrictEqual(near.map((r) => r.start), [279.2, 286.4, 296.5, 306.2], 'the rows within a minute of the moment');
++ const c3 = await autoClip.clipVodMoment({ vod: { ...vod(504), duration: 3600 }, offset: 300, title: 'Desk goes down', desc: 'The streamer walks into the cable and the desk tips over', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });
++ assert.ok(c3 && c3.id);
++ const b3 = created[created.length - 1];
++ assert.deepStrictEqual([b3.start_s, b3.end_s], [279.2, 311], 'from the start of the sentence before to the end of the one after');
++ assert.ok(log.some((l) => /VOD 504 @300s cut .*, on sentences/.test(l)));
++ const c4 = await autoClip.clipVodMoment({ vod: { ...vod(505), duration: 304 }, offset: 300, title: 'At the end', desc: 'The streamer says goodbye as the stream ends abruptly', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });
++ assert.ok(c4 && c4.id);
++ assert.ok(created[created.length - 1].end_s <= 303.5, 'never past the end of the recording');
const entries = JSON.parse(await db.getState('auto_clip_log'));
const e1 = entries.find((e) => e.clip_id === c1.id);
assert.deepStrictEqual(e1.policy, { selector: 'vod-moment', version: '2026-11-03' });