openvibers/openvibe.live Commits

Clips snapped to sentences get a little air at each edge (item 151 C12) (#597)

OP OpenVibers committed 10 days ago · a664db7f2f · 3 files, +29 −9

A row's times are where speech recognition heard the words begin and end, so an edge exactly on
them clips the first breath or the tail of the last syllable. A snapped start moves 0.3 s earlier
and a snapped end 0.5 s later, never into the next sentence or past the recording's edge.

Measured 11-03 on ten AI VOD clips cut with the old fixed window: two ended and one started
mid-word. Snapping to sentences (#592) already puts edges between words, so Live does not ask
the AI for word times when it trims; this closes C12.

Changes

ddiff --git a/server/ai/clip-trim.js b/server/ai/clip-trim.js
iindex c527da01..202bf8b7 100644
---- a/server/ai/clip-trim.js
++++ b/server/ai/clip-trim.js
@@@ -16,10 +16,14 @@
* flushed edge of a live recording).
* - Without speech near the moment, or with rows that would make the clip shorter than `min` or longer than `max`,
* the fixed window stands (clamped), so trimming never makes a clip worse than before.
++ * - A snapped edge gets a little air: `padStart` before the first word and `padEnd` after the last, never into the
++ * sentence next to it. A row's times are where speech recognition heard the words begin and end, so an edge exactly
++ * on them clips the first breath or the last syllable's tail (item 151 C12; measured 11-03 on ten AI clips cut with
++ * the fixed window: two ended and one started mid-word, which snapping to sentences already avoids).
* All times are in one clock (the caller converts between stream and recording time). Pure.
*/
--const DEFAULTS = Object.freeze({ pre: 16, post: 9, min: 12, max: 45, maxLead: 25, maxTail: 20 });
++const DEFAULTS = Object.freeze({ pre: 16, post: 9, min: 12, max: 45, maxLead: 25, maxTail: 20, padStart: 0.3, padEnd: 0.5 });
function rowEnd(rows, i) {
const r = rows[i];
@@@ -42,18 +46,29 @@ function trim({ moment, segments = [], limit = Infinity, ...o } = {}) {
if (!rows.length) return fixed;
const at = (t) => { let idx = -1; for (let i = 0; i < rows.length; i++) { if (rows[i].start <= t && rowEnd(rows, i) > t) idx = i; } return idx; };
++ // A sentence's start, a little earlier, but not into the sentence before it.
++ const airBefore = (i) => {
++ const s = Math.max(0, rows[i].start);
++ const prevEnd = i > 0 ? rowEnd(rows, i - 1) : 0;
++ return Math.min(s, Math.max(0, prevEnd, s - k.padStart));
++ };
let start = fixedStart, end = fixedEnd, snapped = false;
const si = at(fixedStart);
-- if (si >= 0 && rows[si].start >= m - k.maxLead) { start = Math.max(0, rows[si].start); snapped = true; }
++ if (si >= 0 && rows[si].start >= m - k.maxLead) { start = airBefore(si); snapped = true; }
else if (si < 0) {
// The fixed start is in silence: begin at the first sentence that starts between it and the moment.
-- const next = rows.find((r) => r.start > fixedStart && r.start < m);
-- if (next) { start = next.start; snapped = true; }
++ const ni = rows.findIndex((r) => r.start > fixedStart && r.start < m);
++ if (ni >= 0) { start = airBefore(ni); snapped = true; }
}
const ei = at(Math.min(m + k.post, limit - 0.01));
if (ei >= 0) {
const e = Math.min(rowEnd(rows, ei), limit);
-- if (e <= m + k.maxTail && e > start) { end = e; snapped = true; }
++ if (e <= m + k.maxTail && e > start) {
++ // The sentence's end, a little later, but not into the next sentence or past the recording's edge.
++ const nextStart = ei + 1 < rows.length ? rows[ei + 1].start : Infinity;
++ end = Math.max(e, Math.min(e + k.padEnd, nextStart, limit));
++ snapped = true;
++ }
}
const len = end - start;
if (len < k.min || len > k.max) return fixed;
ddiff --git a/test/clip-score-trim.test.js b/test/clip-score-trim.test.js
iindex 024493ca..bb52ba80 100644
---- a/test/clip-score-trim.test.js
++++ b/test/clip-score-trim.test.js
@@@ -54,12 +54,17 @@ const { trim } = require('../server/ai/clip-trim');
{ start: 112, end: 117.4, text: 'And then it held, and we all screamed!' },
{ start: 118, end: 124, text: 'Look at chat right now.' },
];
-- // The moment is the scream at 115: the fixed window is 99..124; snapped to rows it is 100 (the setup) .. 124.
++ // The moment is the scream at 115: the fixed window is 99..124; snapped to rows it is 100 (the setup) .. 124, with
++ // 0.3 s of air before the first word (the end, 124, is the fixed window's: no sentence is spoken at that second).
const t = trim({ moment: 115, segments: rows });
-- assert.deepStrictEqual([t.start, t.end, t.snapped], [100, 124, true], JSON.stringify(t));
++ assert.deepStrictEqual([t.start, t.end, t.snapped], [99.7, 124, true], JSON.stringify(t));
// A row without an end runs to the next row's start.
const noEnd = trim({ moment: 115, segments: rows.map((r) => ({ ...r, end: null })) });
-- assert.strictEqual(noEnd.start, 100);
++ assert.strictEqual(noEnd.start, 99.7);
++ // The air never reaches into the sentence next to an edge.
++ const tight = trim({ moment: 115, segments: [{ start: 98.2, end: 99.9, text: 'Before.' }, ...rows.slice(0, 3), { start: 117.6, end: 124, text: 'Right after.' }], pre: 15, post: 2 });
++ assert.deepStrictEqual([tight.start, tight.end], [99.9, 117.6], `the air stops at the neighbours (${JSON.stringify(tight)})`);
++ assert.deepStrictEqual(trim({ moment: 115, segments: rows, padStart: 0, padEnd: 0 }), { start: 100, end: 124, snapped: true }, 'without air: the rows exactly');
// The live edge caps the end.
const edge = trim({ moment: 115, segments: rows, limit: 119 });
assert.ok(edge.end <= 119, `never past the flushed edge (${edge.end})`);
ddiff --git a/test/vod-moment-evidence.test.js b/test/vod-moment-evidence.test.js
iindex 81c9cf1e..d9031f6d 100644
---- a/test/vod-moment-evidence.test.js
++++ b/test/vod-moment-evidence.test.js
@@@ -86,7 +86,7 @@ console.warn = () => {};
const c3 = await autoClip.clipVodMoment({ vod: { ...vod(504), duration: 3600 }, offset: 300, title: 'Desk goes down', desc: 'The streamer walks into the cable and the desk tips over', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });
assert.ok(c3 && c3.id);
const b3 = created[created.length - 1];
-- assert.deepStrictEqual([b3.start_s, b3.end_s], [279.2, 311], 'from the start of the sentence before to the end of the one after');
++ assert.deepStrictEqual([b3.start_s, b3.end_s], [278.9, 311.5], 'from the start of the sentence before to the end of the one after, with a little air');
assert.ok(log.some((l) => /VOD 504 @300s cut .*, on sentences/.test(l)));
const c4 = await autoClip.clipVodMoment({ vod: { ...vod(505), duration: 304 }, offset: 300, title: 'At the end', desc: 'The streamer says goodbye as the stream ends abruptly', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });
assert.ok(c4 && c4.id);