openvibers/openvibe.live Commits
Clips snapped to sentences get a little air at each edge (item 151 C12) (#597)
Changes
@@@ -16,10 +16,14 @@* flushed edge of a live recording).* - Without speech near the moment, or with rows that would make the clip shorter than `min` or longer than `max`,* the fixed window stands (clamped), so trimming never makes a clip worse than before.++ * - A snapped edge gets a little air: `padStart` before the first word and `padEnd` after the last, never into the++ * sentence next to it. A row's times are where speech recognition heard the words begin and end, so an edge exactly++ * on them clips the first breath or the last syllable's tail (item 151 C12; measured 11-03 on ten AI clips cut with++ * the fixed window: two ended and one started mid-word, which snapping to sentences already avoids).* All times are in one clock (the caller converts between stream and recording time). Pure.*/--const DEFAULTS = Object.freeze({ pre: 16, post: 9, min: 12, max: 45, maxLead: 25, maxTail: 20 });++const DEFAULTS = Object.freeze({ pre: 16, post: 9, min: 12, max: 45, maxLead: 25, maxTail: 20, padStart: 0.3, padEnd: 0.5 });function rowEnd(rows, i) {const r = rows[i];@@@ -42,18 +46,29 @@ function trim({ moment, segments = [], limit = Infinity, ...o } = {}) {if (!rows.length) return fixed;const at = (t) => { let idx = -1; for (let i = 0; i < rows.length; i++) { if (rows[i].start <= t && rowEnd(rows, i) > t) idx = i; } return idx; };++ // A sentence's start, a little earlier, but not into the sentence before it.++ const airBefore = (i) => {++ const s = Math.max(0, rows[i].start);++ const prevEnd = i > 0 ? rowEnd(rows, i - 1) : 0;++ return Math.min(s, Math.max(0, prevEnd, s - k.padStart));++ };let start = fixedStart, end = fixedEnd, snapped = false;const si = at(fixedStart);-- if (si >= 0 && rows[si].start >= m - k.maxLead) { start = Math.max(0, rows[si].start); snapped = true; }++ if (si >= 0 && rows[si].start >= m - k.maxLead) { start = airBefore(si); snapped = true; }else if (si < 0) {// The fixed start is in silence: begin at the first sentence that starts between it and the moment.-- const next = rows.find((r) => r.start > fixedStart && r.start < m);-- if (next) { start = next.start; snapped = true; }++ const ni = rows.findIndex((r) => r.start > fixedStart && r.start < m);++ if (ni >= 0) { start = airBefore(ni); snapped = true; }}const ei = at(Math.min(m + k.post, limit - 0.01));if (ei >= 0) {const e = Math.min(rowEnd(rows, ei), limit);-- if (e <= m + k.maxTail && e > start) { end = e; snapped = true; }++ if (e <= m + k.maxTail && e > start) {++ // The sentence's end, a little later, but not into the next sentence or past the recording's edge.++ const nextStart = ei + 1 < rows.length ? rows[ei + 1].start : Infinity;++ end = Math.max(e, Math.min(e + k.padEnd, nextStart, limit));++ snapped = true;++ }}const len = end - start;if (len < k.min || len > k.max) return fixed;@@@ -54,12 +54,17 @@ const { trim } = require('../server/ai/clip-trim');{ start: 112, end: 117.4, text: 'And then it held, and we all screamed!' },{ start: 118, end: 124, text: 'Look at chat right now.' },];-- // The moment is the scream at 115: the fixed window is 99..124; snapped to rows it is 100 (the setup) .. 124.++ // The moment is the scream at 115: the fixed window is 99..124; snapped to rows it is 100 (the setup) .. 124, with++ // 0.3 s of air before the first word (the end, 124, is the fixed window's: no sentence is spoken at that second).const t = trim({ moment: 115, segments: rows });-- assert.deepStrictEqual([t.start, t.end, t.snapped], [100, 124, true], JSON.stringify(t));++ assert.deepStrictEqual([t.start, t.end, t.snapped], [99.7, 124, true], JSON.stringify(t));// A row without an end runs to the next row's start.const noEnd = trim({ moment: 115, segments: rows.map((r) => ({ ...r, end: null })) });-- assert.strictEqual(noEnd.start, 100);++ assert.strictEqual(noEnd.start, 99.7);++ // The air never reaches into the sentence next to an edge.++ const tight = trim({ moment: 115, segments: [{ start: 98.2, end: 99.9, text: 'Before.' }, ...rows.slice(0, 3), { start: 117.6, end: 124, text: 'Right after.' }], pre: 15, post: 2 });++ assert.deepStrictEqual([tight.start, tight.end], [99.9, 117.6], `the air stops at the neighbours (${JSON.stringify(tight)})`);++ assert.deepStrictEqual(trim({ moment: 115, segments: rows, padStart: 0, padEnd: 0 }), { start: 100, end: 124, snapped: true }, 'without air: the rows exactly');// The live edge caps the end.const edge = trim({ moment: 115, segments: rows, limit: 119 });assert.ok(edge.end <= 119, `never past the flushed edge (${edge.end})`);@@@ -86,7 +86,7 @@ console.warn = () => {};const c3 = await autoClip.clipVodMoment({ vod: { ...vod(504), duration: 3600 }, offset: 300, title: 'Desk goes down', desc: 'The streamer walks into the cable and the desk tips over', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });assert.ok(c3 && c3.id);const b3 = created[created.length - 1];-- assert.deepStrictEqual([b3.start_s, b3.end_s], [279.2, 311], 'from the start of the sentence before to the end of the one after');++ assert.deepStrictEqual([b3.start_s, b3.end_s], [278.9, 311.5], 'from the start of the sentence before to the end of the one after, with a little air');assert.ok(log.some((l) => /VOD 504 @300s cut .*, on sentences/.test(l)));const c4 = await autoClip.clipVodMoment({ vod: { ...vod(505), duration: 304 }, offset: 300, title: 'At the end', desc: 'The streamer says goodbye as the stream ends abruptly', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });assert.ok(c4 && c4.id);
@@ -16,10 +16,14 @@
@@ -16,10 +16,14 @@
* flushed edge of a live recording).
* flushed edge of a live recording).
* - Without speech near the moment, or with rows that would make the clip shorter than `min` or longer than `max`,
* - Without speech near the moment, or with rows that would make the clip shorter than `min` or longer than `max`,
* the fixed window stands (clamped), so trimming never makes a clip worse than before.
* the fixed window stands (clamped), so trimming never makes a clip worse than before.
+ * - A snapped edge gets a little air: `padStart` before the first word and `padEnd` after the last, never into the
+ * sentence next to it. A row's times are where speech recognition heard the words begin and end, so an edge exactly
+ * on them clips the first breath or the last syllable's tail (item 151 C12; measured 11-03 on ten AI clips cut with
+ * the fixed window: two ended and one started mid-word, which snapping to sentences already avoids).
* All times are in one clock (the caller converts between stream and recording time). Pure.
* All times are in one clock (the caller converts between stream and recording time). Pure.
*/
*/
-const DEFAULTS = Object.freeze({ pre: 16, post: 9, min: 12, max: 45, maxLead: 25, maxTail: 20 });
+const DEFAULTS = Object.freeze({ pre: 16, post: 9, min: 12, max: 45, maxLead: 25, maxTail: 20, padStart: 0.3, padEnd: 0.5 });
function rowEnd(rows, i) {
function rowEnd(rows, i) {
const r = rows[i];
const r = rows[i];
@@ -42,18 +46,29 @@ function trim({ moment, segments = [], limit = Infinity, ...o } = {}) {
@@ -42,18 +46,29 @@ function trim({ moment, segments = [], limit = Infinity, ...o } = {}) {
if (!rows.length) return fixed;
if (!rows.length) return fixed;
const at = (t) => { let idx = -1; for (let i = 0; i < rows.length; i++) { if (rows[i].start <= t && rowEnd(rows, i) > t) idx = i; } return idx; };
const at = (t) => { let idx = -1; for (let i = 0; i < rows.length; i++) { if (rows[i].start <= t && rowEnd(rows, i) > t) idx = i; } return idx; };
+ // A sentence's start, a little earlier, but not into the sentence before it.
+ const airBefore = (i) => {
+ const s = Math.max(0, rows[i].start);
+ const prevEnd = i > 0 ? rowEnd(rows, i - 1) : 0;
+ return Math.min(s, Math.max(0, prevEnd, s - k.padStart));
+ };
let start = fixedStart, end = fixedEnd, snapped = false;
let start = fixedStart, end = fixedEnd, snapped = false;
const si = at(fixedStart);
const si = at(fixedStart);
- if (si >= 0 && rows[si].start >= m - k.maxLead) { start = Math.max(0, rows[si].start); snapped = true; }
+ if (si >= 0 && rows[si].start >= m - k.maxLead) { start = airBefore(si); snapped = true; }
else if (si < 0) {
else if (si < 0) {
// The fixed start is in silence: begin at the first sentence that starts between it and the moment.
// The fixed start is in silence: begin at the first sentence that starts between it and the moment.
- const next = rows.find((r) => r.start > fixedStart && r.start < m);
- if (next) { start = next.start; snapped = true; }
+ const ni = rows.findIndex((r) => r.start > fixedStart && r.start < m);
+ if (ni >= 0) { start = airBefore(ni); snapped = true; }
}
}
const ei = at(Math.min(m + k.post, limit - 0.01));
const ei = at(Math.min(m + k.post, limit - 0.01));
if (ei >= 0) {
if (ei >= 0) {
const e = Math.min(rowEnd(rows, ei), limit);
const e = Math.min(rowEnd(rows, ei), limit);
- if (e <= m + k.maxTail && e > start) { end = e; snapped = true; }
+ if (e <= m + k.maxTail && e > start) {
+ // The sentence's end, a little later, but not into the next sentence or past the recording's edge.
+ const nextStart = ei + 1 < rows.length ? rows[ei + 1].start : Infinity;
+ end = Math.max(e, Math.min(e + k.padEnd, nextStart, limit));
+ snapped = true;
+ }
}
}
const len = end - start;
const len = end - start;
if (len < k.min || len > k.max) return fixed;
if (len < k.min || len > k.max) return fixed;
@@ -54,12 +54,17 @@ const { trim } = require('../server/ai/clip-trim');
@@ -54,12 +54,17 @@ const { trim } = require('../server/ai/clip-trim');
{ start: 112, end: 117.4, text: 'And then it held, and we all screamed!' },
{ start: 112, end: 117.4, text: 'And then it held, and we all screamed!' },
{ start: 118, end: 124, text: 'Look at chat right now.' },
{ start: 118, end: 124, text: 'Look at chat right now.' },
];
];
- // The moment is the scream at 115: the fixed window is 99..124; snapped to rows it is 100 (the setup) .. 124.
+ // The moment is the scream at 115: the fixed window is 99..124; snapped to rows it is 100 (the setup) .. 124, with
+ // 0.3 s of air before the first word (the end, 124, is the fixed window's: no sentence is spoken at that second).
const t = trim({ moment: 115, segments: rows });
const t = trim({ moment: 115, segments: rows });
- assert.deepStrictEqual([t.start, t.end, t.snapped], [100, 124, true], JSON.stringify(t));
+ assert.deepStrictEqual([t.start, t.end, t.snapped], [99.7, 124, true], JSON.stringify(t));
// A row without an end runs to the next row's start.
// A row without an end runs to the next row's start.
const noEnd = trim({ moment: 115, segments: rows.map((r) => ({ ...r, end: null })) });
const noEnd = trim({ moment: 115, segments: rows.map((r) => ({ ...r, end: null })) });
- assert.strictEqual(noEnd.start, 100);
+ assert.strictEqual(noEnd.start, 99.7);
+ // The air never reaches into the sentence next to an edge.
+ const tight = trim({ moment: 115, segments: [{ start: 98.2, end: 99.9, text: 'Before.' }, ...rows.slice(0, 3), { start: 117.6, end: 124, text: 'Right after.' }], pre: 15, post: 2 });
+ assert.deepStrictEqual([tight.start, tight.end], [99.9, 117.6], `the air stops at the neighbours (${JSON.stringify(tight)})`);
+ assert.deepStrictEqual(trim({ moment: 115, segments: rows, padStart: 0, padEnd: 0 }), { start: 100, end: 124, snapped: true }, 'without air: the rows exactly');
// The live edge caps the end.
// The live edge caps the end.
const edge = trim({ moment: 115, segments: rows, limit: 119 });
const edge = trim({ moment: 115, segments: rows, limit: 119 });
assert.ok(edge.end <= 119, `never past the flushed edge (${edge.end})`);
assert.ok(edge.end <= 119, `never past the flushed edge (${edge.end})`);
@@ -86,7 +86,7 @@ console.warn = () => {};
@@ -86,7 +86,7 @@ console.warn = () => {};
const c3 = await autoClip.clipVodMoment({ vod: { ...vod(504), duration: 3600 }, offset: 300, title: 'Desk goes down', desc: 'The streamer walks into the cable and the desk tips over', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });
const c3 = await autoClip.clipVodMoment({ vod: { ...vod(504), duration: 3600 }, offset: 300, title: 'Desk goes down', desc: 'The streamer walks into the cable and the desk tips over', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });
assert.ok(c3 && c3.id);
assert.ok(c3 && c3.id);
const b3 = created[created.length - 1];
const b3 = created[created.length - 1];
- assert.deepStrictEqual([b3.start_s, b3.end_s], [279.2, 311], 'from the start of the sentence before to the end of the one after');
+ assert.deepStrictEqual([b3.start_s, b3.end_s], [278.9, 311.5], 'from the start of the sentence before to the end of the one after, with a little air');
assert.ok(log.some((l) => /VOD 504 @300s cut .*, on sentences/.test(l)));
assert.ok(log.some((l) => /VOD 504 @300s cut .*, on sentences/.test(l)));
const c4 = await autoClip.clipVodMoment({ vod: { ...vod(505), duration: 304 }, offset: 300, title: 'At the end', desc: 'The streamer says goodbye as the stream ends abruptly', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });
const c4 = await autoClip.clipVodMoment({ vod: { ...vod(505), duration: 304 }, offset: 300, title: 'At the end', desc: 'The streamer says goodbye as the stream ends abruptly', segments: near, evidence: { picked_by: 'ai', spike: { peak: 22, avg: 1.5 }, chatters: 7, sounds: [], speechWords: 25 } });
assert.ok(c4 && c4.id);
assert.ok(c4 && c4.id);