openvibers/openvibe.mediahub Commits

VOD captions name what was heard (item 151, C15) (#272)

OP OpenVibers committed 10 days ago · 85c1d80cb6 · 3 files, +37 −4

The captions track carried the spoken lines only. Live now gives a VOD's sounds as cues (GET /api/vods/:id/sounds:
[laughter], [applause], [music], each from a classifier label above its own bar, nothing written by a model), and
/v/:id/captions.vtt puts them between the lines in time order. Only bracketed tags with sane times are taken; Live
down or an older Live gives speech only.

Changes

ddiff --git a/server/video/live-client.js b/server/video/live-client.js
iindex cfe1c41..659ab6c 100644
---- a/server/video/live-client.js
++++ b/server/video/live-client.js
@@@ -241,6 +241,10 @@ function createLiveClient({ config, fetchImpl = globalThis.fetch, log = console
vodMemories(id) {
return getJson(`/api/vods/${id}/memories`);
},
++ /** What was heard in a VOD, as caption cues in its own time ("[laughter]"; Live's sound-captions.js), or null. */
++ vodSounds(id) {
++ return getJson(`/api/vods/${id}/sounds`);
++ },
/** A stream's recording parts (Live's history: a broadcaster who reconnected is recorded in parts), or null. */
streamHistory(streamId) {
return isId(streamId) ? getJson(`/api/streams/${streamId}/history`) : Promise.resolve(null);
ddiff --git a/server/video/pages.js b/server/video/pages.js
iindex 7da58d3..4fb2507 100644
---- a/server/video/pages.js
++++ b/server/video/pages.js
@@@ -11,7 +11,7 @@
* /v/:id a VOD watch page (player, AI chapters and captions, its clips)
* /c/:id a clip watch page
* /@:username a channel's recordings (and clips), newest first
-- * /v/:id/captions.vtt the AI transcript as WebVTT captions
++ * /v/:id/captions.vtt the AI transcript as WebVTT captions, with what was heard ("[laughter]")
* /v/:id/chapters.vtt the AI moments as WebVTT chapters
* /v/:id/download/:file the recording, its transcript, chat replay, chapters, details or all of it as files
* /c/:id/download/:file the clip, its transcript and details (server/video/downloads.js)
@@@ -479,6 +479,25 @@ function chapters(mems, durationSeconds) {
return out;
}
++/**
++ * Live's sound cues (GET /api/vods/:id/sounds, item 151 C15): only bracketed tags like "[laughter]" with finite times
++ * are taken, so nothing but a sound's name can reach the captions from there.
++ */
++function soundCues(list) {
++ if (!Array.isArray(list)) return [];
++ return list.map((c) => {
++ if (!c || typeof c !== 'object') return null;
++ const start = Number(c.start), end = Number(c.end), text = String(c.text == null ? '' : c.text);
++ if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start || start < 0) return null;
++ return /^\[[a-z][a-z ]{1,29}\]$/.test(text) ? { start, end, text } : null;
++ }).filter(Boolean).slice(0, 5000);
++}
++
++/** The spoken lines and the sounds, in time order: a sound's cue shows beside the line it falls in. */
++function withSounds(segs, sounds) {
++ return [...segs, ...sounds].sort((a, b) => a.start - b.start || a.end - b.end);
++}
++
function renderCaptionsVtt(segs) {
const lines = ['WEBVTT', ''];
for (const s of segs) {
@@@ -1146,7 +1165,10 @@ ${found ? sections.map((x) => (x.cards.length || kind ? html`<section class="ovv
if (!data || !data.vod) return notFoundVtt(req, res);
const segs = transcriptSegments(data.vod);
if (!segs.length) return notFoundVtt(req, res);
-- return sendVtt(res, renderCaptionsVtt(segs));
++ // What was heard rides along: "[laughter]", "[applause]", "[music]". Live down or an older Live: speech only.
++ let sounds = [];
++ try { const out = await live.vodSounds(id); sounds = soundCues(out && out.cues); } catch { sounds = []; }
++ return sendVtt(res, renderCaptionsVtt(withSounds(segs, sounds)));
});
r.get('/v/:id/chapters.vtt', async (req, res) => {
ddiff --git a/test/video-pages.test.js b/test/video-pages.test.js
iindex 3de4cab..f67a3ac 100644
---- a/test/video-pages.test.js
++++ b/test/video-pages.test.js
@@@ -172,6 +172,12 @@ function startLive() {
if (username === 'streamer') return json(200, { clips: [clipListItem], total: 1, hasMore: false });
return json(200, { clips: [], total: 0, hasMore: false });
}
++ // What was heard (Live's sound cues, C15): a laugh, and two cues that must never reach the captions.
++ const sounds = /^\/api\/vods\/(\d+)\/sounds$/.exec(p);
++ if (sounds) {
++ if (sounds[1] === '101') return json(200, { cues: [{ start: 1, end: 2.5, text: '[laughter]' }, { start: 3, end: 4, text: '<b>not a tag</b>' }, { start: 5, end: 4, text: '[backwards]' }] });
++ return json(200, { cues: [] });
++ }
const memories = /^\/api\/vods\/(\d+)\/memories$/.exec(p);
if (memories) {
if (memories[1] === '103') return json(404, { error: 'not found' });
@@@ -438,11 +444,12 @@ function countOccurrences(haystack, needle) {
assert.ok(!r.text.includes('No recordings yet'));
});
-- await check('captions.vtt is exact, escaping and collapsing cue text', async () => {
++ await check('captions.vtt is exact, escaping and collapsing cue text, with what was heard in time order', async () => {
const r = await getVideo('/v/101/captions.vtt');
assert.strictEqual(r.status, 200);
assert.ok(String(r.headers['content-type']).includes('text/vtt'));
-- assert.strictEqual(r.text, 'WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHi &amp; bye &lt;tag&gt; here\n\n00:00:02.000 --> 00:00:04.000\nSecond line done\n');
++ assert.strictEqual(r.text, 'WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHi &amp; bye &lt;tag&gt; here\n\n00:00:01.000 --> 00:00:02.500\n[laughter]\n\n00:00:02.000 --> 00:00:04.000\nSecond line done\n',
++ 'the laugh sits between the lines; a cue that is not a bracketed tag, or runs backwards, is dropped');
});
await check('chapters.vtt runs each moment to the next or the end', async () => {