openvibers/openvibe.mediahub Commits
VOD captions name what was heard (item 151, C15) (#272)
Changes
@@@ -241,6 +241,10 @@ function createLiveClient({ config, fetchImpl = globalThis.fetch, log = consolevodMemories(id) {return getJson(`/api/vods/${id}/memories`);},++ /** What was heard in a VOD, as caption cues in its own time ("[laughter]"; Live's sound-captions.js), or null. */++ vodSounds(id) {++ return getJson(`/api/vods/${id}/sounds`);++ },/** A stream's recording parts (Live's history: a broadcaster who reconnected is recorded in parts), or null. */streamHistory(streamId) {return isId(streamId) ? getJson(`/api/streams/${streamId}/history`) : Promise.resolve(null);@@@ -11,7 +11,7 @@* /v/:id a VOD watch page (player, AI chapters and captions, its clips)* /c/:id a clip watch page* /@:username a channel's recordings (and clips), newest first-- * /v/:id/captions.vtt the AI transcript as WebVTT captions++ * /v/:id/captions.vtt the AI transcript as WebVTT captions, with what was heard ("[laughter]")* /v/:id/chapters.vtt the AI moments as WebVTT chapters* /v/:id/download/:file the recording, its transcript, chat replay, chapters, details or all of it as files* /c/:id/download/:file the clip, its transcript and details (server/video/downloads.js)@@@ -479,6 +479,25 @@ function chapters(mems, durationSeconds) {return out;}++/**++ * Live's sound cues (GET /api/vods/:id/sounds, item 151 C15): only bracketed tags like "[laughter]" with finite times++ * are taken, so nothing but a sound's name can reach the captions from there.++ */++function soundCues(list) {++ if (!Array.isArray(list)) return [];++ return list.map((c) => {++ if (!c || typeof c !== 'object') return null;++ const start = Number(c.start), end = Number(c.end), text = String(c.text == null ? '' : c.text);++ if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start || start < 0) return null;++ return /^\[[a-z][a-z ]{1,29}\]$/.test(text) ? { start, end, text } : null;++ }).filter(Boolean).slice(0, 5000);++}++++/** The spoken lines and the sounds, in time order: a sound's cue shows beside the line it falls in. */++function withSounds(segs, sounds) {++ return [...segs, ...sounds].sort((a, b) => a.start - b.start || a.end - b.end);++}++function renderCaptionsVtt(segs) {const lines = ['WEBVTT', ''];for (const s of segs) {@@@ -1146,7 +1165,10 @@ ${found ? sections.map((x) => (x.cards.length || kind ? html`<section class="ovvif (!data || !data.vod) return notFoundVtt(req, res);const segs = transcriptSegments(data.vod);if (!segs.length) return notFoundVtt(req, res);-- return sendVtt(res, renderCaptionsVtt(segs));++ // What was heard rides along: "[laughter]", "[applause]", "[music]". Live down or an older Live: speech only.++ let sounds = [];++ try { const out = await live.vodSounds(id); sounds = soundCues(out && out.cues); } catch { sounds = []; }++ return sendVtt(res, renderCaptionsVtt(withSounds(segs, sounds)));});r.get('/v/:id/chapters.vtt', async (req, res) => {@@@ -172,6 +172,12 @@ function startLive() {if (username === 'streamer') return json(200, { clips: [clipListItem], total: 1, hasMore: false });return json(200, { clips: [], total: 0, hasMore: false });}++ // What was heard (Live's sound cues, C15): a laugh, and two cues that must never reach the captions.++ const sounds = /^\/api\/vods\/(\d+)\/sounds$/.exec(p);++ if (sounds) {++ if (sounds[1] === '101') return json(200, { cues: [{ start: 1, end: 2.5, text: '[laughter]' }, { start: 3, end: 4, text: '<b>not a tag</b>' }, { start: 5, end: 4, text: '[backwards]' }] });++ return json(200, { cues: [] });++ }const memories = /^\/api\/vods\/(\d+)\/memories$/.exec(p);if (memories) {if (memories[1] === '103') return json(404, { error: 'not found' });@@@ -438,11 +444,12 @@ function countOccurrences(haystack, needle) {assert.ok(!r.text.includes('No recordings yet'));});-- await check('captions.vtt is exact, escaping and collapsing cue text', async () => {++ await check('captions.vtt is exact, escaping and collapsing cue text, with what was heard in time order', async () => {const r = await getVideo('/v/101/captions.vtt');assert.strictEqual(r.status, 200);assert.ok(String(r.headers['content-type']).includes('text/vtt'));-- assert.strictEqual(r.text, 'WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHi & bye <tag> here\n\n00:00:02.000 --> 00:00:04.000\nSecond line done\n');++ assert.strictEqual(r.text, 'WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHi & bye <tag> here\n\n00:00:01.000 --> 00:00:02.500\n[laughter]\n\n00:00:02.000 --> 00:00:04.000\nSecond line done\n',++ 'the laugh sits between the lines; a cue that is not a bracketed tag, or runs backwards, is dropped');});await check('chapters.vtt runs each moment to the next or the end', async () => {
@@ -241,6 +241,10 @@ function createLiveClient({ config, fetchImpl = globalThis.fetch, log = console
@@ -241,6 +241,10 @@ function createLiveClient({ config, fetchImpl = globalThis.fetch, log = console
vodMemories(id) {
vodMemories(id) {
return getJson(`/api/vods/${id}/memories`);
return getJson(`/api/vods/${id}/memories`);
},
},
+ /** What was heard in a VOD, as caption cues in its own time ("[laughter]"; Live's sound-captions.js), or null. */
+ vodSounds(id) {
+ return getJson(`/api/vods/${id}/sounds`);
+ },
/** A stream's recording parts (Live's history: a broadcaster who reconnected is recorded in parts), or null. */
/** A stream's recording parts (Live's history: a broadcaster who reconnected is recorded in parts), or null. */
streamHistory(streamId) {
streamHistory(streamId) {
return isId(streamId) ? getJson(`/api/streams/${streamId}/history`) : Promise.resolve(null);
return isId(streamId) ? getJson(`/api/streams/${streamId}/history`) : Promise.resolve(null);
@@ -11,7 +11,7 @@
@@ -11,7 +11,7 @@
* /v/:id a VOD watch page (player, AI chapters and captions, its clips)
* /v/:id a VOD watch page (player, AI chapters and captions, its clips)
* /c/:id a clip watch page
* /c/:id a clip watch page
* /@:username a channel's recordings (and clips), newest first
* /@:username a channel's recordings (and clips), newest first
- * /v/:id/captions.vtt the AI transcript as WebVTT captions
+ * /v/:id/captions.vtt the AI transcript as WebVTT captions, with what was heard ("[laughter]")
* /v/:id/chapters.vtt the AI moments as WebVTT chapters
* /v/:id/chapters.vtt the AI moments as WebVTT chapters
* /v/:id/download/:file the recording, its transcript, chat replay, chapters, details or all of it as files
* /v/:id/download/:file the recording, its transcript, chat replay, chapters, details or all of it as files
* /c/:id/download/:file the clip, its transcript and details (server/video/downloads.js)
* /c/:id/download/:file the clip, its transcript and details (server/video/downloads.js)
@@ -479,6 +479,25 @@ function chapters(mems, durationSeconds) {
@@ -479,6 +479,25 @@ function chapters(mems, durationSeconds) {
return out;
return out;
}
}
+/**
+ * Live's sound cues (GET /api/vods/:id/sounds, item 151 C15): only bracketed tags like "[laughter]" with finite times
+ * are taken, so nothing but a sound's name can reach the captions from there.
+ */
+function soundCues(list) {
+ if (!Array.isArray(list)) return [];
+ return list.map((c) => {
+ if (!c || typeof c !== 'object') return null;
+ const start = Number(c.start), end = Number(c.end), text = String(c.text == null ? '' : c.text);
+ if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start || start < 0) return null;
+ return /^\[[a-z][a-z ]{1,29}\]$/.test(text) ? { start, end, text } : null;
+ }).filter(Boolean).slice(0, 5000);
+}
+
+/** The spoken lines and the sounds, in time order: a sound's cue shows beside the line it falls in. */
+function withSounds(segs, sounds) {
+ return [...segs, ...sounds].sort((a, b) => a.start - b.start || a.end - b.end);
+}
+
function renderCaptionsVtt(segs) {
function renderCaptionsVtt(segs) {
const lines = ['WEBVTT', ''];
const lines = ['WEBVTT', ''];
for (const s of segs) {
for (const s of segs) {
@@ -1146,7 +1165,10 @@ ${found ? sections.map((x) => (x.cards.length || kind ? html`<section class="ovv
@@ -1146,7 +1165,10 @@ ${found ? sections.map((x) => (x.cards.length || kind ? html`<section class="ovv
if (!data || !data.vod) return notFoundVtt(req, res);
if (!data || !data.vod) return notFoundVtt(req, res);
const segs = transcriptSegments(data.vod);
const segs = transcriptSegments(data.vod);
if (!segs.length) return notFoundVtt(req, res);
if (!segs.length) return notFoundVtt(req, res);
- return sendVtt(res, renderCaptionsVtt(segs));
+ // What was heard rides along: "[laughter]", "[applause]", "[music]". Live down or an older Live: speech only.
+ let sounds = [];
+ try { const out = await live.vodSounds(id); sounds = soundCues(out && out.cues); } catch { sounds = []; }
+ return sendVtt(res, renderCaptionsVtt(withSounds(segs, sounds)));
});
});
r.get('/v/:id/chapters.vtt', async (req, res) => {
r.get('/v/:id/chapters.vtt', async (req, res) => {
@@ -172,6 +172,12 @@ function startLive() {
@@ -172,6 +172,12 @@ function startLive() {
if (username === 'streamer') return json(200, { clips: [clipListItem], total: 1, hasMore: false });
if (username === 'streamer') return json(200, { clips: [clipListItem], total: 1, hasMore: false });
return json(200, { clips: [], total: 0, hasMore: false });
return json(200, { clips: [], total: 0, hasMore: false });
}
}
+ // What was heard (Live's sound cues, C15): a laugh, and two cues that must never reach the captions.
+ const sounds = /^\/api\/vods\/(\d+)\/sounds$/.exec(p);
+ if (sounds) {
+ if (sounds[1] === '101') return json(200, { cues: [{ start: 1, end: 2.5, text: '[laughter]' }, { start: 3, end: 4, text: '<b>not a tag</b>' }, { start: 5, end: 4, text: '[backwards]' }] });
+ return json(200, { cues: [] });
+ }
const memories = /^\/api\/vods\/(\d+)\/memories$/.exec(p);
const memories = /^\/api\/vods\/(\d+)\/memories$/.exec(p);
if (memories) {
if (memories) {
if (memories[1] === '103') return json(404, { error: 'not found' });
if (memories[1] === '103') return json(404, { error: 'not found' });
@@ -438,11 +444,12 @@ function countOccurrences(haystack, needle) {
@@ -438,11 +444,12 @@ function countOccurrences(haystack, needle) {
assert.ok(!r.text.includes('No recordings yet'));
assert.ok(!r.text.includes('No recordings yet'));
});
});
- await check('captions.vtt is exact, escaping and collapsing cue text', async () => {
+ await check('captions.vtt is exact, escaping and collapsing cue text, with what was heard in time order', async () => {
const r = await getVideo('/v/101/captions.vtt');
const r = await getVideo('/v/101/captions.vtt');
assert.strictEqual(r.status, 200);
assert.strictEqual(r.status, 200);
assert.ok(String(r.headers['content-type']).includes('text/vtt'));
assert.ok(String(r.headers['content-type']).includes('text/vtt'));
- assert.strictEqual(r.text, 'WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHi & bye <tag> here\n\n00:00:02.000 --> 00:00:04.000\nSecond line done\n');
+ assert.strictEqual(r.text, 'WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHi & bye <tag> here\n\n00:00:01.000 --> 00:00:02.500\n[laughter]\n\n00:00:02.000 --> 00:00:04.000\nSecond line done\n',
+ 'the laugh sits between the lines; a cue that is not a bracketed tag, or runs backwards, is dropped');
});
});
await check('chapters.vtt runs each moment to the next or the end', async () => {
await check('chapters.vtt runs each moment to the next or the end', async () => {