openvibers/openvibe.live Commits
Audio description from the AI scene notes (X1) (#650)
Changes
nnew file mode 100644@@@ -0,0 +1,211 @@++/**++ * descriptions.js — audio-description cues for a VOD or clip (X1, consent plan 2026-10-31, "AI and data").++ *++ * Live already writes what a stream shows (stream_memories: the AI's scene notes at an offset) and what it++ * says (stream_timeline_events: speech rows). A blind or low-vision viewer needs the first told in the++ * silences of the second, so this module turns the two into WebVTT `kind="descriptions"` cues: each scene++ * note becomes a short line placed in a real gap between speech, near the moment it describes.++ *++ * The words are the AI's own, so people are described as written (server/ai/test ai-text-describes-people).++ * The platform line stays: sex acts and pornography are never described (server/ai/explicit.js), and a VOD++ * with any explicit scene note gets no track at all rather than a track that talks around it.++ *++ * Times are the video's own. A stream can be recorded in parts, so the callers move rows with++ * media-proxy/vod-time.js before asking for cues.++ */++'use strict';++++const db = require('../db/database');++const { isExplicit } = require('./explicit');++++// A description cue is read aloud at roughly this many characters a second; 2 s is the shortest cue that can++// be heard, 8 s the longest that still feels like a description rather than a monologue.++const CHARS_PER_SEC = 13;++const MIN_CUE_SEC = 2;++const MAX_CUE_SEC = 8;++const GAP_PAD_SEC = 0.35; // air either side of speech, so a cue never starts on the last syllable++const MAX_SHIFT_SEC = 20; // a note waits for the next real silence, but not across the whole video++++/** JSON-wrapped model text and stray markup become one plain line of cue text. */++function plainText(value) {++ let text = String(value == null ? '' : value).trim();++ if (text.startsWith('{')) {++ try {++ const o = JSON.parse(text);++ text = String((o && (o.description || o.text || o.caption)) || text);++ } catch { /* not JSON, keep it */ }++ }++ text = db.publicMemoryText(text)++ .replace(/<[^>]*>/g, ' ')++ .replace(/-->/g, '→')++ .replace(/\s+/g, ' ')++ .trim();++ return text;++}++++/** The speech spans, merged where they touch so a breath between two phrases is not treated as a gap. */++function speechSpans(rows) {++ const out = [];++ for (const r of (Array.isArray(rows) ? rows : [])) {++ const start = Math.max(0, Number(r.start) || 0);++ const end = Math.max(start + 0.2, Number(r.end == null ? r.end_sec : r.end) || start + 2);++ const last = out[out.length - 1];++ if (last && start <= last.end + 0.5) last.end = Math.max(last.end, end);++ else out.push({ start, end });++ }++ return out.sort((a, b) => a.start - b.start);++}++++/**++ * The gap where a cue of `length` seconds fits without touching speech, closest to `at`.++ * Gaps are between speech spans (with GAP_PAD either side); a note that falls inside speech waits++ * for the next gap, and if there is none near it the caller drops the cue rather than speak over++ * the person.++ */++function gapStart(at, length, spans, duration, minStart = 0) {++ const gaps = [];++ let cursor = Math.max(0, Number(minStart) || 0);++ for (const s of (spans || [])) {++ const end = Math.max(cursor, s.start - GAP_PAD_SEC);++ if (end - cursor >= length) gaps.push({ start: cursor, end });++ cursor = Math.max(cursor, s.end + GAP_PAD_SEC);++ }++ if (duration - cursor >= length) gaps.push({ start: cursor, end: duration });++ const note = Math.max(0, Number(at) || 0);++ const place = (g) => Math.min(g.end - length, Math.max(g.start, note));++ // Prefer a gap at or after the note (a scene is described as it appears, or just after), and only++ // fall back to the last silence before it when nothing later can hold the line.++ let best = null, bestD = Infinity;++ for (const g of gaps) {++ if (g.end <= note + 0.01) continue;++ const start = place(g);++ const d = Math.abs(start - note);++ if (d < bestD) { bestD = d; best = start; }++ }++ if (best == null) {++ for (const g of gaps) {++ if (g.start >= note) continue;++ const start = place(g);++ const d = Math.abs(start - note);++ if (d < bestD) { bestD = d; best = start; }++ }++ }++ return best == null ? null : Math.round(best * 1000) / 1000;++}++++/**++ * Cues from scene notes and speech rows (both already in the video's time).++ * notes: [{ at, text }] ; speech: [{ start, end }] ; duration: seconds.++ * Returns [{ start, end, text }] in time order, never overlapping and never on top of speech.++ */++function cueList(notes, speech, duration) {++ const total = Number(duration) || 0;++ if (!(total > 0)) return [];++ const spans = speechSpans(speech);++ const seen = new Set();++ const picked = [];++ const ordered = (Array.isArray(notes) ? notes : [])++ .map((n) => ({ at: Math.max(0, Number(n && n.at) || 0), text: plainText(n && n.text) }))++ .filter((n) => n.text && !isExplicit(n.text))++ .sort((a, b) => a.at - b.at);++ // Any explicit scene note: no track for this video. A sex act is never described or talked around.++ if ((Array.isArray(notes) ? notes : []).some((n) => isExplicit(plainText(n && n.text)))) return [];++++ let lastEnd = 0;++ for (const note of ordered) {++ if (picked.length >= 120) break;++ const key = note.text.toLowerCase();++ if (seen.has(key)) continue;++ const length = Math.min(MAX_CUE_SEC, Math.max(MIN_CUE_SEC, note.text.length / CHARS_PER_SEC));++ const near = Math.min(total - 0.05, note.at);++ // Search at the note, then keep moving to the next silence inside the window around it.++ for (let attempt = 0; attempt < 3; attempt++) {++ const candidate = attempt === 0 ? near : near + attempt * MAX_SHIFT_SEC / 3;++ const start = gapStart(candidate, length, spans, total, lastEnd + (picked.length ? 0.2 : 0));++ if (start == null || Math.abs(start - near) > MAX_SHIFT_SEC) break;++ const end = Math.min(total - 0.05, start + length);++ if (end - start < MIN_CUE_SEC - 0.05) break;++ picked.push({ start: Math.round(start * 1000) / 1000, end: Math.round(end * 1000) / 1000, text: note.text });++ seen.add(key);++ lastEnd = end;++ break;++ }++ }++ return picked;++}++++/** WebVTT timestamp: HH:MM:SS.mmm. */++function timestamp(sec) {++ const s = Math.max(0, Number(sec) || 0);++ const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), whole = Math.floor(s % 60);++ const ms = Math.round((s - Math.floor(s)) * 1000);++ return `${String(h).padStart(2, '0')}:${String(m).padStart(2, '0')}:${String(whole).padStart(2, '0')}.${String(ms).padStart(3, '0')}`;++}++++/** The WebVTT file, or '' when there is nothing to describe. */++function buildVtt(cues) {++ const list = Array.isArray(cues) ? cues : [];++ if (!list.length) return '';++ const lines = [++ 'WEBVTT',++ '',++ 'NOTE AI scene descriptions (audio description). People are described; sex acts never are.',++ '',++ ];++ for (const cue of list) {++ if (!cue || !cue.text) continue;++ lines.push(`${timestamp(cue.start)} --> ${timestamp(cue.end)}`, cue.text, '');++ }++ return lines.join('\n');++}++++/** The VOD's rows in its own time: speech from its linked timeline (or the stream's), scene notes from the stream. */++async function vodVtt(vod) {++ if (!vod || vod.stream_id == null) return '';++ const streamId = Number(vod.stream_id);++ // The linked timeline and the stream's own are two views of the same audio and either can be the more++ // complete (late-transcribed spool rows land after the vod.ready link). Take the one with more speech,++ // exactly as the transcript endpoint does, so a description never talks over a line only one view knows.++ let linked = [], streamRows = [];++ try { linked = await db.getTimelineByVod(vod.id) || []; } catch { linked = []; }++ try { streamRows = await db.getTimeline(streamId, { limit: 20000 }) || []; } catch { streamRows = []; }++ const spoken = (list) => list.reduce((n, r) => n + (r.kind === 'speech' ? String(r.text || '').trim().length : 0), 0);++ const rows = spoken(linked) >= spoken(streamRows) && linked.length ? linked : streamRows;++ let memories = [];++ try { memories = await db.getStreamMemories(streamId) || []; } catch { memories = []; }++ const duration = Number(vod.duration_seconds || vod.duration) || (rows.concat(memories.map((m) => ({ start_sec: m.offset_seconds }))).reduce((max, r) => Math.max(max, Number(r.start_sec || r.offset_seconds) || 0), 0) + 10);++ if (!memories.length || !(duration > 0)) return '';++ const { inVodTime } = require('../media-proxy/vod-time');++ let shiftedRows = rows;++ let shiftedNotes = memories;++ try {++ shiftedRows = await inVodTime(vod, rows, 'start_sec', { round: false });++ shiftedNotes = await inVodTime(vod, memories.map((m) => ({ ...m, at: Number(m.offset_seconds) || 0 })), 'at', { round: false });++ } catch { /* a broken part lookup: keep stream time rather than lose the track */ }++ const speech = shiftedRows.filter((r) => r.kind === 'speech').map((r) => ({ start: r.start_sec, end: r.end_sec }));++ const notes = shiftedNotes.filter((m) => Number(m.offset_seconds) >= 0).map((m) => ({ at: m.at != null ? m.at : m.offset_seconds, text: m.description }));++ return buildVtt(cueList(notes, speech, duration));++}++++/** A clip's window of its source stream, in the clip's own time. */++async function clipVtt(clip) {++ if (!clip || clip.stream_id == null) return '';++ const start = Math.max(0, Number(clip.start_time) || 0);++ const end = Number(clip.end_time) || (start + (Number(clip.duration_seconds) || 0));++ const duration = Number(clip.duration_seconds) || Math.max(0, end - start);++ if (!(duration > 0)) return '';++ let rows = [];++ let memories = [];++ try { rows = await db.getTimeline(Number(clip.stream_id), { limit: 20000 }) || []; } catch { rows = []; }++ try { memories = await db.getStreamMemories(Number(clip.stream_id)) || []; } catch { memories = []; }++ if (!memories.length) return '';++ const speech = rows++ .filter((r) => r.kind === 'speech' && Number(r.start_sec) <= end + 1 && Number(r.end_sec == null ? r.start_sec : r.end_sec) >= start - 1)++ .map((r) => ({ start: Math.max(0, Number(r.start_sec) - start), end: Math.max(0, Number(r.end_sec == null ? r.start_sec + 2 : r.end_sec) - start) }));++ const notes = memories++ .filter((m) => Number(m.offset_seconds) >= start - 1 && Number(m.offset_seconds) <= end + 1)++ .map((m) => ({ at: Math.max(0, Number(m.offset_seconds) - start), text: m.description }));++ return buildVtt(cueList(notes, speech, duration));++}++++module.exports = { cueList, buildVtt, vodVtt, clipVtt, plainText, gapStart, timestamp };@@@ -266,6 +266,37 @@ router.get('/:id/download-url', optionalAuth, async (req, res) => {}});++/**++ * GET /api/clips/:id/descriptions.vtt — the clip's audio description (X1), the source stream's scene notes++ * inside the clip's window, in the clip's own time and in the silences between speech. Same visibility rule++ * as GET /api/clips/:id: public and link clips for anyone, a private clip for its owner or staff only.++ */++router.get('/:id/descriptions.vtt', optionalAuth, async (req, res) => {++ try {++ if (!/^\d+$/.test(String(req.params.id))) return clipNotFound(res);++ let clip;++ try { clip = await media.getClip(req.params.id); }++ catch (err) {++ if (err && err.name === 'MediaApiError' && err.status === 404) return clipNotFound(res);++ return mediaErr(res, err, 'Failed to get clip');++ }++ if (!clip || !await access.canView(req.user, clip)) return clipNotFound(res);++ if (clip.app_id && clip.app_id !== 'live') return clipNotFound(res);++ if (clip.status && clip.status !== 'ready') return clipNotFound(res);++ const vtt = await require('../ai/descriptions').clipVtt(clip);++ if (!vtt) return clipNotFound(res);++ res.set({++ 'Content-Type': 'text/vtt; charset=utf-8',++ 'Cache-Control': access.isPrivate(clip) ? 'private, no-store' : 'public, max-age=60',++ 'X-Content-Type-Options': 'nosniff',++ });++ res.send(vtt);++ } catch (err) {++ console.error('[Clips] descriptions error:', err.message);++ if (!res.headersSent) res.status(500).json({ error: 'Failed to load the descriptions' });++ }++});++router.get('/:id', optionalAuth, async (req, res) => {try {if (!/^\d+$/.test(req.params.id)) return clipNotFound(res);nnew file mode 100644@@@ -0,0 +1,28 @@++/**++ * OpenVibe.Live — a stream's rows in one of its VODs' own time.++ *++ * A stream can be recorded in parts (a broadcaster reconnecting), and each part is its own VOD; the++ * rows Live keeps (scene notes, speech, sound) are in stream time. The first part of a joined recording++ * answers for the whole video (streaming/recordings.js); a VOD that is one part of its stream keeps the++ * rows inside it, moved to its time (its chapters pointed eleven minutes late on part 2 of stream 2795).++ * `round` keeps whole seconds, which is what caption and description cues want.++ */++'use strict';++++async function inVodTime(vod, rows, key, { round = true } = {}) {++ const fix = (v) => (round ? Math.round(v) : Math.round(v * 100) / 100);++ if (vod.recording && Number(vod.recording.vod_id) === Number(vod.id) && vod.recording.hls_url && (vod.recording.parts || []).length > 1) {++ const recordings = require('../streaming/recordings');++ const parts = await recordings.presentParts(vod.recording);++ return rows.map((r) => ({ r, at: recordings.toRecordingTime(parts, Number(r[key])) }))++ .filter((x) => x.at != null).map(({ r, at }) => ({ ...r, [key]: fix(at), ...(r.end != null ? { end: fix(at + (Number(r.end) - Number(r[key]))) } : {}) }));++ }++ const part = await require('../streaming/stream-history').partForVod(vod.id);++ if (!part) return rows;++ const len = part.end != null ? part.end - part.at : Infinity;++ return rows++ .filter((r) => Number(r[key]) >= part.at && Number(r[key]) - part.at <= len)++ .map((r) => ({ ...r, [key]: fix(Number(r[key]) - part.at), ...(r.end != null ? { end: fix(Number(r.end) - part.at) } : {}) }));++}++++module.exports = { inVodTime };@@@ -25,6 +25,7 @@ function chunkOpensVod(stream, buffer) {const { requireAuth, optionalAuth } = require('../auth/auth');const permissions = require('../auth/permissions');const access = require('./access');++const { inVodTime } = require('./vod-time');const purge = require('./purge');const { commentCount } = require('./comments');const contentLabels = require('../labels');@@@ -420,28 +421,6 @@ router.post('/bulk-delete-old', requireAuth, async (req, res) => {}});--/**-- * A stream's rows (stream time, in `key`) in this VOD's own time. The first part of a joined recording answers for the-- * whole video: every row that was recorded, in the video's time (streaming/recordings.js; the gaps between parts were-- * not). A VOD that is one part of its stream (the broadcaster reconnected) keeps the rows inside it, moved to its time:-- * its chapters (OpenVibe.Video) pointed eleven minutes late on part 2 of stream 2795. `round` keeps whole seconds.-- */--async function inVodTime(vod, rows, key, { round = true } = {}) {-- const fix = (v) => (round ? Math.round(v) : Math.round(v * 100) / 100);-- if (vod.recording && Number(vod.recording.vod_id) === Number(vod.id) && vod.recording.hls_url && (vod.recording.parts || []).length > 1) {-- const recordings = require('../streaming/recordings');-- const parts = await recordings.presentParts(vod.recording);-- return rows.map((r) => ({ r, at: recordings.toRecordingTime(parts, Number(r[key])) }))-- .filter((x) => x.at != null).map(({ r, at }) => ({ ...r, [key]: fix(at), ...(r.end != null ? { end: fix(at + (Number(r.end) - Number(r[key]))) } : {}) }));-- }-- const part = await require('../streaming/stream-history').partForVod(vod.id);-- if (!part) return rows;-- const len = part.end != null ? part.end - part.at : Infinity;-- return rows-- .filter((r) => Number(r[key]) >= part.at && Number(r[key]) - part.at <= len)-- .map((r) => ({ ...r, [key]: fix(Number(r[key]) - part.at), ...(r.end != null ? { end: fix(Number(r.end) - part.at) } : {}) }));--}--// AI "memory" timeline for a VOD (stream_memories stays in live.db).router.get('/:id/memories', optionalAuth, async (req, res) => {try {@@@ -481,6 +460,34 @@ router.get('/:id/sounds', optionalAuth, async (req, res) => {}});++/**++ * GET /api/vods/:id/descriptions.vtt — the VOD's audio description as a WebVTT `kind="descriptions"` track++ * (X1, consent plan 2026-10-31): short AI scene notes from Live's timeline, timed into the silences between++ * speech. Public and link VODs are described for anyone; a private VOD only answers its owner (or staff), so++ * the same visibility rule as every other VOD read applies. People are described; sex acts never are++ * (server/ai/descriptions.js + explicit.js). 404 when there is nothing to describe.++ */++router.get('/:id/descriptions.vtt', optionalAuth, async (req, res) => {++ try {++ if (!/^\d+$/.test(String(req.params.id))) return vodNotFound(res);++ const vod = await loadVisibleVod(req, res, req.params.id);++ if (!vod) return;++ if (vod.app_id && vod.app_id !== 'live') return vodNotFound(res);++ if (vod.is_recording || vod.status === 'recording') return vodNotFound(res);++ const vtt = await require('../ai/descriptions').vodVtt(vod);++ if (!vtt) return vodNotFound(res);++ res.set({++ 'Content-Type': 'text/vtt; charset=utf-8',++ 'Cache-Control': access.isPrivate(vod) ? 'private, no-store' : 'public, max-age=60',++ 'X-Content-Type-Options': 'nosniff',++ });++ res.send(vtt);++ } catch (err) {++ console.error('[VODs] descriptions error:', err.message);++ if (!res.headersSent) res.status(500).json({ error: 'Failed to load the descriptions' });++ }++});++/*** GET /api/vods/:id/context — everything around a VOD that isn't the file: the stream it came* from, its activity numbers (chat, viewers, mic moments, follows, clips), the clips cut fromnnew file mode 100644@@@ -0,0 +1,158 @@++'use strict';++/**++ * Audio description (X1, consent plan 2026-10-31, "AI and data"): Live's scene notes become a WebVTT++ * `kind="descriptions"` track, timed into the silences between speech. Public and link VODs/clips are++ * described for anyone; a private one only answers its owner (server/ai/descriptions.js + the routes in++ * media-proxy/vods.js and clips.js). The AI's text may describe people; it never describes sex acts, and a++ * video with an explicit scene note gets no track at all (server/ai/explicit.js).++ *++ * node test/audio-description.test.js++ */++const assert = require('assert');++const http = require('http');++++process.env.NODE_ENV = 'test';++const quiet = console.log;++console.log = (...a) => { if (!/^\[/.test(String(a[0]))) quiet(...a); };++console.warn = () => {};++++const VODS = {++ 100: { id: 100, app_id: 'live', user_id: 3, stream_id: 500, title: 'Public VOD', visibility: 'public', is_public: 1, status: 'ready', duration_seconds: 60 },++ 101: { id: 101, app_id: 'live', user_id: 3, stream_id: 500, title: 'Private VOD', visibility: 'private', is_public: 0, status: 'ready', duration_seconds: 60 },++ 102: { id: 102, app_id: 'live', user_id: 3, stream_id: 501, title: 'Explicit VOD', visibility: 'public', is_public: 1, status: 'ready', duration_seconds: 30 },++ 103: { id: 103, app_id: 'games', user_id: 3, stream_id: 500, title: 'Another app', visibility: 'public', is_public: 1, status: 'ready', duration_seconds: 60 },++};++const CLIPS = {++ 200: { id: 200, app_id: 'live', user_id: 3, stream_id: 500, vod_id: 100, title: 'Public clip', visibility: 'public', is_public: 1, status: 'ready', start_time: 3, end_time: 15, duration_seconds: 12 },++ 201: { id: 201, app_id: 'live', user_id: 3, stream_id: 500, vod_id: 100, title: 'Private clip', visibility: 'private', is_public: 0, status: 'ready', start_time: 3, end_time: 15, duration_seconds: 12 },++};++++function cuesOf(vtt) {++ const out = [];++ const re = /(\d{2}:\d{2}:\d{2}\.\d{3}) --> (\d{2}:\d{2}:\d{2}\.\d{3})\n([^\n]+)/g;++ let m;++ const sec = (t) => {++ const [h, mm, rest] = t.split(':');++ const [s, ms] = rest.split('.');++ return Number(h) * 3600 + Number(mm) * 60 + Number(s) + Number(ms) / 1000;++ };++ while ((m = re.exec(vtt))) out.push({ start: sec(m[1]), end: sec(m[2]), text: m[3] });++ return out;++}++++(async () => {++ const db = require('../server/db/database');++ await db.initDb();++ const raw = db.getDb();++ // Fixed ids: the Media stubs below name them, and the person the owner check expects.++ await raw.prepare(`INSERT INTO users (id, username, display_name, stream_key) OVERRIDING SYSTEM VALUE VALUES (?, ?, ?, ?)`)++ .run(3, 'alice', 'alice', 'a'.repeat(32));++ await raw.prepare(`INSERT INTO users (id, username, display_name, stream_key) OVERRIDING SYSTEM VALUE VALUES (?, ?, ?, ?)`)++ .run(4, 'bob', 'bob', 'b'.repeat(32));++ const alice = 3;++ for (const [id, title] of [[500, 'Described stream'], [501, 'Explicit stream']]) {++ await raw.prepare(`INSERT INTO streams (id, user_id, title, category, protocol, is_live, started_at) OVERRIDING SYSTEM VALUE VALUES (?, ?, ?, 'irl', 'rtmp', 1, ov_now())`)++ .run(id, alice, title);++ }++ const s500 = 500, s501 = 501;++++ // Scene notes: one in a gap, one during speech, and an explicit note on another stream.++ await db.addStreamMemory({ stream_id: s500, user_id: alice, offset_seconds: 5, description: 'A person in a red coat opens a blue door.' });++ await db.addStreamMemory({ stream_id: s500, user_id: alice, offset_seconds: 9, description: 'A dog sleeps on the sofa.' });++ await db.addStreamMemory({ stream_id: s501, user_id: alice, offset_seconds: 5, description: 'A person waves at the camera.' });++ await db.addStreamMemory({ stream_id: s501, user_id: alice, offset_seconds: 12, description: 'The streamer is having sex on camera.' });++ await db.addTimelineEvents([++ { stream_id: s500, user_id: alice, vod_id: 100, kind: 'speech', start_sec: 0, end_sec: 4, text: 'Hello and welcome.' },++ { stream_id: s500, user_id: alice, vod_id: 100, kind: 'speech', start_sec: 8, end_sec: 12, text: 'Today we build a table.' },++ ]);++++ // ── The cue builder on its own: gaps, no overlap, dedupe, and the explicit fail-closed rule ──++ const { cueList, buildVtt, gapStart } = require('../server/ai/descriptions');++ const speech = [{ start: 0, end: 4 }, { start: 8, end: 12 }];++ const list = cueList([++ { at: 5, text: 'A person in a red coat opens a blue door.' },++ { at: 9, text: 'A dog sleeps on the sofa.' },++ { at: 5, text: 'A person in a red coat opens a blue door.' },++ ], speech, 60);++ assert.strictEqual(list.length, 2, 'a repeated note becomes one cue');++ for (const cue of list) {++ assert.ok(cue.end > cue.start, 'a cue has duration');++ for (const s of speech) {++ assert.ok(cue.end <= s.start + 0.001 || cue.start >= s.end - 0.001, `cue "${cue.text}" does not overlap speech ${s.start}-${s.end}`);++ }++ }++ assert.ok(list.find((c) => /dog/.test(c.text)).start >= 12.3, 'a note during speech waits for the next silence');++ assert.ok(gapStart(5, 3, speech, 60) != null, 'a gap can be found at the note');++ assert.ok(buildVtt(list).startsWith('WEBVTT\n'), 'WebVTT header');++ assert.deepStrictEqual(cueList([{ at: 5, text: 'A person waves.' }, { at: 12, text: 'sexual acts on camera' }], speech, 60), [], 'one explicit note means no track');++ assert.deepStrictEqual(cueList([{ at: 5, text: 'A nude figure drawing session.' }], speech, 60).length, 1, 'nudity is art, not explicit');++++ // ── The routes, with Media stubbed ──++ const media = require('../server/media-client');++ const notFound = (what) => new media.MediaApiError(`${what} not found`, 404, { error: `${what} not found` });++ media.getVod = async (id) => { const v = VODS[Number(id)]; if (!v) throw notFound('VOD'); return { ...v }; };++ media.getClip = async (id) => { const c = CLIPS[Number(id)]; if (!c) throw notFound('Clip'); return { ...c }; };++++ const express = require('express');++ const app = express();++ // optionalAuth leaves an already-present req.user alone; this stands in for a signed-in Live session.++ app.use((req, res, next) => {++ const as = req.headers['x-test-user'];++ if (as) req.user = { id: Number(as), username: as === '3' ? 'alice' : 'bob', role: 'user' };++ next();++ });++ app.use('/api/vods', require('../server/media-proxy/vods'));++ app.use('/api/clips', require('../server/media-proxy/clips'));++ const server = http.createServer(app).listen(0);++ await new Promise((r) => (server.listening ? r() : server.once('listening', r)));++ const get = (path, headers = {}) => new Promise((resolve, reject) => {++ http.get({ port: server.address().port, path, headers }, (res) => {++ let body = '';++ res.on('data', (c) => { body += c; });++ res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body }));++ }).on('error', reject);++ });++++ // Public VOD: a descriptions track for anyone.++ const pub = await get('/api/vods/100/descriptions.vtt');++ assert.strictEqual(pub.status, 200);++ assert.match(pub.headers['content-type'], /^text\/vtt/);++ assert.match(pub.headers['cache-control'], /public/);++ const pubCues = cuesOf(pub.body);++ assert.strictEqual(pubCues.length, 2);++ assert.ok(pubCues.some((c) => /red coat/.test(c.text)));++ assert.ok(pubCues.some((c) => /dog/.test(c.text)));++ assert.ok(pubCues.find((c) => /dog/.test(c.text)).start >= 12.3, 'the route keeps the note out of speech');++++ // Private VOD: the missing-VOD answer until its owner asks; the owner and nobody else gets the track.++ const privAnon = await get('/api/vods/101/descriptions.vtt');++ assert.strictEqual(privAnon.status, 404, 'a private VOD is not described for a visitor');++ assert.deepStrictEqual(privAnon.body, JSON.stringify({ error: 'VOD not found' }));++ const privOther = await get('/api/vods/101/descriptions.vtt', { 'x-test-user': '4' });++ assert.strictEqual(privOther.status, 404, 'not for another person');++ const privOwner = await get('/api/vods/101/descriptions.vtt', { 'x-test-user': '3' });++ assert.strictEqual(privOwner.status, 200, 'the owner asks and gets it');++ assert.match(privOwner.headers['cache-control'], /private/);++++ // A video with an explicit scene note: no track, not even for its owner.++ for (const who of [null, '3']) {++ const r = await get('/api/vods/102/descriptions.vtt', who ? { 'x-test-user': who } : {});++ assert.strictEqual(r.status, 404, 'explicit scenes are never described');++ }++++ // Another app's media is not Live's to describe.++ assert.strictEqual((await get('/api/vods/103/descriptions.vtt')).status, 404);++++ // A clip: the source notes inside its window, in the clip's own time, between speech.++ const clip = await get('/api/clips/200/descriptions.vtt');++ assert.strictEqual(clip.status, 200);++ const clipCues = cuesOf(clip.body);++ assert.strictEqual(clipCues.length, 2, 'both notes fall inside the clip window');++ assert.ok(clipCues.every((c) => c.end <= 12.001), 'clip cues stay inside the clip');++ assert.ok(clipCues.find((c) => /red coat/.test(c.text)).start <= 2.1, 'a note moves into the clip time');++ assert.strictEqual((await get('/api/clips/201/descriptions.vtt')).status, 404, 'a private clip needs its owner');++ assert.strictEqual((await get('/api/clips/201/descriptions.vtt', { 'x-test-user': '3' })).status, 200);++++ server.close();++ console.log('audio description: all checks passed');++})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,211 @@
@@ -0,0 +1,211 @@
+/**
+ * descriptions.js — audio-description cues for a VOD or clip (X1, consent plan 2026-10-31, "AI and data").
+ *
+ * Live already writes what a stream shows (stream_memories: the AI's scene notes at an offset) and what it
+ * says (stream_timeline_events: speech rows). A blind or low-vision viewer needs the first told in the
+ * silences of the second, so this module turns the two into WebVTT `kind="descriptions"` cues: each scene
+ * note becomes a short line placed in a real gap between speech, near the moment it describes.
+ *
+ * The words are the AI's own, so people are described as written (server/ai/test ai-text-describes-people).
+ * The platform line stays: sex acts and pornography are never described (server/ai/explicit.js), and a VOD
+ * with any explicit scene note gets no track at all rather than a track that talks around it.
+ *
+ * Times are the video's own. A stream can be recorded in parts, so the callers move rows with
+ * media-proxy/vod-time.js before asking for cues.
+ */
+'use strict';
+
+const db = require('../db/database');
+const { isExplicit } = require('./explicit');
+
+// A description cue is read aloud at roughly this many characters a second; 2 s is the shortest cue that can
+// be heard, 8 s the longest that still feels like a description rather than a monologue.
+const CHARS_PER_SEC = 13;
+const MIN_CUE_SEC = 2;
+const MAX_CUE_SEC = 8;
+const GAP_PAD_SEC = 0.35; // air either side of speech, so a cue never starts on the last syllable
+const MAX_SHIFT_SEC = 20; // a note waits for the next real silence, but not across the whole video
+
+/** JSON-wrapped model text and stray markup become one plain line of cue text. */
+function plainText(value) {
+ let text = String(value == null ? '' : value).trim();
+ if (text.startsWith('{')) {
+ try {
+ const o = JSON.parse(text);
+ text = String((o && (o.description || o.text || o.caption)) || text);
+ } catch { /* not JSON, keep it */ }
+ }
+ text = db.publicMemoryText(text)
+ .replace(/<[^>]*>/g, ' ')
+ .replace(/-->/g, '→')
+ .replace(/\s+/g, ' ')
+ .trim();
+ return text;
+}
+
+/** The speech spans, merged where they touch so a breath between two phrases is not treated as a gap. */
+function speechSpans(rows) {
+ const out = [];
+ for (const r of (Array.isArray(rows) ? rows : [])) {
+ const start = Math.max(0, Number(r.start) || 0);
+ const end = Math.max(start + 0.2, Number(r.end == null ? r.end_sec : r.end) || start + 2);
+ const last = out[out.length - 1];
+ if (last && start <= last.end + 0.5) last.end = Math.max(last.end, end);
+ else out.push({ start, end });
+ }
+ return out.sort((a, b) => a.start - b.start);
+}
+
+/**
+ * The gap where a cue of `length` seconds fits without touching speech, closest to `at`.
+ * Gaps are between speech spans (with GAP_PAD either side); a note that falls inside speech waits
+ * for the next gap, and if there is none near it the caller drops the cue rather than speak over
+ * the person.
+ */
+function gapStart(at, length, spans, duration, minStart = 0) {
+ const gaps = [];
+ let cursor = Math.max(0, Number(minStart) || 0);
+ for (const s of (spans || [])) {
+ const end = Math.max(cursor, s.start - GAP_PAD_SEC);
+ if (end - cursor >= length) gaps.push({ start: cursor, end });
+ cursor = Math.max(cursor, s.end + GAP_PAD_SEC);
+ }
+ if (duration - cursor >= length) gaps.push({ start: cursor, end: duration });
+ const note = Math.max(0, Number(at) || 0);
+ const place = (g) => Math.min(g.end - length, Math.max(g.start, note));
+ // Prefer a gap at or after the note (a scene is described as it appears, or just after), and only
+ // fall back to the last silence before it when nothing later can hold the line.
+ let best = null, bestD = Infinity;
+ for (const g of gaps) {
+ if (g.end <= note + 0.01) continue;
+ const start = place(g);
+ const d = Math.abs(start - note);
+ if (d < bestD) { bestD = d; best = start; }
+ }
+ if (best == null) {
+ for (const g of gaps) {
+ if (g.start >= note) continue;
+ const start = place(g);
+ const d = Math.abs(start - note);
+ if (d < bestD) { bestD = d; best = start; }
+ }
+ }
+ return best == null ? null : Math.round(best * 1000) / 1000;
+}
+
+/**
+ * Cues from scene notes and speech rows (both already in the video's time).
+ * notes: [{ at, text }] ; speech: [{ start, end }] ; duration: seconds.
+ * Returns [{ start, end, text }] in time order, never overlapping and never on top of speech.
+ */
+function cueList(notes, speech, duration) {
+ const total = Number(duration) || 0;
+ if (!(total > 0)) return [];
+ const spans = speechSpans(speech);
+ const seen = new Set();
+ const picked = [];
+ const ordered = (Array.isArray(notes) ? notes : [])
+ .map((n) => ({ at: Math.max(0, Number(n && n.at) || 0), text: plainText(n && n.text) }))
+ .filter((n) => n.text && !isExplicit(n.text))
+ .sort((a, b) => a.at - b.at);
+ // Any explicit scene note: no track for this video. A sex act is never described or talked around.
+ if ((Array.isArray(notes) ? notes : []).some((n) => isExplicit(plainText(n && n.text)))) return [];
+
+ let lastEnd = 0;
+ for (const note of ordered) {
+ if (picked.length >= 120) break;
+ const key = note.text.toLowerCase();
+ if (seen.has(key)) continue;
+ const length = Math.min(MAX_CUE_SEC, Math.max(MIN_CUE_SEC, note.text.length / CHARS_PER_SEC));
+ const near = Math.min(total - 0.05, note.at);
+ // Search at the note, then keep moving to the next silence inside the window around it.
+ for (let attempt = 0; attempt < 3; attempt++) {
+ const candidate = attempt === 0 ? near : near + attempt * MAX_SHIFT_SEC / 3;
+ const start = gapStart(candidate, length, spans, total, lastEnd + (picked.length ? 0.2 : 0));
+ if (start == null || Math.abs(start - near) > MAX_SHIFT_SEC) break;
+ const end = Math.min(total - 0.05, start + length);
+ if (end - start < MIN_CUE_SEC - 0.05) break;
+ picked.push({ start: Math.round(start * 1000) / 1000, end: Math.round(end * 1000) / 1000, text: note.text });
+ seen.add(key);
+ lastEnd = end;
+ break;
+ }
+ }
+ return picked;
+}
+
+/** WebVTT timestamp: HH:MM:SS.mmm. */
+function timestamp(sec) {
+ const s = Math.max(0, Number(sec) || 0);
+ const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), whole = Math.floor(s % 60);
+ const ms = Math.round((s - Math.floor(s)) * 1000);
+ return `${String(h).padStart(2, '0')}:${String(m).padStart(2, '0')}:${String(whole).padStart(2, '0')}.${String(ms).padStart(3, '0')}`;
+}
+
+/** The WebVTT file, or '' when there is nothing to describe. */
+function buildVtt(cues) {
+ const list = Array.isArray(cues) ? cues : [];
+ if (!list.length) return '';
+ const lines = [
+ 'WEBVTT',
+ '',
+ 'NOTE AI scene descriptions (audio description). People are described; sex acts never are.',
+ '',
+ ];
+ for (const cue of list) {
+ if (!cue || !cue.text) continue;
+ lines.push(`${timestamp(cue.start)} --> ${timestamp(cue.end)}`, cue.text, '');
+ }
+ return lines.join('\n');
+}
+
+/** The VOD's rows in its own time: speech from its linked timeline (or the stream's), scene notes from the stream. */
+async function vodVtt(vod) {
+ if (!vod || vod.stream_id == null) return '';
+ const streamId = Number(vod.stream_id);
+ // The linked timeline and the stream's own are two views of the same audio and either can be the more
+ // complete (late-transcribed spool rows land after the vod.ready link). Take the one with more speech,
+ // exactly as the transcript endpoint does, so a description never talks over a line only one view knows.
+ let linked = [], streamRows = [];
+ try { linked = await db.getTimelineByVod(vod.id) || []; } catch { linked = []; }
+ try { streamRows = await db.getTimeline(streamId, { limit: 20000 }) || []; } catch { streamRows = []; }
+ const spoken = (list) => list.reduce((n, r) => n + (r.kind === 'speech' ? String(r.text || '').trim().length : 0), 0);
+ const rows = spoken(linked) >= spoken(streamRows) && linked.length ? linked : streamRows;
+ let memories = [];
+ try { memories = await db.getStreamMemories(streamId) || []; } catch { memories = []; }
+ const duration = Number(vod.duration_seconds || vod.duration) || (rows.concat(memories.map((m) => ({ start_sec: m.offset_seconds }))).reduce((max, r) => Math.max(max, Number(r.start_sec || r.offset_seconds) || 0), 0) + 10);
+ if (!memories.length || !(duration > 0)) return '';
+ const { inVodTime } = require('../media-proxy/vod-time');
+ let shiftedRows = rows;
+ let shiftedNotes = memories;
+ try {
+ shiftedRows = await inVodTime(vod, rows, 'start_sec', { round: false });
+ shiftedNotes = await inVodTime(vod, memories.map((m) => ({ ...m, at: Number(m.offset_seconds) || 0 })), 'at', { round: false });
+ } catch { /* a broken part lookup: keep stream time rather than lose the track */ }
+ const speech = shiftedRows.filter((r) => r.kind === 'speech').map((r) => ({ start: r.start_sec, end: r.end_sec }));
+ const notes = shiftedNotes.filter((m) => Number(m.offset_seconds) >= 0).map((m) => ({ at: m.at != null ? m.at : m.offset_seconds, text: m.description }));
+ return buildVtt(cueList(notes, speech, duration));
+}
+
+/** A clip's window of its source stream, in the clip's own time. */
+async function clipVtt(clip) {
+ if (!clip || clip.stream_id == null) return '';
+ const start = Math.max(0, Number(clip.start_time) || 0);
+ const end = Number(clip.end_time) || (start + (Number(clip.duration_seconds) || 0));
+ const duration = Number(clip.duration_seconds) || Math.max(0, end - start);
+ if (!(duration > 0)) return '';
+ let rows = [];
+ let memories = [];
+ try { rows = await db.getTimeline(Number(clip.stream_id), { limit: 20000 }) || []; } catch { rows = []; }
+ try { memories = await db.getStreamMemories(Number(clip.stream_id)) || []; } catch { memories = []; }
+ if (!memories.length) return '';
+ const speech = rows
+ .filter((r) => r.kind === 'speech' && Number(r.start_sec) <= end + 1 && Number(r.end_sec == null ? r.start_sec : r.end_sec) >= start - 1)
+ .map((r) => ({ start: Math.max(0, Number(r.start_sec) - start), end: Math.max(0, Number(r.end_sec == null ? r.start_sec + 2 : r.end_sec) - start) }));
+ const notes = memories
+ .filter((m) => Number(m.offset_seconds) >= start - 1 && Number(m.offset_seconds) <= end + 1)
+ .map((m) => ({ at: Math.max(0, Number(m.offset_seconds) - start), text: m.description }));
+ return buildVtt(cueList(notes, speech, duration));
+}
+
+module.exports = { cueList, buildVtt, vodVtt, clipVtt, plainText, gapStart, timestamp };
@@ -266,6 +266,37 @@ router.get('/:id/download-url', optionalAuth, async (req, res) => {
@@ -266,6 +266,37 @@ router.get('/:id/download-url', optionalAuth, async (req, res) => {
}
}
});
});
+/**
+ * GET /api/clips/:id/descriptions.vtt — the clip's audio description (X1), the source stream's scene notes
+ * inside the clip's window, in the clip's own time and in the silences between speech. Same visibility rule
+ * as GET /api/clips/:id: public and link clips for anyone, a private clip for its owner or staff only.
+ */
+router.get('/:id/descriptions.vtt', optionalAuth, async (req, res) => {
+ try {
+ if (!/^\d+$/.test(String(req.params.id))) return clipNotFound(res);
+ let clip;
+ try { clip = await media.getClip(req.params.id); }
+ catch (err) {
+ if (err && err.name === 'MediaApiError' && err.status === 404) return clipNotFound(res);
+ return mediaErr(res, err, 'Failed to get clip');
+ }
+ if (!clip || !await access.canView(req.user, clip)) return clipNotFound(res);
+ if (clip.app_id && clip.app_id !== 'live') return clipNotFound(res);
+ if (clip.status && clip.status !== 'ready') return clipNotFound(res);
+ const vtt = await require('../ai/descriptions').clipVtt(clip);
+ if (!vtt) return clipNotFound(res);
+ res.set({
+ 'Content-Type': 'text/vtt; charset=utf-8',
+ 'Cache-Control': access.isPrivate(clip) ? 'private, no-store' : 'public, max-age=60',
+ 'X-Content-Type-Options': 'nosniff',
+ });
+ res.send(vtt);
+ } catch (err) {
+ console.error('[Clips] descriptions error:', err.message);
+ if (!res.headersSent) res.status(500).json({ error: 'Failed to load the descriptions' });
+ }
+});
+
router.get('/:id', optionalAuth, async (req, res) => {
router.get('/:id', optionalAuth, async (req, res) => {
try {
try {
if (!/^\d+$/.test(req.params.id)) return clipNotFound(res);
if (!/^\d+$/.test(req.params.id)) return clipNotFound(res);
new file mode 100644
new file mode 100644
@@ -0,0 +1,28 @@
@@ -0,0 +1,28 @@
+/**
+ * OpenVibe.Live — a stream's rows in one of its VODs' own time.
+ *
+ * A stream can be recorded in parts (a broadcaster reconnecting), and each part is its own VOD; the
+ * rows Live keeps (scene notes, speech, sound) are in stream time. The first part of a joined recording
+ * answers for the whole video (streaming/recordings.js); a VOD that is one part of its stream keeps the
+ * rows inside it, moved to its time (its chapters pointed eleven minutes late on part 2 of stream 2795).
+ * `round` keeps whole seconds, which is what caption and description cues want.
+ */
+'use strict';
+
+async function inVodTime(vod, rows, key, { round = true } = {}) {
+ const fix = (v) => (round ? Math.round(v) : Math.round(v * 100) / 100);
+ if (vod.recording && Number(vod.recording.vod_id) === Number(vod.id) && vod.recording.hls_url && (vod.recording.parts || []).length > 1) {
+ const recordings = require('../streaming/recordings');
+ const parts = await recordings.presentParts(vod.recording);
+ return rows.map((r) => ({ r, at: recordings.toRecordingTime(parts, Number(r[key])) }))
+ .filter((x) => x.at != null).map(({ r, at }) => ({ ...r, [key]: fix(at), ...(r.end != null ? { end: fix(at + (Number(r.end) - Number(r[key]))) } : {}) }));
+ }
+ const part = await require('../streaming/stream-history').partForVod(vod.id);
+ if (!part) return rows;
+ const len = part.end != null ? part.end - part.at : Infinity;
+ return rows
+ .filter((r) => Number(r[key]) >= part.at && Number(r[key]) - part.at <= len)
+ .map((r) => ({ ...r, [key]: fix(Number(r[key]) - part.at), ...(r.end != null ? { end: fix(Number(r.end) - part.at) } : {}) }));
+}
+
+module.exports = { inVodTime };
@@ -25,6 +25,7 @@ function chunkOpensVod(stream, buffer) {
@@ -25,6 +25,7 @@ function chunkOpensVod(stream, buffer) {
const { requireAuth, optionalAuth } = require('../auth/auth');
const { requireAuth, optionalAuth } = require('../auth/auth');
const permissions = require('../auth/permissions');
const permissions = require('../auth/permissions');
const access = require('./access');
const access = require('./access');
+const { inVodTime } = require('./vod-time');
const purge = require('./purge');
const purge = require('./purge');
const { commentCount } = require('./comments');
const { commentCount } = require('./comments');
const contentLabels = require('../labels');
const contentLabels = require('../labels');
@@ -420,28 +421,6 @@ router.post('/bulk-delete-old', requireAuth, async (req, res) => {
@@ -420,28 +421,6 @@ router.post('/bulk-delete-old', requireAuth, async (req, res) => {
}
}
});
});
-/**
- * A stream's rows (stream time, in `key`) in this VOD's own time. The first part of a joined recording answers for the
- * whole video: every row that was recorded, in the video's time (streaming/recordings.js; the gaps between parts were
- * not). A VOD that is one part of its stream (the broadcaster reconnected) keeps the rows inside it, moved to its time:
- * its chapters (OpenVibe.Video) pointed eleven minutes late on part 2 of stream 2795. `round` keeps whole seconds.
- */
-async function inVodTime(vod, rows, key, { round = true } = {}) {
- const fix = (v) => (round ? Math.round(v) : Math.round(v * 100) / 100);
- if (vod.recording && Number(vod.recording.vod_id) === Number(vod.id) && vod.recording.hls_url && (vod.recording.parts || []).length > 1) {
- const recordings = require('../streaming/recordings');
- const parts = await recordings.presentParts(vod.recording);
- return rows.map((r) => ({ r, at: recordings.toRecordingTime(parts, Number(r[key])) }))
- .filter((x) => x.at != null).map(({ r, at }) => ({ ...r, [key]: fix(at), ...(r.end != null ? { end: fix(at + (Number(r.end) - Number(r[key]))) } : {}) }));
- }
- const part = await require('../streaming/stream-history').partForVod(vod.id);
- if (!part) return rows;
- const len = part.end != null ? part.end - part.at : Infinity;
- return rows
- .filter((r) => Number(r[key]) >= part.at && Number(r[key]) - part.at <= len)
- .map((r) => ({ ...r, [key]: fix(Number(r[key]) - part.at), ...(r.end != null ? { end: fix(Number(r.end) - part.at) } : {}) }));
-}
-
// AI "memory" timeline for a VOD (stream_memories stays in live.db).
// AI "memory" timeline for a VOD (stream_memories stays in live.db).
router.get('/:id/memories', optionalAuth, async (req, res) => {
router.get('/:id/memories', optionalAuth, async (req, res) => {
try {
try {
@@ -481,6 +460,34 @@ router.get('/:id/sounds', optionalAuth, async (req, res) => {
@@ -481,6 +460,34 @@ router.get('/:id/sounds', optionalAuth, async (req, res) => {
}
}
});
});
+/**
+ * GET /api/vods/:id/descriptions.vtt — the VOD's audio description as a WebVTT `kind="descriptions"` track
+ * (X1, consent plan 2026-10-31): short AI scene notes from Live's timeline, timed into the silences between
+ * speech. Public and link VODs are described for anyone; a private VOD only answers its owner (or staff), so
+ * the same visibility rule as every other VOD read applies. People are described; sex acts never are
+ * (server/ai/descriptions.js + explicit.js). 404 when there is nothing to describe.
+ */
+router.get('/:id/descriptions.vtt', optionalAuth, async (req, res) => {
+ try {
+ if (!/^\d+$/.test(String(req.params.id))) return vodNotFound(res);
+ const vod = await loadVisibleVod(req, res, req.params.id);
+ if (!vod) return;
+ if (vod.app_id && vod.app_id !== 'live') return vodNotFound(res);
+ if (vod.is_recording || vod.status === 'recording') return vodNotFound(res);
+ const vtt = await require('../ai/descriptions').vodVtt(vod);
+ if (!vtt) return vodNotFound(res);
+ res.set({
+ 'Content-Type': 'text/vtt; charset=utf-8',
+ 'Cache-Control': access.isPrivate(vod) ? 'private, no-store' : 'public, max-age=60',
+ 'X-Content-Type-Options': 'nosniff',
+ });
+ res.send(vtt);
+ } catch (err) {
+ console.error('[VODs] descriptions error:', err.message);
+ if (!res.headersSent) res.status(500).json({ error: 'Failed to load the descriptions' });
+ }
+});
+
/**
/**
* GET /api/vods/:id/context — everything around a VOD that isn't the file: the stream it came
* GET /api/vods/:id/context — everything around a VOD that isn't the file: the stream it came
* from, its activity numbers (chat, viewers, mic moments, follows, clips), the clips cut from
* from, its activity numbers (chat, viewers, mic moments, follows, clips), the clips cut from
new file mode 100644
new file mode 100644
@@ -0,0 +1,158 @@
@@ -0,0 +1,158 @@
+'use strict';
+/**
+ * Audio description (X1, consent plan 2026-10-31, "AI and data"): Live's scene notes become a WebVTT
+ * `kind="descriptions"` track, timed into the silences between speech. Public and link VODs/clips are
+ * described for anyone; a private one only answers its owner (server/ai/descriptions.js + the routes in
+ * media-proxy/vods.js and clips.js). The AI's text may describe people; it never describes sex acts, and a
+ * video with an explicit scene note gets no track at all (server/ai/explicit.js).
+ *
+ * node test/audio-description.test.js
+ */
+const assert = require('assert');
+const http = require('http');
+
+process.env.NODE_ENV = 'test';
+const quiet = console.log;
+console.log = (...a) => { if (!/^\[/.test(String(a[0]))) quiet(...a); };
+console.warn = () => {};
+
+const VODS = {
+ 100: { id: 100, app_id: 'live', user_id: 3, stream_id: 500, title: 'Public VOD', visibility: 'public', is_public: 1, status: 'ready', duration_seconds: 60 },
+ 101: { id: 101, app_id: 'live', user_id: 3, stream_id: 500, title: 'Private VOD', visibility: 'private', is_public: 0, status: 'ready', duration_seconds: 60 },
+ 102: { id: 102, app_id: 'live', user_id: 3, stream_id: 501, title: 'Explicit VOD', visibility: 'public', is_public: 1, status: 'ready', duration_seconds: 30 },
+ 103: { id: 103, app_id: 'games', user_id: 3, stream_id: 500, title: 'Another app', visibility: 'public', is_public: 1, status: 'ready', duration_seconds: 60 },
+};
+const CLIPS = {
+ 200: { id: 200, app_id: 'live', user_id: 3, stream_id: 500, vod_id: 100, title: 'Public clip', visibility: 'public', is_public: 1, status: 'ready', start_time: 3, end_time: 15, duration_seconds: 12 },
+ 201: { id: 201, app_id: 'live', user_id: 3, stream_id: 500, vod_id: 100, title: 'Private clip', visibility: 'private', is_public: 0, status: 'ready', start_time: 3, end_time: 15, duration_seconds: 12 },
+};
+
+function cuesOf(vtt) {
+ const out = [];
+ const re = /(\d{2}:\d{2}:\d{2}\.\d{3}) --> (\d{2}:\d{2}:\d{2}\.\d{3})\n([^\n]+)/g;
+ let m;
+ const sec = (t) => {
+ const [h, mm, rest] = t.split(':');
+ const [s, ms] = rest.split('.');
+ return Number(h) * 3600 + Number(mm) * 60 + Number(s) + Number(ms) / 1000;
+ };
+ while ((m = re.exec(vtt))) out.push({ start: sec(m[1]), end: sec(m[2]), text: m[3] });
+ return out;
+}
+
+(async () => {
+ const db = require('../server/db/database');
+ await db.initDb();
+ const raw = db.getDb();
+ // Fixed ids: the Media stubs below name them, and the person the owner check expects.
+ await raw.prepare(`INSERT INTO users (id, username, display_name, stream_key) OVERRIDING SYSTEM VALUE VALUES (?, ?, ?, ?)`)
+ .run(3, 'alice', 'alice', 'a'.repeat(32));
+ await raw.prepare(`INSERT INTO users (id, username, display_name, stream_key) OVERRIDING SYSTEM VALUE VALUES (?, ?, ?, ?)`)
+ .run(4, 'bob', 'bob', 'b'.repeat(32));
+ const alice = 3;
+ for (const [id, title] of [[500, 'Described stream'], [501, 'Explicit stream']]) {
+ await raw.prepare(`INSERT INTO streams (id, user_id, title, category, protocol, is_live, started_at) OVERRIDING SYSTEM VALUE VALUES (?, ?, ?, 'irl', 'rtmp', 1, ov_now())`)
+ .run(id, alice, title);
+ }
+ const s500 = 500, s501 = 501;
+
+ // Scene notes: one in a gap, one during speech, and an explicit note on another stream.
+ await db.addStreamMemory({ stream_id: s500, user_id: alice, offset_seconds: 5, description: 'A person in a red coat opens a blue door.' });
+ await db.addStreamMemory({ stream_id: s500, user_id: alice, offset_seconds: 9, description: 'A dog sleeps on the sofa.' });
+ await db.addStreamMemory({ stream_id: s501, user_id: alice, offset_seconds: 5, description: 'A person waves at the camera.' });
+ await db.addStreamMemory({ stream_id: s501, user_id: alice, offset_seconds: 12, description: 'The streamer is having sex on camera.' });
+ await db.addTimelineEvents([
+ { stream_id: s500, user_id: alice, vod_id: 100, kind: 'speech', start_sec: 0, end_sec: 4, text: 'Hello and welcome.' },
+ { stream_id: s500, user_id: alice, vod_id: 100, kind: 'speech', start_sec: 8, end_sec: 12, text: 'Today we build a table.' },
+ ]);
+
+ // ── The cue builder on its own: gaps, no overlap, dedupe, and the explicit fail-closed rule ──
+ const { cueList, buildVtt, gapStart } = require('../server/ai/descriptions');
+ const speech = [{ start: 0, end: 4 }, { start: 8, end: 12 }];
+ const list = cueList([
+ { at: 5, text: 'A person in a red coat opens a blue door.' },
+ { at: 9, text: 'A dog sleeps on the sofa.' },
+ { at: 5, text: 'A person in a red coat opens a blue door.' },
+ ], speech, 60);
+ assert.strictEqual(list.length, 2, 'a repeated note becomes one cue');
+ for (const cue of list) {
+ assert.ok(cue.end > cue.start, 'a cue has duration');
+ for (const s of speech) {
+ assert.ok(cue.end <= s.start + 0.001 || cue.start >= s.end - 0.001, `cue "${cue.text}" does not overlap speech ${s.start}-${s.end}`);
+ }
+ }
+ assert.ok(list.find((c) => /dog/.test(c.text)).start >= 12.3, 'a note during speech waits for the next silence');
+ assert.ok(gapStart(5, 3, speech, 60) != null, 'a gap can be found at the note');
+ assert.ok(buildVtt(list).startsWith('WEBVTT\n'), 'WebVTT header');
+ assert.deepStrictEqual(cueList([{ at: 5, text: 'A person waves.' }, { at: 12, text: 'sexual acts on camera' }], speech, 60), [], 'one explicit note means no track');
+ assert.deepStrictEqual(cueList([{ at: 5, text: 'A nude figure drawing session.' }], speech, 60).length, 1, 'nudity is art, not explicit');
+
+ // ── The routes, with Media stubbed ──
+ const media = require('../server/media-client');
+ const notFound = (what) => new media.MediaApiError(`${what} not found`, 404, { error: `${what} not found` });
+ media.getVod = async (id) => { const v = VODS[Number(id)]; if (!v) throw notFound('VOD'); return { ...v }; };
+ media.getClip = async (id) => { const c = CLIPS[Number(id)]; if (!c) throw notFound('Clip'); return { ...c }; };
+
+ const express = require('express');
+ const app = express();
+ // optionalAuth leaves an already-present req.user alone; this stands in for a signed-in Live session.
+ app.use((req, res, next) => {
+ const as = req.headers['x-test-user'];
+ if (as) req.user = { id: Number(as), username: as === '3' ? 'alice' : 'bob', role: 'user' };
+ next();
+ });
+ app.use('/api/vods', require('../server/media-proxy/vods'));
+ app.use('/api/clips', require('../server/media-proxy/clips'));
+ const server = http.createServer(app).listen(0);
+ await new Promise((r) => (server.listening ? r() : server.once('listening', r)));
+ const get = (path, headers = {}) => new Promise((resolve, reject) => {
+ http.get({ port: server.address().port, path, headers }, (res) => {
+ let body = '';
+ res.on('data', (c) => { body += c; });
+ res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body }));
+ }).on('error', reject);
+ });
+
+ // Public VOD: a descriptions track for anyone.
+ const pub = await get('/api/vods/100/descriptions.vtt');
+ assert.strictEqual(pub.status, 200);
+ assert.match(pub.headers['content-type'], /^text\/vtt/);
+ assert.match(pub.headers['cache-control'], /public/);
+ const pubCues = cuesOf(pub.body);
+ assert.strictEqual(pubCues.length, 2);
+ assert.ok(pubCues.some((c) => /red coat/.test(c.text)));
+ assert.ok(pubCues.some((c) => /dog/.test(c.text)));
+ assert.ok(pubCues.find((c) => /dog/.test(c.text)).start >= 12.3, 'the route keeps the note out of speech');
+
+ // Private VOD: the missing-VOD answer until its owner asks; the owner and nobody else gets the track.
+ const privAnon = await get('/api/vods/101/descriptions.vtt');
+ assert.strictEqual(privAnon.status, 404, 'a private VOD is not described for a visitor');
+ assert.deepStrictEqual(privAnon.body, JSON.stringify({ error: 'VOD not found' }));
+ const privOther = await get('/api/vods/101/descriptions.vtt', { 'x-test-user': '4' });
+ assert.strictEqual(privOther.status, 404, 'not for another person');
+ const privOwner = await get('/api/vods/101/descriptions.vtt', { 'x-test-user': '3' });
+ assert.strictEqual(privOwner.status, 200, 'the owner asks and gets it');
+ assert.match(privOwner.headers['cache-control'], /private/);
+
+ // A video with an explicit scene note: no track, not even for its owner.
+ for (const who of [null, '3']) {
+ const r = await get('/api/vods/102/descriptions.vtt', who ? { 'x-test-user': who } : {});
+ assert.strictEqual(r.status, 404, 'explicit scenes are never described');
+ }
+
+ // Another app's media is not Live's to describe.
+ assert.strictEqual((await get('/api/vods/103/descriptions.vtt')).status, 404);
+
+ // A clip: the source notes inside its window, in the clip's own time, between speech.
+ const clip = await get('/api/clips/200/descriptions.vtt');
+ assert.strictEqual(clip.status, 200);
+ const clipCues = cuesOf(clip.body);
+ assert.strictEqual(clipCues.length, 2, 'both notes fall inside the clip window');
+ assert.ok(clipCues.every((c) => c.end <= 12.001), 'clip cues stay inside the clip');
+ assert.ok(clipCues.find((c) => /red coat/.test(c.text)).start <= 2.1, 'a note moves into the clip time');
+ assert.strictEqual((await get('/api/clips/201/descriptions.vtt')).status, 404, 'a private clip needs its owner');
+ assert.strictEqual((await get('/api/clips/201/descriptions.vtt', { 'x-test-user': '3' })).status, 200);
+
+ server.close();
+ console.log('audio description: all checks passed');
+})().catch((e) => { console.error(e); process.exit(1); });