openvibers/openvibe.mediahub Commits

Replay agent steps and chapters on Video VODs (#395)

OP OpenVibers committed 2 days ago · 0b604515aa · 5 files, +84 −2

Changes

ddiff --git a/public/css/video.css b/public/css/video.css
iindex 19112d2..c3dae11 100644
---- a/public/css/video.css
++++ b/public/css/video.css
@@@ -110,6 +110,16 @@ body:has(> main.ovv-page) .openvibe-navbar.ovnav-top { background-color: var(--b
.ovv-player-wrap > .ov-player-ambient { top: 24px; }
/* The player clips itself to the corners, so the wrapper can let the ambient glow (shared player.js) spill around it. */
.ovv-player-wrap > .ov-player, .ovv-player-wrap > video { border-radius: inherit; overflow: hidden; }
++.ovv-agent-overlay { position: absolute; z-index: 4; left: 12px; right: 12px; bottom: 48px; display: grid; gap: 3px; padding: 10px 13px; border-radius: 10px; background: rgba(7, 13, 26, .86); color: #fff; font-size: 13px; pointer-events: none; }
++.ovv-agent-overlay strong { font-size: 11px; letter-spacing: .06em; text-transform: uppercase; color: #a9c6ff; }
++.ovv-agent-task { font-weight: 650; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; }
++.ovv-agent-current { overflow: hidden; white-space: nowrap; text-overflow: ellipsis; }
++.ovv-agent-current::before { content: ''; display: inline-block; width: 7px; height: 7px; margin-right: 7px; border-radius: 50%; background: #60a5fa; }
++.ovv-agent-current[data-status="passed"]::before, .ovv-agent-tick[data-status="passed"] { background: #4ade80; }
++.ovv-agent-current[data-status="failed"]::before, .ovv-agent-tick[data-status="failed"] { background: #fb7185; }
++.ovv-agent-ticks { display: flex; gap: 3px; height: 3px; }
++.ovv-agent-tick { flex: 1; background: #60a5fa; border-radius: 2px; }
++@media (max-width: 600px) { .ovv-agent-overlay { bottom: 40px; padding: 7px 9px; } .ovv-agent-overlay strong, .ovv-agent-task, .ovv-agent-ticks { display: none; } }
.ovv-player-wrap.ov-player-host { box-shadow: none; border-color: transparent; }
/* The picture's shape once it plays (video.js followShape: --ovv-ar), eased when a recording turns mid-stream; 16:9
before that. The height is capped so the title under the player stays on screen. */
ddiff --git a/public/js/video.js b/public/js/video.js
iindex 840db78..14c89ac 100644
---- a/public/js/video.js
++++ b/public/js/video.js
@@@ -22,6 +22,40 @@
window.history.replaceState(null, '', u.pathname + '?' + u.searchParams.toString());
}
++ function agentOverlay(video) {
++ const box = document.querySelector('[data-ovv-agent]');
++ if (!box || !video) return;
++ let steps;
++ try { steps = JSON.parse(box.getAttribute('data-ovv-agent')); } catch { return; }
++ if (!Array.isArray(steps)) return;
++ const current = box.querySelector('.ovv-agent-current');
++ const timeline = box.querySelector('.ovv-agent-ticks');
++ let shown = -2;
++ function sync() {
++ let index = -1;
++ for (let i = 0; i < steps.length; i++) {
++ if (steps[i].at <= video.currentTime + .25) index = i;
++ else break;
++ }
++ if (index === shown) return;
++ shown = index;
++ const step = index >= 0 ? steps[index] : null;
++ current.textContent = step ? step.title : 'Waiting for the first step';
++ current.setAttribute('data-status', step ? step.status : 'started');
++ timeline.replaceChildren();
++ steps.slice(Math.max(0, index - 11), index + 1).forEach(function (item) {
++ const tick = document.createElement('span');
++ tick.className = 'ovv-agent-tick';
++ tick.setAttribute('data-status', item.status);
++ tick.title = item.title;
++ timeline.appendChild(tick);
++ });
++ }
++ video.addEventListener('timeupdate', sync);
++ video.addEventListener('seeked', sync);
++ sync();
++ }
++
// Media's segment timeline (HLS) when the recording has one: native where the browser plays HLS (Safari, iOS),
// otherwise hls.js (self-hosted, loaded only then). The file in src stays the fallback: a playlist that fails
// falls back to it once, at the same position.
@@@ -564,6 +598,7 @@
const video = document.querySelector('video[data-ovv-video]');
transcript(video);
if (!video) return;
++ agentOverlay(video);
// Keep one local resume point for the app home. The URL and time stay on this device.
let lastResumeWrite = 0;
video.addEventListener('timeupdate', function () {
ddiff --git a/server/video/live-client.js b/server/video/live-client.js
iindex 659ab6c..c47ff1e 100644
---- a/server/video/live-client.js
++++ b/server/video/live-client.js
@@@ -241,6 +241,10 @@ function createLiveClient({ config, fetchImpl = globalThis.fetch, log = console
vodMemories(id) {
return getJson(`/api/vods/${id}/memories`);
},
++ agentSteps(id) {
++ return getJson(`/api/vods/${id}/agent-steps`);
++ },
++ agentStepsAs: (token, id) => asPerson(token, 'GET', `/api/vods/${id}/agent-steps`),
/** What was heard in a VOD, as caption cues in its own time ("[laughter]"; Live's sound-captions.js), or null. */
vodSounds(id) {
return getJson(`/api/vods/${id}/sounds`);
ddiff --git a/server/video/pages.js b/server/video/pages.js
iindex 73ffdc8..3e1c594 100644
---- a/server/video/pages.js
++++ b/server/video/pages.js
@@@ -1345,6 +1345,12 @@ ${found ? sections.map((x) => (x.cards.length || kind ? html`<section class="ovv
if (!live.isId(id)) return notFoundVtt(req, res);
const data = await live.vod(id);
if (!data || !data.vod) return notFoundVtt(req, res);
++ const agent = await settle(live.agentSteps(id));
++ if (agent && Array.isArray(agent.steps)) {
++ const list = agent.steps.filter((s) => s.status === 'started' && Number.isFinite(Number(s.offset_seconds)))
++ .map((s) => ({ offset_seconds: Number(s.offset_seconds), description: s.title }));
++ return list.length ? sendVtt(res, renderChaptersVtt(list, data.vod.duration_seconds)) : notFoundVtt(req, res);
++ }
let mems = [];
try {
const m = await live.vodMemories(id);
@@@ -1427,7 +1433,7 @@ ${found ? sections.map((x) => (x.cards.length || kind ? html`<section class="ovv
// Chapters, comments, chat replay, the playlist form and up next do not depend on each other: asked together
// (one after another, five services' answers in turn, the page's p95 was 581 ms on 2026-10-17).
const fail = Symbol('failed');
-- const [m, cm, replay, saveForm, up, hist, more] = await Promise.all([
++ const [m, cm, replay, saveForm, up, hist, more, agent] = await Promise.all([
// A private recording's moments are Live's answer to its owner only.
Promise.resolve().then(() => (ownOnly ? asPerson(req, res, (token) => live.vodMemoriesAs(token, id)) : live.vodMemories(id))).catch(() => fail),
Promise.resolve().then(() => live.comments('vod', id)).catch(() => fail),
@@@ -1438,9 +1444,13 @@ ${found ? sections.map((x) => (x.cards.length || kind ? html`<section class="ovv
// parts (its broadcaster reconnected) the other parts, to move between.
Promise.resolve().then(() => (live.isId(vod.stream_id) ? live.streamHistory(vod.stream_id) : null)).catch(() => fail),
moreFromChannel(vod.username, vod.display_name, 'vod', id).catch(() => ''),
++ Promise.resolve().then(() => (ownOnly ? asPerson(req, res, (token) => live.agentStepsAs(token, id)) : live.agentSteps(id))).catch(() => fail),
]);
let mems = [];
try { if (m !== fail) mems = chapters(moments(m && m.memories), vod.duration_seconds); } catch { mems = []; }
++ const agentSteps = agent !== fail && agent && Array.isArray(agent.steps) ? agent.steps : null;
++ if (agentSteps) mems = agentSteps.filter((s) => s.status === 'started' && Number.isFinite(Number(s.offset_seconds)))
++ .map((s) => ({ offset_seconds: Number(s.offset_seconds), description: String(s.title || '').slice(0, 120) }));
let threadUrl = null;
let commentRows = [];
let commentsOk = false;
@@@ -1505,7 +1515,7 @@ ${found ? sections.map((x) => (x.cards.length || kind ? html`<section class="ovv
<div class="ovv-col ovv-col--main">
<div class="ovv-stage">
${savedNotice(req)}${ownOnly ? notice('Private: only you (and OpenVibe staff) can see this recording.', 'info') : ''}
--<div class="ovv-player-wrap"${labelsUi.attrs(labelsUi.inForce(vod), { owner: vod.username, opened: true })}><video class="ovv-player" data-ov-player${miniAttr(vod.title || 'Stream recording', channelName || vod.username, watchPath, fileOf(vod))} controls preload="metadata" playsinline${posterAttr(httpsUrl(vod.thumbnail_url))}${avatarAttr(vod.username, config.networkUrl)}${shapeAttr} src="${fileOf(vod) || ''}" data-ovv-video${nextAttr}${viewAttr}${hlsAttr}>${segs.length ? html`<track kind="captions" label="Transcript (AI)" src="/v/${id}/captions.vtt" default>` : ''}${mems.length ? html`<track kind="chapters" src="/v/${id}/chapters.vtt">` : ''}</video>${pipsTag(pips)}</div>
++<div class="ovv-player-wrap"${labelsUi.attrs(labelsUi.inForce(vod), { owner: vod.username, opened: true })}><video class="ovv-player" data-ov-player${miniAttr(vod.title || 'Stream recording', channelName || vod.username, watchPath, fileOf(vod))} controls preload="metadata" playsinline${posterAttr(httpsUrl(vod.thumbnail_url))}${avatarAttr(vod.username, config.networkUrl)}${shapeAttr} src="${fileOf(vod) || ''}" data-ovv-video${nextAttr}${viewAttr}${hlsAttr}>${segs.length ? html`<track kind="captions" label="Transcript (AI)" src="/v/${id}/captions.vtt" default>` : ''}${mems.length && !ownOnly ? html`<track kind="chapters" src="/v/${id}/chapters.vtt">` : ''}</video>${pipsTag(pips)}${agentSteps ? html`<div class="ovv-agent-overlay" data-ovv-agent="${JSON.stringify(agentSteps.map((s) => ({ at: Number(s.offset_seconds), title: s.title, status: s.status, tests: s.tests, cost: s.cost_usd })))}"><strong>AI agent</strong><span class="ovv-agent-task">${agent.task || vod.title}</span><span class="ovv-agent-current" aria-live="polite">Waiting for the first step</span><span class="ovv-agent-ticks" aria-label="Step timeline"></span></div>` : ''}</div>
<div class="ovv-head">
${titleLine('Stream recording', vod.title || 'Stream recording')}
${labelsUi.chips(labelsUi.inForce(vod))}
ddiff --git a/test/video-pages.test.js b/test/video-pages.test.js
iindex 6d20891..2a1ea52 100644
---- a/test/video-pages.test.js
++++ b/test/video-pages.test.js
@@@ -185,6 +185,15 @@ function startLive() {
if (memories[1] === '101') return json(200, { memories: [{ offset_seconds: 10, description: 'Intro & setup' }, { offset_seconds: 30, description: 'The <main> part' }] });
return json(200, { memories: [] });
}
++ const agentSteps = /^\/api\/vods\/(\d+)\/agent-steps$/.exec(p);
++ if (agentSteps) {
++ if (agentSteps[1] === '104') return json(200, { task: 'Fix checkout', steps: [
++ { offset_seconds: 2, title: 'Run checkout tests', status: 'started' },
++ { offset_seconds: 8, title: 'A test failed', status: 'failed' },
++ { offset_seconds: 15, title: 'Try again', status: 'started' },
++ ] });
++ return json(404, { error: 'not an agent recording' });
++ }
// Live's stream history: stream 1001 (VOD 101) was streamed upright from a phone, then turned sideways later.
const history = /^\/api\/streams\/(\d+)\/history$/.exec(p);
if (history) {
@@@ -398,6 +407,20 @@ function countOccurrences(haystack, needle) {
assert.strictEqual(q.get('title'), 'A public recording');
});
++ await check('an agent recording replays its steps and takes chapters from started steps', async () => {
++ const page = await getVideo('/v/104');
++ assert.strictEqual(page.status, 200);
++ assert.ok(page.text.includes('data-ovv-agent='));
++ assert.ok(page.text.includes('Fix checkout'));
++ assert.ok(page.text.includes('Run checkout tests'));
++ assert.ok(page.text.includes('Try again'));
++ const track = await getVideo('/v/104/chapters.vtt');
++ assert.strictEqual(track.status, 200);
++ assert.ok(track.text.includes('Run checkout tests'));
++ assert.ok(track.text.includes('Try again'));
++ assert.ok(!track.text.includes('A test failed'));
++ });
++
await check('the clip watch page attributes its maker and links the full recording', async () => {
const r = await getVideo('/c/201');
assert.strictEqual(r.status, 200);