openvibers/openvibe.live Commits
CLIP-2: 48-hour freshness on stream end, Live-stream quietness, atomic archive reservation (#434)
Changes
@@@ -21,6 +21,8 @@ const ai = require('./ai-analysis');const aiService = require('./ai-service');const media = require('../media-client');const registry = require('./moment-registry');++const clipPolicy = require('./clip-policy');++const dedupe = require('./clip-dedupe');const explicit = require('./explicit');let cfg = null; try { cfg = require('../config'); } catch { /* */ }let thumb = null; try { thumb = require('../media-proxy/live-thumbs'); } catch { /* optional */ }@@@ -195,6 +197,14 @@ function _nearestMemory(memories, offset) {for (const m of memories) { const d = Math.abs((m.offset_seconds || 0) - offset); if (d < bestD) { bestD = d; best = m; } }return best;}++/** Transcript text around a moment, so dedupe can tell a new conversation from a repeated scene. */++function _speechNear(transcript, offset, windowSec = 45) {++ try {++ return (transcript || [])++ .filter(x => Math.abs(Number(x.start || 0) - Number(offset || 0)) <= windowSec)++ .map(x => String(x.text || '')).join(' ').replace(/\s+/g, ' ').trim().slice(0, 600);++ } catch { return ''; }++}// flavor: 'paste' → a frame that is striking on its own; 'clip' → a beat that plays out over ~25 s.// avoid: offsets (seconds) already turned into something — the picker must stay ≥ 2 min away.async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {@@@ -225,7 +235,7 @@ async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {t = clamp(t);if (!farFromUsed(t)) { console.log(`[AI-Moments] VOD ${vod.vod_id}: pick at ${_mmss(t)} is next to a used moment — trying the objective signals instead`); throw new Error('near used'); }const near = _nearestMemory(ctx.memories, t);-- const result = { offset: t, title: _publicTitle(j.title, vod, t, j.desc), desc: db.noLooks(_cleanText(_deJson(j.desc), 400) || (near && _deJson(near.description)) || ''), tags: _memTags(near) };++ const result = { offset: t, title: _publicTitle(j.title, vod, t, j.desc), desc: db.noLooks(_cleanText(_deJson(j.desc), 400) || (near && _deJson(near.description)) || ''), speech: _speechNear(ctx.transcript, t), tags: _memTags(near) };if (explicit.isExplicit(j.desc, j.title, near && near.description, result.desc)) { console.log(`[AI-Moments] VOD ${vod.vod_id}: the pick at ${_mmss(t)} shows sexual content — no moment from it`); return null; }return _isEmptyScene(result.desc) ? null : result;}@@@ -244,7 +254,7 @@ async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {const near = _nearestMemory(ctx.memories, offset);const desc = near ? db.noLooks(_deJson(near.description)) : '';if (near && explicit.isExplicit(near.description)) return null; // never a moment of sexual content (server/ai/explicit.js)-- return _isEmptyScene(desc) ? null : { offset, title: _fallbackTitle(vod, offset, desc), desc, tags: _memTags(near) };++ return _isEmptyScene(desc) ? null : { offset, title: _fallbackTitle(vod, offset, desc), desc, speech: _speechNear(ctx.transcript, offset), tags: _memTags(near) };}// True when a description says the frame is essentially empty — black/dark/loading/blank — so we// never turn "nothing" into a clip or paste.@@@ -273,27 +283,54 @@ async function _frameTooDark(imgPath) {} catch { return false; }}--// Coarse content signature for dedup — the first few significant words of a scene description.--// Near-identical scenes ("A split-image close-up of an older man…") collapse to one signature.--function _sig(desc) {-- return String(desc || '').toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim().split(' ').filter(Boolean).slice(0, 5).join(' ');--}--// TODO(contract): the old duplicate-paste sweep pruned the local pastes table, which is// frozen for the OpenVibe.Media migration; Media has no metadata-filtered paste listing// yet, so scene-dedup is handled purely via the usedSigs log below.function _dedupePastes() { /* moved to Media — sig-dedup happens pre-post */ }--// Build the moment-ranking VOD pool: Media list (most-viewed public VODs) joined with--// local AI state + stream memories. Media down → an empty pool (the run makes nothing).++// Build the moment-ranking VOD pool: newest public VODs from Media joined with local AI state +++// stream memories, filtered by the shared freshness policy (clip-policy.js): footage from the++// last 48h normally; the archive only after 72h of site quietness, one archival clip per UTC day.++// Media down → an empty pool (and never "quiet": an outage does not open the archive).async function _momentPool(limit) {try {-- const r = await media.listVods({ limit, order: 'views' });-- // A channel that turned AI Moments off (channels.ai_derivation_enabled) is never picked from.++ const now = Date.now();++ const r = await media.listVods({ limit: Math.max(24, limit * 2), order: 'newest' });++ const all = r?.vods || (Array.isArray(r) ? r : []);++ const history = require('../streaming/stream-history');++ const site = clipPolicy.siteStatus({ now, ...await clipPolicy.siteActivity(db, { now }), upstreamHealthy: true });++ let clipLog = [];++ try { const l = JSON.parse(await db.getState('auto_clip_log') || '[]'); if (Array.isArray(l)) clipLog = l; } catch { /* */ }++ // The newest listed VOD per channel gives us the row (for `newest`), but the channel's++ // latest ACTIVITY comes from Live's streams: a channel can be live right now without a new++ // VOD, and that live stream must keep it out of the quiet/archive exception.++ const firstByUser = new Map();++ for (const v of all) if (v.user_id != null && !firstByUser.has(v.user_id)) firstByUser.set(v.user_id, v);++ const channelByUser = new Map();++ for (const [userId, first] of firstByUser) {++ const latest = await clipPolicy.latestStreamForUser(db, userId);++ const lastAt = clipPolicy.streamActivityAt(latest, now);++ channelByUser.set(userId, {++ quiet: clipPolicy.channelStatus({ now, lastStreamAt: lastAt }).quiet,++ stream_id: latest ? latest.id : null, vod_id: first.id,++ });++ }++ const anyQuiet = site.quiet || [...channelByUser.values()].some((c) => c.quiet);++ const archivalAllowed = anyQuiet && clipPolicy.archiveAllowed(clipLog, now);const rows = [];-- for (const v of r?.vods || (Array.isArray(r) ? r : [])) {-- try { if (await db.isAiDerivationEnabled(v.user_id)) rows.push(v); }-- catch { rows.push(v); }++ for (const v of all) {++ try { if (!await db.isAiDerivationEnabled(v.user_id)) continue; } catch { /* opt-out unreadable: keep it */ }++ let stream = null, part = null;++ if (v.stream_id) {++ try { stream = await db.get('SELECT started_at, ended_at, is_live, peak_viewers FROM streams WHERE id = ?', [v.stream_id]); } catch { /* unknown age */ }++ try { part = await history.partForVod(v.id); } catch { /* no part row */ }++ }++ const ch = channelByUser.get(v.user_id);++ const channel = ch ? { quiet: ch.quiet, newest: ch.stream_id ? v.stream_id === ch.stream_id : v.id === ch.vod_id } : null;++ const verdict = clipPolicy.classifySource({ vod: v, stream, part, now, site, channel });++ if (!verdict.ok || (verdict.archival && !archivalAllowed)) continue;++ rows.push({ ...v, archival: verdict.archival, captured_at: verdict.capturedAt, age_hours: verdict.ageHours, peak_viewers: stream ? stream.peak_viewers : 0 });++ if (rows.length >= limit) break;}if (rows.length) {return (await Promise.all(rows.map(async (v) => {@@@ -302,10 +339,9 @@ async function _momentPool(limit) {// (pastes read "@null's stream" without it).let username = v.username || null;if (!username && v.user_id) { try { username = (await db.getUserById(v.user_id))?.username || null; } catch { /* */ } }-- let memory_count = 0, peak_viewers = 0;++ let memory_count = 0;if (v.stream_id) {try { memory_count = (await db.get('SELECT COUNT(*) AS c FROM stream_memories WHERE stream_id = ?', [v.stream_id]))?.c || 0; } catch { /* */ }-- try { peak_viewers = (await db.getStreamById(v.stream_id))?.peak_viewers || 0; } catch { /* */ }}return {vod_id: v.id, stream_id: v.stream_id || null, user_id: v.user_id || null,@@@ -314,9 +350,10 @@ async function _momentPool(limit) {ai_overview_short: state.ai_overview_short || v.ai_overview_short || '',view_count: Number(v.view_count) || 0,duration: Number(v.duration_seconds ?? v.duration) || 0,-- peak_viewers, created_at: v.created_at || null,++ peak_viewers: Number(v.peak_viewers) || 0, created_at: v.created_at || null,clip_count: Number(v.clip_count) || 0,-- memory_count,++ memory_count, archival: !!v.archival,++ captured_at: v.captured_at, age_hours: v.age_hours,};})));}@@@ -336,10 +373,12 @@ async function tick(opts = {}) {try {const prev = await _load();const usedVods = new Set(ignoreUsed ? [] : (prev.usedVods || []));-- // Content-signature dedup: even different VODs from the same streamer (same setup) yield-- // near-identical scenes → we skip a moment whose description signature was used recently,-- // so the hero/pastes never show visual duplicates. Respected even on --fresh.-- const thisSigs = new Set(); // scenes picked in THIS run (cross-run dedupe lives in moment-registry)++ // Content dedupe inside THIS run: even different VODs from the same streamer (same setup)++ // can yield near-identical scenes. The registry is the only durable "used" log; this++ // per-run list is the reservation for moments accepted before the next registry write.++ const thisMoments = []; // { desc, speech } picked in THIS run (cross-run dedupe lives in moment-registry)++ const seenThisRun = (desc, speech) => thisMoments.some((m) =>++ dedupe.isSameScene(m.desc, desc) && (!(m.speech && speech) || dedupe.isSameSpeech(m.speech, speech)));// Stage 1: consider every eligible VOD, ranked by its AI overview + popularity prior.// The VOD pool comes from OpenVibe.Media (most-viewed public VODs), overlaid with@@@ -380,14 +419,13 @@ async function tick(opts = {}) {// Stage 2: find the best moment within this VOD.const moment = await _findBestMoment(v);if (!moment) continue;-- // Skip near-duplicate scenes / spots — the registry is shared with the clip job.-- const sig = _sig(moment.desc || v.ai_overview_short || v.title);-- const why0 = await registry.usedReason({ vod_id: v.vod_id, stream_id: v.stream_id, offset: moment.offset, sig });-- if (why0 || (sig && thisSigs.has(sig))) {-- console.log(`[AI-Moments] Skipped VOD ${v.vod_id} — ${why0 || `duplicate scene ("${sig}")`}`);++ // Skip near-duplicate scenes / spots — the registry is shared with the clip job, and++ // seenThisRun closes the gap before this run's first registry write.++ const why0 = await registry.usedReason({ vod_id: v.vod_id, stream_id: v.stream_id, offset: moment.offset, desc: moment.desc, title: moment.title, speech: moment.speech });++ if (why0 || seenThisRun(moment.desc, moment.speech)) {++ console.log(`[AI-Moments] Skipped VOD ${v.vod_id} — ${why0 || 'duplicate scene in this run'}`);continue;}-- if (sig) thisSigs.add(sig);const offset = Math.floor(moment.offset || 0);let desc = moment.desc || '';let title = moment.title || _fallbackTitle(v, offset, desc);@@@ -455,14 +493,13 @@ async function tick(opts = {}) {}if (!desc) desc = _cleanText(v.ai_overview_short || v.ai_overview, 400) || 'A standout moment from this stream.';-- // Re-check the signature against the FINAL (vision-verified) description — this is-- // what actually catches same-scene duplicates (e.g. a split-image close-up re-picked).-- const finalSig = _sig(desc);-- if (finalSig && finalSig !== sig && (await registry.isUsed({ sig: finalSig }) || thisSigs.has(finalSig))) {-- console.log(`[AI-Moments] Skipped VOD ${v.vod_id} — duplicate scene after vision ("${finalSig}")`);++ // Re-check against the FINAL (vision-verified) description — this is what actually++ // catches same-scene duplicates (e.g. a split-image close-up re-picked).++ if ((await registry.isUsed({ desc, title, speech: moment.speech })) || seenThisRun(desc, moment.speech)) {++ console.log(`[AI-Moments] Skipped VOD ${v.vod_id} — duplicate scene after vision`);continue;}-- if (finalSig) thisSigs.add(finalSig);++ thisMoments.push({ desc, speech: moment.speech });const content = `${desc}\n\n▶ Watch this moment on ${v.username ? `@${v.username}'s stream` : 'the stream'}: ${vodLink}`;const metadata = JSON.stringify({ ai_moment: true, vod_id: v.vod_id, offset, vod_link: vodPath, username: v.username, why: r.why || null });let pasteSlug = slug;@@@ -497,15 +534,15 @@ async function tick(opts = {}) {try { fs.unlinkSync(screenshotPath); } catch { /* tmp frame */ }moments.push({ vodId: v.vod_id, offset, title: title.slice(0, 80), thumbnail: img, username: v.username, pasteSlug });-- await registry.record({ kind: 'paste', vod_id: v.vod_id, stream_id: v.stream_id, offset, sig: finalSig || sig, title });++ await registry.record({ kind: 'paste', vod_id: v.vod_id, stream_id: v.stream_id, offset, title, desc, speech: moment.speech });// Also cut a VOD clip from the SAME VOD — but a DIFFERENT beat (clip flavor, ≥ 2 min away),// so the clip and the paste never show the identical second with the identical title.if (opts.clip !== false && momentSource) {try {const second = await _findBestMoment(v, { flavor: 'clip', avoid: [offset] });-- if (second && !await registry.isUsed({ vod_id: v.vod_id, stream_id: v.stream_id, offset: second.offset, desc: second.desc, title: second.title })) {-- const clip = await require('./auto-clip-job').clipVodMoment({ vod: v, offset: second.offset, title: second.title, desc: second.desc, source: momentSource });++ if (second && !await registry.isUsed({ vod_id: v.vod_id, stream_id: v.stream_id, offset: second.offset, desc: second.desc, title: second.title, speech: second.speech })) {++ const clip = await require('./auto-clip-job').clipVodMoment({ vod: v, offset: second.offset, title: second.title, desc: second.desc, speech: second.speech, archival: v.archival, source: momentSource });if (clip) console.log(`[AI-Moments] Auto-clip cut for VOD ${v.vod_id} at ${_mmss(second.offset)} ("${String(second.title || '').slice(0, 60)}") — paste was at ${_mmss(offset)}`);} else console.log(`[AI-Moments] VOD ${v.vod_id}: no second, distinct beat for a clip — skipped`);} catch { /* clipping is best-effort */ }@@@ -22,6 +22,8 @@ const ai = require('./ai-analysis');const media = require('../media-client');const recorder = require('../streaming/recorder');const registry = require('./moment-registry');++const clipPolicy = require('./clip-policy');++const dedupe = require('./clip-dedupe');// ── Tunables ────────────────────────────────────────────────const CHECK_INTERVAL_MS = 90 * 1000;@@@ -56,7 +58,11 @@ async function _logClip(entry) {log.unshift({ ...entry, ts: Date.now() });await db.setState(CLIP_LOG_SETTING, JSON.stringify(log.slice(0, 300)));// Shared with the paste job, so a paste never lands on a clipped second (and vice versa).-- await registry.record({ kind: 'clip', vod_id: entry.vod_id || null, stream_id: entry.stream_id || null, offset: (entry.start_time || 0) + CLIP_PRE, sig: entry.sig || null, title: entry.title || null });++ await registry.record({++ kind: 'clip', vod_id: entry.vod_id || null, stream_id: entry.stream_id || null,++ offset: (entry.start_time || 0) + CLIP_PRE, sig: entry.sig || null, title: entry.title || null,++ desc: entry.desc || null, speech: entry.speech || null,++ });} catch { /* */ }}async function _countClipsSince(streamId, minutes) {@@@ -70,7 +76,7 @@ async function _confirmLiveMoment(stream) {const memories = (await db.getStreamMemories(stream.id) || []).filter(m => m.description).slice(-5);const transcript = (await db.getStreamTranscriptSegments(stream.id) || []).slice(-14);const chat = await chatReads.recentChatText(stream.id, WINDOW_SEC, 40) || [];-- if (!await _aiOn()) return { clip: null }; // caller decides via the stricter no-AI threshold++ if (!await _aiOn()) return { clip: null, transcript }; // caller decides via the stricter no-AI threshold// Sound events are a strong clip signal on their own — a burst of gunfire, an explosion or laughter is exactly// the kind of thing viewers clip, and it is often the reason chat spiked in the first place.let sounds = [];@@@ -88,9 +94,18 @@ async function _confirmLiveMoment(stream) {sounds: sounds.map(e => ({ t: Math.max(0, Number(e.start_sec) || 0), label: String(e.label || '').slice(0, 120), confidence: Number(e.confidence || 0) })),chat: chat.slice(-30).map(c => _clean(c, 100)).filter(Boolean),}, { target: { service: 'live', type: 'stream', id: String(stream.id) }, meter: { kind: 'auto_clip_confirm', role: 'legacy', ownerUserId: stream.user_id } });-- if (!j) return { clip: false };-- return { clip: j.clip === true, title: _clean(j.title, 80), desc: _clean(j.desc, 400) };-- } catch { return { clip: false }; }++ if (!j) return { clip: false, transcript };++ return { clip: j.clip === true, title: _clean(j.title, 80), desc: _clean(j.desc, 400), transcript };++ } catch { return { clip: false, transcript }; }++}++++/** What was said around a stream second: the registry uses it so a static camera alone is not a repeat. */++function _speechNear(transcript, sec, windowSec = 45) {++ try {++ return (transcript || [])++ .filter(x => Math.abs(Number(x.start || 0) - Number(sec || 0)) <= windowSec)++ .map(x => String(x.text || '')).join(' ').replace(/\s+/g, ' ').trim().slice(0, 600);++ } catch { return ''; }}async function _checkLiveStream(stream) {@@@ -136,6 +151,13 @@ async function _checkLiveStream(stream) {// A moment the AI describes as sexual content is never clipped (server/ai/explicit.js).if (require('./explicit').isExplicit(verdict.desc, verdict.title)) { console.log(`[AutoClip] stream ${streamId}: skipped a moment the AI describes as sexual content`); return; }const title = verdict.title ? require('./ai-moments-job').publicTitle(verdict.title, stream, momentOffset, verdict.desc) : 'Chat-Hyped Moment';++ // The live path used to skip the shared registry and record only AFTER cutting, so a moment++ // already turned into a paste/clip could be cut again here. Ask before spending a clip.++ let streamSec = momentOffset; // transcripts are in stream time; momentOffset is in the VOD's time++ try { const part = await require('../streaming/stream-history').partForVod(rec.vodId); if (part && Number.isFinite(Number(part.at))) streamSec = momentOffset + Number(part.at); } catch { /* one-part recording */ }++ const speech = _speechNear(verdict.transcript, streamSec);++ const why = await registry.usedReason({ vod_id: rec.vodId, stream_id: streamId, offset: momentOffset, desc: verdict.desc, title, speech });++ if (why) { console.log(`[AutoClip] stream ${streamId}: ${why}`); return; }const clip = await media.createClip({vod_id: rec.vodId, start_s: start, end_s: end,title, user_id: stream.user_id,@@@ -144,7 +166,7 @@ async function _checkLiveStream(stream) {if (clip) {// deduplicated: Media handed back a clip someone already cut of this window. It stays in// the log for the caps, but it is theirs, not the AI's (syncAutoClipFlags skips it).-- await _logClip({ stream_id: streamId, vod_id: rec.vodId, start_time: start, title, clip_id: clip.id, dedup: !!clip.deduplicated || undefined, sig: _sceneSig(verdict.desc || title) });++ await _logClip({ stream_id: streamId, vod_id: rec.vodId, start_time: start, title, clip_id: clip.id, dedup: !!clip.deduplicated || undefined, desc: verdict.desc || '', speech, sig: _sceneSig(verdict.desc || title) });console.log(`[AutoClip] LIVE clip for stream ${streamId} @${Math.round(momentOffset)}s ("${title}") — spike ${peak.count} msgs`);}} catch (e) {@@@ -162,45 +184,77 @@ async function _tick() {} finally { _busy = false; }}--// ── Historical VOD backfill: a few auto-clips per day (daily-gated, like the moments/paste--// jobs). Processes the most-watched un-clipped VODs first; idempotent (skips VODs that already--// have an auto-clip), so it steadily works through the back catalogue over days. ─────────────++// ── Recent VOD backfill: a few auto-clips per run (daily-gated, like the moments/paste jobs).++// Only footage from the last 48 hours is eligible; the old back catalogue is mined ONLY after the++// site has been quiet for 72 hours and then at most one archive clip per UTC day (clip-policy.js).++// Newest-first, idempotent (skips VODs that already have an auto-clip). ─────────────────────const BACKFILL_SETTING = 'auto_clip_backfill';--const BACKFILL_PER_RUN = 3; // a few historical clips…--const BACKFILL_INTERVAL_MS = 6 * 60 * 60 * 1000; // …every 6h (guarantees ≥1 clip / 6h)++const BACKFILL_PER_RUN = 3; // a few recent clips…++const BACKFILL_INTERVAL_MS = 6 * 60 * 60 * 1000; // …every 6hasync function _backfillDue() {try { const p = JSON.parse(await db.getState(BACKFILL_SETTING) || '{}'); return !p.updated_at || (Date.now() - p.updated_at) >= BACKFILL_INTERVAL_MS; }catch { return true; }}--// Historical-backfill candidate pool: most-viewed public VODs from OpenVibe.Media--// that have stream memories and no auto-clip in the local log yet. Media down → an--// empty pool (the clips are cut by Media anyway).++/**++ * Backfill candidate pool: newest public VODs from OpenVibe.Media that have stream memories and no++ * auto-clip in the local log yet, filtered by the shared freshness policy. `archival` is set on a++ * row only when the quiet-site archive window is open. Media down → an empty pool (the site is not++ * "quiet", it is unreachable — an outage never opens the archive).++ */async function _backfillPool(limit) {try {-- const r = await media.listVods({ limit: limit * 2, order: 'views' });++ const now = Date.now();++ const r = await media.listVods({ limit: Math.max(24, limit * 4), order: 'newest' });const rows = r?.vods || (Array.isArray(r) ? r : []);-- if (rows.length) {-- const clipped = new Set();-- for (const c of await _clipLog()) { if (c.vod_id) clipped.add(c.vod_id); if (c.stream_id) clipped.add(`s${c.stream_id}`); }-- const candidates = [];-- for (const v of rows) {-- if (!v.stream_id || clipped.has(v.id) || clipped.has(`s${v.stream_id}`)) continue;-- if (!await _derivationOn(v.user_id)) continue;-- try {-- if ((await db.get('SELECT COUNT(*) AS c FROM stream_memories WHERE stream_id = ?', [v.stream_id]))?.c > 0) candidates.push(v);-- } catch { /* skip an unreadable stream */ }-- if (candidates.length >= limit) break;-- }-- return (await Promise.all(candidates.map(async v => ({-- vod_id: v.id, stream_id: v.stream_id, user_id: v.user_id, username: v.username,-- title: v.title || '',-- ai_overview: (db.getVodAiState && (await db.getVodAiState(v.id))?.ai_overview_short) || '',-- ai_overview_short: (db.getVodAiState && (await db.getVodAiState(v.id))?.ai_overview_short) || '',-- duration: Number(v.duration_seconds ?? v.duration) || 0,-- view_count: Number(v.view_count) || 0,-- }))));++ if (!rows.length) return [];++ const history = require('../streaming/stream-history');++ const site = clipPolicy.siteStatus({ now, ...await clipPolicy.siteActivity(db, { now }), upstreamHealthy: true });++ // The newest listed VOD per channel gives us the row (for `newest`), but the channel's++ // latest ACTIVITY comes from Live's streams: a channel can be live right now without a new++ // VOD, and that live stream must keep it out of the quiet/archive exception.++ const firstByUser = new Map();++ for (const v of rows) if (v.user_id != null && !firstByUser.has(v.user_id)) firstByUser.set(v.user_id, v);++ const channelByUser = new Map();++ for (const [userId, first] of firstByUser) {++ const latest = await clipPolicy.latestStreamForUser(db, userId);++ const lastAt = clipPolicy.streamActivityAt(latest, now);++ channelByUser.set(userId, {++ quiet: clipPolicy.channelStatus({ now, lastStreamAt: lastAt }).quiet,++ stream_id: latest ? latest.id : null, vod_id: first.id,++ });}++ const anyQuiet = site.quiet || [...channelByUser.values()].some((c) => c.quiet);++ const archivalAllowed = anyQuiet && clipPolicy.archiveAllowed(await _clipLog(), now);++ const clipped = new Set();++ for (const c of await _clipLog()) { if (c.vod_id) clipped.add(c.vod_id); if (c.stream_id) clipped.add(`s${c.stream_id}`); }++ const candidates = [];++ for (const v of rows) {++ if (candidates.length >= limit) break;++ if (!v.stream_id || clipped.has(v.id) || clipped.has(`s${v.stream_id}`)) continue;++ if (!await _derivationOn(v.user_id)) continue;++ let stream = null, part = null;++ try { stream = await db.get('SELECT started_at, ended_at, is_live FROM streams WHERE id = ?', [v.stream_id]); } catch { /* unknown age */ }++ try { part = await history.partForVod(v.id); } catch { /* no part row */ }++ const ch = channelByUser.get(v.user_id);++ const channel = ch ? { quiet: ch.quiet, newest: ch.stream_id ? v.stream_id === ch.stream_id : v.id === ch.vod_id } : null;++ const verdict = clipPolicy.classifySource({ vod: v, stream, part, now, site, channel });++ if (!verdict.ok || (verdict.archival && !archivalAllowed)) continue;++ try {++ if ((await db.get('SELECT COUNT(*) AS c FROM stream_memories WHERE stream_id = ?', [v.stream_id]))?.c > 0) {++ candidates.push({ ...v, archival: verdict.archival, captured_at: verdict.capturedAt, age_hours: verdict.ageHours });++ }++ } catch { /* skip an unreadable stream */ }++ }++ return (await Promise.all(candidates.map(async v => ({++ vod_id: v.id, stream_id: v.stream_id, user_id: v.user_id, username: v.username,++ title: v.title || '',++ ai_overview: (db.getVodAiState && (await db.getVodAiState(v.id))?.ai_overview_short) || '',++ ai_overview_short: (db.getVodAiState && (await db.getVodAiState(v.id))?.ai_overview_short) || '',++ duration: Number(v.duration_seconds ?? v.duration) || 0,++ view_count: Number(v.view_count) || 0,++ archival: !!v.archival, captured_at: v.captured_at, age_hours: v.age_hours,++ }))));} catch { /* Media down */ }return [];}@@@ -227,9 +281,11 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}// Over-fetch so skipped/dead VODs don't starve a batch.const pool = (await _backfillPool(Math.max(1, limit) * 6)).filter(v => !skip.has(v.vod_id));let made = 0;++ let archiveMade = 0;const newSkip = [];for (const v of pool) {if (made >= limit) break;++ if (v.archival && archiveMade >= clipPolicy.ARCHIVE_PER_DAY) continue; // one archive clip per UTC daytry {const source = await _resolveVodSource(v.vod_id);if (!source) { newSkip.push(v.vod_id); continue; } // media gone / unreadable@@@ -246,8 +302,9 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}if (dark) { console.log(`[AutoClip] backfill skip VOD ${v.vod_id} — dark/empty frame`); newSkip.push(v.vod_id); continue; }}} catch { /* probe optional */ }-- const clip = await clipVodMoment({ vod: v, offset: moment.offset, title: moment.title, desc: moment.desc, source });-- if (clip) { made++; console.log(`[AutoClip] Backfilled VOD ${v.vod_id} ("${String(moment.title || '').slice(0, 60)}")`); }++ const clip = await clipVodMoment({ vod: v, offset: moment.offset, title: moment.title, desc: moment.desc, speech: moment.speech, archival: v.archival, source });++ if (clip) { made++; if (v.archival) archiveMade++; console.log(`[AutoClip] Backfilled VOD ${v.vod_id}${v.archival ? ' (archive)' : ''} ("${String(moment.title || '').slice(0, 60)}")`); }++ else if (v.archival && !clipPolicy.archiveAllowed(await _clipLog(), Date.now())) continue; // another cutter took today's archive slot: not this VOD's fault, don't skip it foreverelse newSkip.push(v.vod_id); // cut failed (e.g. unseekable/short) — don't retry forever} catch (e) { newSkip.push(v.vod_id); console.warn(`[AutoClip] backfill VOD ${v.vod_id} failed:`, e.message); }}@@@ -259,45 +316,61 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}/*** Cut a clip around a moment the AI-moments pipeline already chose for a finished VOD.-- * @param {object} o { vod (row w/ user_id, stream_id, vod_id/id), offset, title, desc }++ * @param {object} o { vod (row w/ user_id, stream_id, vod_id/id), offset, title, desc, speech, archival, source }*/function _sceneSig(text) {-- return String(text || '').toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim().split(' ').filter(Boolean).slice(0, 5).join(' ');++ return dedupe.shortSignature(text, 5);}// True if a recent auto-clip is basically the same as (vod, offset, scene) we're about to cut —--// same VOD spot, or a near-identical scene signature (same streamer's repeated intro/setup).--async function _isDuplicateAutoClip(vod, offset, desc, title) {++// same VOD spot, or a near-identical scene signature (same streamer's repeated intro/setup). What++// was said is part of the comparison, so a static camera with a new conversation is not a repeat.++async function _isDuplicateAutoClip(vod, offset, desc, title, speech) {try {const vId = vod.vod_id || vod.id;-- const why = await registry.usedReason({ vod_id: vId, stream_id: vod.stream_id, offset: Math.floor(offset), sig: _sceneSig(desc || title) });++ const why = await registry.usedReason({ vod_id: vId, stream_id: vod.stream_id, offset: Math.floor(offset), desc: desc || title, speech });if (why) { console.log(`[AutoClip] VOD ${vId}: ${why}`); return true; }} catch { /* */ }return false;}async function clipVodMoment(o) {++ let archiveHeld = false;try {-- const { vod, offset, title, desc } = o || {};++ const { vod, offset, title, desc, speech, archival } = o || {};if (!vod || !(offset >= 0)) return null;if (!await _derivationOn(vod.user_id)) return null; // the channel turned AI Moments offif (require('./explicit').isExplicit(desc, title)) { // never a clip of sexual content (server/ai/explicit.js)console.log(`[AutoClip] skip VOD ${vod.vod_id || vod.id} — the AI describes the moment as sexual content`);return null;}-- if (await _isDuplicateAutoClip(vod, offset, desc, title)) {++ if (await _isDuplicateAutoClip(vod, offset, desc, title, speech)) {console.log(`[AutoClip] skip VOD ${vod.vod_id || vod.id} — duplicate/near-identical auto-clip already exists`);return null;}const vodId = vod.vod_id || vod.id;const start = Math.max(0, Math.floor(offset) - CLIP_PRE);++ // One archive clip per UTC day is shared with the AI moments job. Both may pass the pool++ // gate before either writes auto_clip_log, so claim the slot atomically before cutting and++ // give it back if no clip results.++ if (archival) {++ if (!clipPolicy.reserveArchive(await _clipLog(), Date.now())) {++ console.log(`[AutoClip] VOD ${vodId}: today's archival clip is already taken`);++ return null;++ }++ archiveHeld = true;++ }const clip = await media.createClip({vod_id: vodId, start_s: start, end_s: start + CLIP_PRE + CLIP_POST,title: title || 'Standout Moment', user_id: vod.user_id,stream_id: vod.stream_id || undefined, auto_generated: true, description: db.noLooks(desc || ''),});-- if (clip) await _logClip({ stream_id: vod.stream_id || null, vod_id: vodId, start_time: start, title: title || 'Standout Moment', clip_id: clip.id, dedup: !!clip.deduplicated || undefined, sig: _sceneSig(desc || title) });++ if (clip) {++ archiveHeld = false; // the clip exists: keep the day reserved even if the log write fails++ await _logClip({ stream_id: vod.stream_id || null, vod_id: vodId, start_time: start, title: title || 'Standout Moment', clip_id: clip.id, dedup: !!clip.deduplicated || undefined, desc: desc || '', speech, archival: archival || undefined, sig: _sceneSig(desc || title) });++ }return clip || null;} catch { return null; }++ finally { if (archiveHeld) clipPolicy.releaseArchive(); }}// ── Tell Media which clips the AI cut ────────────────────────────────────────────────────nnew file mode 100644@@@ -0,0 +1,137 @@++/**++ * clip-dedupe.js — text/visual identity for moments, shared by the moment registry and both cutters.++ *++ * The old registry compared only the first five ASCII words of a description: paraphrases got++ * through, Japanese collapsed to an empty signature, and unrelated scenes that opened with the++ * same boilerplate collided. This module keeps Unicode, compares token sets (Dice similarity, so++ * a re-worded or longer description of the same scene still matches) and keeps the visual scene++ * separate from what was said. A static webcam with a NEW conversation must not be treated as a++ * repeated moment: when both sides have speech, the speech has to be similar too.++ *++ * normalize(text) lower-case NFKC text, punctuation removed, spaces kept++ * tokens(text) {word} tokens; CJK uses character bigrams++ * contentTokens(text) tokens without articles/boilerplate media words++ * shortSignature(text, words = 5) the legacy five-token signature (registry `sig`)++ * isSameScene(a, b, opts) true for the same visual scene (near-duplicate wording)++ * isSameSpeech(a, b) true for a near-duplicate transcript excerpt (Dice ≥ 0.8)++ * isSameEmbedding(a, b) true for near-identical frame embeddings, when both exist++ * similarity(a, b) Dice coefficient over token sets++ * cosine(a, b) cosine similarity over numeric vectors++ *++ * No model or network: a cheap baseline that replaces exact string equality. Frame embeddings are++ * optional — the registry stores them when a caller has them and otherwise relies on text.++ */++'use strict';++++const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u;++const WORD_RE = /[\p{L}\p{N}]+/gu;++++// Words that describe the medium rather than the scene. They must not be the only reason two++// different moments look alike ("a live streaming session with chat" vs "… with music").++const BOILERPLATE = new Set((++ 'a an the and or of to in on at for with from by as is are was were be been being ' +++ 'it its this that these those there here ' +++ 'video videos stream streams live streaming session screen screenshot image images frame ' +++ 'footage scene clip moment appears appear shown showing visible view camera close ' +++ 'background foreground'++).split(' '));++++function normalize(text) {++ return String(text == null ? '' : text).normalize('NFKC').toLowerCase()++ .replace(/[^\p{L}\p{N}]+/gu, ' ').replace(/\s+/g, ' ').trim();++}++++/** Token set: words for spaced languages, overlapping bigrams for CJK (which has no spaces). */++function tokens(text) {++ const norm = normalize(text);++ if (!norm) return [];++ if (CJK_RE.test(norm)) {++ const compact = norm.replace(/\s+/g, '');++ const out = [];++ for (let i = 0; i + 1 < compact.length; i++) out.push(compact.slice(i, i + 2));++ // Mixed Japanese/English streams: reuse the Latin words as well.++ for (const w of norm.match(/[a-z0-9]+/g) || []) if (w.length > 1) out.push(w);++ return [...new Set(out)];++ }++ return [...new Set((norm.match(WORD_RE) || []))];++}++++/** Tokens that actually describe content; articles and media boilerplate are dropped. */++function contentTokens(text) {++ return tokens(text).filter((t) => !BOILERPLATE.has(t) && !/^\d+$/.test(t));++}++++function similarity(a, b) {++ const A = new Set(a), B = new Set(b);++ if (!A.size || !B.size) return 0;++ let shared = 0;++ for (const x of A) if (B.has(x)) shared++;++ return (2 * shared) / (A.size + B.size);++}++++/** Cosine similarity of two numeric vectors; 0 for a missing/zero/mismatched pair. */++function cosine(a, b) {++ if (!Array.isArray(a) || !Array.isArray(b) || !a.length || a.length !== b.length) return 0;++ let dot = 0, na = 0, nb = 0;++ for (let i = 0; i < a.length; i++) {++ const x = Number(a[i]), y = Number(b[i]);++ if (!Number.isFinite(x) || !Number.isFinite(y)) return 0;++ dot += x * y; na += x * x; nb += y * y;++ }++ if (!na || !nb) return 0;++ return dot / (Math.sqrt(na) * Math.sqrt(nb));++}++++/** The legacy registry signature: the first `words` tokens, joined. */++function shortSignature(text, words = 5) {++ return tokens(text).slice(0, words).join(' ');++}++++/**++ * Same scene when the token sets are near-identical. A match may not rest on boilerplate alone:++ * the two descriptions must share at least one content token, so "a live stream with chat" does++ * not match "a live stream with music". Short texts only match exactly.++ */++function isSameScene(a, b, { threshold = 0.85, minTokens = 4 } = {}) {++ const A = tokens(a), B = tokens(b);++ if (!A.length || !B.length) return false;++ if (normalize(a) === normalize(b)) return true;++ const CA = contentTokens(a), CB = contentTokens(b);++ const cbSet = new Set(CB);++ const sharedContent = CA.filter((t) => cbSet.has(t));++ if (CA.length || CB.length) {++ if (!sharedContent.length) return false;++ if (CA.length >= 2 && CB.length >= 2) return similarity(CA, CB) >= threshold;++ }++ if (A.length < minTokens || B.length < minTokens) return false;++ return similarity(A, B) >= threshold;++}++++/**++ * A transcript excerpt is the same speech even when speech-to-text differs a little. Dice ≥ 0.8++ * over Unicode-safe token sets; a match that shares only boilerplate ("this is a live stream" vs++ * "this is a live chat") is not enough.++ */++function isSameSpeech(a, b, { threshold = 0.8 } = {}) {++ const A = tokens(a), B = tokens(b);++ if (!A.length || !B.length) return false;++ if (normalize(a) === normalize(b)) return true;++ if (similarity(A, B) < threshold) return false;++ const CA = contentTokens(a), CB = contentTokens(b);++ if (CA.length || CB.length) {++ const cbSet = new Set(CB);++ return CA.some((t) => cbSet.has(t));++ }++ return true;++}++++/** Frame embeddings, when the caller has them: the same footage scores near 1. */++function isSameEmbedding(a, b, { threshold = 0.92 } = {}) {++ if (!Array.isArray(a) || !Array.isArray(b) || !a.length || a.length !== b.length) return false;++ return cosine(a, b) >= threshold;++}++++module.exports = {++ normalize, tokens, contentTokens, similarity, cosine, shortSignature,++ isSameScene, isSameSpeech, isSameEmbedding,++};nnew file mode 100644@@@ -0,0 +1,213 @@++/**++ * clip-policy.js — ONE freshness policy for every automated cutter.++ *++ * The owner (2026-10-29): "the AI clipping system seems to clip old videos. Maybe only make++ * clips of fresh stuff, unless the site's dead." Both the live clip job and the AI moments job++ * mined the most-VIEWED back catalogue with no age cutoff, so old VODs kept being clipped.++ *++ * fresh the producing stream ended (or is still running) in the last 48 hours++ * archive only when the site has been quiet for 72 hours (no new footage AND no human++ * pageviews) OR the channel itself has had no new stream in 48 hours (then only its++ * newest VODs), and the upstream service is healthy; then at most one archival clip++ * per UTC day, from the previous 90 days, newest first, labelled archival.++ *++ * The 48-hour clock starts when the producing STREAM ended, not when it started and not when++ * the VOD was uploaded: a five-day stream that finished an hour ago is fresh even with no part++ * row. A still-running stream is fresh; a source with no ended_at falls back to its capture/part++ * time, and an old imported recording is never fresh just because it was uploaded recently.++ *++ * footageStart({ vod, stream, part }) → epoch ms of the source's first frame, or null++ * streamEndedAt({ vod, stream, now }) → epoch ms the producing stream last had footage++ * classifySource({ vod, stream, part, now, site, channel }) → { ok, archival, ageHours, … }++ * siteActivity(db, { now }) → { lastFootageAt, lastHumanActivityAt }++ * siteStatus({ now, … }) → { quiet, hours, why }++ * latestStreamForUser(db, userId) → the user's newest stream row (live first)++ * streamActivityAt(stream, now) → epoch ms of that stream's latest activity++ * channelStatus({ now, lastStreamAt }) → { quiet, hours, why }++ * archiveAllowed(log, now) → one archival clip per UTC day++ * reserveArchive(log, now) / releaseArchive(now) → atomic in-process reservation of that slot++ *++ * An outage is never quietness: without a healthy upstream (`upstreamHealthy: false`) or a known++ * activity timestamp, the archive stays closed.++ */++'use strict';++++const HOUR_MS = 3600_000;++const FRESH_HOURS = 48;++const QUIET_HOURS = 72;++const CHANNEL_QUIET_HOURS = 48; // a channel with no new stream in the freshness window is quiet++const ARCHIVE_DAYS = 90;++const ARCHIVE_PER_DAY = 1;++++/** Parse the DB's `YYYY-MM-DD HH:MM:SS` UTC text, ISO text, Date, or epoch (s/ms) → ms or null. */++function parseTs(value) {++ if (value == null || value === '') return null;++ if (value instanceof Date) return Number.isFinite(value.getTime()) ? value.getTime() : null;++ if (typeof value === 'number') {++ if (!Number.isFinite(value)) return null;++ return value > 1e12 ? value : value * 1000; // epoch seconds vs milliseconds++ }++ const s = String(value).trim();++ if (!s) return null;++ const iso = /[zZ]|[+-]\d\d:?\d\d$/.test(s) ? s : `${s.replace(' ', 'T')}Z`;++ const ms = Date.parse(iso);++ return Number.isFinite(ms) ? ms : null;++}++++/**++ * When the footage was captured. Media may name it directly; otherwise the stream's part row++ * gives the exact start of this recording (stream start + part offset), which is why a long++ * stream recorded in parts does not look old. `created_at` is the last resort — for recordings++ * Live itself opened it is the recorder's start time.++ */++function footageStart({ vod, stream, part } = {}) {++ for (const field of ['captured_at', 'recorded_at', 'recording_started_at']) {++ const t = parseTs(vod && vod[field]);++ if (t != null) return t;++ }++ const streamStart = parseTs(stream && stream.started_at);++ if (streamStart != null && part && Number.isFinite(Number(part.at))) return streamStart + Number(part.at) * 1000;++ const created = parseTs(vod && (vod.created_at || vod.createdAt));++ if (created != null) return created;++ return streamStart;++}++++/**++ * When the stream that produced a VOD last had footage — the reference for the 48-hour window.++ * A finished stream's ended_at governs; a still-live stream is "now". Without a trusted end++ * (an import or a stream row that never got ended_at), the capture/part time stands in, and only++ * then the VOD's created_at: an old file uploaded today must not look fresh because of upload time.++ */++function streamEndedAt({ vod, stream, now = Date.now() } = {}) {++ if (stream) {++ if (Number(stream.is_live) > 0) return now;++ const ended = parseTs(stream.ended_at);++ if (ended != null) return ended;++ }++ // Capture metadata before created_at: created_at is the upload/import time for a recording++ // Live did not open itself.++ const captured = parseTs(vod && (vod.captured_at || vod.recorded_at || vod.recording_started_at));++ if (captured != null) return captured;++ if (stream) {++ const started = parseTs(stream.started_at);++ if (started != null) return started;++ }++ return parseTs(vod && (vod.created_at || vod.createdAt));++}++++/**++ * Site quietness from the two signals we have in Live: new footage (a live stream, a stream that++ * ended, a VOD capture) and human pageviews. Both are required; an unknown signal is not quiet.++ * `upstreamHealthy` comes from the caller (its Media listing answered).++ */++function siteStatus({ now = Date.now(), lastFootageAt = null, lastHumanActivityAt = null, upstreamHealthy = false } = {}) {++ if (!upstreamHealthy) return { quiet: false, hours: null, lastAt: null, why: 'upstream not healthy' };++ if (lastFootageAt == null || lastHumanActivityAt == null) return { quiet: false, hours: null, lastAt: null, why: 'site activity unknown' };++ const lastAt = Math.max(lastFootageAt, lastHumanActivityAt);++ const hours = (now - lastAt) / HOUR_MS;++ return { quiet: hours >= QUIET_HOURS, hours, lastAt, why: hours >= QUIET_HOURS ? 'site quiet' : 'recent activity' };++}++++/**++ * A channel's quietness: no new stream in the freshness window (48h). Unknown activity is not++ * quietness, and an outage is never quietness. `newest` on the per-VOD channel object limits the++ * channel exception to the newest VODs, so a quiet channel does not reopen its whole archive.++ */++function channelStatus({ now = Date.now(), lastStreamAt = null } = {}) {++ if (lastStreamAt == null) return { quiet: false, hours: null, lastAt: null, why: 'channel activity unknown' };++ const hours = (now - lastStreamAt) / HOUR_MS;++ return { quiet: hours >= CHANNEL_QUIET_HOURS, hours, lastAt: lastStreamAt, why: hours >= CHANNEL_QUIET_HOURS ? 'channel quiet' : 'channel active' };++}++++async function siteActivity(db, { now = Date.now() } = {}) {++ let lastFootageAt = null, lastHumanActivityAt = null;++ try {++ const s = await db.get(`SELECT MAX(started_at) AS started, MAX(ended_at) AS ended,++ MAX(CASE WHEN is_live = 1 THEN 1 ELSE 0 END) AS live++ FROM streams`);++ const started = parseTs(s && s.started), ended = parseTs(s && s.ended);++ if (Number(s && s.live) > 0) lastFootageAt = now; // footage is being recorded right now++ else lastFootageAt = Math.max(started || 0, ended || 0) || null;++ } catch { /* unknown is not quiet */ }++ try {++ const a = await db.get(`SELECT MAX(created_at) AS t FROM analytics_events WHERE COALESCE(is_bot, 0) = 0`);++ lastHumanActivityAt = parseTs(a && a.t);++ } catch { /* unknown is not quiet */ }++ return { lastFootageAt, lastHumanActivityAt };++}++++/** A user's newest stream row (an active one wins), for the channel-quiet exception. */++async function latestStreamForUser(db, userId) {++ try {++ return await db.get(`SELECT id, started_at, ended_at, is_live++ FROM streams WHERE user_id = ?++ ORDER BY (CASE WHEN is_live = 1 THEN 1 ELSE 0 END) DESC,++ COALESCE(ended_at, started_at, created_at) DESC, id DESC++ LIMIT 1`, [userId]);++ } catch { return null; } // unknown activity is not quietness++}++++/** Epoch ms of a stream's latest activity: now while it is live, otherwise its last end/start. */++function streamActivityAt(stream, now = Date.now()) {++ if (!stream) return null;++ if (Number(stream.is_live) > 0) return now;++ return Math.max(parseTs(stream.ended_at) || 0, parseTs(stream.started_at) || 0) || null;++}++++/**++ * Fresh, or archival when the site (or this channel) is quiet and the source is inside the++ * archive window. A channel exception applies only to the channel's newest VODs (`channel.newest`),++ * so a quiet channel does not reopen its whole back catalogue.++ */++function classifySource({ vod, stream, part, now = Date.now(), site = { quiet: false }, channel = null } = {}) {++ const capturedAt = footageStart({ vod, stream, part });++ const endedAt = streamEndedAt({ vod, stream, now });++ // Freshness follows the stream's end (see streamEndedAt), not the first frame: a long stream++ // that finished recently is eligible even when its VOD was opened five days ago.++ const ageHours = endedAt == null ? null : (now - endedAt) / HOUR_MS;++ if (ageHours == null) return { ok: false, archival: false, capturedAt, endedAt, ageHours, why: 'unknown source age' };++ if (ageHours <= FRESH_HOURS) return { ok: true, archival: false, capturedAt, endedAt, ageHours, why: 'fresh' };++ // The 90-day archive window is about the footage itself, not an upload that happened later.++ const archiveHours = capturedAt == null ? ageHours : (now - capturedAt) / HOUR_MS;++ const channelQuiet = !!(channel && channel.quiet && channel.newest !== false);++ const quiet = !!(site && site.quiet) || channelQuiet;++ if (quiet && archiveHours <= ARCHIVE_DAYS * 24) {++ return { ok: true, archival: true, capturedAt, endedAt, ageHours, why: site && site.quiet ? 'quiet site: archive' : 'quiet channel: archive' };++ }++ return { ok: false, archival: false, capturedAt, endedAt, ageHours, why: quiet ? 'older than the archive window' : 'stale source' };++}++++const _day = (ms) => new Date(ms).toISOString().slice(0, 10);++++// One archive clip per UTC day, shared by the AI moments job and the VOD backfill. Both can pass++// archiveAllowed() before either writes auto_clip_log, so the slot is reserved synchronously here:++// reserveArchive() is a single check-and-set (no await between), and archiveAllowed() sees the++// reservation before the clip log is written.++let _archiveReservedDay = null;++++/** The clip log (or a reservation this process already holds) has spent today's archive clip? */++function archiveAllowed(log, now = Date.now()) {++ const day = _day(now);++ if (_archiveReservedDay === day) return false;++ const used = (Array.isArray(log) ? log : []).filter((e) => e && e.archival && e.ts && _day(e.ts) === day).length;++ return used < ARCHIVE_PER_DAY;++}++++/** Atomically claim today's archival clip slot for this process; false when it is taken. */++function reserveArchive(log, now = Date.now()) {++ if (!archiveAllowed(log, now)) return false;++ _archiveReservedDay = _day(now);++ return true;++}++++/** Give the slot back after a cut that did not happen. */++function releaseArchive(now = Date.now()) {++ if (_archiveReservedDay === _day(now)) _archiveReservedDay = null;++}++++module.exports = {++ FRESH_HOURS, QUIET_HOURS, CHANNEL_QUIET_HOURS, ARCHIVE_DAYS, ARCHIVE_PER_DAY,++ parseTs, footageStart, streamEndedAt, siteStatus, siteActivity, latestStreamForUser, streamActivityAt,++ channelStatus, classifySource, archiveAllowed, reserveArchive, releaseArchive,++ _resetArchiveReservation: () => { _archiveReservedDay = null; },++};@@@ -14,6 +14,7 @@'use strict';const db = require('../db/database');++const dedupe = require('./clip-dedupe');const KEY = 'ai_used_moments';const MAX = 800;@@@ -39,26 +40,58 @@ async function _importLegacy() {return out.slice(0, MAX);}--/** Coarse scene signature — the first five significant words of a description/title. */++/** Coarse scene signature — the first five significant tokens of a description/title. */function sig(text) {-- return String(text || '').toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim().split(' ').filter(Boolean).slice(0, 5).join(' ');++ return dedupe.shortSignature(text, 5);}--async function record({ kind, vod_id = null, stream_id = null, offset = null, sig: s = null, title = null, desc = null }) {++/** A caller's frame embedding, kept only when it is a usable finite vector. */++function _embedding(v) {++ if (!Array.isArray(v) || !v.length || v.length > 1024) return null;++ const out = v.map(Number);++ return out.every(Number.isFinite) ? out : null;++}++++async function record({ kind, vod_id = null, stream_id = null, offset = null, sig: s = null, title = null, desc = null, speech = null, embedding = null }) {const list = await _load();-- list.unshift({ kind: kind || 'moment', vod_id: vod_id || null, stream_id: stream_id || null, offset: offset == null ? null : Math.floor(offset), sig: s || sig(desc || title), title: title ? String(title).slice(0, 120) : null, ts: Date.now() });++ const scene = dedupe.normalize(desc || title);++ list.unshift({++ kind: kind || 'moment', vod_id: vod_id || null, stream_id: stream_id || null,++ offset: offset == null ? null : Math.floor(offset),++ sig: s || sig(desc || title),++ // Full normalized scene + what was said, so a re-worded description still matches and a++ // static webcam with a NEW conversation does not. Legacy entries keep only `sig`.++ scene: scene ? scene.slice(0, 600) : null,++ speech: speech ? dedupe.normalize(speech).slice(0, 600) : null,++ // Optional frame embedding: the same footage in another upload scores near 1.++ embedding: _embedding(embedding),++ title: title ? String(title).slice(0, 120) : null, ts: Date.now(),++ });await _save(list);}/** Why a moment is blocked, or null when it is free. */--async function usedReason({ vod_id = null, stream_id = null, offset = null, sig: s = null, desc = null, title = null } = {}, { gapSec = GAP_SEC } = {}) {++async function usedReason({ vod_id = null, stream_id = null, offset = null, sig: s = null, desc = null, title = null, speech = null, embedding = null } = {}, { gapSec = GAP_SEC } = {}) {const now = Date.now();const scene = s || sig(desc || title);++ const full = dedupe.normalize(desc || title);++ const said = dedupe.normalize(speech);++ const emb = _embedding(embedding);for (const e of await _load()) {const age = now - (e.ts || 0);if (age > OFFSET_TTL_MS) continue;if (offset != null && e.offset != null && ((vod_id && e.vod_id === vod_id) || (stream_id && e.stream_id === stream_id)) && Math.abs(e.offset - offset) < gapSec) return `${e.kind} already made at ${Math.floor(e.offset)}s of this ${e.vod_id === vod_id ? 'VOD' : 'stream'}`;-- if (scene && e.sig && e.sig === scene && age <= SIG_TTL_MS) return `same scene as a recent ${e.kind} ("${scene}")`;++ if (age > SIG_TTL_MS) continue;++ const shortMatch = scene && e.sig && e.sig === scene;++ const sceneMatch = full && e.scene && dedupe.isSameScene(e.scene, full);++ // Frame embeddings (when both sides have them) are independent evidence: the same footage++ // is a repeat even if it was re-described or re-uploaded.++ const visualMatch = emb && e.embedding && dedupe.isSameEmbedding(e.embedding, emb);++ if (!shortMatch && !sceneMatch && !visualMatch) continue;++ if (visualMatch) return `same footage as a recent ${e.kind}`;++ // Same camera/description but a different conversation is a new moment, not a repeat.++ if (said && e.speech && !dedupe.isSameSpeech(e.speech, said)) continue;++ return `same scene as a recent ${e.kind} ("${scene || e.sig}")`;}return null;}nnew file mode 100644@@@ -0,0 +1,96 @@++/**++ * Content dedupe for moments (owner, 2026-10-29): repeats were getting clipped over and over.++ * The old registry compared the first five ASCII words, so paraphrases slipped through and++ * Japanese became an empty signature. clip-dedupe.js keeps Unicode and compares token sets,++ * and the live cutter now asks the shared registry BEFORE cutting.++ *++ * node test/clip-dedupe.test.js++ */++'use strict';++const assert = require('assert');++const db = require('../server/db/database');++(async () => {++await db.initDb();++const dedupe = require('../server/ai/clip-dedupe');++const registry = require('../server/ai/moment-registry');++const media = require('../server/media-client');++const autoClip = require('../server/ai/auto-clip-job');++++// ── Text identity ──────────────────────────────────────────++assert.strictEqual(dedupe.normalize('びっくり！？'), dedupe.normalize('びっくり'), 'Unicode punctuation is dropped, not the text');++assert.ok(dedupe.shortSignature('配信者が大きな声で叫ぶ').length > 0, 'Japanese produces a signature instead of an empty one');++assert.ok(dedupe.isSameScene('A player is fighting a boss in a dark cave', 'The player is fighting a boss in a dark cave'), 're-worded description is the same scene');++assert.ok(!dedupe.isSameScene('A player is fighting a boss in a dark cave', 'A player is fighting a boss in a bright castle'), 'same topic with a different scene is not blocked');++assert.ok(dedupe.isSameScene('配信者が大きな声で叫ぶ', '配信者が大きな声で叫んだ'), 'Japanese near-duplicate matches');++assert.ok(!dedupe.isSameScene('配信者が静かにゲームをする', '猫がテーブルの上で寝ている'), 'unrelated Japanese is not blocked');++assert.ok(dedupe.isSameSpeech('today we are playing a horror game and it is terrifying', 'today we are playing a horror game and it is very scary'), 'near-duplicate speech matches');++assert.ok(!dedupe.isSameSpeech('today we are playing a horror game', 'we just beat the final boss on the first try'), 'a different conversation is not the same speech');++assert.ok(dedupe.similarity(dedupe.tokens('we are going to play the game now'), dedupe.tokens('now we are going to play the game')) >= 0.8, 're-ordered speech still clears the 0.8 Dice bar');++assert.ok(!dedupe.isSameSpeech('the red car explodes', 'the red car is parked'), 'a below-0.8 token overlap is not the same speech');++assert.ok(!dedupe.isSameScene('The video appears to be a live streaming session with a chat overlay', 'The video appears to be a live streaming session with music'), 'shared boilerplate alone is not the same scene');++assert.ok(!dedupe.isSameSpeech('this is a live stream', 'this is a live chat'), 'shared boilerplate alone is not the same speech');++assert.ok(dedupe.isSameEmbedding([1, 0, 0], [0.99, 0.01, 0]), 'near-identical frame embeddings match');++assert.ok(!dedupe.isSameEmbedding([1, 0, 0], [0, 1, 0]), 'different frame embeddings do not');++console.log('✅ clip-dedupe: Unicode, paraphrases, 0.8 Dice, boilerplate and embedding negatives');++++// ── The shared registry ────────────────────────────────────++await db.setState('ai_used_moments', '[]');++registry._reset();++await registry.record({++ kind: 'paste', vod_id: 30, stream_id: 5, offset: 100,++ desc: 'A person sits at a desk in front of a microphone',++ speech: 'today we are playing a horror game and it is terrifying',++});++assert.strictEqual(await registry.usedReason({++ vod_id: 31, offset: 5, desc: 'A person sits at a desk in front of a microphone',++ speech: 'we just beat the final boss on the first try',++}), null, 'static webcam + a NEW conversation is not a repeat');++assert.ok(await registry.usedReason({++ vod_id: 31, offset: 5, desc: 'A person sits at a desk in front of a microphone',++ speech: 'today we are playing a horror game and it is very scary',++}), 'static webcam + the SAME conversation is a repeat');++++await registry.record({ kind: 'clip', vod_id: 40, stream_id: 6, offset: 200, desc: 'A red car explodes on the highway' });++assert.ok(await registry.usedReason({ vod_id: 41, offset: 10, desc: 'The red car explodes on the highway' }), 'a silent re-upload is caught by the visual description');++++await registry.record({ kind: 'clip', vod_id: 50, stream_id: 7, offset: 300, desc: '配信者が大きな声で叫ぶ瞬間' });++assert.ok(await registry.usedReason({ vod_id: 51, offset: 10, desc: '配信者が大きな声で叫んだ瞬間' }), 'Japanese description is preserved across VODs');++++await registry.record({ kind: 'clip', vod_id: 52, stream_id: 8, offset: 400, desc: 'A person sits at a desk', embedding: [1, 0, 0] });++assert.ok(await registry.usedReason({ vod_id: 53, offset: 10, desc: 'A completely different caption', embedding: [0.99, 0.01, 0] }), 'a matching frame embedding blocks even a different description');++assert.strictEqual(await registry.usedReason({ vod_id: 54, offset: 10, desc: 'A completely different caption', embedding: [0, 1, 0] }), null, 'a different frame embedding is free');++console.log('✅ registry: new speech is free, repeats, silent re-uploads and frame embeddings are blocked');++++// ── The live cutter asks the registry before spending a clip ──++const created = [];++media.createClip = async (body) => { created.push(body); return { id: 900 + created.length }; };++const recorder = require('../server/streaming/recorder');++recorder.getActiveRecording = () => ({ vodId: 60, startedAt: Date.now() - 100_000 });++const chatReads = require('../server/chat/chat-reads');++chatReads.liveChatBuckets = async () => [++ { count: 1, tsEpoch: Math.floor(Date.now() / 1000) },++ { count: 1 }, { count: 10, tsEpoch: Math.floor(Date.now() / 1000) }, { count: 1 },++];++chatReads.recentChatText = async () => [];++const ai = require('../server/ai/ai-analysis');++ai.isEnabled = () => true;++ai.withinBudget = () => true;++require('../server/ai/ai-service').structured = async () => ({ clip: true, title: 'A moment', desc: 'A person sits at a desk in front of a microphone' });++db.isStreamClipRecordingEnabled = () => true;++await db.setState('auto_clip_log', '[]');++++const liveStream = { id: 5, user_id: 3, started_at: new Date(Date.now() - 2 * 3600_000).toISOString().replace('T', ' ').slice(0, 19), title: 'Alice live' };++await registry.record({ kind: 'paste', vod_id: 60, stream_id: 5, offset: 100, desc: 'A person sits at a desk' });++await autoClip._internals.checkLiveStream(liveStream);++assert.strictEqual(created.length, 0, 'the live path never cuts a moment the registry already used');++++registry._reset();++await db.setState('ai_used_moments', '[]');++registry._reset();++await autoClip._internals.checkLiveStream(liveStream);++assert.strictEqual(created.length, 1, 'a free moment is still cut (the check does not block everything)');++console.log('✅ live cutter: registry rejection happens before Media.createClip');++++registry._reset();++console.log('\n✅ All clip-dedupe tests passed');++})().catch((e) => { console.error(e); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,277 @@++/**++ * Freshness + dedupe policy for automated clips (owner, 2026-10-29: "the AI clipping system++ * seems to clip old videos. Maybe only make clips of fresh stuff, unless the site's dead").++ *++ * - footage from the last 48 hours is eligible, counting from when the stream ENDED;++ * - the archive (≤ 90 days) opens only after 72h with no new footage AND no human pageviews,++ * and then at most one archival clip per UTC day;++ * - an upstream outage is never quietness;++ * - a long/reconnected stream uses the VOD part's capture time, not the stream start.++ *++ * node test/clip-freshness.test.js++ */++'use strict';++const assert = require('assert');++const db = require('../server/db/database');++(async () => {++await db.initDb();++const raw = db.getDb();++const policy = require('../server/ai/clip-policy');++const registry = require('../server/ai/moment-registry');++const media = require('../server/media-client');++const autoClip = require('../server/ai/auto-clip-job');++const moments = require('../server/ai/ai-moments-job');++++const HOUR = 3600_000, DAY = 24 * HOUR;++const text = (ms) => new Date(ms).toISOString().replace('T', ' ').slice(0, 19);++const now = Math.floor(Date.now() / 1000) * 1000;++++// ── Pure policy ────────────────────────────────────────────++assert.strictEqual(policy.parseTs('2026-10-28 01:02:03'), Date.UTC(2026, 9, 28, 1, 2, 3), 'DB UTC text parses');++assert.strictEqual(policy.parseTs('2026-10-28T01:02:03Z'), Date.UTC(2026, 9, 28, 1, 2, 3), 'ISO parses');++assert.strictEqual(policy.parseTs(1_700_000_000), 1_700_000_000_000, 'epoch seconds parse');++++const stream = { started_at: text(now - 5 * DAY) };++assert.strictEqual(policy.footageStart({ vod: { captured_at: text(now - HOUR) }, stream }), now - HOUR, 'explicit capture time wins');++assert.strictEqual(policy.footageStart({ vod: { created_at: text(now - 9 * DAY) }, stream, part: { at: 4 * DAY / 1000 } }), now - DAY, 'part capture time wins over created_at');++assert.strictEqual(policy.footageStart({ vod: { created_at: text(now - 3 * DAY) }, stream }), now - 3 * DAY, 'created_at fallback for a one-part VOD');++assert.strictEqual(policy.footageStart({}), null, 'unknown source age');++++const fresh = policy.classifySource({ vod: { captured_at: text(now - 47 * HOUR) }, now });++assert.ok(fresh.ok && !fresh.archival, '47h old is fresh');++const stale = policy.classifySource({ vod: { captured_at: text(now - 49 * HOUR) }, now, site: { quiet: false } });++assert.ok(!stale.ok, '49h old is stale while the site is active');++const archive = policy.classifySource({ vod: { captured_at: text(now - 5 * DAY) }, now, site: { quiet: true } });++assert.ok(archive.ok && archive.archival, '5 days old is archival on a quiet site');++const tooOld = policy.classifySource({ vod: { captured_at: text(now - 100 * DAY) }, now, site: { quiet: true } });++assert.ok(!tooOld.ok, 'older than the 90-day archive window is never eligible');++++// The 48h clock is the stream's END, not the VOD's creation: a five-day stream that finished an++// hour ago is fresh even with no part row; one that ended 49h ago is stale.++const endedFresh = policy.classifySource({++ vod: { created_at: text(now - 5 * DAY) },++ stream: { started_at: text(now - 5 * DAY), ended_at: text(now - HOUR) },++ now,++ site: { quiet: false },++});++assert.ok(endedFresh.ok && !endedFresh.archival, 'a long stream that ended an hour ago is fresh without a part row');++assert.ok(endedFresh.ageHours < 2, 'its freshness age comes from ended_at, not created_at');++const endedStale = policy.classifySource({++ vod: { created_at: text(now - 5 * DAY) },++ stream: { started_at: text(now - 5 * DAY), ended_at: text(now - 49 * HOUR) },++ now,++ site: { quiet: false },++});++assert.ok(!endedStale.ok && !endedStale.archival, 'a stream that ended 49h ago is stale');++const stillLive = policy.classifySource({++ vod: { created_at: text(now - 5 * DAY) },++ stream: { started_at: text(now - 5 * DAY), is_live: 1 },++ now,++ site: { quiet: false },++});++assert.ok(stillLive.ok && !stillLive.archival, 'a still-running stream is fresh');++++// An old imported recording is judged by its capture time, never by a recent upload.++const imported = policy.classifySource({++ vod: { created_at: text(now - HOUR), captured_at: text(now - 10 * DAY) },++ now,++ site: { quiet: false },++});++assert.ok(!imported.ok && !imported.archival, 'old recording uploaded an hour ago is stale, not fresh');++const importedArchive = policy.classifySource({++ vod: { created_at: text(now - HOUR), captured_at: text(now - 10 * DAY) },++ now,++ site: { quiet: true },++});++assert.ok(importedArchive.ok && importedArchive.archival, 'on a quiet site it is archival, still not fresh');++const importedLinked = policy.classifySource({++ vod: { created_at: text(now - HOUR) },++ stream: { started_at: text(now - 10 * DAY), ended_at: text(now - 10 * DAY) },++ now,++ site: { quiet: false },++});++assert.ok(!importedLinked.ok, 'a recent upload of an old stream is still stale');++++const quiet = policy.siteStatus({ now, lastFootageAt: now - 73 * HOUR, lastHumanActivityAt: now - 73 * HOUR, upstreamHealthy: true });++assert.strictEqual(quiet.quiet, true, '72h without footage or pageviews is quiet');++assert.strictEqual(policy.siteStatus({ now, lastFootageAt: now - 73 * HOUR, lastHumanActivityAt: now - 73 * HOUR, upstreamHealthy: false }).quiet, false, 'an outage is not quietness');++assert.strictEqual(policy.siteStatus({ now, lastFootageAt: now - 73 * HOUR, lastHumanActivityAt: null, upstreamHealthy: true }).quiet, false, 'unknown activity is not quietness');++assert.strictEqual(policy.siteStatus({ now, lastFootageAt: now - 73 * HOUR, lastHumanActivityAt: now - 1 * HOUR, upstreamHealthy: true }).quiet, false, 'a human pageview keeps the archive closed');++++assert.strictEqual(policy.archiveAllowed([{ archival: true, ts: Date.now() }]), false, 'one archive clip per day');++assert.strictEqual(policy.archiveAllowed([{ archival: true, ts: Date.now() - DAY }]), true, "yesterday's archive clip does not block today");++// Both candidate pools can pass the gate before either writes auto_clip_log. The in-process++// reservation is a synchronous check-and-set, so the second cutter loses even with an empty log.++policy._resetArchiveReservation();++assert.strictEqual(policy.reserveArchive([], now), true, 'first simultaneous archive cut claims the slot');++assert.strictEqual(policy.archiveAllowed([], now), false, 'the reservation closes the gate before the log write');++assert.strictEqual(policy.reserveArchive([], now), false, 'a second simultaneous archive cut is refused');++policy.releaseArchive(now);++assert.strictEqual(policy.archiveAllowed([], now), true, 'a failed cut gives the slot back');++++// A stale VOD is also allowed when its own channel has been quiet for 48h (the brief's channel++// exception) — but only its newest VODs, and never outside the 90-day window.++assert.strictEqual(policy.channelStatus({ now, lastStreamAt: now - 49 * HOUR }).quiet, true, 'a channel with no new stream in 48h is quiet');++assert.strictEqual(policy.channelStatus({ now, lastStreamAt: now - HOUR }).quiet, false, 'a channel still streaming is active');++const quietChannel = policy.classifySource({ vod: { captured_at: text(now - 5 * DAY) }, now, site: { quiet: false }, channel: { quiet: true, newest: true } });++assert.ok(quietChannel.ok && quietChannel.archival, 'stale VOD on a quiet channel: archive allowed');++assert.ok(!policy.classifySource({ vod: { captured_at: text(now - 5 * DAY) }, now, site: { quiet: false }, channel: { quiet: true, newest: false } }).ok, 'quiet channel: only its newest VODs');++assert.ok(!policy.classifySource({ vod: { captured_at: text(now - 5 * DAY) }, now, site: { quiet: false }, channel: { quiet: false } }).ok, 'stale VOD on an active channel: refused');++console.log('✅ freshness policy: 48h, 72h quiet gate, channel quiet, outage, archive window and cap');++++// ── Integration fixtures ───────────────────────────────────++const addUser = async (id, username) => await raw.prepare(++ `INSERT INTO users (id, username, display_name, email, password_hash, role, created_at) OVERRIDING SYSTEM VALUE++ VALUES (?, ?, ?, ?, 'x', 'streamer', '2025-01-01 00:00:00')`).run(id, username, username, `${username}@x`);++if (!await db.getUserById(3)) { await addUser(3, 'alice'); await db.ensureChannel(3); }++const ch = await db.getChannelByUserId(3);++++const mkStream = async (startedMs, { live = false, endedMs = null } = {}) => {++ const id = Number((await db.createStream({ user_id: 3, channel_id: ch.id, title: 'stream', protocol: 'rtmp' })).lastInsertRowid);++ await db.run('UPDATE streams SET started_at = ?, is_live = ?, ended_at = ? WHERE id = ?', [text(startedMs), live ? 1 : 0, endedMs == null ? null : text(endedMs), id]);++ return id;++};++const vod = (id, streamId, extra = {}) => ({ id, stream_id: streamId, user_id: 3, title: `VOD ${id}`, visibility: 'public', is_public: true, view_count: 5, duration_seconds: 600, status: 'ready', ...extra });++let currentVods = [];++media.listVods = async () => ({ vods: currentVods });++const addMemory = async (streamId) => await db.addStreamMemory({ stream_id: streamId, user_id: 3, offset_seconds: 60, description: 'a scene' });++++async function reset() {++ await raw.prepare('DELETE FROM stream_history').run();++ await raw.prepare('DELETE FROM stream_memories').run();++ await raw.prepare('DELETE FROM analytics_events').run();++ await raw.prepare('DELETE FROM streams').run();++ await db.setState('auto_clip_log', '[]');++ await db.setState('ai_used_moments', '[]');++ require('../server/ai/moment-registry')._reset();++ require('../server/ai/clip-policy')._resetArchiveReservation();++ currentVods = [];++}++++// Active site: only the VOD captured in the last 48h is eligible.++await reset();++const sFresh = await mkStream(now - HOUR, { endedMs: now - HOUR });++const sOld = await mkStream(now - 10 * DAY, { endedMs: now - 10 * DAY });++const sMid = await mkStream(now - 5 * DAY, { endedMs: now - 5 * DAY });++await addMemory(sFresh); await addMemory(sOld); await addMemory(sMid);++await db.run(`INSERT INTO analytics_events (service, event_type, is_bot, created_at) VALUES ('live', 'pageview', 0, ?)`, [text(now - HOUR)]);++currentVods = [vod(10, sFresh), vod(11, sOld), vod(12, sMid)];++let pool = await autoClip._internals.backfillPool(10);++assert.deepStrictEqual(pool.map(v => v.vod_id), [10], 'active site: only fresh footage is clipped');++assert.strictEqual(pool[0].archival, false);++console.log('✅ backfill pool: old popular VODs are not clipped while the site is active');++++// Review case: VOD created when a long stream started five days ago, stream.ended_at one hour++// ago, no part row. The ended_at clock makes it fresh; one that ended 49h ago stays stale.++await reset();++const sEndedRecent = await mkStream(now - 5 * DAY, { endedMs: now - HOUR });++const sEndedStale = await mkStream(now - 5 * DAY, { endedMs: now - 49 * HOUR });++await addMemory(sEndedRecent); await addMemory(sEndedStale);++await db.run(`INSERT INTO analytics_events (service, event_type, is_bot, created_at) VALUES ('live', 'pageview', 0, ?)`, [text(now - HOUR)]);++currentVods = [++ vod(14, sEndedRecent, { created_at: text(now - 5 * DAY) }),++ vod(15, sEndedStale, { created_at: text(now - 5 * DAY) }),++];++pool = await autoClip._internals.backfillPool(10);++assert.deepStrictEqual(pool.map(v => v.vod_id), [14], 'a stream that ended an hour ago is eligible; 49h old is not');++assert.strictEqual(pool[0].archival, false);++console.log('✅ backfill pool: 48h runs from stream end, not VOD creation');++++// A channel live now has no new VOD yet. The old VOD must not reopen the quiet-channel exception++// just because Media's newest row for that channel is the old stream.++await reset();++const sChannelOld = await mkStream(now - 10 * DAY, { endedMs: now - 10 * DAY });++const sChannelLive = await mkStream(now - HOUR, { live: true });++await addMemory(sChannelOld);++await db.run(`INSERT INTO analytics_events (service, event_type, is_bot, created_at) VALUES ('live', 'pageview', 0, ?)`, [text(now - HOUR)]);++currentVods = [vod(30, sChannelOld)];++assert.deepStrictEqual(await autoClip._internals.backfillPool(10), [], 'a current live stream closes the quiet-channel exception (backfill)');++assert.deepStrictEqual(await moments._internals.momentPool(10), [], 'a current live stream closes the quiet-channel exception (moments)');++assert.ok(sChannelLive, 'the live stream exists without a listed VOD');++console.log('✅ candidate pools: latest stream activity comes from Live streams, not the newest VOD');++++// Long/reconnected stream: capture time is streams.started_at + its part offset.++await reset();++const sLong = await mkStream(now - 5 * DAY, { live: true });++await db.run(`INSERT INTO stream_history (stream_id, kind, at_sec, end_sec, vod_id, detail) VALUES (?, 'part', ?, NULL, ?, '{}')`, [sLong, (5 * DAY - HOUR) / 1000, 13]);++await addMemory(sLong);++currentVods = [vod(13, sLong)];++pool = await autoClip._internals.backfillPool(10);++assert.deepStrictEqual(pool.map(v => v.vod_id), [13], 'a five-day stream still recording is fresh');++assert.strictEqual(pool[0].captured_at, now - HOUR, 'its VOD part captured one hour ago, not five days ago');++assert.ok(pool[0].age_hours < 2, 'a still-running stream has a now-based freshness age');++console.log('✅ backfill pool: long/reconnected streams use the part capture time');++++// Quiet site: the archive opens, up to the daily cap.++await reset();++const sArchiveA = await mkStream(now - 10 * DAY, { endedMs: now - 10 * DAY });++const sArchiveB = await mkStream(now - 5 * DAY, { endedMs: now - 5 * DAY });++await addMemory(sArchiveA); await addMemory(sArchiveB);++await db.run(`INSERT INTO analytics_events (service, event_type, is_bot, created_at) VALUES ('live', 'pageview', 0, ?)`, [text(now - 4 * DAY)]);++currentVods = [vod(20, sArchiveA), vod(21, sArchiveB)];++pool = await autoClip._internals.backfillPool(10);++assert.deepStrictEqual(pool.map(v => v.vod_id), [20, 21], 'quiet site: archive candidates inside 90 days');++assert.ok(pool.every(v => v.archival), 'they are labelled archival');++++await db.setState('auto_clip_log', JSON.stringify([{ archival: true, ts: Date.now(), vod_id: 999 }]));++pool = await autoClip._internals.backfillPool(10);++assert.deepStrictEqual(pool, [], "today's archival clip closes the archive for the rest of the day");++console.log('✅ backfill pool: quiet site opens the archive, one archival clip per day');++++// A Media outage is not quietness.++await db.setState('auto_clip_log', '[]');++media.listVods = async () => { throw new Error('Media down'); };++pool = await autoClip._internals.backfillPool(10);++assert.deepStrictEqual(pool, [], 'Media down: no candidates, and no archive opened');++media.listVods = async () => ({ vods: currentVods });++++// The home-page moments pool follows the same policy.++const mpool = await moments._internals.momentPool(10);++assert.deepStrictEqual(mpool.map(v => v.vod_id), [20, 21], 'moments pool: same quiet-site archive');++assert.ok(mpool.every(v => v.archival));++await db.setState('auto_clip_log', JSON.stringify([{ archival: true, ts: Date.now(), vod_id: 999 }]));++assert.deepStrictEqual(await moments._internals.momentPool(10), [], 'moments pool: archive cap is shared with the clip job');++console.log('✅ moments pool: one freshness policy for every automated cutter');++++// Both cutters (AI moments and VOD backfill) can see an empty auto_clip_log before either writes.++// The in-process reservation lets exactly one reach Media; a failed cut releases the slot.++await reset();++const cutCalls = [];++let releaseFirst = null;++media.createClip = async (body) => {++ cutCalls.push(body);++ if (cutCalls.length === 1) await new Promise((resolve) => { releaseFirst = resolve; });++ return { id: 7000 + cutCalls.length };++};++const archiveCut = (id) => autoClip.clipVodMoment({++ vod: { vod_id: id, user_id: 3, stream_id: id, title: 'archive' },++ offset: 60,++ title: `Archive moment ${id}`,++ desc: `A distinct archival scene number ${id} with enough words`,++ archival: true,++});++const firstCut = archiveCut(70);++const secondCut = archiveCut(71);++for (let i = 0; i < 200 && !releaseFirst; i++) await new Promise((r) => setTimeout(r, 5));++assert.ok(releaseFirst, 'the first archival cutter reached Media');++assert.strictEqual(cutCalls.length, 1, 'the second simultaneous archival cutter never reaches Media');++releaseFirst();++const cuts = await Promise.all([firstCut, secondCut]);++assert.strictEqual(cuts.filter(Boolean).length, 1, 'exactly one archival clip is cut');++const clipLog = JSON.parse(await db.getState('auto_clip_log') || '[]');++assert.strictEqual(clipLog.filter((e) => e.archival).length, 1, 'exactly one archival clip is logged');++assert.strictEqual(policy.archiveAllowed(clipLog, Date.now()), false, "today's archive slot stays closed");++++// A cut that fails must not burn the day.++policy._resetArchiveReservation();++await db.setState('auto_clip_log', '[]');++await db.setState('ai_used_moments', '[]');++registry._reset();++media.createClip = async () => null;++assert.strictEqual(await archiveCut(72), null, 'the failed archival cut returns nothing');++assert.strictEqual(policy.archiveAllowed([], Date.now()), true, 'its reservation was released');++console.log('✅ archive gate: simultaneous cutters share one atomic slot');++++registry._reset();++console.log('\n✅ All clip-freshness tests passed');++})().catch((e) => { console.error(e); process.exit(1); });
@@ -21,6 +21,8 @@ const ai = require('./ai-analysis');
@@ -21,6 +21,8 @@ const ai = require('./ai-analysis');
const aiService = require('./ai-service');
const aiService = require('./ai-service');
const media = require('../media-client');
const media = require('../media-client');
const registry = require('./moment-registry');
const registry = require('./moment-registry');
+const clipPolicy = require('./clip-policy');
+const dedupe = require('./clip-dedupe');
const explicit = require('./explicit');
const explicit = require('./explicit');
let cfg = null; try { cfg = require('../config'); } catch { /* */ }
let cfg = null; try { cfg = require('../config'); } catch { /* */ }
let thumb = null; try { thumb = require('../media-proxy/live-thumbs'); } catch { /* optional */ }
let thumb = null; try { thumb = require('../media-proxy/live-thumbs'); } catch { /* optional */ }
@@ -195,6 +197,14 @@ function _nearestMemory(memories, offset) {
@@ -195,6 +197,14 @@ function _nearestMemory(memories, offset) {
for (const m of memories) { const d = Math.abs((m.offset_seconds || 0) - offset); if (d < bestD) { bestD = d; best = m; } }
for (const m of memories) { const d = Math.abs((m.offset_seconds || 0) - offset); if (d < bestD) { bestD = d; best = m; } }
return best;
return best;
}
}
+/** Transcript text around a moment, so dedupe can tell a new conversation from a repeated scene. */
+function _speechNear(transcript, offset, windowSec = 45) {
+ try {
+ return (transcript || [])
+ .filter(x => Math.abs(Number(x.start || 0) - Number(offset || 0)) <= windowSec)
+ .map(x => String(x.text || '')).join(' ').replace(/\s+/g, ' ').trim().slice(0, 600);
+ } catch { return ''; }
+}
// flavor: 'paste' → a frame that is striking on its own; 'clip' → a beat that plays out over ~25 s.
// flavor: 'paste' → a frame that is striking on its own; 'clip' → a beat that plays out over ~25 s.
// avoid: offsets (seconds) already turned into something — the picker must stay ≥ 2 min away.
// avoid: offsets (seconds) already turned into something — the picker must stay ≥ 2 min away.
async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {
async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {
@@ -225,7 +235,7 @@ async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {
@@ -225,7 +235,7 @@ async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {
t = clamp(t);
t = clamp(t);
if (!farFromUsed(t)) { console.log(`[AI-Moments] VOD ${vod.vod_id}: pick at ${_mmss(t)} is next to a used moment — trying the objective signals instead`); throw new Error('near used'); }
if (!farFromUsed(t)) { console.log(`[AI-Moments] VOD ${vod.vod_id}: pick at ${_mmss(t)} is next to a used moment — trying the objective signals instead`); throw new Error('near used'); }
const near = _nearestMemory(ctx.memories, t);
const near = _nearestMemory(ctx.memories, t);
- const result = { offset: t, title: _publicTitle(j.title, vod, t, j.desc), desc: db.noLooks(_cleanText(_deJson(j.desc), 400) || (near && _deJson(near.description)) || ''), tags: _memTags(near) };
+ const result = { offset: t, title: _publicTitle(j.title, vod, t, j.desc), desc: db.noLooks(_cleanText(_deJson(j.desc), 400) || (near && _deJson(near.description)) || ''), speech: _speechNear(ctx.transcript, t), tags: _memTags(near) };
if (explicit.isExplicit(j.desc, j.title, near && near.description, result.desc)) { console.log(`[AI-Moments] VOD ${vod.vod_id}: the pick at ${_mmss(t)} shows sexual content — no moment from it`); return null; }
if (explicit.isExplicit(j.desc, j.title, near && near.description, result.desc)) { console.log(`[AI-Moments] VOD ${vod.vod_id}: the pick at ${_mmss(t)} shows sexual content — no moment from it`); return null; }
return _isEmptyScene(result.desc) ? null : result;
return _isEmptyScene(result.desc) ? null : result;
}
}
@@ -244,7 +254,7 @@ async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {
@@ -244,7 +254,7 @@ async function _findBestMoment(vod, { flavor = 'paste', avoid = [] } = {}) {
const near = _nearestMemory(ctx.memories, offset);
const near = _nearestMemory(ctx.memories, offset);
const desc = near ? db.noLooks(_deJson(near.description)) : '';
const desc = near ? db.noLooks(_deJson(near.description)) : '';
if (near && explicit.isExplicit(near.description)) return null; // never a moment of sexual content (server/ai/explicit.js)
if (near && explicit.isExplicit(near.description)) return null; // never a moment of sexual content (server/ai/explicit.js)
- return _isEmptyScene(desc) ? null : { offset, title: _fallbackTitle(vod, offset, desc), desc, tags: _memTags(near) };
+ return _isEmptyScene(desc) ? null : { offset, title: _fallbackTitle(vod, offset, desc), desc, speech: _speechNear(ctx.transcript, offset), tags: _memTags(near) };
}
}
// True when a description says the frame is essentially empty — black/dark/loading/blank — so we
// True when a description says the frame is essentially empty — black/dark/loading/blank — so we
// never turn "nothing" into a clip or paste.
// never turn "nothing" into a clip or paste.
@@ -273,27 +283,54 @@ async function _frameTooDark(imgPath) {
@@ -273,27 +283,54 @@ async function _frameTooDark(imgPath) {
} catch { return false; }
} catch { return false; }
}
}
-// Coarse content signature for dedup — the first few significant words of a scene description.
-// Near-identical scenes ("A split-image close-up of an older man…") collapse to one signature.
-function _sig(desc) {
- return String(desc || '').toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim().split(' ').filter(Boolean).slice(0, 5).join(' ');
-}
-
// TODO(contract): the old duplicate-paste sweep pruned the local pastes table, which is
// TODO(contract): the old duplicate-paste sweep pruned the local pastes table, which is
// frozen for the OpenVibe.Media migration; Media has no metadata-filtered paste listing
// frozen for the OpenVibe.Media migration; Media has no metadata-filtered paste listing
// yet, so scene-dedup is handled purely via the usedSigs log below.
// yet, so scene-dedup is handled purely via the usedSigs log below.
function _dedupePastes() { /* moved to Media — sig-dedup happens pre-post */ }
function _dedupePastes() { /* moved to Media — sig-dedup happens pre-post */ }
-// Build the moment-ranking VOD pool: Media list (most-viewed public VODs) joined with
-// local AI state + stream memories. Media down → an empty pool (the run makes nothing).
+// Build the moment-ranking VOD pool: newest public VODs from Media joined with local AI state +
+// stream memories, filtered by the shared freshness policy (clip-policy.js): footage from the
+// last 48h normally; the archive only after 72h of site quietness, one archival clip per UTC day.
+// Media down → an empty pool (and never "quiet": an outage does not open the archive).
async function _momentPool(limit) {
async function _momentPool(limit) {
try {
try {
- const r = await media.listVods({ limit, order: 'views' });
- // A channel that turned AI Moments off (channels.ai_derivation_enabled) is never picked from.
+ const now = Date.now();
+ const r = await media.listVods({ limit: Math.max(24, limit * 2), order: 'newest' });
+ const all = r?.vods || (Array.isArray(r) ? r : []);
+ const history = require('../streaming/stream-history');
+ const site = clipPolicy.siteStatus({ now, ...await clipPolicy.siteActivity(db, { now }), upstreamHealthy: true });
+ let clipLog = [];
+ try { const l = JSON.parse(await db.getState('auto_clip_log') || '[]'); if (Array.isArray(l)) clipLog = l; } catch { /* */ }
+ // The newest listed VOD per channel gives us the row (for `newest`), but the channel's
+ // latest ACTIVITY comes from Live's streams: a channel can be live right now without a new
+ // VOD, and that live stream must keep it out of the quiet/archive exception.
+ const firstByUser = new Map();
+ for (const v of all) if (v.user_id != null && !firstByUser.has(v.user_id)) firstByUser.set(v.user_id, v);
+ const channelByUser = new Map();
+ for (const [userId, first] of firstByUser) {
+ const latest = await clipPolicy.latestStreamForUser(db, userId);
+ const lastAt = clipPolicy.streamActivityAt(latest, now);
+ channelByUser.set(userId, {
+ quiet: clipPolicy.channelStatus({ now, lastStreamAt: lastAt }).quiet,
+ stream_id: latest ? latest.id : null, vod_id: first.id,
+ });
+ }
+ const anyQuiet = site.quiet || [...channelByUser.values()].some((c) => c.quiet);
+ const archivalAllowed = anyQuiet && clipPolicy.archiveAllowed(clipLog, now);
const rows = [];
const rows = [];
- for (const v of r?.vods || (Array.isArray(r) ? r : [])) {
- try { if (await db.isAiDerivationEnabled(v.user_id)) rows.push(v); }
- catch { rows.push(v); }
+ for (const v of all) {
+ try { if (!await db.isAiDerivationEnabled(v.user_id)) continue; } catch { /* opt-out unreadable: keep it */ }
+ let stream = null, part = null;
+ if (v.stream_id) {
+ try { stream = await db.get('SELECT started_at, ended_at, is_live, peak_viewers FROM streams WHERE id = ?', [v.stream_id]); } catch { /* unknown age */ }
+ try { part = await history.partForVod(v.id); } catch { /* no part row */ }
+ }
+ const ch = channelByUser.get(v.user_id);
+ const channel = ch ? { quiet: ch.quiet, newest: ch.stream_id ? v.stream_id === ch.stream_id : v.id === ch.vod_id } : null;
+ const verdict = clipPolicy.classifySource({ vod: v, stream, part, now, site, channel });
+ if (!verdict.ok || (verdict.archival && !archivalAllowed)) continue;
+ rows.push({ ...v, archival: verdict.archival, captured_at: verdict.capturedAt, age_hours: verdict.ageHours, peak_viewers: stream ? stream.peak_viewers : 0 });
+ if (rows.length >= limit) break;
}
}
if (rows.length) {
if (rows.length) {
return (await Promise.all(rows.map(async (v) => {
return (await Promise.all(rows.map(async (v) => {
@@ -302,10 +339,9 @@ async function _momentPool(limit) {
@@ -302,10 +339,9 @@ async function _momentPool(limit) {
// (pastes read "@null's stream" without it).
// (pastes read "@null's stream" without it).
let username = v.username || null;
let username = v.username || null;
if (!username && v.user_id) { try { username = (await db.getUserById(v.user_id))?.username || null; } catch { /* */ } }
if (!username && v.user_id) { try { username = (await db.getUserById(v.user_id))?.username || null; } catch { /* */ } }
- let memory_count = 0, peak_viewers = 0;
+ let memory_count = 0;
if (v.stream_id) {
if (v.stream_id) {
try { memory_count = (await db.get('SELECT COUNT(*) AS c FROM stream_memories WHERE stream_id = ?', [v.stream_id]))?.c || 0; } catch { /* */ }
try { memory_count = (await db.get('SELECT COUNT(*) AS c FROM stream_memories WHERE stream_id = ?', [v.stream_id]))?.c || 0; } catch { /* */ }
- try { peak_viewers = (await db.getStreamById(v.stream_id))?.peak_viewers || 0; } catch { /* */ }
}
}
return {
return {
vod_id: v.id, stream_id: v.stream_id || null, user_id: v.user_id || null,
vod_id: v.id, stream_id: v.stream_id || null, user_id: v.user_id || null,
@@ -314,9 +350,10 @@ async function _momentPool(limit) {
@@ -314,9 +350,10 @@ async function _momentPool(limit) {
ai_overview_short: state.ai_overview_short || v.ai_overview_short || '',
ai_overview_short: state.ai_overview_short || v.ai_overview_short || '',
view_count: Number(v.view_count) || 0,
view_count: Number(v.view_count) || 0,
duration: Number(v.duration_seconds ?? v.duration) || 0,
duration: Number(v.duration_seconds ?? v.duration) || 0,
- peak_viewers, created_at: v.created_at || null,
+ peak_viewers: Number(v.peak_viewers) || 0, created_at: v.created_at || null,
clip_count: Number(v.clip_count) || 0,
clip_count: Number(v.clip_count) || 0,
- memory_count,
+ memory_count, archival: !!v.archival,
+ captured_at: v.captured_at, age_hours: v.age_hours,
};
};
})));
})));
}
}
@@ -336,10 +373,12 @@ async function tick(opts = {}) {
@@ -336,10 +373,12 @@ async function tick(opts = {}) {
try {
try {
const prev = await _load();
const prev = await _load();
const usedVods = new Set(ignoreUsed ? [] : (prev.usedVods || []));
const usedVods = new Set(ignoreUsed ? [] : (prev.usedVods || []));
- // Content-signature dedup: even different VODs from the same streamer (same setup) yield
- // near-identical scenes → we skip a moment whose description signature was used recently,
- // so the hero/pastes never show visual duplicates. Respected even on --fresh.
- const thisSigs = new Set(); // scenes picked in THIS run (cross-run dedupe lives in moment-registry)
+ // Content dedupe inside THIS run: even different VODs from the same streamer (same setup)
+ // can yield near-identical scenes. The registry is the only durable "used" log; this
+ // per-run list is the reservation for moments accepted before the next registry write.
+ const thisMoments = []; // { desc, speech } picked in THIS run (cross-run dedupe lives in moment-registry)
+ const seenThisRun = (desc, speech) => thisMoments.some((m) =>
+ dedupe.isSameScene(m.desc, desc) && (!(m.speech && speech) || dedupe.isSameSpeech(m.speech, speech)));
// Stage 1: consider every eligible VOD, ranked by its AI overview + popularity prior.
// Stage 1: consider every eligible VOD, ranked by its AI overview + popularity prior.
// The VOD pool comes from OpenVibe.Media (most-viewed public VODs), overlaid with
// The VOD pool comes from OpenVibe.Media (most-viewed public VODs), overlaid with
@@ -380,14 +419,13 @@ async function tick(opts = {}) {
@@ -380,14 +419,13 @@ async function tick(opts = {}) {
// Stage 2: find the best moment within this VOD.
// Stage 2: find the best moment within this VOD.
const moment = await _findBestMoment(v);
const moment = await _findBestMoment(v);
if (!moment) continue;
if (!moment) continue;
- // Skip near-duplicate scenes / spots — the registry is shared with the clip job.
- const sig = _sig(moment.desc || v.ai_overview_short || v.title);
- const why0 = await registry.usedReason({ vod_id: v.vod_id, stream_id: v.stream_id, offset: moment.offset, sig });
- if (why0 || (sig && thisSigs.has(sig))) {
- console.log(`[AI-Moments] Skipped VOD ${v.vod_id} — ${why0 || `duplicate scene ("${sig}")`}`);
+ // Skip near-duplicate scenes / spots — the registry is shared with the clip job, and
+ // seenThisRun closes the gap before this run's first registry write.
+ const why0 = await registry.usedReason({ vod_id: v.vod_id, stream_id: v.stream_id, offset: moment.offset, desc: moment.desc, title: moment.title, speech: moment.speech });
+ if (why0 || seenThisRun(moment.desc, moment.speech)) {
+ console.log(`[AI-Moments] Skipped VOD ${v.vod_id} — ${why0 || 'duplicate scene in this run'}`);
continue;
continue;
}
}
- if (sig) thisSigs.add(sig);
const offset = Math.floor(moment.offset || 0);
const offset = Math.floor(moment.offset || 0);
let desc = moment.desc || '';
let desc = moment.desc || '';
let title = moment.title || _fallbackTitle(v, offset, desc);
let title = moment.title || _fallbackTitle(v, offset, desc);
@@ -455,14 +493,13 @@ async function tick(opts = {}) {
@@ -455,14 +493,13 @@ async function tick(opts = {}) {
}
}
if (!desc) desc = _cleanText(v.ai_overview_short || v.ai_overview, 400) || 'A standout moment from this stream.';
if (!desc) desc = _cleanText(v.ai_overview_short || v.ai_overview, 400) || 'A standout moment from this stream.';
- // Re-check the signature against the FINAL (vision-verified) description — this is
- // what actually catches same-scene duplicates (e.g. a split-image close-up re-picked).
- const finalSig = _sig(desc);
- if (finalSig && finalSig !== sig && (await registry.isUsed({ sig: finalSig }) || thisSigs.has(finalSig))) {
- console.log(`[AI-Moments] Skipped VOD ${v.vod_id} — duplicate scene after vision ("${finalSig}")`);
+ // Re-check against the FINAL (vision-verified) description — this is what actually
+ // catches same-scene duplicates (e.g. a split-image close-up re-picked).
+ if ((await registry.isUsed({ desc, title, speech: moment.speech })) || seenThisRun(desc, moment.speech)) {
+ console.log(`[AI-Moments] Skipped VOD ${v.vod_id} — duplicate scene after vision`);
continue;
continue;
}
}
- if (finalSig) thisSigs.add(finalSig);
+ thisMoments.push({ desc, speech: moment.speech });
const content = `${desc}\n\n▶ Watch this moment on ${v.username ? `@${v.username}'s stream` : 'the stream'}: ${vodLink}`;
const content = `${desc}\n\n▶ Watch this moment on ${v.username ? `@${v.username}'s stream` : 'the stream'}: ${vodLink}`;
const metadata = JSON.stringify({ ai_moment: true, vod_id: v.vod_id, offset, vod_link: vodPath, username: v.username, why: r.why || null });
const metadata = JSON.stringify({ ai_moment: true, vod_id: v.vod_id, offset, vod_link: vodPath, username: v.username, why: r.why || null });
let pasteSlug = slug;
let pasteSlug = slug;
@@ -497,15 +534,15 @@ async function tick(opts = {}) {
@@ -497,15 +534,15 @@ async function tick(opts = {}) {
try { fs.unlinkSync(screenshotPath); } catch { /* tmp frame */ }
try { fs.unlinkSync(screenshotPath); } catch { /* tmp frame */ }
moments.push({ vodId: v.vod_id, offset, title: title.slice(0, 80), thumbnail: img, username: v.username, pasteSlug });
moments.push({ vodId: v.vod_id, offset, title: title.slice(0, 80), thumbnail: img, username: v.username, pasteSlug });
- await registry.record({ kind: 'paste', vod_id: v.vod_id, stream_id: v.stream_id, offset, sig: finalSig || sig, title });
+ await registry.record({ kind: 'paste', vod_id: v.vod_id, stream_id: v.stream_id, offset, title, desc, speech: moment.speech });
// Also cut a VOD clip from the SAME VOD — but a DIFFERENT beat (clip flavor, ≥ 2 min away),
// Also cut a VOD clip from the SAME VOD — but a DIFFERENT beat (clip flavor, ≥ 2 min away),
// so the clip and the paste never show the identical second with the identical title.
// so the clip and the paste never show the identical second with the identical title.
if (opts.clip !== false && momentSource) {
if (opts.clip !== false && momentSource) {
try {
try {
const second = await _findBestMoment(v, { flavor: 'clip', avoid: [offset] });
const second = await _findBestMoment(v, { flavor: 'clip', avoid: [offset] });
- if (second && !await registry.isUsed({ vod_id: v.vod_id, stream_id: v.stream_id, offset: second.offset, desc: second.desc, title: second.title })) {
- const clip = await require('./auto-clip-job').clipVodMoment({ vod: v, offset: second.offset, title: second.title, desc: second.desc, source: momentSource });
+ if (second && !await registry.isUsed({ vod_id: v.vod_id, stream_id: v.stream_id, offset: second.offset, desc: second.desc, title: second.title, speech: second.speech })) {
+ const clip = await require('./auto-clip-job').clipVodMoment({ vod: v, offset: second.offset, title: second.title, desc: second.desc, speech: second.speech, archival: v.archival, source: momentSource });
if (clip) console.log(`[AI-Moments] Auto-clip cut for VOD ${v.vod_id} at ${_mmss(second.offset)} ("${String(second.title || '').slice(0, 60)}") — paste was at ${_mmss(offset)}`);
if (clip) console.log(`[AI-Moments] Auto-clip cut for VOD ${v.vod_id} at ${_mmss(second.offset)} ("${String(second.title || '').slice(0, 60)}") — paste was at ${_mmss(offset)}`);
} else console.log(`[AI-Moments] VOD ${v.vod_id}: no second, distinct beat for a clip — skipped`);
} else console.log(`[AI-Moments] VOD ${v.vod_id}: no second, distinct beat for a clip — skipped`);
} catch { /* clipping is best-effort */ }
} catch { /* clipping is best-effort */ }
@@ -22,6 +22,8 @@ const ai = require('./ai-analysis');
@@ -22,6 +22,8 @@ const ai = require('./ai-analysis');
const media = require('../media-client');
const media = require('../media-client');
const recorder = require('../streaming/recorder');
const recorder = require('../streaming/recorder');
const registry = require('./moment-registry');
const registry = require('./moment-registry');
+const clipPolicy = require('./clip-policy');
+const dedupe = require('./clip-dedupe');
// ── Tunables ────────────────────────────────────────────────
// ── Tunables ────────────────────────────────────────────────
const CHECK_INTERVAL_MS = 90 * 1000;
const CHECK_INTERVAL_MS = 90 * 1000;
@@ -56,7 +58,11 @@ async function _logClip(entry) {
@@ -56,7 +58,11 @@ async function _logClip(entry) {
log.unshift({ ...entry, ts: Date.now() });
log.unshift({ ...entry, ts: Date.now() });
await db.setState(CLIP_LOG_SETTING, JSON.stringify(log.slice(0, 300)));
await db.setState(CLIP_LOG_SETTING, JSON.stringify(log.slice(0, 300)));
// Shared with the paste job, so a paste never lands on a clipped second (and vice versa).
// Shared with the paste job, so a paste never lands on a clipped second (and vice versa).
- await registry.record({ kind: 'clip', vod_id: entry.vod_id || null, stream_id: entry.stream_id || null, offset: (entry.start_time || 0) + CLIP_PRE, sig: entry.sig || null, title: entry.title || null });
+ await registry.record({
+ kind: 'clip', vod_id: entry.vod_id || null, stream_id: entry.stream_id || null,
+ offset: (entry.start_time || 0) + CLIP_PRE, sig: entry.sig || null, title: entry.title || null,
+ desc: entry.desc || null, speech: entry.speech || null,
+ });
} catch { /* */ }
} catch { /* */ }
}
}
async function _countClipsSince(streamId, minutes) {
async function _countClipsSince(streamId, minutes) {
@@ -70,7 +76,7 @@ async function _confirmLiveMoment(stream) {
@@ -70,7 +76,7 @@ async function _confirmLiveMoment(stream) {
const memories = (await db.getStreamMemories(stream.id) || []).filter(m => m.description).slice(-5);
const memories = (await db.getStreamMemories(stream.id) || []).filter(m => m.description).slice(-5);
const transcript = (await db.getStreamTranscriptSegments(stream.id) || []).slice(-14);
const transcript = (await db.getStreamTranscriptSegments(stream.id) || []).slice(-14);
const chat = await chatReads.recentChatText(stream.id, WINDOW_SEC, 40) || [];
const chat = await chatReads.recentChatText(stream.id, WINDOW_SEC, 40) || [];
- if (!await _aiOn()) return { clip: null }; // caller decides via the stricter no-AI threshold
+ if (!await _aiOn()) return { clip: null, transcript }; // caller decides via the stricter no-AI threshold
// Sound events are a strong clip signal on their own — a burst of gunfire, an explosion or laughter is exactly
// Sound events are a strong clip signal on their own — a burst of gunfire, an explosion or laughter is exactly
// the kind of thing viewers clip, and it is often the reason chat spiked in the first place.
// the kind of thing viewers clip, and it is often the reason chat spiked in the first place.
let sounds = [];
let sounds = [];
@@ -88,9 +94,18 @@ async function _confirmLiveMoment(stream) {
@@ -88,9 +94,18 @@ async function _confirmLiveMoment(stream) {
sounds: sounds.map(e => ({ t: Math.max(0, Number(e.start_sec) || 0), label: String(e.label || '').slice(0, 120), confidence: Number(e.confidence || 0) })),
sounds: sounds.map(e => ({ t: Math.max(0, Number(e.start_sec) || 0), label: String(e.label || '').slice(0, 120), confidence: Number(e.confidence || 0) })),
chat: chat.slice(-30).map(c => _clean(c, 100)).filter(Boolean),
chat: chat.slice(-30).map(c => _clean(c, 100)).filter(Boolean),
}, { target: { service: 'live', type: 'stream', id: String(stream.id) }, meter: { kind: 'auto_clip_confirm', role: 'legacy', ownerUserId: stream.user_id } });
}, { target: { service: 'live', type: 'stream', id: String(stream.id) }, meter: { kind: 'auto_clip_confirm', role: 'legacy', ownerUserId: stream.user_id } });
- if (!j) return { clip: false };
- return { clip: j.clip === true, title: _clean(j.title, 80), desc: _clean(j.desc, 400) };
- } catch { return { clip: false }; }
+ if (!j) return { clip: false, transcript };
+ return { clip: j.clip === true, title: _clean(j.title, 80), desc: _clean(j.desc, 400), transcript };
+ } catch { return { clip: false, transcript }; }
+}
+
+/** What was said around a stream second: the registry uses it so a static camera alone is not a repeat. */
+function _speechNear(transcript, sec, windowSec = 45) {
+ try {
+ return (transcript || [])
+ .filter(x => Math.abs(Number(x.start || 0) - Number(sec || 0)) <= windowSec)
+ .map(x => String(x.text || '')).join(' ').replace(/\s+/g, ' ').trim().slice(0, 600);
+ } catch { return ''; }
}
}
async function _checkLiveStream(stream) {
async function _checkLiveStream(stream) {
@@ -136,6 +151,13 @@ async function _checkLiveStream(stream) {
@@ -136,6 +151,13 @@ async function _checkLiveStream(stream) {
// A moment the AI describes as sexual content is never clipped (server/ai/explicit.js).
// A moment the AI describes as sexual content is never clipped (server/ai/explicit.js).
if (require('./explicit').isExplicit(verdict.desc, verdict.title)) { console.log(`[AutoClip] stream ${streamId}: skipped a moment the AI describes as sexual content`); return; }
if (require('./explicit').isExplicit(verdict.desc, verdict.title)) { console.log(`[AutoClip] stream ${streamId}: skipped a moment the AI describes as sexual content`); return; }
const title = verdict.title ? require('./ai-moments-job').publicTitle(verdict.title, stream, momentOffset, verdict.desc) : 'Chat-Hyped Moment';
const title = verdict.title ? require('./ai-moments-job').publicTitle(verdict.title, stream, momentOffset, verdict.desc) : 'Chat-Hyped Moment';
+ // The live path used to skip the shared registry and record only AFTER cutting, so a moment
+ // already turned into a paste/clip could be cut again here. Ask before spending a clip.
+ let streamSec = momentOffset; // transcripts are in stream time; momentOffset is in the VOD's time
+ try { const part = await require('../streaming/stream-history').partForVod(rec.vodId); if (part && Number.isFinite(Number(part.at))) streamSec = momentOffset + Number(part.at); } catch { /* one-part recording */ }
+ const speech = _speechNear(verdict.transcript, streamSec);
+ const why = await registry.usedReason({ vod_id: rec.vodId, stream_id: streamId, offset: momentOffset, desc: verdict.desc, title, speech });
+ if (why) { console.log(`[AutoClip] stream ${streamId}: ${why}`); return; }
const clip = await media.createClip({
const clip = await media.createClip({
vod_id: rec.vodId, start_s: start, end_s: end,
vod_id: rec.vodId, start_s: start, end_s: end,
title, user_id: stream.user_id,
title, user_id: stream.user_id,
@@ -144,7 +166,7 @@ async function _checkLiveStream(stream) {
@@ -144,7 +166,7 @@ async function _checkLiveStream(stream) {
if (clip) {
if (clip) {
// deduplicated: Media handed back a clip someone already cut of this window. It stays in
// deduplicated: Media handed back a clip someone already cut of this window. It stays in
// the log for the caps, but it is theirs, not the AI's (syncAutoClipFlags skips it).
// the log for the caps, but it is theirs, not the AI's (syncAutoClipFlags skips it).
- await _logClip({ stream_id: streamId, vod_id: rec.vodId, start_time: start, title, clip_id: clip.id, dedup: !!clip.deduplicated || undefined, sig: _sceneSig(verdict.desc || title) });
+ await _logClip({ stream_id: streamId, vod_id: rec.vodId, start_time: start, title, clip_id: clip.id, dedup: !!clip.deduplicated || undefined, desc: verdict.desc || '', speech, sig: _sceneSig(verdict.desc || title) });
console.log(`[AutoClip] LIVE clip for stream ${streamId} @${Math.round(momentOffset)}s ("${title}") — spike ${peak.count} msgs`);
console.log(`[AutoClip] LIVE clip for stream ${streamId} @${Math.round(momentOffset)}s ("${title}") — spike ${peak.count} msgs`);
}
}
} catch (e) {
} catch (e) {
@@ -162,45 +184,77 @@ async function _tick() {
@@ -162,45 +184,77 @@ async function _tick() {
} finally { _busy = false; }
} finally { _busy = false; }
}
}
-// ── Historical VOD backfill: a few auto-clips per day (daily-gated, like the moments/paste
-// jobs). Processes the most-watched un-clipped VODs first; idempotent (skips VODs that already
-// have an auto-clip), so it steadily works through the back catalogue over days. ─────────────
+// ── Recent VOD backfill: a few auto-clips per run (daily-gated, like the moments/paste jobs).
+// Only footage from the last 48 hours is eligible; the old back catalogue is mined ONLY after the
+// site has been quiet for 72 hours and then at most one archive clip per UTC day (clip-policy.js).
+// Newest-first, idempotent (skips VODs that already have an auto-clip). ─────────────────────
const BACKFILL_SETTING = 'auto_clip_backfill';
const BACKFILL_SETTING = 'auto_clip_backfill';
-const BACKFILL_PER_RUN = 3; // a few historical clips…
-const BACKFILL_INTERVAL_MS = 6 * 60 * 60 * 1000; // …every 6h (guarantees ≥1 clip / 6h)
+const BACKFILL_PER_RUN = 3; // a few recent clips…
+const BACKFILL_INTERVAL_MS = 6 * 60 * 60 * 1000; // …every 6h
async function _backfillDue() {
async function _backfillDue() {
try { const p = JSON.parse(await db.getState(BACKFILL_SETTING) || '{}'); return !p.updated_at || (Date.now() - p.updated_at) >= BACKFILL_INTERVAL_MS; }
try { const p = JSON.parse(await db.getState(BACKFILL_SETTING) || '{}'); return !p.updated_at || (Date.now() - p.updated_at) >= BACKFILL_INTERVAL_MS; }
catch { return true; }
catch { return true; }
}
}
-// Historical-backfill candidate pool: most-viewed public VODs from OpenVibe.Media
-// that have stream memories and no auto-clip in the local log yet. Media down → an
-// empty pool (the clips are cut by Media anyway).
+/**
+ * Backfill candidate pool: newest public VODs from OpenVibe.Media that have stream memories and no
+ * auto-clip in the local log yet, filtered by the shared freshness policy. `archival` is set on a
+ * row only when the quiet-site archive window is open. Media down → an empty pool (the site is not
+ * "quiet", it is unreachable — an outage never opens the archive).
+ */
async function _backfillPool(limit) {
async function _backfillPool(limit) {
try {
try {
- const r = await media.listVods({ limit: limit * 2, order: 'views' });
+ const now = Date.now();
+ const r = await media.listVods({ limit: Math.max(24, limit * 4), order: 'newest' });
const rows = r?.vods || (Array.isArray(r) ? r : []);
const rows = r?.vods || (Array.isArray(r) ? r : []);
- if (rows.length) {
- const clipped = new Set();
- for (const c of await _clipLog()) { if (c.vod_id) clipped.add(c.vod_id); if (c.stream_id) clipped.add(`s${c.stream_id}`); }
- const candidates = [];
- for (const v of rows) {
- if (!v.stream_id || clipped.has(v.id) || clipped.has(`s${v.stream_id}`)) continue;
- if (!await _derivationOn(v.user_id)) continue;
- try {
- if ((await db.get('SELECT COUNT(*) AS c FROM stream_memories WHERE stream_id = ?', [v.stream_id]))?.c > 0) candidates.push(v);
- } catch { /* skip an unreadable stream */ }
- if (candidates.length >= limit) break;
- }
- return (await Promise.all(candidates.map(async v => ({
- vod_id: v.id, stream_id: v.stream_id, user_id: v.user_id, username: v.username,
- title: v.title || '',
- ai_overview: (db.getVodAiState && (await db.getVodAiState(v.id))?.ai_overview_short) || '',
- ai_overview_short: (db.getVodAiState && (await db.getVodAiState(v.id))?.ai_overview_short) || '',
- duration: Number(v.duration_seconds ?? v.duration) || 0,
- view_count: Number(v.view_count) || 0,
- }))));
+ if (!rows.length) return [];
+ const history = require('../streaming/stream-history');
+ const site = clipPolicy.siteStatus({ now, ...await clipPolicy.siteActivity(db, { now }), upstreamHealthy: true });
+ // The newest listed VOD per channel gives us the row (for `newest`), but the channel's
+ // latest ACTIVITY comes from Live's streams: a channel can be live right now without a new
+ // VOD, and that live stream must keep it out of the quiet/archive exception.
+ const firstByUser = new Map();
+ for (const v of rows) if (v.user_id != null && !firstByUser.has(v.user_id)) firstByUser.set(v.user_id, v);
+ const channelByUser = new Map();
+ for (const [userId, first] of firstByUser) {
+ const latest = await clipPolicy.latestStreamForUser(db, userId);
+ const lastAt = clipPolicy.streamActivityAt(latest, now);
+ channelByUser.set(userId, {
+ quiet: clipPolicy.channelStatus({ now, lastStreamAt: lastAt }).quiet,
+ stream_id: latest ? latest.id : null, vod_id: first.id,
+ });
}
}
+ const anyQuiet = site.quiet || [...channelByUser.values()].some((c) => c.quiet);
+ const archivalAllowed = anyQuiet && clipPolicy.archiveAllowed(await _clipLog(), now);
+ const clipped = new Set();
+ for (const c of await _clipLog()) { if (c.vod_id) clipped.add(c.vod_id); if (c.stream_id) clipped.add(`s${c.stream_id}`); }
+ const candidates = [];
+ for (const v of rows) {
+ if (candidates.length >= limit) break;
+ if (!v.stream_id || clipped.has(v.id) || clipped.has(`s${v.stream_id}`)) continue;
+ if (!await _derivationOn(v.user_id)) continue;
+ let stream = null, part = null;
+ try { stream = await db.get('SELECT started_at, ended_at, is_live FROM streams WHERE id = ?', [v.stream_id]); } catch { /* unknown age */ }
+ try { part = await history.partForVod(v.id); } catch { /* no part row */ }
+ const ch = channelByUser.get(v.user_id);
+ const channel = ch ? { quiet: ch.quiet, newest: ch.stream_id ? v.stream_id === ch.stream_id : v.id === ch.vod_id } : null;
+ const verdict = clipPolicy.classifySource({ vod: v, stream, part, now, site, channel });
+ if (!verdict.ok || (verdict.archival && !archivalAllowed)) continue;
+ try {
+ if ((await db.get('SELECT COUNT(*) AS c FROM stream_memories WHERE stream_id = ?', [v.stream_id]))?.c > 0) {
+ candidates.push({ ...v, archival: verdict.archival, captured_at: verdict.capturedAt, age_hours: verdict.ageHours });
+ }
+ } catch { /* skip an unreadable stream */ }
+ }
+ return (await Promise.all(candidates.map(async v => ({
+ vod_id: v.id, stream_id: v.stream_id, user_id: v.user_id, username: v.username,
+ title: v.title || '',
+ ai_overview: (db.getVodAiState && (await db.getVodAiState(v.id))?.ai_overview_short) || '',
+ ai_overview_short: (db.getVodAiState && (await db.getVodAiState(v.id))?.ai_overview_short) || '',
+ duration: Number(v.duration_seconds ?? v.duration) || 0,
+ view_count: Number(v.view_count) || 0,
+ archival: !!v.archival, captured_at: v.captured_at, age_hours: v.age_hours,
+ }))));
} catch { /* Media down */ }
} catch { /* Media down */ }
return [];
return [];
}
}
@@ -227,9 +281,11 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}
@@ -227,9 +281,11 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}
// Over-fetch so skipped/dead VODs don't starve a batch.
// Over-fetch so skipped/dead VODs don't starve a batch.
const pool = (await _backfillPool(Math.max(1, limit) * 6)).filter(v => !skip.has(v.vod_id));
const pool = (await _backfillPool(Math.max(1, limit) * 6)).filter(v => !skip.has(v.vod_id));
let made = 0;
let made = 0;
+ let archiveMade = 0;
const newSkip = [];
const newSkip = [];
for (const v of pool) {
for (const v of pool) {
if (made >= limit) break;
if (made >= limit) break;
+ if (v.archival && archiveMade >= clipPolicy.ARCHIVE_PER_DAY) continue; // one archive clip per UTC day
try {
try {
const source = await _resolveVodSource(v.vod_id);
const source = await _resolveVodSource(v.vod_id);
if (!source) { newSkip.push(v.vod_id); continue; } // media gone / unreadable
if (!source) { newSkip.push(v.vod_id); continue; } // media gone / unreadable
@@ -246,8 +302,9 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}
@@ -246,8 +302,9 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}
if (dark) { console.log(`[AutoClip] backfill skip VOD ${v.vod_id} — dark/empty frame`); newSkip.push(v.vod_id); continue; }
if (dark) { console.log(`[AutoClip] backfill skip VOD ${v.vod_id} — dark/empty frame`); newSkip.push(v.vod_id); continue; }
}
}
} catch { /* probe optional */ }
} catch { /* probe optional */ }
- const clip = await clipVodMoment({ vod: v, offset: moment.offset, title: moment.title, desc: moment.desc, source });
- if (clip) { made++; console.log(`[AutoClip] Backfilled VOD ${v.vod_id} ("${String(moment.title || '').slice(0, 60)}")`); }
+ const clip = await clipVodMoment({ vod: v, offset: moment.offset, title: moment.title, desc: moment.desc, speech: moment.speech, archival: v.archival, source });
+ if (clip) { made++; if (v.archival) archiveMade++; console.log(`[AutoClip] Backfilled VOD ${v.vod_id}${v.archival ? ' (archive)' : ''} ("${String(moment.title || '').slice(0, 60)}")`); }
+ else if (v.archival && !clipPolicy.archiveAllowed(await _clipLog(), Date.now())) continue; // another cutter took today's archive slot: not this VOD's fault, don't skip it forever
else newSkip.push(v.vod_id); // cut failed (e.g. unseekable/short) — don't retry forever
else newSkip.push(v.vod_id); // cut failed (e.g. unseekable/short) — don't retry forever
} catch (e) { newSkip.push(v.vod_id); console.warn(`[AutoClip] backfill VOD ${v.vod_id} failed:`, e.message); }
} catch (e) { newSkip.push(v.vod_id); console.warn(`[AutoClip] backfill VOD ${v.vod_id} failed:`, e.message); }
}
}
@@ -259,45 +316,61 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}
@@ -259,45 +316,61 @@ async function backfillVodClips({ limit = BACKFILL_PER_RUN, force = false } = {}
/**
/**
* Cut a clip around a moment the AI-moments pipeline already chose for a finished VOD.
* Cut a clip around a moment the AI-moments pipeline already chose for a finished VOD.
- * @param {object} o { vod (row w/ user_id, stream_id, vod_id/id), offset, title, desc }
+ * @param {object} o { vod (row w/ user_id, stream_id, vod_id/id), offset, title, desc, speech, archival, source }
*/
*/
function _sceneSig(text) {
function _sceneSig(text) {
- return String(text || '').toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim().split(' ').filter(Boolean).slice(0, 5).join(' ');
+ return dedupe.shortSignature(text, 5);
}
}
// True if a recent auto-clip is basically the same as (vod, offset, scene) we're about to cut —
// True if a recent auto-clip is basically the same as (vod, offset, scene) we're about to cut —
-// same VOD spot, or a near-identical scene signature (same streamer's repeated intro/setup).
-async function _isDuplicateAutoClip(vod, offset, desc, title) {
+// same VOD spot, or a near-identical scene signature (same streamer's repeated intro/setup). What
+// was said is part of the comparison, so a static camera with a new conversation is not a repeat.
+async function _isDuplicateAutoClip(vod, offset, desc, title, speech) {
try {
try {
const vId = vod.vod_id || vod.id;
const vId = vod.vod_id || vod.id;
- const why = await registry.usedReason({ vod_id: vId, stream_id: vod.stream_id, offset: Math.floor(offset), sig: _sceneSig(desc || title) });
+ const why = await registry.usedReason({ vod_id: vId, stream_id: vod.stream_id, offset: Math.floor(offset), desc: desc || title, speech });
if (why) { console.log(`[AutoClip] VOD ${vId}: ${why}`); return true; }
if (why) { console.log(`[AutoClip] VOD ${vId}: ${why}`); return true; }
} catch { /* */ }
} catch { /* */ }
return false;
return false;
}
}
async function clipVodMoment(o) {
async function clipVodMoment(o) {
+ let archiveHeld = false;
try {
try {
- const { vod, offset, title, desc } = o || {};
+ const { vod, offset, title, desc, speech, archival } = o || {};
if (!vod || !(offset >= 0)) return null;
if (!vod || !(offset >= 0)) return null;
if (!await _derivationOn(vod.user_id)) return null; // the channel turned AI Moments off
if (!await _derivationOn(vod.user_id)) return null; // the channel turned AI Moments off
if (require('./explicit').isExplicit(desc, title)) { // never a clip of sexual content (server/ai/explicit.js)
if (require('./explicit').isExplicit(desc, title)) { // never a clip of sexual content (server/ai/explicit.js)
console.log(`[AutoClip] skip VOD ${vod.vod_id || vod.id} — the AI describes the moment as sexual content`);
console.log(`[AutoClip] skip VOD ${vod.vod_id || vod.id} — the AI describes the moment as sexual content`);
return null;
return null;
}
}
- if (await _isDuplicateAutoClip(vod, offset, desc, title)) {
+ if (await _isDuplicateAutoClip(vod, offset, desc, title, speech)) {
console.log(`[AutoClip] skip VOD ${vod.vod_id || vod.id} — duplicate/near-identical auto-clip already exists`);
console.log(`[AutoClip] skip VOD ${vod.vod_id || vod.id} — duplicate/near-identical auto-clip already exists`);
return null;
return null;
}
}
const vodId = vod.vod_id || vod.id;
const vodId = vod.vod_id || vod.id;
const start = Math.max(0, Math.floor(offset) - CLIP_PRE);
const start = Math.max(0, Math.floor(offset) - CLIP_PRE);
+ // One archive clip per UTC day is shared with the AI moments job. Both may pass the pool
+ // gate before either writes auto_clip_log, so claim the slot atomically before cutting and
+ // give it back if no clip results.
+ if (archival) {
+ if (!clipPolicy.reserveArchive(await _clipLog(), Date.now())) {
+ console.log(`[AutoClip] VOD ${vodId}: today's archival clip is already taken`);
+ return null;
+ }
+ archiveHeld = true;
+ }
const clip = await media.createClip({
const clip = await media.createClip({
vod_id: vodId, start_s: start, end_s: start + CLIP_PRE + CLIP_POST,
vod_id: vodId, start_s: start, end_s: start + CLIP_PRE + CLIP_POST,
title: title || 'Standout Moment', user_id: vod.user_id,
title: title || 'Standout Moment', user_id: vod.user_id,
stream_id: vod.stream_id || undefined, auto_generated: true, description: db.noLooks(desc || ''),
stream_id: vod.stream_id || undefined, auto_generated: true, description: db.noLooks(desc || ''),
});
});
- if (clip) await _logClip({ stream_id: vod.stream_id || null, vod_id: vodId, start_time: start, title: title || 'Standout Moment', clip_id: clip.id, dedup: !!clip.deduplicated || undefined, sig: _sceneSig(desc || title) });
+ if (clip) {
+ archiveHeld = false; // the clip exists: keep the day reserved even if the log write fails
+ await _logClip({ stream_id: vod.stream_id || null, vod_id: vodId, start_time: start, title: title || 'Standout Moment', clip_id: clip.id, dedup: !!clip.deduplicated || undefined, desc: desc || '', speech, archival: archival || undefined, sig: _sceneSig(desc || title) });
+ }
return clip || null;
return clip || null;
} catch { return null; }
} catch { return null; }
+ finally { if (archiveHeld) clipPolicy.releaseArchive(); }
}
}
// ── Tell Media which clips the AI cut ────────────────────────────────────────────────────
// ── Tell Media which clips the AI cut ────────────────────────────────────────────────────
new file mode 100644
new file mode 100644
@@ -0,0 +1,137 @@
@@ -0,0 +1,137 @@
+/**
+ * clip-dedupe.js — text/visual identity for moments, shared by the moment registry and both cutters.
+ *
+ * The old registry compared only the first five ASCII words of a description: paraphrases got
+ * through, Japanese collapsed to an empty signature, and unrelated scenes that opened with the
+ * same boilerplate collided. This module keeps Unicode, compares token sets (Dice similarity, so
+ * a re-worded or longer description of the same scene still matches) and keeps the visual scene
+ * separate from what was said. A static webcam with a NEW conversation must not be treated as a
+ * repeated moment: when both sides have speech, the speech has to be similar too.
+ *
+ * normalize(text) lower-case NFKC text, punctuation removed, spaces kept
+ * tokens(text) {word} tokens; CJK uses character bigrams
+ * contentTokens(text) tokens without articles/boilerplate media words
+ * shortSignature(text, words = 5) the legacy five-token signature (registry `sig`)
+ * isSameScene(a, b, opts) true for the same visual scene (near-duplicate wording)
+ * isSameSpeech(a, b) true for a near-duplicate transcript excerpt (Dice ≥ 0.8)
+ * isSameEmbedding(a, b) true for near-identical frame embeddings, when both exist
+ * similarity(a, b) Dice coefficient over token sets
+ * cosine(a, b) cosine similarity over numeric vectors
+ *
+ * No model or network: a cheap baseline that replaces exact string equality. Frame embeddings are
+ * optional — the registry stores them when a caller has them and otherwise relies on text.
+ */
+'use strict';
+
+const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u;
+const WORD_RE = /[\p{L}\p{N}]+/gu;
+
+// Words that describe the medium rather than the scene. They must not be the only reason two
+// different moments look alike ("a live streaming session with chat" vs "… with music").
+const BOILERPLATE = new Set((
+ 'a an the and or of to in on at for with from by as is are was were be been being ' +
+ 'it its this that these those there here ' +
+ 'video videos stream streams live streaming session screen screenshot image images frame ' +
+ 'footage scene clip moment appears appear shown showing visible view camera close ' +
+ 'background foreground'
+).split(' '));
+
+function normalize(text) {
+ return String(text == null ? '' : text).normalize('NFKC').toLowerCase()
+ .replace(/[^\p{L}\p{N}]+/gu, ' ').replace(/\s+/g, ' ').trim();
+}
+
+/** Token set: words for spaced languages, overlapping bigrams for CJK (which has no spaces). */
+function tokens(text) {
+ const norm = normalize(text);
+ if (!norm) return [];
+ if (CJK_RE.test(norm)) {
+ const compact = norm.replace(/\s+/g, '');
+ const out = [];
+ for (let i = 0; i + 1 < compact.length; i++) out.push(compact.slice(i, i + 2));
+ // Mixed Japanese/English streams: reuse the Latin words as well.
+ for (const w of norm.match(/[a-z0-9]+/g) || []) if (w.length > 1) out.push(w);
+ return [...new Set(out)];
+ }
+ return [...new Set((norm.match(WORD_RE) || []))];
+}
+
+/** Tokens that actually describe content; articles and media boilerplate are dropped. */
+function contentTokens(text) {
+ return tokens(text).filter((t) => !BOILERPLATE.has(t) && !/^\d+$/.test(t));
+}
+
+function similarity(a, b) {
+ const A = new Set(a), B = new Set(b);
+ if (!A.size || !B.size) return 0;
+ let shared = 0;
+ for (const x of A) if (B.has(x)) shared++;
+ return (2 * shared) / (A.size + B.size);
+}
+
+/** Cosine similarity of two numeric vectors; 0 for a missing/zero/mismatched pair. */
+function cosine(a, b) {
+ if (!Array.isArray(a) || !Array.isArray(b) || !a.length || a.length !== b.length) return 0;
+ let dot = 0, na = 0, nb = 0;
+ for (let i = 0; i < a.length; i++) {
+ const x = Number(a[i]), y = Number(b[i]);
+ if (!Number.isFinite(x) || !Number.isFinite(y)) return 0;
+ dot += x * y; na += x * x; nb += y * y;
+ }
+ if (!na || !nb) return 0;
+ return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+
+/** The legacy registry signature: the first `words` tokens, joined. */
+function shortSignature(text, words = 5) {
+ return tokens(text).slice(0, words).join(' ');
+}
+
+/**
+ * Same scene when the token sets are near-identical. A match may not rest on boilerplate alone:
+ * the two descriptions must share at least one content token, so "a live stream with chat" does
+ * not match "a live stream with music". Short texts only match exactly.
+ */
+function isSameScene(a, b, { threshold = 0.85, minTokens = 4 } = {}) {
+ const A = tokens(a), B = tokens(b);
+ if (!A.length || !B.length) return false;
+ if (normalize(a) === normalize(b)) return true;
+ const CA = contentTokens(a), CB = contentTokens(b);
+ const cbSet = new Set(CB);
+ const sharedContent = CA.filter((t) => cbSet.has(t));
+ if (CA.length || CB.length) {
+ if (!sharedContent.length) return false;
+ if (CA.length >= 2 && CB.length >= 2) return similarity(CA, CB) >= threshold;
+ }
+ if (A.length < minTokens || B.length < minTokens) return false;
+ return similarity(A, B) >= threshold;
+}
+
+/**
+ * A transcript excerpt is the same speech even when speech-to-text differs a little. Dice ≥ 0.8
+ * over Unicode-safe token sets; a match that shares only boilerplate ("this is a live stream" vs
+ * "this is a live chat") is not enough.
+ */
+function isSameSpeech(a, b, { threshold = 0.8 } = {}) {
+ const A = tokens(a), B = tokens(b);
+ if (!A.length || !B.length) return false;
+ if (normalize(a) === normalize(b)) return true;
+ if (similarity(A, B) < threshold) return false;
+ const CA = contentTokens(a), CB = contentTokens(b);
+ if (CA.length || CB.length) {
+ const cbSet = new Set(CB);
+ return CA.some((t) => cbSet.has(t));
+ }
+ return true;
+}
+
+/** Frame embeddings, when the caller has them: the same footage scores near 1. */
+function isSameEmbedding(a, b, { threshold = 0.92 } = {}) {
+ if (!Array.isArray(a) || !Array.isArray(b) || !a.length || a.length !== b.length) return false;
+ return cosine(a, b) >= threshold;
+}
+
+module.exports = {
+ normalize, tokens, contentTokens, similarity, cosine, shortSignature,
+ isSameScene, isSameSpeech, isSameEmbedding,
+};
new file mode 100644
new file mode 100644
@@ -0,0 +1,213 @@
@@ -0,0 +1,213 @@
+/**
+ * clip-policy.js — ONE freshness policy for every automated cutter.
+ *
+ * The owner (2026-10-29): "the AI clipping system seems to clip old videos. Maybe only make
+ * clips of fresh stuff, unless the site's dead." Both the live clip job and the AI moments job
+ * mined the most-VIEWED back catalogue with no age cutoff, so old VODs kept being clipped.
+ *
+ * fresh the producing stream ended (or is still running) in the last 48 hours
+ * archive only when the site has been quiet for 72 hours (no new footage AND no human
+ * pageviews) OR the channel itself has had no new stream in 48 hours (then only its
+ * newest VODs), and the upstream service is healthy; then at most one archival clip
+ * per UTC day, from the previous 90 days, newest first, labelled archival.
+ *
+ * The 48-hour clock starts when the producing STREAM ended, not when it started and not when
+ * the VOD was uploaded: a five-day stream that finished an hour ago is fresh even with no part
+ * row. A still-running stream is fresh; a source with no ended_at falls back to its capture/part
+ * time, and an old imported recording is never fresh just because it was uploaded recently.
+ *
+ * footageStart({ vod, stream, part }) → epoch ms of the source's first frame, or null
+ * streamEndedAt({ vod, stream, now }) → epoch ms the producing stream last had footage
+ * classifySource({ vod, stream, part, now, site, channel }) → { ok, archival, ageHours, … }
+ * siteActivity(db, { now }) → { lastFootageAt, lastHumanActivityAt }
+ * siteStatus({ now, … }) → { quiet, hours, why }
+ * latestStreamForUser(db, userId) → the user's newest stream row (live first)
+ * streamActivityAt(stream, now) → epoch ms of that stream's latest activity
+ * channelStatus({ now, lastStreamAt }) → { quiet, hours, why }
+ * archiveAllowed(log, now) → one archival clip per UTC day
+ * reserveArchive(log, now) / releaseArchive(now) → atomic in-process reservation of that slot
+ *
+ * An outage is never quietness: without a healthy upstream (`upstreamHealthy: false`) or a known
+ * activity timestamp, the archive stays closed.
+ */
+'use strict';
+
+const HOUR_MS = 3600_000;
+const FRESH_HOURS = 48;
+const QUIET_HOURS = 72;
+const CHANNEL_QUIET_HOURS = 48; // a channel with no new stream in the freshness window is quiet
+const ARCHIVE_DAYS = 90;
+const ARCHIVE_PER_DAY = 1;
+
+/** Parse the DB's `YYYY-MM-DD HH:MM:SS` UTC text, ISO text, Date, or epoch (s/ms) → ms or null. */
+function parseTs(value) {
+ if (value == null || value === '') return null;
+ if (value instanceof Date) return Number.isFinite(value.getTime()) ? value.getTime() : null;
+ if (typeof value === 'number') {
+ if (!Number.isFinite(value)) return null;
+ return value > 1e12 ? value : value * 1000; // epoch seconds vs milliseconds
+ }
+ const s = String(value).trim();
+ if (!s) return null;
+ const iso = /[zZ]|[+-]\d\d:?\d\d$/.test(s) ? s : `${s.replace(' ', 'T')}Z`;
+ const ms = Date.parse(iso);
+ return Number.isFinite(ms) ? ms : null;
+}
+
+/**
+ * When the footage was captured. Media may name it directly; otherwise the stream's part row
+ * gives the exact start of this recording (stream start + part offset), which is why a long
+ * stream recorded in parts does not look old. `created_at` is the last resort — for recordings
+ * Live itself opened it is the recorder's start time.
+ */
+function footageStart({ vod, stream, part } = {}) {
+ for (const field of ['captured_at', 'recorded_at', 'recording_started_at']) {
+ const t = parseTs(vod && vod[field]);
+ if (t != null) return t;
+ }
+ const streamStart = parseTs(stream && stream.started_at);
+ if (streamStart != null && part && Number.isFinite(Number(part.at))) return streamStart + Number(part.at) * 1000;
+ const created = parseTs(vod && (vod.created_at || vod.createdAt));
+ if (created != null) return created;
+ return streamStart;
+}
+
+/**
+ * When the stream that produced a VOD last had footage — the reference for the 48-hour window.
+ * A finished stream's ended_at governs; a still-live stream is "now". Without a trusted end
+ * (an import or a stream row that never got ended_at), the capture/part time stands in, and only
+ * then the VOD's created_at: an old file uploaded today must not look fresh because of upload time.
+ */
+function streamEndedAt({ vod, stream, now = Date.now() } = {}) {
+ if (stream) {
+ if (Number(stream.is_live) > 0) return now;
+ const ended = parseTs(stream.ended_at);
+ if (ended != null) return ended;
+ }
+ // Capture metadata before created_at: created_at is the upload/import time for a recording
+ // Live did not open itself.
+ const captured = parseTs(vod && (vod.captured_at || vod.recorded_at || vod.recording_started_at));
+ if (captured != null) return captured;
+ if (stream) {
+ const started = parseTs(stream.started_at);
+ if (started != null) return started;
+ }
+ return parseTs(vod && (vod.created_at || vod.createdAt));
+}
+
+/**
+ * Site quietness from the two signals we have in Live: new footage (a live stream, a stream that
+ * ended, a VOD capture) and human pageviews. Both are required; an unknown signal is not quiet.
+ * `upstreamHealthy` comes from the caller (its Media listing answered).
+ */
+function siteStatus({ now = Date.now(), lastFootageAt = null, lastHumanActivityAt = null, upstreamHealthy = false } = {}) {
+ if (!upstreamHealthy) return { quiet: false, hours: null, lastAt: null, why: 'upstream not healthy' };
+ if (lastFootageAt == null || lastHumanActivityAt == null) return { quiet: false, hours: null, lastAt: null, why: 'site activity unknown' };
+ const lastAt = Math.max(lastFootageAt, lastHumanActivityAt);
+ const hours = (now - lastAt) / HOUR_MS;
+ return { quiet: hours >= QUIET_HOURS, hours, lastAt, why: hours >= QUIET_HOURS ? 'site quiet' : 'recent activity' };
+}
+
+/**
+ * A channel's quietness: no new stream in the freshness window (48h). Unknown activity is not
+ * quietness, and an outage is never quietness. `newest` on the per-VOD channel object limits the
+ * channel exception to the newest VODs, so a quiet channel does not reopen its whole archive.
+ */
+function channelStatus({ now = Date.now(), lastStreamAt = null } = {}) {
+ if (lastStreamAt == null) return { quiet: false, hours: null, lastAt: null, why: 'channel activity unknown' };
+ const hours = (now - lastStreamAt) / HOUR_MS;
+ return { quiet: hours >= CHANNEL_QUIET_HOURS, hours, lastAt: lastStreamAt, why: hours >= CHANNEL_QUIET_HOURS ? 'channel quiet' : 'channel active' };
+}
+
+async function siteActivity(db, { now = Date.now() } = {}) {
+ let lastFootageAt = null, lastHumanActivityAt = null;
+ try {
+ const s = await db.get(`SELECT MAX(started_at) AS started, MAX(ended_at) AS ended,
+ MAX(CASE WHEN is_live = 1 THEN 1 ELSE 0 END) AS live
+ FROM streams`);
+ const started = parseTs(s && s.started), ended = parseTs(s && s.ended);
+ if (Number(s && s.live) > 0) lastFootageAt = now; // footage is being recorded right now
+ else lastFootageAt = Math.max(started || 0, ended || 0) || null;
+ } catch { /* unknown is not quiet */ }
+ try {
+ const a = await db.get(`SELECT MAX(created_at) AS t FROM analytics_events WHERE COALESCE(is_bot, 0) = 0`);
+ lastHumanActivityAt = parseTs(a && a.t);
+ } catch { /* unknown is not quiet */ }
+ return { lastFootageAt, lastHumanActivityAt };
+}
+
+/** A user's newest stream row (an active one wins), for the channel-quiet exception. */
+async function latestStreamForUser(db, userId) {
+ try {
+ return await db.get(`SELECT id, started_at, ended_at, is_live
+ FROM streams WHERE user_id = ?
+ ORDER BY (CASE WHEN is_live = 1 THEN 1 ELSE 0 END) DESC,
+ COALESCE(ended_at, started_at, created_at) DESC, id DESC
+ LIMIT 1`, [userId]);
+ } catch { return null; } // unknown activity is not quietness
+}
+
+/** Epoch ms of a stream's latest activity: now while it is live, otherwise its last end/start. */
+function streamActivityAt(stream, now = Date.now()) {
+ if (!stream) return null;
+ if (Number(stream.is_live) > 0) return now;
+ return Math.max(parseTs(stream.ended_at) || 0, parseTs(stream.started_at) || 0) || null;
+}
+
+/**
+ * Fresh, or archival when the site (or this channel) is quiet and the source is inside the
+ * archive window. A channel exception applies only to the channel's newest VODs (`channel.newest`),
+ * so a quiet channel does not reopen its whole back catalogue.
+ */
+function classifySource({ vod, stream, part, now = Date.now(), site = { quiet: false }, channel = null } = {}) {
+ const capturedAt = footageStart({ vod, stream, part });
+ const endedAt = streamEndedAt({ vod, stream, now });
+ // Freshness follows the stream's end (see streamEndedAt), not the first frame: a long stream
+ // that finished recently is eligible even when its VOD was opened five days ago.
+ const ageHours = endedAt == null ? null : (now - endedAt) / HOUR_MS;
+ if (ageHours == null) return { ok: false, archival: false, capturedAt, endedAt, ageHours, why: 'unknown source age' };
+ if (ageHours <= FRESH_HOURS) return { ok: true, archival: false, capturedAt, endedAt, ageHours, why: 'fresh' };
+ // The 90-day archive window is about the footage itself, not an upload that happened later.
+ const archiveHours = capturedAt == null ? ageHours : (now - capturedAt) / HOUR_MS;
+ const channelQuiet = !!(channel && channel.quiet && channel.newest !== false);
+ const quiet = !!(site && site.quiet) || channelQuiet;
+ if (quiet && archiveHours <= ARCHIVE_DAYS * 24) {
+ return { ok: true, archival: true, capturedAt, endedAt, ageHours, why: site && site.quiet ? 'quiet site: archive' : 'quiet channel: archive' };
+ }
+ return { ok: false, archival: false, capturedAt, endedAt, ageHours, why: quiet ? 'older than the archive window' : 'stale source' };
+}
+
+const _day = (ms) => new Date(ms).toISOString().slice(0, 10);
+
+// One archive clip per UTC day, shared by the AI moments job and the VOD backfill. Both can pass
+// archiveAllowed() before either writes auto_clip_log, so the slot is reserved synchronously here:
+// reserveArchive() is a single check-and-set (no await between), and archiveAllowed() sees the
+// reservation before the clip log is written.
+let _archiveReservedDay = null;
+
+/** The clip log (or a reservation this process already holds) has spent today's archive clip? */
+function archiveAllowed(log, now = Date.now()) {
+ const day = _day(now);
+ if (_archiveReservedDay === day) return false;
+ const used = (Array.isArray(log) ? log : []).filter((e) => e && e.archival && e.ts && _day(e.ts) === day).length;
+ return used < ARCHIVE_PER_DAY;
+}
+
+/** Atomically claim today's archival clip slot for this process; false when it is taken. */
+function reserveArchive(log, now = Date.now()) {
+ if (!archiveAllowed(log, now)) return false;
+ _archiveReservedDay = _day(now);
+ return true;
+}
+
+/** Give the slot back after a cut that did not happen. */
+function releaseArchive(now = Date.now()) {
+ if (_archiveReservedDay === _day(now)) _archiveReservedDay = null;
+}
+
+module.exports = {
+ FRESH_HOURS, QUIET_HOURS, CHANNEL_QUIET_HOURS, ARCHIVE_DAYS, ARCHIVE_PER_DAY,
+ parseTs, footageStart, streamEndedAt, siteStatus, siteActivity, latestStreamForUser, streamActivityAt,
+ channelStatus, classifySource, archiveAllowed, reserveArchive, releaseArchive,
+ _resetArchiveReservation: () => { _archiveReservedDay = null; },
+};
@@ -14,6 +14,7 @@
@@ -14,6 +14,7 @@
'use strict';
'use strict';
const db = require('../db/database');
const db = require('../db/database');
+const dedupe = require('./clip-dedupe');
const KEY = 'ai_used_moments';
const KEY = 'ai_used_moments';
const MAX = 800;
const MAX = 800;
@@ -39,26 +40,58 @@ async function _importLegacy() {
@@ -39,26 +40,58 @@ async function _importLegacy() {
return out.slice(0, MAX);
return out.slice(0, MAX);
}
}
-/** Coarse scene signature — the first five significant words of a description/title. */
+/** Coarse scene signature — the first five significant tokens of a description/title. */
function sig(text) {
function sig(text) {
- return String(text || '').toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim().split(' ').filter(Boolean).slice(0, 5).join(' ');
+ return dedupe.shortSignature(text, 5);
}
}
-async function record({ kind, vod_id = null, stream_id = null, offset = null, sig: s = null, title = null, desc = null }) {
+/** A caller's frame embedding, kept only when it is a usable finite vector. */
+function _embedding(v) {
+ if (!Array.isArray(v) || !v.length || v.length > 1024) return null;
+ const out = v.map(Number);
+ return out.every(Number.isFinite) ? out : null;
+}
+
+async function record({ kind, vod_id = null, stream_id = null, offset = null, sig: s = null, title = null, desc = null, speech = null, embedding = null }) {
const list = await _load();
const list = await _load();
- list.unshift({ kind: kind || 'moment', vod_id: vod_id || null, stream_id: stream_id || null, offset: offset == null ? null : Math.floor(offset), sig: s || sig(desc || title), title: title ? String(title).slice(0, 120) : null, ts: Date.now() });
+ const scene = dedupe.normalize(desc || title);
+ list.unshift({
+ kind: kind || 'moment', vod_id: vod_id || null, stream_id: stream_id || null,
+ offset: offset == null ? null : Math.floor(offset),
+ sig: s || sig(desc || title),
+ // Full normalized scene + what was said, so a re-worded description still matches and a
+ // static webcam with a NEW conversation does not. Legacy entries keep only `sig`.
+ scene: scene ? scene.slice(0, 600) : null,
+ speech: speech ? dedupe.normalize(speech).slice(0, 600) : null,
+ // Optional frame embedding: the same footage in another upload scores near 1.
+ embedding: _embedding(embedding),
+ title: title ? String(title).slice(0, 120) : null, ts: Date.now(),
+ });
await _save(list);
await _save(list);
}
}
/** Why a moment is blocked, or null when it is free. */
/** Why a moment is blocked, or null when it is free. */
-async function usedReason({ vod_id = null, stream_id = null, offset = null, sig: s = null, desc = null, title = null } = {}, { gapSec = GAP_SEC } = {}) {
+async function usedReason({ vod_id = null, stream_id = null, offset = null, sig: s = null, desc = null, title = null, speech = null, embedding = null } = {}, { gapSec = GAP_SEC } = {}) {
const now = Date.now();
const now = Date.now();
const scene = s || sig(desc || title);
const scene = s || sig(desc || title);
+ const full = dedupe.normalize(desc || title);
+ const said = dedupe.normalize(speech);
+ const emb = _embedding(embedding);
for (const e of await _load()) {
for (const e of await _load()) {
const age = now - (e.ts || 0);
const age = now - (e.ts || 0);
if (age > OFFSET_TTL_MS) continue;
if (age > OFFSET_TTL_MS) continue;
if (offset != null && e.offset != null && ((vod_id && e.vod_id === vod_id) || (stream_id && e.stream_id === stream_id)) && Math.abs(e.offset - offset) < gapSec) return `${e.kind} already made at ${Math.floor(e.offset)}s of this ${e.vod_id === vod_id ? 'VOD' : 'stream'}`;
if (offset != null && e.offset != null && ((vod_id && e.vod_id === vod_id) || (stream_id && e.stream_id === stream_id)) && Math.abs(e.offset - offset) < gapSec) return `${e.kind} already made at ${Math.floor(e.offset)}s of this ${e.vod_id === vod_id ? 'VOD' : 'stream'}`;
- if (scene && e.sig && e.sig === scene && age <= SIG_TTL_MS) return `same scene as a recent ${e.kind} ("${scene}")`;
+ if (age > SIG_TTL_MS) continue;
+ const shortMatch = scene && e.sig && e.sig === scene;
+ const sceneMatch = full && e.scene && dedupe.isSameScene(e.scene, full);
+ // Frame embeddings (when both sides have them) are independent evidence: the same footage
+ // is a repeat even if it was re-described or re-uploaded.
+ const visualMatch = emb && e.embedding && dedupe.isSameEmbedding(e.embedding, emb);
+ if (!shortMatch && !sceneMatch && !visualMatch) continue;
+ if (visualMatch) return `same footage as a recent ${e.kind}`;
+ // Same camera/description but a different conversation is a new moment, not a repeat.
+ if (said && e.speech && !dedupe.isSameSpeech(e.speech, said)) continue;
+ return `same scene as a recent ${e.kind} ("${scene || e.sig}")`;
}
}
return null;
return null;
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,96 @@
@@ -0,0 +1,96 @@
+/**
+ * Content dedupe for moments (owner, 2026-10-29): repeats were getting clipped over and over.
+ * The old registry compared the first five ASCII words, so paraphrases slipped through and
+ * Japanese became an empty signature. clip-dedupe.js keeps Unicode and compares token sets,
+ * and the live cutter now asks the shared registry BEFORE cutting.
+ *
+ * node test/clip-dedupe.test.js
+ */
+'use strict';
+const assert = require('assert');
+const db = require('../server/db/database');
+(async () => {
+await db.initDb();
+const dedupe = require('../server/ai/clip-dedupe');
+const registry = require('../server/ai/moment-registry');
+const media = require('../server/media-client');
+const autoClip = require('../server/ai/auto-clip-job');
+
+// ── Text identity ──────────────────────────────────────────
+assert.strictEqual(dedupe.normalize('びっくり！？'), dedupe.normalize('びっくり'), 'Unicode punctuation is dropped, not the text');
+assert.ok(dedupe.shortSignature('配信者が大きな声で叫ぶ').length > 0, 'Japanese produces a signature instead of an empty one');
+assert.ok(dedupe.isSameScene('A player is fighting a boss in a dark cave', 'The player is fighting a boss in a dark cave'), 're-worded description is the same scene');
+assert.ok(!dedupe.isSameScene('A player is fighting a boss in a dark cave', 'A player is fighting a boss in a bright castle'), 'same topic with a different scene is not blocked');
+assert.ok(dedupe.isSameScene('配信者が大きな声で叫ぶ', '配信者が大きな声で叫んだ'), 'Japanese near-duplicate matches');
+assert.ok(!dedupe.isSameScene('配信者が静かにゲームをする', '猫がテーブルの上で寝ている'), 'unrelated Japanese is not blocked');
+assert.ok(dedupe.isSameSpeech('today we are playing a horror game and it is terrifying', 'today we are playing a horror game and it is very scary'), 'near-duplicate speech matches');
+assert.ok(!dedupe.isSameSpeech('today we are playing a horror game', 'we just beat the final boss on the first try'), 'a different conversation is not the same speech');
+assert.ok(dedupe.similarity(dedupe.tokens('we are going to play the game now'), dedupe.tokens('now we are going to play the game')) >= 0.8, 're-ordered speech still clears the 0.8 Dice bar');
+assert.ok(!dedupe.isSameSpeech('the red car explodes', 'the red car is parked'), 'a below-0.8 token overlap is not the same speech');
+assert.ok(!dedupe.isSameScene('The video appears to be a live streaming session with a chat overlay', 'The video appears to be a live streaming session with music'), 'shared boilerplate alone is not the same scene');
+assert.ok(!dedupe.isSameSpeech('this is a live stream', 'this is a live chat'), 'shared boilerplate alone is not the same speech');
+assert.ok(dedupe.isSameEmbedding([1, 0, 0], [0.99, 0.01, 0]), 'near-identical frame embeddings match');
+assert.ok(!dedupe.isSameEmbedding([1, 0, 0], [0, 1, 0]), 'different frame embeddings do not');
+console.log('✅ clip-dedupe: Unicode, paraphrases, 0.8 Dice, boilerplate and embedding negatives');
+
+// ── The shared registry ────────────────────────────────────
+await db.setState('ai_used_moments', '[]');
+registry._reset();
+await registry.record({
+ kind: 'paste', vod_id: 30, stream_id: 5, offset: 100,
+ desc: 'A person sits at a desk in front of a microphone',
+ speech: 'today we are playing a horror game and it is terrifying',
+});
+assert.strictEqual(await registry.usedReason({
+ vod_id: 31, offset: 5, desc: 'A person sits at a desk in front of a microphone',
+ speech: 'we just beat the final boss on the first try',
+}), null, 'static webcam + a NEW conversation is not a repeat');
+assert.ok(await registry.usedReason({
+ vod_id: 31, offset: 5, desc: 'A person sits at a desk in front of a microphone',
+ speech: 'today we are playing a horror game and it is very scary',
+}), 'static webcam + the SAME conversation is a repeat');
+
+await registry.record({ kind: 'clip', vod_id: 40, stream_id: 6, offset: 200, desc: 'A red car explodes on the highway' });
+assert.ok(await registry.usedReason({ vod_id: 41, offset: 10, desc: 'The red car explodes on the highway' }), 'a silent re-upload is caught by the visual description');
+
+await registry.record({ kind: 'clip', vod_id: 50, stream_id: 7, offset: 300, desc: '配信者が大きな声で叫ぶ瞬間' });
+assert.ok(await registry.usedReason({ vod_id: 51, offset: 10, desc: '配信者が大きな声で叫んだ瞬間' }), 'Japanese description is preserved across VODs');
+
+await registry.record({ kind: 'clip', vod_id: 52, stream_id: 8, offset: 400, desc: 'A person sits at a desk', embedding: [1, 0, 0] });
+assert.ok(await registry.usedReason({ vod_id: 53, offset: 10, desc: 'A completely different caption', embedding: [0.99, 0.01, 0] }), 'a matching frame embedding blocks even a different description');
+assert.strictEqual(await registry.usedReason({ vod_id: 54, offset: 10, desc: 'A completely different caption', embedding: [0, 1, 0] }), null, 'a different frame embedding is free');
+console.log('✅ registry: new speech is free, repeats, silent re-uploads and frame embeddings are blocked');
+
+// ── The live cutter asks the registry before spending a clip ──
+const created = [];
+media.createClip = async (body) => { created.push(body); return { id: 900 + created.length }; };
+const recorder = require('../server/streaming/recorder');
+recorder.getActiveRecording = () => ({ vodId: 60, startedAt: Date.now() - 100_000 });
+const chatReads = require('../server/chat/chat-reads');
+chatReads.liveChatBuckets = async () => [
+ { count: 1, tsEpoch: Math.floor(Date.now() / 1000) },
+ { count: 1 }, { count: 10, tsEpoch: Math.floor(Date.now() / 1000) }, { count: 1 },
+];
+chatReads.recentChatText = async () => [];
+const ai = require('../server/ai/ai-analysis');
+ai.isEnabled = () => true;
+ai.withinBudget = () => true;
+require('../server/ai/ai-service').structured = async () => ({ clip: true, title: 'A moment', desc: 'A person sits at a desk in front of a microphone' });
+db.isStreamClipRecordingEnabled = () => true;
+await db.setState('auto_clip_log', '[]');
+
+const liveStream = { id: 5, user_id: 3, started_at: new Date(Date.now() - 2 * 3600_000).toISOString().replace('T', ' ').slice(0, 19), title: 'Alice live' };
+await registry.record({ kind: 'paste', vod_id: 60, stream_id: 5, offset: 100, desc: 'A person sits at a desk' });
+await autoClip._internals.checkLiveStream(liveStream);
+assert.strictEqual(created.length, 0, 'the live path never cuts a moment the registry already used');
+
+registry._reset();
+await db.setState('ai_used_moments', '[]');
+registry._reset();
+await autoClip._internals.checkLiveStream(liveStream);
+assert.strictEqual(created.length, 1, 'a free moment is still cut (the check does not block everything)');
+console.log('✅ live cutter: registry rejection happens before Media.createClip');
+
+registry._reset();
+console.log('\n✅ All clip-dedupe tests passed');
+})().catch((e) => { console.error(e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,277 @@
@@ -0,0 +1,277 @@
+/**
+ * Freshness + dedupe policy for automated clips (owner, 2026-10-29: "the AI clipping system
+ * seems to clip old videos. Maybe only make clips of fresh stuff, unless the site's dead").
+ *
+ * - footage from the last 48 hours is eligible, counting from when the stream ENDED;
+ * - the archive (≤ 90 days) opens only after 72h with no new footage AND no human pageviews,
+ * and then at most one archival clip per UTC day;
+ * - an upstream outage is never quietness;
+ * - a long/reconnected stream uses the VOD part's capture time, not the stream start.
+ *
+ * node test/clip-freshness.test.js
+ */
+'use strict';
+const assert = require('assert');
+const db = require('../server/db/database');
+(async () => {
+await db.initDb();
+const raw = db.getDb();
+const policy = require('../server/ai/clip-policy');
+const registry = require('../server/ai/moment-registry');
+const media = require('../server/media-client');
+const autoClip = require('../server/ai/auto-clip-job');
+const moments = require('../server/ai/ai-moments-job');
+
+const HOUR = 3600_000, DAY = 24 * HOUR;
+const text = (ms) => new Date(ms).toISOString().replace('T', ' ').slice(0, 19);
+const now = Math.floor(Date.now() / 1000) * 1000;
+
+// ── Pure policy ────────────────────────────────────────────
+assert.strictEqual(policy.parseTs('2026-10-28 01:02:03'), Date.UTC(2026, 9, 28, 1, 2, 3), 'DB UTC text parses');
+assert.strictEqual(policy.parseTs('2026-10-28T01:02:03Z'), Date.UTC(2026, 9, 28, 1, 2, 3), 'ISO parses');
+assert.strictEqual(policy.parseTs(1_700_000_000), 1_700_000_000_000, 'epoch seconds parse');
+
+const stream = { started_at: text(now - 5 * DAY) };
+assert.strictEqual(policy.footageStart({ vod: { captured_at: text(now - HOUR) }, stream }), now - HOUR, 'explicit capture time wins');
+assert.strictEqual(policy.footageStart({ vod: { created_at: text(now - 9 * DAY) }, stream, part: { at: 4 * DAY / 1000 } }), now - DAY, 'part capture time wins over created_at');
+assert.strictEqual(policy.footageStart({ vod: { created_at: text(now - 3 * DAY) }, stream }), now - 3 * DAY, 'created_at fallback for a one-part VOD');
+assert.strictEqual(policy.footageStart({}), null, 'unknown source age');
+
+const fresh = policy.classifySource({ vod: { captured_at: text(now - 47 * HOUR) }, now });
+assert.ok(fresh.ok && !fresh.archival, '47h old is fresh');
+const stale = policy.classifySource({ vod: { captured_at: text(now - 49 * HOUR) }, now, site: { quiet: false } });
+assert.ok(!stale.ok, '49h old is stale while the site is active');
+const archive = policy.classifySource({ vod: { captured_at: text(now - 5 * DAY) }, now, site: { quiet: true } });
+assert.ok(archive.ok && archive.archival, '5 days old is archival on a quiet site');
+const tooOld = policy.classifySource({ vod: { captured_at: text(now - 100 * DAY) }, now, site: { quiet: true } });
+assert.ok(!tooOld.ok, 'older than the 90-day archive window is never eligible');
+
+// The 48h clock is the stream's END, not the VOD's creation: a five-day stream that finished an
+// hour ago is fresh even with no part row; one that ended 49h ago is stale.
+const endedFresh = policy.classifySource({
+ vod: { created_at: text(now - 5 * DAY) },
+ stream: { started_at: text(now - 5 * DAY), ended_at: text(now - HOUR) },
+ now,
+ site: { quiet: false },
+});
+assert.ok(endedFresh.ok && !endedFresh.archival, 'a long stream that ended an hour ago is fresh without a part row');
+assert.ok(endedFresh.ageHours < 2, 'its freshness age comes from ended_at, not created_at');
+const endedStale = policy.classifySource({
+ vod: { created_at: text(now - 5 * DAY) },
+ stream: { started_at: text(now - 5 * DAY), ended_at: text(now - 49 * HOUR) },
+ now,
+ site: { quiet: false },
+});
+assert.ok(!endedStale.ok && !endedStale.archival, 'a stream that ended 49h ago is stale');
+const stillLive = policy.classifySource({
+ vod: { created_at: text(now - 5 * DAY) },
+ stream: { started_at: text(now - 5 * DAY), is_live: 1 },
+ now,
+ site: { quiet: false },
+});
+assert.ok(stillLive.ok && !stillLive.archival, 'a still-running stream is fresh');
+
+// An old imported recording is judged by its capture time, never by a recent upload.
+const imported = policy.classifySource({
+ vod: { created_at: text(now - HOUR), captured_at: text(now - 10 * DAY) },
+ now,
+ site: { quiet: false },
+});
+assert.ok(!imported.ok && !imported.archival, 'old recording uploaded an hour ago is stale, not fresh');
+const importedArchive = policy.classifySource({
+ vod: { created_at: text(now - HOUR), captured_at: text(now - 10 * DAY) },
+ now,
+ site: { quiet: true },
+});
+assert.ok(importedArchive.ok && importedArchive.archival, 'on a quiet site it is archival, still not fresh');
+const importedLinked = policy.classifySource({
+ vod: { created_at: text(now - HOUR) },
+ stream: { started_at: text(now - 10 * DAY), ended_at: text(now - 10 * DAY) },
+ now,
+ site: { quiet: false },
+});
+assert.ok(!importedLinked.ok, 'a recent upload of an old stream is still stale');
+
+const quiet = policy.siteStatus({ now, lastFootageAt: now - 73 * HOUR, lastHumanActivityAt: now - 73 * HOUR, upstreamHealthy: true });
+assert.strictEqual(quiet.quiet, true, '72h without footage or pageviews is quiet');
+assert.strictEqual(policy.siteStatus({ now, lastFootageAt: now - 73 * HOUR, lastHumanActivityAt: now - 73 * HOUR, upstreamHealthy: false }).quiet, false, 'an outage is not quietness');
+assert.strictEqual(policy.siteStatus({ now, lastFootageAt: now - 73 * HOUR, lastHumanActivityAt: null, upstreamHealthy: true }).quiet, false, 'unknown activity is not quietness');
+assert.strictEqual(policy.siteStatus({ now, lastFootageAt: now - 73 * HOUR, lastHumanActivityAt: now - 1 * HOUR, upstreamHealthy: true }).quiet, false, 'a human pageview keeps the archive closed');
+
+assert.strictEqual(policy.archiveAllowed([{ archival: true, ts: Date.now() }]), false, 'one archive clip per day');
+assert.strictEqual(policy.archiveAllowed([{ archival: true, ts: Date.now() - DAY }]), true, "yesterday's archive clip does not block today");
+// Both candidate pools can pass the gate before either writes auto_clip_log. The in-process
+// reservation is a synchronous check-and-set, so the second cutter loses even with an empty log.
+policy._resetArchiveReservation();
+assert.strictEqual(policy.reserveArchive([], now), true, 'first simultaneous archive cut claims the slot');
+assert.strictEqual(policy.archiveAllowed([], now), false, 'the reservation closes the gate before the log write');
+assert.strictEqual(policy.reserveArchive([], now), false, 'a second simultaneous archive cut is refused');
+policy.releaseArchive(now);
+assert.strictEqual(policy.archiveAllowed([], now), true, 'a failed cut gives the slot back');
+
+// A stale VOD is also allowed when its own channel has been quiet for 48h (the brief's channel
+// exception) — but only its newest VODs, and never outside the 90-day window.
+assert.strictEqual(policy.channelStatus({ now, lastStreamAt: now - 49 * HOUR }).quiet, true, 'a channel with no new stream in 48h is quiet');
+assert.strictEqual(policy.channelStatus({ now, lastStreamAt: now - HOUR }).quiet, false, 'a channel still streaming is active');
+const quietChannel = policy.classifySource({ vod: { captured_at: text(now - 5 * DAY) }, now, site: { quiet: false }, channel: { quiet: true, newest: true } });
+assert.ok(quietChannel.ok && quietChannel.archival, 'stale VOD on a quiet channel: archive allowed');
+assert.ok(!policy.classifySource({ vod: { captured_at: text(now - 5 * DAY) }, now, site: { quiet: false }, channel: { quiet: true, newest: false } }).ok, 'quiet channel: only its newest VODs');
+assert.ok(!policy.classifySource({ vod: { captured_at: text(now - 5 * DAY) }, now, site: { quiet: false }, channel: { quiet: false } }).ok, 'stale VOD on an active channel: refused');
+console.log('✅ freshness policy: 48h, 72h quiet gate, channel quiet, outage, archive window and cap');
+
+// ── Integration fixtures ───────────────────────────────────
+const addUser = async (id, username) => await raw.prepare(
+ `INSERT INTO users (id, username, display_name, email, password_hash, role, created_at) OVERRIDING SYSTEM VALUE
+ VALUES (?, ?, ?, ?, 'x', 'streamer', '2025-01-01 00:00:00')`).run(id, username, username, `${username}@x`);
+if (!await db.getUserById(3)) { await addUser(3, 'alice'); await db.ensureChannel(3); }
+const ch = await db.getChannelByUserId(3);
+
+const mkStream = async (startedMs, { live = false, endedMs = null } = {}) => {
+ const id = Number((await db.createStream({ user_id: 3, channel_id: ch.id, title: 'stream', protocol: 'rtmp' })).lastInsertRowid);
+ await db.run('UPDATE streams SET started_at = ?, is_live = ?, ended_at = ? WHERE id = ?', [text(startedMs), live ? 1 : 0, endedMs == null ? null : text(endedMs), id]);
+ return id;
+};
+const vod = (id, streamId, extra = {}) => ({ id, stream_id: streamId, user_id: 3, title: `VOD ${id}`, visibility: 'public', is_public: true, view_count: 5, duration_seconds: 600, status: 'ready', ...extra });
+let currentVods = [];
+media.listVods = async () => ({ vods: currentVods });
+const addMemory = async (streamId) => await db.addStreamMemory({ stream_id: streamId, user_id: 3, offset_seconds: 60, description: 'a scene' });
+
+async function reset() {
+ await raw.prepare('DELETE FROM stream_history').run();
+ await raw.prepare('DELETE FROM stream_memories').run();
+ await raw.prepare('DELETE FROM analytics_events').run();
+ await raw.prepare('DELETE FROM streams').run();
+ await db.setState('auto_clip_log', '[]');
+ await db.setState('ai_used_moments', '[]');
+ require('../server/ai/moment-registry')._reset();
+ require('../server/ai/clip-policy')._resetArchiveReservation();
+ currentVods = [];
+}
+
+// Active site: only the VOD captured in the last 48h is eligible.
+await reset();
+const sFresh = await mkStream(now - HOUR, { endedMs: now - HOUR });
+const sOld = await mkStream(now - 10 * DAY, { endedMs: now - 10 * DAY });
+const sMid = await mkStream(now - 5 * DAY, { endedMs: now - 5 * DAY });
+await addMemory(sFresh); await addMemory(sOld); await addMemory(sMid);
+await db.run(`INSERT INTO analytics_events (service, event_type, is_bot, created_at) VALUES ('live', 'pageview', 0, ?)`, [text(now - HOUR)]);
+currentVods = [vod(10, sFresh), vod(11, sOld), vod(12, sMid)];
+let pool = await autoClip._internals.backfillPool(10);
+assert.deepStrictEqual(pool.map(v => v.vod_id), [10], 'active site: only fresh footage is clipped');
+assert.strictEqual(pool[0].archival, false);
+console.log('✅ backfill pool: old popular VODs are not clipped while the site is active');
+
+// Review case: VOD created when a long stream started five days ago, stream.ended_at one hour
+// ago, no part row. The ended_at clock makes it fresh; one that ended 49h ago stays stale.
+await reset();
+const sEndedRecent = await mkStream(now - 5 * DAY, { endedMs: now - HOUR });
+const sEndedStale = await mkStream(now - 5 * DAY, { endedMs: now - 49 * HOUR });
+await addMemory(sEndedRecent); await addMemory(sEndedStale);
+await db.run(`INSERT INTO analytics_events (service, event_type, is_bot, created_at) VALUES ('live', 'pageview', 0, ?)`, [text(now - HOUR)]);
+currentVods = [
+ vod(14, sEndedRecent, { created_at: text(now - 5 * DAY) }),
+ vod(15, sEndedStale, { created_at: text(now - 5 * DAY) }),
+];
+pool = await autoClip._internals.backfillPool(10);
+assert.deepStrictEqual(pool.map(v => v.vod_id), [14], 'a stream that ended an hour ago is eligible; 49h old is not');
+assert.strictEqual(pool[0].archival, false);
+console.log('✅ backfill pool: 48h runs from stream end, not VOD creation');
+
+// A channel live now has no new VOD yet. The old VOD must not reopen the quiet-channel exception
+// just because Media's newest row for that channel is the old stream.
+await reset();
+const sChannelOld = await mkStream(now - 10 * DAY, { endedMs: now - 10 * DAY });
+const sChannelLive = await mkStream(now - HOUR, { live: true });
+await addMemory(sChannelOld);
+await db.run(`INSERT INTO analytics_events (service, event_type, is_bot, created_at) VALUES ('live', 'pageview', 0, ?)`, [text(now - HOUR)]);
+currentVods = [vod(30, sChannelOld)];
+assert.deepStrictEqual(await autoClip._internals.backfillPool(10), [], 'a current live stream closes the quiet-channel exception (backfill)');
+assert.deepStrictEqual(await moments._internals.momentPool(10), [], 'a current live stream closes the quiet-channel exception (moments)');
+assert.ok(sChannelLive, 'the live stream exists without a listed VOD');
+console.log('✅ candidate pools: latest stream activity comes from Live streams, not the newest VOD');
+
+// Long/reconnected stream: capture time is streams.started_at + its part offset.
+await reset();
+const sLong = await mkStream(now - 5 * DAY, { live: true });
+await db.run(`INSERT INTO stream_history (stream_id, kind, at_sec, end_sec, vod_id, detail) VALUES (?, 'part', ?, NULL, ?, '{}')`, [sLong, (5 * DAY - HOUR) / 1000, 13]);
+await addMemory(sLong);
+currentVods = [vod(13, sLong)];
+pool = await autoClip._internals.backfillPool(10);
+assert.deepStrictEqual(pool.map(v => v.vod_id), [13], 'a five-day stream still recording is fresh');
+assert.strictEqual(pool[0].captured_at, now - HOUR, 'its VOD part captured one hour ago, not five days ago');
+assert.ok(pool[0].age_hours < 2, 'a still-running stream has a now-based freshness age');
+console.log('✅ backfill pool: long/reconnected streams use the part capture time');
+
+// Quiet site: the archive opens, up to the daily cap.
+await reset();
+const sArchiveA = await mkStream(now - 10 * DAY, { endedMs: now - 10 * DAY });
+const sArchiveB = await mkStream(now - 5 * DAY, { endedMs: now - 5 * DAY });
+await addMemory(sArchiveA); await addMemory(sArchiveB);
+await db.run(`INSERT INTO analytics_events (service, event_type, is_bot, created_at) VALUES ('live', 'pageview', 0, ?)`, [text(now - 4 * DAY)]);
+currentVods = [vod(20, sArchiveA), vod(21, sArchiveB)];
+pool = await autoClip._internals.backfillPool(10);
+assert.deepStrictEqual(pool.map(v => v.vod_id), [20, 21], 'quiet site: archive candidates inside 90 days');
+assert.ok(pool.every(v => v.archival), 'they are labelled archival');
+
+await db.setState('auto_clip_log', JSON.stringify([{ archival: true, ts: Date.now(), vod_id: 999 }]));
+pool = await autoClip._internals.backfillPool(10);
+assert.deepStrictEqual(pool, [], "today's archival clip closes the archive for the rest of the day");
+console.log('✅ backfill pool: quiet site opens the archive, one archival clip per day');
+
+// A Media outage is not quietness.
+await db.setState('auto_clip_log', '[]');
+media.listVods = async () => { throw new Error('Media down'); };
+pool = await autoClip._internals.backfillPool(10);
+assert.deepStrictEqual(pool, [], 'Media down: no candidates, and no archive opened');
+media.listVods = async () => ({ vods: currentVods });
+
+// The home-page moments pool follows the same policy.
+const mpool = await moments._internals.momentPool(10);
+assert.deepStrictEqual(mpool.map(v => v.vod_id), [20, 21], 'moments pool: same quiet-site archive');
+assert.ok(mpool.every(v => v.archival));
+await db.setState('auto_clip_log', JSON.stringify([{ archival: true, ts: Date.now(), vod_id: 999 }]));
+assert.deepStrictEqual(await moments._internals.momentPool(10), [], 'moments pool: archive cap is shared with the clip job');
+console.log('✅ moments pool: one freshness policy for every automated cutter');
+
+// Both cutters (AI moments and VOD backfill) can see an empty auto_clip_log before either writes.
+// The in-process reservation lets exactly one reach Media; a failed cut releases the slot.
+await reset();
+const cutCalls = [];
+let releaseFirst = null;
+media.createClip = async (body) => {
+ cutCalls.push(body);
+ if (cutCalls.length === 1) await new Promise((resolve) => { releaseFirst = resolve; });
+ return { id: 7000 + cutCalls.length };
+};
+const archiveCut = (id) => autoClip.clipVodMoment({
+ vod: { vod_id: id, user_id: 3, stream_id: id, title: 'archive' },
+ offset: 60,
+ title: `Archive moment ${id}`,
+ desc: `A distinct archival scene number ${id} with enough words`,
+ archival: true,
+});
+const firstCut = archiveCut(70);
+const secondCut = archiveCut(71);
+for (let i = 0; i < 200 && !releaseFirst; i++) await new Promise((r) => setTimeout(r, 5));
+assert.ok(releaseFirst, 'the first archival cutter reached Media');
+assert.strictEqual(cutCalls.length, 1, 'the second simultaneous archival cutter never reaches Media');
+releaseFirst();
+const cuts = await Promise.all([firstCut, secondCut]);
+assert.strictEqual(cuts.filter(Boolean).length, 1, 'exactly one archival clip is cut');
+const clipLog = JSON.parse(await db.getState('auto_clip_log') || '[]');
+assert.strictEqual(clipLog.filter((e) => e.archival).length, 1, 'exactly one archival clip is logged');
+assert.strictEqual(policy.archiveAllowed(clipLog, Date.now()), false, "today's archive slot stays closed");
+
+// A cut that fails must not burn the day.
+policy._resetArchiveReservation();
+await db.setState('auto_clip_log', '[]');
+await db.setState('ai_used_moments', '[]');
+registry._reset();
+media.createClip = async () => null;
+assert.strictEqual(await archiveCut(72), null, 'the failed archival cut returns nothing');
+assert.strictEqual(policy.archiveAllowed([], Date.now()), true, 'its reservation was released');
+console.log('✅ archive gate: simultaneous cutters share one atomic slot');
+
+registry._reset();
+console.log('\n✅ All clip-freshness tests passed');
+})().catch((e) => { console.error(e); process.exit(1); });