openvibers/openvibe.live Commits
The AI's stale clips leave the lists: unlisted, never deleted (#567)
Changes
@@@ -427,12 +427,15 @@ function start() {// clips). Jittered so it does not start in lockstep with every other boot-time job._backfillTimer = require('../utils/jobs').every('auto-clip-vod-backfill', 5 * 60 * 1000, () => backfillVodClips(), { initialDelayMs: 30 * 1000, jitterMs: 30 * 1000 });_flagTimer = require('../utils/jobs').every('auto-clip-flag-sync', FLAG_SYNC_INTERVAL_MS, () => syncAutoClipFlags(), { initialDelayMs: 90 * 1000, jitterMs: 60 * 1000 });++ // Once a day, the AI's old clips no person ever opened leave the lists (unlisted, never deleted).++ require('./auto-clip-sweep').start();console.log('[AutoClip] Live auto-clip job started (selective chat-spike + AI agreement, cuts via OpenVibe.Media) + persistent VOD backfill');}function stop() {if (_timer) { clearInterval(_timer); _timer = null; }if (_backfillTimer) { _backfillTimer(); _backfillTimer = null; }if (_flagTimer) { _flagTimer(); _flagTimer = null; }++ require('./auto-clip-sweep').stop();}module.exports = { start, stop, clipVodMoment, backfillVodClips, syncAutoClipFlags, _tick, _internals: { checkLiveStream: _checkLiveStream, backfillPool: _backfillPool } };nnew file mode 100644@@@ -0,0 +1,84 @@++'use strict';++/**++ * auto-clip-sweep.js — the AI's stale clips leave the lists (open item 151: "prune stale auto-clips, hide first").++ *++ * An automatic clip that is public, was cut more than 30 days ago, and that no person ever opened is made unlisted:++ * its page and link keep working, and it leaves the lists, search and the home. Nothing is deleted.++ *++ * "No person ever opened it" is Media's content_visits (a visitor row; GET /clips?unvisited=1, Media#292), the++ * measure of Live's clip audit (Live#443). The play counter is not used: crawlers move it, so by plays only 2 of the++ * 190 old public auto-clips looked unwatched, while 29 had never been opened by a person.++ * Only clips Media marks auto_generated: a person's clip is never touched.++ * At most 25 a run, once a day. Each hidden clip is remembered (state auto_clip_swept: id and when), newest first,++ * so it can be shown and put back (restore()).++ * A Media that does not know unvisited=1 answers every old clip. When the "unvisited" total is not below the plain++ * total, the sweep does nothing that run.++ *++ * sweep({ now, max, dryRun }) → { hidden: [ids], skipped, why }++ * restore(id) → puts one back to public and forgets it++ * start() / stop()++ */++const db = require('../db/database');++const media = require('../media-client');++const { parseTs } = require('./clip-policy');++++const STALE_DAYS = 30;++const MAX_PER_RUN = 25;++const SETTING = 'auto_clip_swept';++const KEEP = 2000;++const DAY_MS = 24 * 3600 * 1000;++++async function _swept() {++ try { const d = JSON.parse(await db.getState(SETTING) || '[]'); return Array.isArray(d) ? d : []; } catch { return []; }++}++++async function sweep({ now = Date.now(), max = MAX_PER_RUN, dryRun = false } = {}) {++ const before = new Date(now - STALE_DAYS * DAY_MS).toISOString();++ const query = { auto_generated: 1, before, order: 'oldest', status: 'ready' };++ const stale = await media.listClips({ ...query, unvisited: 1, limit: max }, { timeoutMs: 15_000 });++ const all = await media.listClips({ ...query, limit: 1 }, { timeoutMs: 15_000 });++ if (!stale || !Array.isArray(stale.clips) || !all || all.total == null) return { hidden: [], skipped: 0, why: 'Media did not answer' };++ if (Number(stale.total) >= Number(all.total) && Number(all.total) > 0) {++ return { hidden: [], skipped: 0, why: `Media did not apply unvisited=1 (${stale.total} of ${all.total} old auto-clips)` };++ }++ const hidden = [];++ let skipped = 0;++ for (const clip of stale.clips) {++ // The list is public and AI-made by its query; each one is checked again before it changes.++ const cut = clip && parseTs(clip.created_at);++ if (!clip || clip.auto_generated !== true || clip.visibility !== 'public' || !(cut && cut < now - STALE_DAYS * DAY_MS)) { skipped++; continue; }++ if (dryRun) { hidden.push(clip.id); continue; }++ try {++ const out = await media.updateClip(clip.id, { visibility: 'unlisted' }, { timeoutMs: 10_000 });++ const after = (out && out.clip) || out;++ if (after && after.visibility && after.visibility !== 'unlisted') { skipped++; continue; }++ hidden.push(clip.id);++ } catch (e) {++ if (e && (e.status === 404 || e.status === 410)) { skipped++; continue; }++ break; // Media down or refusing: the rest wait for tomorrow++ }++ }++ if (!dryRun && hidden.length) {++ const at = new Date(now).toISOString();++ const log = [...hidden.map((id) => ({ id, at })), ...await _swept()].slice(0, KEEP);++ await db.setState(SETTING, JSON.stringify(log));++ console.log(`[AutoClip] Unlisted ${hidden.length} AI clip(s) no person opened in their first ${STALE_DAYS} days or since: ${hidden.join(', ')}`);++ }++ return { hidden, skipped, why: null };++}++++/** Put one swept clip back in the lists, and forget it. */++async function restore(id) {++ const out = await media.updateClip(id, { visibility: 'public' }, { timeoutMs: 10_000 });++ await db.setState(SETTING, JSON.stringify((await _swept()).filter((e) => String(e.id) !== String(id))));++ return (out && out.clip) || out;++}++++let _stop = null;++function start() {++ if (_stop) return;++ _stop = require('../utils/jobs').every('auto-clip-sweep', DAY_MS, () => sweep(), { initialDelayMs: 20 * 60 * 1000, jitterMs: 10 * 60 * 1000 });++}++function stop() { if (_stop) { _stop(); _stop = null; } }++++module.exports = { sweep, restore, start, stop, STALE_DAYS, MAX_PER_RUN, SETTING };nnew file mode 100644@@@ -0,0 +1,95 @@++/**++ * The AI's stale clips leave the lists (server/ai/auto-clip-sweep.js; open item 151, "prune stale auto-clips, hide++ * first"): a public automatic clip cut more than 30 days ago that no person ever opened (Media's unvisited=1) is made++ * unlisted, never deleted, and remembered so it can be put back. A person's clip is never touched; a Media that does++ * not apply unvisited=1 makes the sweep do nothing.++ *++ * node test/run.js auto-clip-sweep++ */++'use strict';++const assert = require('assert');++++const media = require('../server/media-client');++const db = require('../server/db/database');++const state = new Map();++db.getState = async (k) => state.get(k) || null;++db.setState = async (k, v) => { state.set(k, v); };++++const NOW = Date.parse('2026-11-02T20:00:00Z');++const old = '2026-09-20 10:00:00', recent = '2026-10-25 10:00:00';++let clips, applies, calls;++const reset = () => {++ clips = [++ { id: 1, auto_generated: true, visibility: 'public', created_at: old, visited: false },++ { id: 2, auto_generated: true, visibility: 'public', created_at: old, visited: true },++ { id: 3, auto_generated: false, visibility: 'public', created_at: old, visited: false },++ { id: 4, auto_generated: true, visibility: 'public', created_at: recent, visited: false },++ { id: 5, auto_generated: true, visibility: 'public', created_at: '2026-09-01T00:00:00Z', visited: false },++ ];++ applies = true;++ calls = [];++};++media.listClips = async (q) => {++ calls.push({ list: q });++ const before = Date.parse(q.before);++ let rows = clips.filter((c) => c.visibility === 'public' && (q.auto_generated !== 1 || c.auto_generated)++ && Date.parse(c.created_at.includes('T') ? c.created_at : c.created_at.replace(' ', 'T') + 'Z') < before);++ if (q.unvisited && applies) rows = rows.filter((c) => !c.visited);++ return { clips: rows.slice(0, q.limit), total: rows.length };++};++media.updateClip = async (id, fields) => {++ calls.push({ update: id, fields });++ const c = clips.find((x) => x.id === id);++ Object.assign(c, fields);++ return { clip: { ...c } };++};++const sweep = require('../server/ai/auto-clip-sweep');++++(async () => {++ reset();++ const out = await sweep.sweep({ now: NOW });++ assert.deepStrictEqual(out.hidden.sort(), [1, 5], 'old, AI-made, never opened: unlisted');++ assert.strictEqual(clips.find((c) => c.id === 1).visibility, 'unlisted');++ assert.strictEqual(clips.find((c) => c.id === 2).visibility, 'public', 'opened by a person: kept');++ assert.strictEqual(clips.find((c) => c.id === 3).visibility, 'public', 'a person\'s clip: never touched');++ assert.strictEqual(clips.find((c) => c.id === 4).visibility, 'public', 'younger than 30 days: kept');++ assert.ok(calls.every((c) => !c.fields || c.fields.visibility === 'unlisted'), 'unlisted, never deleted');++ const list = calls.find((c) => c.list && c.list.unvisited);++ assert.strictEqual(list.list.auto_generated, 1);++ assert.strictEqual(list.list.before, '2026-10-03T20:00:00.000Z', '30 days before now');++ const log = JSON.parse(state.get(sweep.SETTING));++ assert.deepStrictEqual(log.map((e) => e.id).sort(), [1, 5], 'remembered');++++ // Put one back.++ await sweep.restore(1);++ assert.strictEqual(clips.find((c) => c.id === 1).visibility, 'public');++ assert.deepStrictEqual(JSON.parse(state.get(sweep.SETTING)).map((e) => e.id), [5], 'and forgotten');++++ // A dry run says what it would hide and changes nothing.++ reset(); state.clear();++ const dry = await sweep.sweep({ now: NOW, dryRun: true });++ assert.deepStrictEqual(dry.hidden.sort(), [1, 5]);++ assert.ok(!calls.some((c) => c.update), 'nothing changed');++ assert.strictEqual(state.get(sweep.SETTING), undefined);++++ // A Media that ignores unvisited=1 answers every old clip: the sweep does nothing.++ reset(); applies = false;++ const blind = await sweep.sweep({ now: NOW });++ assert.deepStrictEqual(blind.hidden, []);++ assert.match(blind.why, /did not apply unvisited/);++ assert.ok(!calls.some((c) => c.update));++++ // At most `max` a run.++ reset();++ const one = await sweep.sweep({ now: NOW, max: 1 });++ assert.strictEqual(one.hidden.length, 1);++++ // Media down: nothing, no throw.++ reset();++ const listClips = media.listClips;++ media.listClips = async () => { throw Object.assign(new Error('down'), { status: 503 }); };++ await assert.rejects(sweep.sweep({ now: NOW }), /down/, 'the job runner records the failure');++ media.listClips = listClips;++++ console.log('auto-clip sweep: all checks passed');++})().catch((e) => { console.error(e); process.exit(1); });
@@ -427,12 +427,15 @@ function start() {
@@ -427,12 +427,15 @@ function start() {
// clips). Jittered so it does not start in lockstep with every other boot-time job.
// clips). Jittered so it does not start in lockstep with every other boot-time job.
_backfillTimer = require('../utils/jobs').every('auto-clip-vod-backfill', 5 * 60 * 1000, () => backfillVodClips(), { initialDelayMs: 30 * 1000, jitterMs: 30 * 1000 });
_backfillTimer = require('../utils/jobs').every('auto-clip-vod-backfill', 5 * 60 * 1000, () => backfillVodClips(), { initialDelayMs: 30 * 1000, jitterMs: 30 * 1000 });
_flagTimer = require('../utils/jobs').every('auto-clip-flag-sync', FLAG_SYNC_INTERVAL_MS, () => syncAutoClipFlags(), { initialDelayMs: 90 * 1000, jitterMs: 60 * 1000 });
_flagTimer = require('../utils/jobs').every('auto-clip-flag-sync', FLAG_SYNC_INTERVAL_MS, () => syncAutoClipFlags(), { initialDelayMs: 90 * 1000, jitterMs: 60 * 1000 });
+ // Once a day, the AI's old clips no person ever opened leave the lists (unlisted, never deleted).
+ require('./auto-clip-sweep').start();
console.log('[AutoClip] Live auto-clip job started (selective chat-spike + AI agreement, cuts via OpenVibe.Media) + persistent VOD backfill');
console.log('[AutoClip] Live auto-clip job started (selective chat-spike + AI agreement, cuts via OpenVibe.Media) + persistent VOD backfill');
}
}
function stop() {
function stop() {
if (_timer) { clearInterval(_timer); _timer = null; }
if (_timer) { clearInterval(_timer); _timer = null; }
if (_backfillTimer) { _backfillTimer(); _backfillTimer = null; }
if (_backfillTimer) { _backfillTimer(); _backfillTimer = null; }
if (_flagTimer) { _flagTimer(); _flagTimer = null; }
if (_flagTimer) { _flagTimer(); _flagTimer = null; }
+ require('./auto-clip-sweep').stop();
}
}
module.exports = { start, stop, clipVodMoment, backfillVodClips, syncAutoClipFlags, _tick, _internals: { checkLiveStream: _checkLiveStream, backfillPool: _backfillPool } };
module.exports = { start, stop, clipVodMoment, backfillVodClips, syncAutoClipFlags, _tick, _internals: { checkLiveStream: _checkLiveStream, backfillPool: _backfillPool } };
new file mode 100644
new file mode 100644
@@ -0,0 +1,84 @@
@@ -0,0 +1,84 @@
+'use strict';
+/**
+ * auto-clip-sweep.js — the AI's stale clips leave the lists (open item 151: "prune stale auto-clips, hide first").
+ *
+ * An automatic clip that is public, was cut more than 30 days ago, and that no person ever opened is made unlisted:
+ * its page and link keep working, and it leaves the lists, search and the home. Nothing is deleted.
+ *
+ * "No person ever opened it" is Media's content_visits (a visitor row; GET /clips?unvisited=1, Media#292), the
+ * measure of Live's clip audit (Live#443). The play counter is not used: crawlers move it, so by plays only 2 of the
+ * 190 old public auto-clips looked unwatched, while 29 had never been opened by a person.
+ * Only clips Media marks auto_generated: a person's clip is never touched.
+ * At most 25 a run, once a day. Each hidden clip is remembered (state auto_clip_swept: id and when), newest first,
+ * so it can be shown and put back (restore()).
+ * A Media that does not know unvisited=1 answers every old clip. When the "unvisited" total is not below the plain
+ * total, the sweep does nothing that run.
+ *
+ * sweep({ now, max, dryRun }) → { hidden: [ids], skipped, why }
+ * restore(id) → puts one back to public and forgets it
+ * start() / stop()
+ */
+const db = require('../db/database');
+const media = require('../media-client');
+const { parseTs } = require('./clip-policy');
+
+const STALE_DAYS = 30;
+const MAX_PER_RUN = 25;
+const SETTING = 'auto_clip_swept';
+const KEEP = 2000;
+const DAY_MS = 24 * 3600 * 1000;
+
+async function _swept() {
+ try { const d = JSON.parse(await db.getState(SETTING) || '[]'); return Array.isArray(d) ? d : []; } catch { return []; }
+}
+
+async function sweep({ now = Date.now(), max = MAX_PER_RUN, dryRun = false } = {}) {
+ const before = new Date(now - STALE_DAYS * DAY_MS).toISOString();
+ const query = { auto_generated: 1, before, order: 'oldest', status: 'ready' };
+ const stale = await media.listClips({ ...query, unvisited: 1, limit: max }, { timeoutMs: 15_000 });
+ const all = await media.listClips({ ...query, limit: 1 }, { timeoutMs: 15_000 });
+ if (!stale || !Array.isArray(stale.clips) || !all || all.total == null) return { hidden: [], skipped: 0, why: 'Media did not answer' };
+ if (Number(stale.total) >= Number(all.total) && Number(all.total) > 0) {
+ return { hidden: [], skipped: 0, why: `Media did not apply unvisited=1 (${stale.total} of ${all.total} old auto-clips)` };
+ }
+ const hidden = [];
+ let skipped = 0;
+ for (const clip of stale.clips) {
+ // The list is public and AI-made by its query; each one is checked again before it changes.
+ const cut = clip && parseTs(clip.created_at);
+ if (!clip || clip.auto_generated !== true || clip.visibility !== 'public' || !(cut && cut < now - STALE_DAYS * DAY_MS)) { skipped++; continue; }
+ if (dryRun) { hidden.push(clip.id); continue; }
+ try {
+ const out = await media.updateClip(clip.id, { visibility: 'unlisted' }, { timeoutMs: 10_000 });
+ const after = (out && out.clip) || out;
+ if (after && after.visibility && after.visibility !== 'unlisted') { skipped++; continue; }
+ hidden.push(clip.id);
+ } catch (e) {
+ if (e && (e.status === 404 || e.status === 410)) { skipped++; continue; }
+ break; // Media down or refusing: the rest wait for tomorrow
+ }
+ }
+ if (!dryRun && hidden.length) {
+ const at = new Date(now).toISOString();
+ const log = [...hidden.map((id) => ({ id, at })), ...await _swept()].slice(0, KEEP);
+ await db.setState(SETTING, JSON.stringify(log));
+ console.log(`[AutoClip] Unlisted ${hidden.length} AI clip(s) no person opened in their first ${STALE_DAYS} days or since: ${hidden.join(', ')}`);
+ }
+ return { hidden, skipped, why: null };
+}
+
+/** Put one swept clip back in the lists, and forget it. */
+async function restore(id) {
+ const out = await media.updateClip(id, { visibility: 'public' }, { timeoutMs: 10_000 });
+ await db.setState(SETTING, JSON.stringify((await _swept()).filter((e) => String(e.id) !== String(id))));
+ return (out && out.clip) || out;
+}
+
+let _stop = null;
+function start() {
+ if (_stop) return;
+ _stop = require('../utils/jobs').every('auto-clip-sweep', DAY_MS, () => sweep(), { initialDelayMs: 20 * 60 * 1000, jitterMs: 10 * 60 * 1000 });
+}
+function stop() { if (_stop) { _stop(); _stop = null; } }
+
+module.exports = { sweep, restore, start, stop, STALE_DAYS, MAX_PER_RUN, SETTING };
new file mode 100644
new file mode 100644
@@ -0,0 +1,95 @@
@@ -0,0 +1,95 @@
+/**
+ * The AI's stale clips leave the lists (server/ai/auto-clip-sweep.js; open item 151, "prune stale auto-clips, hide
+ * first"): a public automatic clip cut more than 30 days ago that no person ever opened (Media's unvisited=1) is made
+ * unlisted, never deleted, and remembered so it can be put back. A person's clip is never touched; a Media that does
+ * not apply unvisited=1 makes the sweep do nothing.
+ *
+ * node test/run.js auto-clip-sweep
+ */
+'use strict';
+const assert = require('assert');
+
+const media = require('../server/media-client');
+const db = require('../server/db/database');
+const state = new Map();
+db.getState = async (k) => state.get(k) || null;
+db.setState = async (k, v) => { state.set(k, v); };
+
+const NOW = Date.parse('2026-11-02T20:00:00Z');
+const old = '2026-09-20 10:00:00', recent = '2026-10-25 10:00:00';
+let clips, applies, calls;
+const reset = () => {
+ clips = [
+ { id: 1, auto_generated: true, visibility: 'public', created_at: old, visited: false },
+ { id: 2, auto_generated: true, visibility: 'public', created_at: old, visited: true },
+ { id: 3, auto_generated: false, visibility: 'public', created_at: old, visited: false },
+ { id: 4, auto_generated: true, visibility: 'public', created_at: recent, visited: false },
+ { id: 5, auto_generated: true, visibility: 'public', created_at: '2026-09-01T00:00:00Z', visited: false },
+ ];
+ applies = true;
+ calls = [];
+};
+media.listClips = async (q) => {
+ calls.push({ list: q });
+ const before = Date.parse(q.before);
+ let rows = clips.filter((c) => c.visibility === 'public' && (q.auto_generated !== 1 || c.auto_generated)
+ && Date.parse(c.created_at.includes('T') ? c.created_at : c.created_at.replace(' ', 'T') + 'Z') < before);
+ if (q.unvisited && applies) rows = rows.filter((c) => !c.visited);
+ return { clips: rows.slice(0, q.limit), total: rows.length };
+};
+media.updateClip = async (id, fields) => {
+ calls.push({ update: id, fields });
+ const c = clips.find((x) => x.id === id);
+ Object.assign(c, fields);
+ return { clip: { ...c } };
+};
+const sweep = require('../server/ai/auto-clip-sweep');
+
+(async () => {
+ reset();
+ const out = await sweep.sweep({ now: NOW });
+ assert.deepStrictEqual(out.hidden.sort(), [1, 5], 'old, AI-made, never opened: unlisted');
+ assert.strictEqual(clips.find((c) => c.id === 1).visibility, 'unlisted');
+ assert.strictEqual(clips.find((c) => c.id === 2).visibility, 'public', 'opened by a person: kept');
+ assert.strictEqual(clips.find((c) => c.id === 3).visibility, 'public', 'a person\'s clip: never touched');
+ assert.strictEqual(clips.find((c) => c.id === 4).visibility, 'public', 'younger than 30 days: kept');
+ assert.ok(calls.every((c) => !c.fields || c.fields.visibility === 'unlisted'), 'unlisted, never deleted');
+ const list = calls.find((c) => c.list && c.list.unvisited);
+ assert.strictEqual(list.list.auto_generated, 1);
+ assert.strictEqual(list.list.before, '2026-10-03T20:00:00.000Z', '30 days before now');
+ const log = JSON.parse(state.get(sweep.SETTING));
+ assert.deepStrictEqual(log.map((e) => e.id).sort(), [1, 5], 'remembered');
+
+ // Put one back.
+ await sweep.restore(1);
+ assert.strictEqual(clips.find((c) => c.id === 1).visibility, 'public');
+ assert.deepStrictEqual(JSON.parse(state.get(sweep.SETTING)).map((e) => e.id), [5], 'and forgotten');
+
+ // A dry run says what it would hide and changes nothing.
+ reset(); state.clear();
+ const dry = await sweep.sweep({ now: NOW, dryRun: true });
+ assert.deepStrictEqual(dry.hidden.sort(), [1, 5]);
+ assert.ok(!calls.some((c) => c.update), 'nothing changed');
+ assert.strictEqual(state.get(sweep.SETTING), undefined);
+
+ // A Media that ignores unvisited=1 answers every old clip: the sweep does nothing.
+ reset(); applies = false;
+ const blind = await sweep.sweep({ now: NOW });
+ assert.deepStrictEqual(blind.hidden, []);
+ assert.match(blind.why, /did not apply unvisited/);
+ assert.ok(!calls.some((c) => c.update));
+
+ // At most `max` a run.
+ reset();
+ const one = await sweep.sweep({ now: NOW, max: 1 });
+ assert.strictEqual(one.hidden.length, 1);
+
+ // Media down: nothing, no throw.
+ reset();
+ const listClips = media.listClips;
+ media.listClips = async () => { throw Object.assign(new Error('down'), { status: 503 }); };
+ await assert.rejects(sweep.sweep({ now: NOW }), /down/, 'the job runner records the failure');
+ media.listClips = listClips;
+
+ console.log('auto-clip sweep: all checks passed');
+})().catch((e) => { console.error(e); process.exit(1); });