openvibers/openvibe.live Commits

The AI's stale clips leave the lists: unlisted, never deleted (#567)

OP OpenVibers committed 11 days ago · 658ba00ddb · 3 files, +182 −0

Once a day, a public automatic clip cut more than 30 days ago that no person ever opened (Media's
content_visits, GET /clips?unvisited=1, Media#292; the measure of the Live#443 audit, not the play counter
crawlers move) is made unlisted: its page and link still work, it leaves lists, search and the home. In
production that is 29 of the 190 old public auto-clips. A person's clip is never touched; at most 25 a run;
each hidden clip is remembered (state auto_clip_swept) and restore(id) puts one back. A Media that ignores
unvisited=1 answers every old clip, and then the sweep does nothing.

Changes

ddiff --git a/server/ai/auto-clip-job.js b/server/ai/auto-clip-job.js
iindex ebe54826..70f81e03 100644
---- a/server/ai/auto-clip-job.js
++++ b/server/ai/auto-clip-job.js
@@@ -427,12 +427,15 @@ function start() {
// clips). Jittered so it does not start in lockstep with every other boot-time job.
_backfillTimer = require('../utils/jobs').every('auto-clip-vod-backfill', 5 * 60 * 1000, () => backfillVodClips(), { initialDelayMs: 30 * 1000, jitterMs: 30 * 1000 });
_flagTimer = require('../utils/jobs').every('auto-clip-flag-sync', FLAG_SYNC_INTERVAL_MS, () => syncAutoClipFlags(), { initialDelayMs: 90 * 1000, jitterMs: 60 * 1000 });
++ // Once a day, the AI's old clips no person ever opened leave the lists (unlisted, never deleted).
++ require('./auto-clip-sweep').start();
console.log('[AutoClip] Live auto-clip job started (selective chat-spike + AI agreement, cuts via OpenVibe.Media) + persistent VOD backfill');
}
function stop() {
if (_timer) { clearInterval(_timer); _timer = null; }
if (_backfillTimer) { _backfillTimer(); _backfillTimer = null; }
if (_flagTimer) { _flagTimer(); _flagTimer = null; }
++ require('./auto-clip-sweep').stop();
}
module.exports = { start, stop, clipVodMoment, backfillVodClips, syncAutoClipFlags, _tick, _internals: { checkLiveStream: _checkLiveStream, backfillPool: _backfillPool } };
ddiff --git a/server/ai/auto-clip-sweep.js b/server/ai/auto-clip-sweep.js
nnew file mode 100644
iindex 00000000..ee32bfdb
---- /dev/null
++++ b/server/ai/auto-clip-sweep.js
@@@ -0,0 +1,84 @@
++'use strict';
++/**
++ * auto-clip-sweep.js — the AI's stale clips leave the lists (open item 151: "prune stale auto-clips, hide first").
++ *
++ * An automatic clip that is public, was cut more than 30 days ago, and that no person ever opened is made unlisted:
++ * its page and link keep working, and it leaves the lists, search and the home. Nothing is deleted.
++ *
++ * "No person ever opened it" is Media's content_visits (a visitor row; GET /clips?unvisited=1, Media#292), the
++ * measure of Live's clip audit (Live#443). The play counter is not used: crawlers move it, so by plays only 2 of the
++ * 190 old public auto-clips looked unwatched, while 29 had never been opened by a person.
++ * Only clips Media marks auto_generated: a person's clip is never touched.
++ * At most 25 a run, once a day. Each hidden clip is remembered (state auto_clip_swept: id and when), newest first,
++ * so it can be shown and put back (restore()).
++ * A Media that does not know unvisited=1 answers every old clip. When the "unvisited" total is not below the plain
++ * total, the sweep does nothing that run.
++ *
++ * sweep({ now, max, dryRun }) → { hidden: [ids], skipped, why }
++ * restore(id) → puts one back to public and forgets it
++ * start() / stop()
++ */
++const db = require('../db/database');
++const media = require('../media-client');
++const { parseTs } = require('./clip-policy');
++
++const STALE_DAYS = 30;
++const MAX_PER_RUN = 25;
++const SETTING = 'auto_clip_swept';
++const KEEP = 2000;
++const DAY_MS = 24 * 3600 * 1000;
++
++async function _swept() {
++ try { const d = JSON.parse(await db.getState(SETTING) || '[]'); return Array.isArray(d) ? d : []; } catch { return []; }
++}
++
++async function sweep({ now = Date.now(), max = MAX_PER_RUN, dryRun = false } = {}) {
++ const before = new Date(now - STALE_DAYS * DAY_MS).toISOString();
++ const query = { auto_generated: 1, before, order: 'oldest', status: 'ready' };
++ const stale = await media.listClips({ ...query, unvisited: 1, limit: max }, { timeoutMs: 15_000 });
++ const all = await media.listClips({ ...query, limit: 1 }, { timeoutMs: 15_000 });
++ if (!stale || !Array.isArray(stale.clips) || !all || all.total == null) return { hidden: [], skipped: 0, why: 'Media did not answer' };
++ if (Number(stale.total) >= Number(all.total) && Number(all.total) > 0) {
++ return { hidden: [], skipped: 0, why: `Media did not apply unvisited=1 (${stale.total} of ${all.total} old auto-clips)` };
++ }
++ const hidden = [];
++ let skipped = 0;
++ for (const clip of stale.clips) {
++ // The list is public and AI-made by its query; each one is checked again before it changes.
++ const cut = clip && parseTs(clip.created_at);
++ if (!clip || clip.auto_generated !== true || clip.visibility !== 'public' || !(cut && cut < now - STALE_DAYS * DAY_MS)) { skipped++; continue; }
++ if (dryRun) { hidden.push(clip.id); continue; }
++ try {
++ const out = await media.updateClip(clip.id, { visibility: 'unlisted' }, { timeoutMs: 10_000 });
++ const after = (out && out.clip) || out;
++ if (after && after.visibility && after.visibility !== 'unlisted') { skipped++; continue; }
++ hidden.push(clip.id);
++ } catch (e) {
++ if (e && (e.status === 404 || e.status === 410)) { skipped++; continue; }
++ break; // Media down or refusing: the rest wait for tomorrow
++ }
++ }
++ if (!dryRun && hidden.length) {
++ const at = new Date(now).toISOString();
++ const log = [...hidden.map((id) => ({ id, at })), ...await _swept()].slice(0, KEEP);
++ await db.setState(SETTING, JSON.stringify(log));
++ console.log(`[AutoClip] Unlisted ${hidden.length} AI clip(s) no person opened in their first ${STALE_DAYS} days or since: ${hidden.join(', ')}`);
++ }
++ return { hidden, skipped, why: null };
++}
++
++/** Put one swept clip back in the lists, and forget it. */
++async function restore(id) {
++ const out = await media.updateClip(id, { visibility: 'public' }, { timeoutMs: 10_000 });
++ await db.setState(SETTING, JSON.stringify((await _swept()).filter((e) => String(e.id) !== String(id))));
++ return (out && out.clip) || out;
++}
++
++let _stop = null;
++function start() {
++ if (_stop) return;
++ _stop = require('../utils/jobs').every('auto-clip-sweep', DAY_MS, () => sweep(), { initialDelayMs: 20 * 60 * 1000, jitterMs: 10 * 60 * 1000 });
++}
++function stop() { if (_stop) { _stop(); _stop = null; } }
++
++module.exports = { sweep, restore, start, stop, STALE_DAYS, MAX_PER_RUN, SETTING };
ddiff --git a/test/auto-clip-sweep.test.js b/test/auto-clip-sweep.test.js
nnew file mode 100644
iindex 00000000..b97ad571
---- /dev/null
++++ b/test/auto-clip-sweep.test.js
@@@ -0,0 +1,95 @@
++/**
++ * The AI's stale clips leave the lists (server/ai/auto-clip-sweep.js; open item 151, "prune stale auto-clips, hide
++ * first"): a public automatic clip cut more than 30 days ago that no person ever opened (Media's unvisited=1) is made
++ * unlisted, never deleted, and remembered so it can be put back. A person's clip is never touched; a Media that does
++ * not apply unvisited=1 makes the sweep do nothing.
++ *
++ * node test/run.js auto-clip-sweep
++ */
++'use strict';
++const assert = require('assert');
++
++const media = require('../server/media-client');
++const db = require('../server/db/database');
++const state = new Map();
++db.getState = async (k) => state.get(k) || null;
++db.setState = async (k, v) => { state.set(k, v); };
++
++const NOW = Date.parse('2026-11-02T20:00:00Z');
++const old = '2026-09-20 10:00:00', recent = '2026-10-25 10:00:00';
++let clips, applies, calls;
++const reset = () => {
++ clips = [
++ { id: 1, auto_generated: true, visibility: 'public', created_at: old, visited: false },
++ { id: 2, auto_generated: true, visibility: 'public', created_at: old, visited: true },
++ { id: 3, auto_generated: false, visibility: 'public', created_at: old, visited: false },
++ { id: 4, auto_generated: true, visibility: 'public', created_at: recent, visited: false },
++ { id: 5, auto_generated: true, visibility: 'public', created_at: '2026-09-01T00:00:00Z', visited: false },
++ ];
++ applies = true;
++ calls = [];
++};
++media.listClips = async (q) => {
++ calls.push({ list: q });
++ const before = Date.parse(q.before);
++ let rows = clips.filter((c) => c.visibility === 'public' && (q.auto_generated !== 1 || c.auto_generated)
++ && Date.parse(c.created_at.includes('T') ? c.created_at : c.created_at.replace(' ', 'T') + 'Z') < before);
++ if (q.unvisited && applies) rows = rows.filter((c) => !c.visited);
++ return { clips: rows.slice(0, q.limit), total: rows.length };
++};
++media.updateClip = async (id, fields) => {
++ calls.push({ update: id, fields });
++ const c = clips.find((x) => x.id === id);
++ Object.assign(c, fields);
++ return { clip: { ...c } };
++};
++const sweep = require('../server/ai/auto-clip-sweep');
++
++(async () => {
++ reset();
++ const out = await sweep.sweep({ now: NOW });
++ assert.deepStrictEqual(out.hidden.sort(), [1, 5], 'old, AI-made, never opened: unlisted');
++ assert.strictEqual(clips.find((c) => c.id === 1).visibility, 'unlisted');
++ assert.strictEqual(clips.find((c) => c.id === 2).visibility, 'public', 'opened by a person: kept');
++ assert.strictEqual(clips.find((c) => c.id === 3).visibility, 'public', 'a person\'s clip: never touched');
++ assert.strictEqual(clips.find((c) => c.id === 4).visibility, 'public', 'younger than 30 days: kept');
++ assert.ok(calls.every((c) => !c.fields || c.fields.visibility === 'unlisted'), 'unlisted, never deleted');
++ const list = calls.find((c) => c.list && c.list.unvisited);
++ assert.strictEqual(list.list.auto_generated, 1);
++ assert.strictEqual(list.list.before, '2026-10-03T20:00:00.000Z', '30 days before now');
++ const log = JSON.parse(state.get(sweep.SETTING));
++ assert.deepStrictEqual(log.map((e) => e.id).sort(), [1, 5], 'remembered');
++
++ // Put one back.
++ await sweep.restore(1);
++ assert.strictEqual(clips.find((c) => c.id === 1).visibility, 'public');
++ assert.deepStrictEqual(JSON.parse(state.get(sweep.SETTING)).map((e) => e.id), [5], 'and forgotten');
++
++ // A dry run says what it would hide and changes nothing.
++ reset(); state.clear();
++ const dry = await sweep.sweep({ now: NOW, dryRun: true });
++ assert.deepStrictEqual(dry.hidden.sort(), [1, 5]);
++ assert.ok(!calls.some((c) => c.update), 'nothing changed');
++ assert.strictEqual(state.get(sweep.SETTING), undefined);
++
++ // A Media that ignores unvisited=1 answers every old clip: the sweep does nothing.
++ reset(); applies = false;
++ const blind = await sweep.sweep({ now: NOW });
++ assert.deepStrictEqual(blind.hidden, []);
++ assert.match(blind.why, /did not apply unvisited/);
++ assert.ok(!calls.some((c) => c.update));
++
++ // At most `max` a run.
++ reset();
++ const one = await sweep.sweep({ now: NOW, max: 1 });
++ assert.strictEqual(one.hidden.length, 1);
++
++ // Media down: nothing, no throw.
++ reset();
++ const listClips = media.listClips;
++ media.listClips = async () => { throw Object.assign(new Error('down'), { status: 503 }); };
++ await assert.rejects(sweep.sweep({ now: NOW }), /down/, 'the job runner records the failure');
++ media.listClips = listClips;
++
++ console.log('auto-clip sweep: all checks passed');
++})().catch((e) => { console.error(e); process.exit(1); });