openvibers/openvibe.live Commits
C01: an aggregate audit of the automatic clips (the data behind the clip-score bars) (#443)
Changes
nnew file mode 100644@@@ -0,0 +1,47 @@++-- C01 clip audit — OpenVibe.Chat (ov_chat).++-- The old cutter cut a clip when a chat bucket spiked; this file gives the buckets (counts only,++-- never who wrote them) so the spike behind every cut can be reconstructed and the cuts can be++-- grouped by spike size.++--++-- Read-only: one READ ONLY transaction, 20 s statement timeout. Aggregate-only: counts per bucket++-- and the summary percentiles. No usernames, no user ids, no IPs.++--++-- scp clip-audit-chat.sql host:/tmp && sudo -u postgres psql -X -q -At -d ov_chat -f /tmp/clip-audit-chat.sql++++SET statement_timeout = '20s';++BEGIN READ ONLY;++++-- 1. The buckets themselves: 15-second buckets (the size the old rule used), last 30 days.++-- Join to the cut list (ov_media clips: stream_id, start_time) to get the spike at each cut.++SELECT 'chat_buckets' AS section,++ stream_id,++ floor(extract(epoch from timestamp::timestamptz) / 15)::bigint AS bucket,++ count(*) AS messages,++ count(DISTINCT COALESCE(user_id::text, anon_id)) AS people++FROM chat_messages++WHERE is_deleted = 0++ AND is_filtered = 0++ AND timestamp::timestamptz >= now() - interval '30 days'++GROUP BY 2, 3++ORDER BY 2, 3;++++-- 2. The same buckets as a summary: how busy a bucket is, and how big the spikes are.++WITH buckets AS (++ SELECT stream_id,++ floor(extract(epoch from timestamp::timestamptz) / 15)::bigint AS bucket,++ count(*) AS messages++ FROM chat_messages++ WHERE is_deleted = 0++ AND is_filtered = 0++ AND timestamp::timestamptz >= now() - interval '30 days'++ GROUP BY 1, 2++)++SELECT 'chat_bucket_summary' AS section,++ count(*) AS occupied_buckets,++ count(DISTINCT stream_id) AS streams,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY messages)) AS p50_messages,++ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY messages)) AS p90_messages,++ round(percentile_cont(0.99) WITHIN GROUP (ORDER BY messages)) AS p99_messages++FROM buckets;++++COMMIT;nnew file mode 100644@@@ -0,0 +1,74 @@++-- C01 clip audit — OpenVibe.Live (ov_live).++-- Aggregate-only diagnostics for the automatic clipper: what the old rule cut, how the new++-- clip-score rule (Live#442, 2026-10-29 18:52 UTC) is doing, and how busy the site was.++--++-- Read-only: the whole file runs in one READ ONLY transaction with a 20 s statement timeout.++-- Private by construction: no titles, no user ids, no emails, no IPs — counts and percentiles only.++--++-- scp clip-audit-live.sql host:/tmp && sudo -u postgres psql -X -q -At -d ov_live -f /tmp/clip-audit-live.sql++++SET statement_timeout = '20s';++BEGIN READ ONLY;++++-- 1. Automatic cuts per day, from Live's own clip log (the state table the cutter writes).++SELECT 'auto_clip_log_per_day' AS section,++ to_char(to_timestamp((e->>'ts')::double precision / 1000), 'YYYY-MM-DD') AS bucket,++ count(*) AS clips++FROM app_state s, LATERAL jsonb_array_elements(s.value::jsonb) e++WHERE s.key = 'auto_clip_log'++ AND (e->>'ts')::double precision >= extract(epoch from now() - interval '30 days') * 1000++GROUP BY 2++ORDER BY 2;++++-- 2. The log's shape: how many cuts it holds, its span, and how many carry the new score fields.++SELECT 'auto_clip_log_shape' AS section,++ count(*) AS entries,++ min(to_timestamp((e->>'ts')::double precision / 1000))::date AS oldest_day,++ max(to_timestamp((e->>'ts')::double precision / 1000))::date AS newest_day,++ count(*) FILTER (WHERE jsonb_exists(e, 'score')) AS with_score,++ count(*) FILTER (WHERE jsonb_exists(e, 'visibility')) AS with_visibility,++ count(*) FILTER (WHERE jsonb_exists(e, 'clip_id')) AS with_clip_id,++ count(*) FILTER (WHERE COALESCE((e->>'dedup')::boolean, false)) AS deduplicated++FROM app_state s, LATERAL jsonb_array_elements(s.value::jsonb) e++WHERE s.key = 'auto_clip_log';++++-- 3. The clip-score the new rule gave the cuts it made (empty until the new rule has cuts).++SELECT 'auto_clip_log_scores' AS section,++ COALESCE(e->>'score', 'none') AS score,++ COALESCE(e->>'visibility', 'none') AS visibility,++ count(*) AS clips++FROM app_state s, LATERAL jsonb_array_elements(s.value::jsonb) e++WHERE s.key = 'auto_clip_log'++GROUP BY 2, 3++ORDER BY 2, 3;++++-- 4. Interest in the streams that produced clips: chatters, peak viewers, chat volume.++SELECT 'clip_stream_interest' AS section,++ count(*) AS stream_rows,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY unique_chatters))::bigint AS median_chatters,++ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY unique_chatters))::bigint AS p90_chatters,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY peak_viewers))::bigint AS median_peak_viewers,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY total_messages))::bigint AS median_messages,++ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY total_messages))::bigint AS p90_messages++FROM stream_analytics++WHERE COALESCE(clips_created, 0) > 0;++++-- 5. The AI verdicts behind the clips: how many clips the AI has worked on.++SELECT 'clip_ai_state' AS section,++ count(*) AS clips,++ count(*) FILTER (WHERE transcript_status = 'ready') AS with_transcript,++ count(*) FILTER (WHERE COALESCE(ai_overview_short, '') <> '') AS with_overview,++ count(*) FILTER (WHERE COALESCE(ai_transcript_json, '') <> '') AS with_transcript_json++FROM clip_ai_state;++++-- 6. Streams and their audience over the last 30 days (the traffic the cutter worked with).++SELECT 'streams_30d' AS section,++ count(*) AS streams,++ count(*) FILTER (WHERE is_live <> 0) AS live_now,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY peak_viewers))::bigint AS median_peak_viewers,++ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY peak_viewers))::bigint AS p90_peak_viewers,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY viewer_count))::bigint AS median_viewers++FROM streams++WHERE started_at::timestamptz >= now() - interval '30 days';++++COMMIT;nnew file mode 100644@@@ -0,0 +1,108 @@++-- C01 clip audit — OpenVibe.Media (ov_media).++-- The canonical clip table. Automatic clips are the ones the AI clipper cut (auto_generated = 1);++-- the rest were cut by people. This file says how many automatic clips there are, what they are++-- worth, how many nobody ever watched, and how they repeat (overlap and semantic repeats counted++-- separately).++--++-- Read-only: one READ ONLY transaction, 20 s statement timeout. Aggregate-only: counts, medians++-- and percentiles. No titles, no user ids, no emails, no IPs.++--++-- scp clip-audit-media.sql host:/tmp && sudo -u postgres psql -X -q -At -d ov_media -f /tmp/clip-audit-media.sql++++SET statement_timeout = '20s';++BEGIN READ ONLY;++++-- 1. Automatic clips per day over the last 30 days.++SELECT 'auto_clips_per_day' AS section,++ to_char(date_trunc('day', created_at::timestamptz), 'YYYY-MM-DD') AS bucket,++ count(*) AS clips++FROM public.clips++WHERE auto_generated = 1++ AND created_at::timestamptz >= now() - interval '30 days'++GROUP BY 2++ORDER BY 2;++++-- 2. Automatic versus human clips, by kind (visibility x status): counts, median length, median plays.++SELECT 'clips_by_kind' AS section,++ CASE WHEN auto_generated = 1 THEN 'auto' ELSE 'human' END AS kind,++ visibility,++ status,++ count(*) AS clips,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_seconds)) AS median_seconds,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY view_count)) AS median_plays,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY unique_views)) AS median_viewers++FROM public.clips++GROUP BY 2, 3, 4++ORDER BY 2, 3, 4;++++-- 3. What the old rule produced: plays, viewers and length of the automatic clips, in percentiles.++SELECT 'auto_clips_percentiles' AS section,++ count(*) AS clips,++ round(percentile_cont(0.25) WITHIN GROUP (ORDER BY view_count)) AS p25_plays,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY view_count)) AS p50_plays,++ round(percentile_cont(0.75) WITHIN GROUP (ORDER BY view_count)) AS p75_plays,++ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY view_count)) AS p90_plays,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY unique_views)) AS p50_viewers,++ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_seconds)) AS p50_seconds,++ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY duration_seconds)) AS p90_seconds++FROM public.clips++WHERE auto_generated = 1;++++-- 4. Never watched by a person. A person is a visitor row (content_visits counts visitors, never++-- names one); a play counter alone is not a person. 7 and 30 days after the clip was cut.++WITH auto_clips AS (++ SELECT c.id,++ c.created_at::timestamptz AS cut_at,++ extract(epoch from now() - c.created_at::timestamptz) / 86400 AS age_days,++ (SELECT count(*) FROM content_visits v++ WHERE v.content_type = 'clip' AND v.content_id = c.id) AS visitors,++ (SELECT count(*) FROM content_visits v++ WHERE v.content_type = 'clip' AND v.content_id = c.id++ AND v.last_at::timestamptz >= c.created_at::timestamptz + interval '7 days') AS visitors_7d,++ (SELECT count(*) FROM content_visits v++ WHERE v.content_type = 'clip' AND v.content_id = c.id++ AND v.last_at::timestamptz >= c.created_at::timestamptz + interval '30 days') AS visitors_30d++ FROM public.clips c++ WHERE c.auto_generated = 1++)++SELECT 'auto_clips_never_watched' AS section,++ count(*) AS clips,++ count(*) FILTER (WHERE visitors = 0) AS never_watched,++ count(*) FILTER (WHERE age_days >= 7) AS clips_at_least_7d_old,++ count(*) FILTER (WHERE age_days >= 7 AND visitors_7d = 0) AS never_watched_after_7d,++ count(*) FILTER (WHERE age_days >= 30) AS clips_at_least_30d_old,++ count(*) FILTER (WHERE age_days >= 30 AND visitors_30d = 0) AS never_watched_after_30d++FROM auto_clips;++++-- 5. Overlap: two automatic cuts of the same source that cover the same seconds.++SELECT 'auto_clip_overlap' AS section,++ count(*) AS overlapping_pairs,++ count(DISTINCT a.id) AS clips_involved++FROM public.clips a++JOIN public.clips b++ ON a.id < b.id++ AND a.vod_id = b.vod_id++ AND a.auto_generated = 1++ AND b.auto_generated = 1++ AND a.start_time < b.end_time++ AND b.start_time < a.end_time;++++-- 6. Semantic repeats, counted separately from the overlap above: the same words cut from++-- different footage. Only hashes are compared; no text is printed.++WITH fingerprints AS (++ SELECT md5(lower(regexp_replace(COALESCE(ai_transcript, ''), '[^a-z0-9]+', ' ', 'g'))) AS fp,++ count(*) AS clips,++ count(DISTINCT vod_id) AS sources++ FROM public.clips++ WHERE auto_generated = 1++ AND COALESCE(ai_transcript, '') <> ''++ GROUP BY 1++)++SELECT 'auto_clip_semantic_repeats' AS section,++ count(*) AS distinct_fingerprints,++ count(*) FILTER (WHERE clips > 1) AS fingerprints_used_twice,++ count(*) FILTER (WHERE sources > 1) AS fingerprints_on_several_sources,++ COALESCE(sum(clips) FILTER (WHERE clips > 1), 0) AS clips_in_repeat_pairs++FROM fingerprints;++++COMMIT;nnew file mode 100644@@@ -0,0 +1,181 @@++'use strict';++++// C01: the clip audit's SQL must stay a read-only, aggregate-only look at production.++//++// The three files in scripts/diagnostics/ are run by hand against production (ov_live, ov_media,++// ov_chat). They may only ever read: one READ ONLY transaction per file, a statement timeout, and++// nothing that names a person — no titles, no user ids, no emails, no IPs. This test runs them on++// the test database (PGlite) against fixture rows and proves all three things.++++const assert = require('assert');++const fs = require('fs');++const path = require('path');++++const ROOT = path.join(__dirname, '..');++const DIR = path.join(ROOT, 'scripts', 'diagnostics');++const FILES = ['clip-audit-live.sql', 'clip-audit-media.sql', 'clip-audit-chat.sql'];++++// A marker planted in the fixtures: it must never come back out of a query.++const MARKER = 'PERSONAL-MARKER-DO-NOT-LEAK';++++const read = (f) => fs.readFileSync(path.join(DIR, f), 'utf8');++const code = (sql) => sql.split('\n').filter((l) => l.trim() && !/^\s*--/.test(l)).join('\n');++++// ── 1. shape of every file ────────────────────────────────────────────────────++for (const f of FILES) {++ const sql = read(f);++ assert.ok(/SET\s+statement_timeout\s*=\s*'20s'\s*;/i.test(sql), `${f}: a 20s statement timeout`);++ assert.ok(/BEGIN\s+READ\s+ONLY\s*;/i.test(sql), `${f}: a READ ONLY transaction`);++ assert.ok(/\bCOMMIT\s*;/i.test(sql), `${f}: it commits`);++ assert.ok(!/(?:^|[^a-z_])(insert|update|delete|create|alter|drop|truncate|copy|grant|revoke|vacuum)\s/i.test(code(sql)),++ `${f}: the file only reads`);++ // Personal columns never leave. A user id may appear, but only inside a count.++ const banned = ['email', 'username', 'title', 'description', 'password', 'secret', 'token', 'api_key',++ 'subject_id', 'user_agent', 'referer', 'country', 'city'];++ for (const col of banned) {++ const re = new RegExp('\\b' + col + '\\b', 'i');++ assert.ok(!re.test(code(sql)), `${f}: no ${col} column`);++ }++ for (const col of ['user_id', 'channel_user_id', 'anon_id']) {++ const re = new RegExp('\\b' + col + '\\b', 'i');++ for (const line of code(sql).split('\n')) {++ if (!re.test(line)) continue;++ assert.ok(/count\s*\(\s*distinct/i.test(line), `${f}: ${col} only inside count(distinct ...)`);++ }++ }++}++++// ── 2. run them on the test database ──────────────────────────────────────────++const db = globalThis.__ovLiveTestDb;++++// PGlite returns one result for a single statement, an array for several.++const asRows = (res) => (Array.isArray(res) ? res.flatMap((r) => (r && r.rows) || []) : (res && res.rows) || []);++++async function run(sql) {++ // One statement per exec, so BEGIN READ ONLY really is a read-only transaction.++ const statements = sql.split('\n').filter((l) => l.trim() && !/^\s*--/.test(l)).join('\n').split(';')++ .map((s) => s.trim()).filter(Boolean);++ const rows = [];++ for (const stmt of statements) {++ if (/^BEGIN\s+READ\s+ONLY/i.test(stmt)) { await db.exec(stmt); continue; }++ const isSelect = /^\s*(select|with)\b/i.test(stmt);++ if (isSelect) rows.push(...((await db.prepare(stmt).all()) || []));++ else await db.exec(stmt);++ }++ return rows;++}++++async function main() {++ if (!db || typeof db.exec !== 'function') {++ console.log('clip audit SQL: skipped (no PGlite database in this run)');++ return;++ }++ // Fixtures: Media's clip table, Media's visitor counter and Chat's messages. Live's own tables++ // (app_state, streams, stream_analytics, clip_ai_state) already exist in the test database.++ await db.exec(`++ DROP TABLE IF EXISTS public.clips;++ DROP TABLE IF EXISTS public.content_visits;++ DROP TABLE IF EXISTS public.chat_messages;++ CREATE TABLE public.clips (++ id bigint PRIMARY KEY, vod_id bigint, stream_id bigint, user_id bigint,++ title text, description text, start_time double precision, end_time double precision,++ duration_seconds double precision, is_public bigint, visibility text, status text,++ view_count bigint, auto_generated bigint, created_at text, unique_views bigint,++ ai_transcript text, ai_overview text);++ CREATE TABLE public.content_visits (++ content_type text, content_id bigint, visitor text, first_at text, last_at text, visits bigint);++ CREATE TABLE public.chat_messages (++ id bigint PRIMARY KEY, stream_id bigint, user_id bigint, anon_id text, message text,++ message_type text, is_deleted bigint, is_filtered bigint, timestamp text);++ INSERT INTO public.clips VALUES++ (1, 10, 100, 7, 'MARKER-title', 'MARKER-description', 0, 25, 25, 1, 'public', 'ready', 12, 1, '2026-10-01 10:00:00', 12, 'hello there friends', 'a short clip'),++ (2, 10, 100, 7, 'MARKER-title', 'MARKER-description', 20, 45, 25, 1, 'public', 'ready', 3, 1, '2026-10-02 10:00:00', 3, 'hello there friends', 'a short clip'),++ (3, 11, 101, 8, 'MARKER-title', 'MARKER-description', 0, 25, 25, 1, 'public', 'ready', 0, 1, '2026-10-03 10:00:00', 0, NULL, NULL),++ (4, 12, 102, 9, 'MARKER-title', 'MARKER-description', 0, 30, 30, 1, 'public', 'ready', 5, 0, '2026-10-04 10:00:00', 5, NULL, NULL);++ INSERT INTO public.content_visits VALUES++ ('clip', 1, 'visitor-1', '2026-10-01 11:00:00', '2026-10-02 11:00:00', 2),++ ('clip', 1, 'visitor-2', '2026-10-02 11:00:00', '2026-10-02 12:00:00', 1),++ ('clip', 4, 'visitor-3', '2026-10-04 11:00:00', '2026-10-05 11:00:00', 1);++ INSERT INTO public.chat_messages VALUES++ (1, 100, 7, 'anon-1', 'hi', 'message', 0, 0, '2026-10-01 10:00:00'),++ (2, 100, 8, 'anon-2', 'MARKER-message', 'message', 0, 0, '2026-10-01 10:00:05'),++ (3, 101, 9, 'anon-3', 'yo', 'message', 1, 0, '2026-10-01 10:00:10');++ INSERT INTO app_state (key, value) VALUES ('auto_clip_log',++ '[{"ts": 1767225600000, "vod_id": 10, "stream_id": 100, "start_time": 0, "title": "MARKER-title", "clip_id": 1},++ {"ts": 1767225600000, "vod_id": 11, "stream_id": 101, "start_time": 0, "title": "MARKER-title", "clip_id": 2}]')++ ON CONFLICT (key) DO UPDATE SET value = excluded.value;++ `);++++ // The live file reads Live's own tables; give them a row each.++ await db.exec(`++ INSERT INTO users (username, password_hash)++ SELECT 'clip-audit-fixture', 'not-a-password'++ WHERE NOT EXISTS (SELECT 1 FROM users WHERE username = 'clip-audit-fixture');++ INSERT INTO streams (user_id, title, is_live, viewer_count, peak_viewers, started_at)++ SELECT id, 'MARKER-title', 0, 2, 5, '2026-10-01 09:00:00' FROM users WHERE username = 'clip-audit-fixture';++ INSERT INTO stream_analytics (stream_id, avg_viewers, peak_viewers, unique_chatters, total_messages, clips_created)++ SELECT id, 2, 5, 3, 120, 1 FROM streams WHERE title = 'MARKER-title' LIMIT 1++ ON CONFLICT (stream_id) DO NOTHING;++ INSERT INTO clip_ai_state (clip_id, ai_overview_short, transcript_status)++ VALUES (900, 'MARKER-description', 'ready')++ ON CONFLICT (clip_id) DO NOTHING;++ `);++++ const before = await db.prepare('SELECT count(*)::int AS n FROM public.clips').all();++ const beforeClips = before[0].n;++++ const live = await run(read('clip-audit-live.sql'));++ const media = await run(read('clip-audit-media.sql'));++ const chat = await run(read('clip-audit-chat.sql'));++ const all = [...live, ...media, ...chat];++++ // Nothing personal came out.++ const dumped = JSON.stringify(all);++ assert.ok(!dumped.includes(MARKER), 'no personal text came out of the audit');++ for (const bad of ['MARKER-title', 'MARKER-description', 'MARKER-message']) {++ assert.ok(!dumped.includes(bad), `no ${bad} came out of the audit`);++ }++++ // The answers the audit exists for are there.++ const perDay = all.filter((r) => r.section === 'auto_clips_per_day');++ assert.strictEqual(perDay.length, 3, 'the automatic clips per day are counted');++ const kind = all.find((r) => r.section === 'clips_by_kind');++ assert.ok(kind && Number(kind.clips) >= 3, 'automatic and human clips are counted by kind');++ const never = all.find((r) => r.section === 'auto_clips_never_watched');++ assert.ok(never && Number(never.never_watched) >= 1, 'the clips nobody watched are counted');++ const overlap = all.find((r) => r.section === 'auto_clip_overlap');++ assert.ok(overlap && Number(overlap.overlapping_pairs) >= 1, 'overlapping cuts are counted');++ const repeats = all.find((r) => r.section === 'auto_clip_semantic_repeats');++ assert.ok(repeats && Number(repeats.distinct_fingerprints) >= 1, 'semantic repeats are counted separately');++ assert.ok(all.filter((r) => r.section === 'chat_buckets').length >= 1, 'the chat buckets are counted');++ const log = all.find((r) => r.section === 'auto_clip_log_shape');++ assert.ok(log && Number(log.entries) === 2, 'the clip log is counted');++++ // The files' own transaction shape: a READ ONLY transaction refuses a write.++ await db.exec('BEGIN READ ONLY');++ let refused = false;++ try { await db.exec('CREATE TABLE clip_audit_write_probe (id int)'); }++ catch (e) { refused = /read-only/i.test(String(e && e.message)); }++ await db.exec('ROLLBACK').catch(() => {});++ assert.ok(refused, 'a READ ONLY transaction refuses a write');++++ // Nothing was written: the fixtures are exactly as they were.++ const after = await db.prepare('SELECT count(*)::int AS n FROM public.clips').all();++ assert.strictEqual(after[0].n, beforeClips, 'the audit wrote nothing');++++ // Clean up the fixtures this file added.++ await db.exec(`++ DROP TABLE IF EXISTS public.clips;++ DROP TABLE IF EXISTS public.content_visits;++ DROP TABLE IF EXISTS public.chat_messages;++ DELETE FROM app_state WHERE key = 'auto_clip_log';++ DELETE FROM stream_analytics WHERE stream_id = 900;++ DELETE FROM clip_ai_state WHERE clip_id = 900;++ DELETE FROM streams WHERE title = 'MARKER-title';++ DELETE FROM users WHERE username = 'clip-audit-fixture';++ `);++++ console.log('clip audit SQL: read-only, aggregate-only, no personal columns — all checks passed');++}++++main().catch((e) => { console.error(e && e.stack || e); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,47 @@
@@ -0,0 +1,47 @@
+-- C01 clip audit — OpenVibe.Chat (ov_chat).
+-- The old cutter cut a clip when a chat bucket spiked; this file gives the buckets (counts only,
+-- never who wrote them) so the spike behind every cut can be reconstructed and the cuts can be
+-- grouped by spike size.
+--
+-- Read-only: one READ ONLY transaction, 20 s statement timeout. Aggregate-only: counts per bucket
+-- and the summary percentiles. No usernames, no user ids, no IPs.
+--
+-- scp clip-audit-chat.sql host:/tmp && sudo -u postgres psql -X -q -At -d ov_chat -f /tmp/clip-audit-chat.sql
+
+SET statement_timeout = '20s';
+BEGIN READ ONLY;
+
+-- 1. The buckets themselves: 15-second buckets (the size the old rule used), last 30 days.
+-- Join to the cut list (ov_media clips: stream_id, start_time) to get the spike at each cut.
+SELECT 'chat_buckets' AS section,
+ stream_id,
+ floor(extract(epoch from timestamp::timestamptz) / 15)::bigint AS bucket,
+ count(*) AS messages,
+ count(DISTINCT COALESCE(user_id::text, anon_id)) AS people
+FROM chat_messages
+WHERE is_deleted = 0
+ AND is_filtered = 0
+ AND timestamp::timestamptz >= now() - interval '30 days'
+GROUP BY 2, 3
+ORDER BY 2, 3;
+
+-- 2. The same buckets as a summary: how busy a bucket is, and how big the spikes are.
+WITH buckets AS (
+ SELECT stream_id,
+ floor(extract(epoch from timestamp::timestamptz) / 15)::bigint AS bucket,
+ count(*) AS messages
+ FROM chat_messages
+ WHERE is_deleted = 0
+ AND is_filtered = 0
+ AND timestamp::timestamptz >= now() - interval '30 days'
+ GROUP BY 1, 2
+)
+SELECT 'chat_bucket_summary' AS section,
+ count(*) AS occupied_buckets,
+ count(DISTINCT stream_id) AS streams,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY messages)) AS p50_messages,
+ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY messages)) AS p90_messages,
+ round(percentile_cont(0.99) WITHIN GROUP (ORDER BY messages)) AS p99_messages
+FROM buckets;
+
+COMMIT;
new file mode 100644
new file mode 100644
@@ -0,0 +1,74 @@
@@ -0,0 +1,74 @@
+-- C01 clip audit — OpenVibe.Live (ov_live).
+-- Aggregate-only diagnostics for the automatic clipper: what the old rule cut, how the new
+-- clip-score rule (Live#442, 2026-10-29 18:52 UTC) is doing, and how busy the site was.
+--
+-- Read-only: the whole file runs in one READ ONLY transaction with a 20 s statement timeout.
+-- Private by construction: no titles, no user ids, no emails, no IPs — counts and percentiles only.
+--
+-- scp clip-audit-live.sql host:/tmp && sudo -u postgres psql -X -q -At -d ov_live -f /tmp/clip-audit-live.sql
+
+SET statement_timeout = '20s';
+BEGIN READ ONLY;
+
+-- 1. Automatic cuts per day, from Live's own clip log (the state table the cutter writes).
+SELECT 'auto_clip_log_per_day' AS section,
+ to_char(to_timestamp((e->>'ts')::double precision / 1000), 'YYYY-MM-DD') AS bucket,
+ count(*) AS clips
+FROM app_state s, LATERAL jsonb_array_elements(s.value::jsonb) e
+WHERE s.key = 'auto_clip_log'
+ AND (e->>'ts')::double precision >= extract(epoch from now() - interval '30 days') * 1000
+GROUP BY 2
+ORDER BY 2;
+
+-- 2. The log's shape: how many cuts it holds, its span, and how many carry the new score fields.
+SELECT 'auto_clip_log_shape' AS section,
+ count(*) AS entries,
+ min(to_timestamp((e->>'ts')::double precision / 1000))::date AS oldest_day,
+ max(to_timestamp((e->>'ts')::double precision / 1000))::date AS newest_day,
+ count(*) FILTER (WHERE jsonb_exists(e, 'score')) AS with_score,
+ count(*) FILTER (WHERE jsonb_exists(e, 'visibility')) AS with_visibility,
+ count(*) FILTER (WHERE jsonb_exists(e, 'clip_id')) AS with_clip_id,
+ count(*) FILTER (WHERE COALESCE((e->>'dedup')::boolean, false)) AS deduplicated
+FROM app_state s, LATERAL jsonb_array_elements(s.value::jsonb) e
+WHERE s.key = 'auto_clip_log';
+
+-- 3. The clip-score the new rule gave the cuts it made (empty until the new rule has cuts).
+SELECT 'auto_clip_log_scores' AS section,
+ COALESCE(e->>'score', 'none') AS score,
+ COALESCE(e->>'visibility', 'none') AS visibility,
+ count(*) AS clips
+FROM app_state s, LATERAL jsonb_array_elements(s.value::jsonb) e
+WHERE s.key = 'auto_clip_log'
+GROUP BY 2, 3
+ORDER BY 2, 3;
+
+-- 4. Interest in the streams that produced clips: chatters, peak viewers, chat volume.
+SELECT 'clip_stream_interest' AS section,
+ count(*) AS stream_rows,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY unique_chatters))::bigint AS median_chatters,
+ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY unique_chatters))::bigint AS p90_chatters,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY peak_viewers))::bigint AS median_peak_viewers,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY total_messages))::bigint AS median_messages,
+ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY total_messages))::bigint AS p90_messages
+FROM stream_analytics
+WHERE COALESCE(clips_created, 0) > 0;
+
+-- 5. The AI verdicts behind the clips: how many clips the AI has worked on.
+SELECT 'clip_ai_state' AS section,
+ count(*) AS clips,
+ count(*) FILTER (WHERE transcript_status = 'ready') AS with_transcript,
+ count(*) FILTER (WHERE COALESCE(ai_overview_short, '') <> '') AS with_overview,
+ count(*) FILTER (WHERE COALESCE(ai_transcript_json, '') <> '') AS with_transcript_json
+FROM clip_ai_state;
+
+-- 6. Streams and their audience over the last 30 days (the traffic the cutter worked with).
+SELECT 'streams_30d' AS section,
+ count(*) AS streams,
+ count(*) FILTER (WHERE is_live <> 0) AS live_now,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY peak_viewers))::bigint AS median_peak_viewers,
+ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY peak_viewers))::bigint AS p90_peak_viewers,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY viewer_count))::bigint AS median_viewers
+FROM streams
+WHERE started_at::timestamptz >= now() - interval '30 days';
+
+COMMIT;
new file mode 100644
new file mode 100644
@@ -0,0 +1,108 @@
@@ -0,0 +1,108 @@
+-- C01 clip audit — OpenVibe.Media (ov_media).
+-- The canonical clip table. Automatic clips are the ones the AI clipper cut (auto_generated = 1);
+-- the rest were cut by people. This file says how many automatic clips there are, what they are
+-- worth, how many nobody ever watched, and how they repeat (overlap and semantic repeats counted
+-- separately).
+--
+-- Read-only: one READ ONLY transaction, 20 s statement timeout. Aggregate-only: counts, medians
+-- and percentiles. No titles, no user ids, no emails, no IPs.
+--
+-- scp clip-audit-media.sql host:/tmp && sudo -u postgres psql -X -q -At -d ov_media -f /tmp/clip-audit-media.sql
+
+SET statement_timeout = '20s';
+BEGIN READ ONLY;
+
+-- 1. Automatic clips per day over the last 30 days.
+SELECT 'auto_clips_per_day' AS section,
+ to_char(date_trunc('day', created_at::timestamptz), 'YYYY-MM-DD') AS bucket,
+ count(*) AS clips
+FROM public.clips
+WHERE auto_generated = 1
+ AND created_at::timestamptz >= now() - interval '30 days'
+GROUP BY 2
+ORDER BY 2;
+
+-- 2. Automatic versus human clips, by kind (visibility x status): counts, median length, median plays.
+SELECT 'clips_by_kind' AS section,
+ CASE WHEN auto_generated = 1 THEN 'auto' ELSE 'human' END AS kind,
+ visibility,
+ status,
+ count(*) AS clips,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_seconds)) AS median_seconds,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY view_count)) AS median_plays,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY unique_views)) AS median_viewers
+FROM public.clips
+GROUP BY 2, 3, 4
+ORDER BY 2, 3, 4;
+
+-- 3. What the old rule produced: plays, viewers and length of the automatic clips, in percentiles.
+SELECT 'auto_clips_percentiles' AS section,
+ count(*) AS clips,
+ round(percentile_cont(0.25) WITHIN GROUP (ORDER BY view_count)) AS p25_plays,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY view_count)) AS p50_plays,
+ round(percentile_cont(0.75) WITHIN GROUP (ORDER BY view_count)) AS p75_plays,
+ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY view_count)) AS p90_plays,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY unique_views)) AS p50_viewers,
+ round(percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_seconds)) AS p50_seconds,
+ round(percentile_cont(0.9) WITHIN GROUP (ORDER BY duration_seconds)) AS p90_seconds
+FROM public.clips
+WHERE auto_generated = 1;
+
+-- 4. Never watched by a person. A person is a visitor row (content_visits counts visitors, never
+-- names one); a play counter alone is not a person. 7 and 30 days after the clip was cut.
+WITH auto_clips AS (
+ SELECT c.id,
+ c.created_at::timestamptz AS cut_at,
+ extract(epoch from now() - c.created_at::timestamptz) / 86400 AS age_days,
+ (SELECT count(*) FROM content_visits v
+ WHERE v.content_type = 'clip' AND v.content_id = c.id) AS visitors,
+ (SELECT count(*) FROM content_visits v
+ WHERE v.content_type = 'clip' AND v.content_id = c.id
+ AND v.last_at::timestamptz >= c.created_at::timestamptz + interval '7 days') AS visitors_7d,
+ (SELECT count(*) FROM content_visits v
+ WHERE v.content_type = 'clip' AND v.content_id = c.id
+ AND v.last_at::timestamptz >= c.created_at::timestamptz + interval '30 days') AS visitors_30d
+ FROM public.clips c
+ WHERE c.auto_generated = 1
+)
+SELECT 'auto_clips_never_watched' AS section,
+ count(*) AS clips,
+ count(*) FILTER (WHERE visitors = 0) AS never_watched,
+ count(*) FILTER (WHERE age_days >= 7) AS clips_at_least_7d_old,
+ count(*) FILTER (WHERE age_days >= 7 AND visitors_7d = 0) AS never_watched_after_7d,
+ count(*) FILTER (WHERE age_days >= 30) AS clips_at_least_30d_old,
+ count(*) FILTER (WHERE age_days >= 30 AND visitors_30d = 0) AS never_watched_after_30d
+FROM auto_clips;
+
+-- 5. Overlap: two automatic cuts of the same source that cover the same seconds.
+SELECT 'auto_clip_overlap' AS section,
+ count(*) AS overlapping_pairs,
+ count(DISTINCT a.id) AS clips_involved
+FROM public.clips a
+JOIN public.clips b
+ ON a.id < b.id
+ AND a.vod_id = b.vod_id
+ AND a.auto_generated = 1
+ AND b.auto_generated = 1
+ AND a.start_time < b.end_time
+ AND b.start_time < a.end_time;
+
+-- 6. Semantic repeats, counted separately from the overlap above: the same words cut from
+-- different footage. Only hashes are compared; no text is printed.
+WITH fingerprints AS (
+ SELECT md5(lower(regexp_replace(COALESCE(ai_transcript, ''), '[^a-z0-9]+', ' ', 'g'))) AS fp,
+ count(*) AS clips,
+ count(DISTINCT vod_id) AS sources
+ FROM public.clips
+ WHERE auto_generated = 1
+ AND COALESCE(ai_transcript, '') <> ''
+ GROUP BY 1
+)
+SELECT 'auto_clip_semantic_repeats' AS section,
+ count(*) AS distinct_fingerprints,
+ count(*) FILTER (WHERE clips > 1) AS fingerprints_used_twice,
+ count(*) FILTER (WHERE sources > 1) AS fingerprints_on_several_sources,
+ COALESCE(sum(clips) FILTER (WHERE clips > 1), 0) AS clips_in_repeat_pairs
+FROM fingerprints;
+
+COMMIT;
new file mode 100644
new file mode 100644
@@ -0,0 +1,181 @@
@@ -0,0 +1,181 @@
+'use strict';
+
+// C01: the clip audit's SQL must stay a read-only, aggregate-only look at production.
+//
+// The three files in scripts/diagnostics/ are run by hand against production (ov_live, ov_media,
+// ov_chat). They may only ever read: one READ ONLY transaction per file, a statement timeout, and
+// nothing that names a person — no titles, no user ids, no emails, no IPs. This test runs them on
+// the test database (PGlite) against fixture rows and proves all three things.
+
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.join(__dirname, '..');
+const DIR = path.join(ROOT, 'scripts', 'diagnostics');
+const FILES = ['clip-audit-live.sql', 'clip-audit-media.sql', 'clip-audit-chat.sql'];
+
+// A marker planted in the fixtures: it must never come back out of a query.
+const MARKER = 'PERSONAL-MARKER-DO-NOT-LEAK';
+
+const read = (f) => fs.readFileSync(path.join(DIR, f), 'utf8');
+const code = (sql) => sql.split('\n').filter((l) => l.trim() && !/^\s*--/.test(l)).join('\n');
+
+// ── 1. shape of every file ────────────────────────────────────────────────────
+for (const f of FILES) {
+ const sql = read(f);
+ assert.ok(/SET\s+statement_timeout\s*=\s*'20s'\s*;/i.test(sql), `${f}: a 20s statement timeout`);
+ assert.ok(/BEGIN\s+READ\s+ONLY\s*;/i.test(sql), `${f}: a READ ONLY transaction`);
+ assert.ok(/\bCOMMIT\s*;/i.test(sql), `${f}: it commits`);
+ assert.ok(!/(?:^|[^a-z_])(insert|update|delete|create|alter|drop|truncate|copy|grant|revoke|vacuum)\s/i.test(code(sql)),
+ `${f}: the file only reads`);
+ // Personal columns never leave. A user id may appear, but only inside a count.
+ const banned = ['email', 'username', 'title', 'description', 'password', 'secret', 'token', 'api_key',
+ 'subject_id', 'user_agent', 'referer', 'country', 'city'];
+ for (const col of banned) {
+ const re = new RegExp('\\b' + col + '\\b', 'i');
+ assert.ok(!re.test(code(sql)), `${f}: no ${col} column`);
+ }
+ for (const col of ['user_id', 'channel_user_id', 'anon_id']) {
+ const re = new RegExp('\\b' + col + '\\b', 'i');
+ for (const line of code(sql).split('\n')) {
+ if (!re.test(line)) continue;
+ assert.ok(/count\s*\(\s*distinct/i.test(line), `${f}: ${col} only inside count(distinct ...)`);
+ }
+ }
+}
+
+// ── 2. run them on the test database ──────────────────────────────────────────
+const db = globalThis.__ovLiveTestDb;
+
+// PGlite returns one result for a single statement, an array for several.
+const asRows = (res) => (Array.isArray(res) ? res.flatMap((r) => (r && r.rows) || []) : (res && res.rows) || []);
+
+async function run(sql) {
+ // One statement per exec, so BEGIN READ ONLY really is a read-only transaction.
+ const statements = sql.split('\n').filter((l) => l.trim() && !/^\s*--/.test(l)).join('\n').split(';')
+ .map((s) => s.trim()).filter(Boolean);
+ const rows = [];
+ for (const stmt of statements) {
+ if (/^BEGIN\s+READ\s+ONLY/i.test(stmt)) { await db.exec(stmt); continue; }
+ const isSelect = /^\s*(select|with)\b/i.test(stmt);
+ if (isSelect) rows.push(...((await db.prepare(stmt).all()) || []));
+ else await db.exec(stmt);
+ }
+ return rows;
+}
+
+async function main() {
+ if (!db || typeof db.exec !== 'function') {
+ console.log('clip audit SQL: skipped (no PGlite database in this run)');
+ return;
+ }
+ // Fixtures: Media's clip table, Media's visitor counter and Chat's messages. Live's own tables
+ // (app_state, streams, stream_analytics, clip_ai_state) already exist in the test database.
+ await db.exec(`
+ DROP TABLE IF EXISTS public.clips;
+ DROP TABLE IF EXISTS public.content_visits;
+ DROP TABLE IF EXISTS public.chat_messages;
+ CREATE TABLE public.clips (
+ id bigint PRIMARY KEY, vod_id bigint, stream_id bigint, user_id bigint,
+ title text, description text, start_time double precision, end_time double precision,
+ duration_seconds double precision, is_public bigint, visibility text, status text,
+ view_count bigint, auto_generated bigint, created_at text, unique_views bigint,
+ ai_transcript text, ai_overview text);
+ CREATE TABLE public.content_visits (
+ content_type text, content_id bigint, visitor text, first_at text, last_at text, visits bigint);
+ CREATE TABLE public.chat_messages (
+ id bigint PRIMARY KEY, stream_id bigint, user_id bigint, anon_id text, message text,
+ message_type text, is_deleted bigint, is_filtered bigint, timestamp text);
+ INSERT INTO public.clips VALUES
+ (1, 10, 100, 7, 'MARKER-title', 'MARKER-description', 0, 25, 25, 1, 'public', 'ready', 12, 1, '2026-10-01 10:00:00', 12, 'hello there friends', 'a short clip'),
+ (2, 10, 100, 7, 'MARKER-title', 'MARKER-description', 20, 45, 25, 1, 'public', 'ready', 3, 1, '2026-10-02 10:00:00', 3, 'hello there friends', 'a short clip'),
+ (3, 11, 101, 8, 'MARKER-title', 'MARKER-description', 0, 25, 25, 1, 'public', 'ready', 0, 1, '2026-10-03 10:00:00', 0, NULL, NULL),
+ (4, 12, 102, 9, 'MARKER-title', 'MARKER-description', 0, 30, 30, 1, 'public', 'ready', 5, 0, '2026-10-04 10:00:00', 5, NULL, NULL);
+ INSERT INTO public.content_visits VALUES
+ ('clip', 1, 'visitor-1', '2026-10-01 11:00:00', '2026-10-02 11:00:00', 2),
+ ('clip', 1, 'visitor-2', '2026-10-02 11:00:00', '2026-10-02 12:00:00', 1),
+ ('clip', 4, 'visitor-3', '2026-10-04 11:00:00', '2026-10-05 11:00:00', 1);
+ INSERT INTO public.chat_messages VALUES
+ (1, 100, 7, 'anon-1', 'hi', 'message', 0, 0, '2026-10-01 10:00:00'),
+ (2, 100, 8, 'anon-2', 'MARKER-message', 'message', 0, 0, '2026-10-01 10:00:05'),
+ (3, 101, 9, 'anon-3', 'yo', 'message', 1, 0, '2026-10-01 10:00:10');
+ INSERT INTO app_state (key, value) VALUES ('auto_clip_log',
+ '[{"ts": 1767225600000, "vod_id": 10, "stream_id": 100, "start_time": 0, "title": "MARKER-title", "clip_id": 1},
+ {"ts": 1767225600000, "vod_id": 11, "stream_id": 101, "start_time": 0, "title": "MARKER-title", "clip_id": 2}]')
+ ON CONFLICT (key) DO UPDATE SET value = excluded.value;
+ `);
+
+ // The live file reads Live's own tables; give them a row each.
+ await db.exec(`
+ INSERT INTO users (username, password_hash)
+ SELECT 'clip-audit-fixture', 'not-a-password'
+ WHERE NOT EXISTS (SELECT 1 FROM users WHERE username = 'clip-audit-fixture');
+ INSERT INTO streams (user_id, title, is_live, viewer_count, peak_viewers, started_at)
+ SELECT id, 'MARKER-title', 0, 2, 5, '2026-10-01 09:00:00' FROM users WHERE username = 'clip-audit-fixture';
+ INSERT INTO stream_analytics (stream_id, avg_viewers, peak_viewers, unique_chatters, total_messages, clips_created)
+ SELECT id, 2, 5, 3, 120, 1 FROM streams WHERE title = 'MARKER-title' LIMIT 1
+ ON CONFLICT (stream_id) DO NOTHING;
+ INSERT INTO clip_ai_state (clip_id, ai_overview_short, transcript_status)
+ VALUES (900, 'MARKER-description', 'ready')
+ ON CONFLICT (clip_id) DO NOTHING;
+ `);
+
+ const before = await db.prepare('SELECT count(*)::int AS n FROM public.clips').all();
+ const beforeClips = before[0].n;
+
+ const live = await run(read('clip-audit-live.sql'));
+ const media = await run(read('clip-audit-media.sql'));
+ const chat = await run(read('clip-audit-chat.sql'));
+ const all = [...live, ...media, ...chat];
+
+ // Nothing personal came out.
+ const dumped = JSON.stringify(all);
+ assert.ok(!dumped.includes(MARKER), 'no personal text came out of the audit');
+ for (const bad of ['MARKER-title', 'MARKER-description', 'MARKER-message']) {
+ assert.ok(!dumped.includes(bad), `no ${bad} came out of the audit`);
+ }
+
+ // The answers the audit exists for are there.
+ const perDay = all.filter((r) => r.section === 'auto_clips_per_day');
+ assert.strictEqual(perDay.length, 3, 'the automatic clips per day are counted');
+ const kind = all.find((r) => r.section === 'clips_by_kind');
+ assert.ok(kind && Number(kind.clips) >= 3, 'automatic and human clips are counted by kind');
+ const never = all.find((r) => r.section === 'auto_clips_never_watched');
+ assert.ok(never && Number(never.never_watched) >= 1, 'the clips nobody watched are counted');
+ const overlap = all.find((r) => r.section === 'auto_clip_overlap');
+ assert.ok(overlap && Number(overlap.overlapping_pairs) >= 1, 'overlapping cuts are counted');
+ const repeats = all.find((r) => r.section === 'auto_clip_semantic_repeats');
+ assert.ok(repeats && Number(repeats.distinct_fingerprints) >= 1, 'semantic repeats are counted separately');
+ assert.ok(all.filter((r) => r.section === 'chat_buckets').length >= 1, 'the chat buckets are counted');
+ const log = all.find((r) => r.section === 'auto_clip_log_shape');
+ assert.ok(log && Number(log.entries) === 2, 'the clip log is counted');
+
+ // The files' own transaction shape: a READ ONLY transaction refuses a write.
+ await db.exec('BEGIN READ ONLY');
+ let refused = false;
+ try { await db.exec('CREATE TABLE clip_audit_write_probe (id int)'); }
+ catch (e) { refused = /read-only/i.test(String(e && e.message)); }
+ await db.exec('ROLLBACK').catch(() => {});
+ assert.ok(refused, 'a READ ONLY transaction refuses a write');
+
+ // Nothing was written: the fixtures are exactly as they were.
+ const after = await db.prepare('SELECT count(*)::int AS n FROM public.clips').all();
+ assert.strictEqual(after[0].n, beforeClips, 'the audit wrote nothing');
+
+ // Clean up the fixtures this file added.
+ await db.exec(`
+ DROP TABLE IF EXISTS public.clips;
+ DROP TABLE IF EXISTS public.content_visits;
+ DROP TABLE IF EXISTS public.chat_messages;
+ DELETE FROM app_state WHERE key = 'auto_clip_log';
+ DELETE FROM stream_analytics WHERE stream_id = 900;
+ DELETE FROM clip_ai_state WHERE clip_id = 900;
+ DELETE FROM streams WHERE title = 'MARKER-title';
+ DELETE FROM users WHERE username = 'clip-audit-fixture';
+ `);
+
+ console.log('clip audit SQL: read-only, aggregate-only, no personal columns — all checks passed');
+}
+
+main().catch((e) => { console.error(e && e.stack || e); process.exit(1); });