openvibers/openvibe.live Commits

Arena: hard blocklist for slurs/hate/profanity on transcript quotes, persona context and hot-mic lines

OP OpenVibers committed 29 Aug 2026 · b9c457e554 · 2 files, +59 −6

The AI quote picker surfaced transcript lines containing slurs on production. Filter
candidates server-side before the model sees them (and again after selection, in the
persona prompt's 'things they said', and the live hot-mic line) with a broad,
variant-aware pattern list. Regression test: banned lines are never candidates.

Co-Authored-By: Claude Fable 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_017AyeSFu7qdLdR64Zv99QNm

Changes

ddiff --git a/server/arena/arena-service.js b/server/arena/arena-service.js
iindex cf721858..96f0b8ef 100644
---- a/server/arena/arena-service.js
++++ b/server/arena/arena-service.js
@@@ -63,6 +63,40 @@ const ROUNDS = [
// Words that count as "hyped" when they show up in the transcript.
const HYPE_PATTERNS = ["let's go", 'lets go', 'no way', 'oh my god', 'insane', 'clutch', 'holy', 'gg', 'unreal', 'what the', 'bro', 'chat,', 'chat ', 'yo ', 'welcome', 'lfg', 'poggers', 'pog'];
++// Transcript text that must never be surfaced on the site, no matter what a model
++// "picks": slurs, hate terms, threats and hard sexual profanity. Applied to quote
++// candidates BEFORE the AI sees them, to the persona prompt's "things they said", and to
++// the live hot-mic line. Deliberately broad — a false positive costs one quote, a false
++// negative puts a slur on a public page. (Spelling variants: repeated letters, common
++// leetspeak, and word-boundary aware so "hell" does not catch "shell".)
++const BANNED_PATTERNS = [
++ /n+[i1!]+g+(?:a+|e+r+|u+h+|r+)s?\b/i,
++ /\bf+[a@]+g+(?:o+t+|s|z)?\b/i,
++ /\br+[e3]+t+[a@]+r+d+(?:ed|s)?\b/i,
++ /\bt+r+[a@]+n+n+(?:y|ie|ies)\b/i,
++ /\bk+[i1]+k+e+s?\b/i,
++ /\bs+p+[i1]+c+k?s?\b/i,
++ /\bc+h+[i1]+n+k+s?\b/i,
++ /\bg+[o0]+[o0]+k+s?\b/i,
++ /\bw+e+t+b+a+c+k+s?\b/i,
++ /\bd+[y1]+k+e+s?\b/i,
++ /\bc+u+n+t+s?\b/i,
++ /\bw+h+[o0]+r+e+s?\b/i,
++ /\bs+l+u+t+s?\b/i,
++ /\bp+u+s+s+(?:y|ies)\b/i,
++ /\b(?:d+[i1]+c+k|c+[o0]+c+k)s?\b/i,
++ /\br+a+p+(?:e|ed|ing|ist)\b/i,
++ /\bk+y+s+\b/i,
++ /\bkill\s+(?:yourself|urself|himself|herself|themselves)\b/i,
++ /\b(?:hitler|nazi|nazis)\b/i,
++ /\bmolest/i,
++ /\bpedo/i,
++];
++function isBannedText(text) {
++ const t = String(text || '');
++ return BANNED_PATTERNS.some(re => re.test(t));
++}
++
// ── Tables ───────────────────────────────────────────────────
let _tablesReady = false;
@@@ -419,7 +453,7 @@ function gatherContext(userId) {
try { ctx.titles = db.all('SELECT DISTINCT title FROM streams WHERE user_id = ? AND duration_seconds > 0 ORDER BY started_at DESC LIMIT 8', [userId]).map(r => r.title).filter(Boolean); } catch { ctx.titles = []; }
try {
// A taste of how they actually talk — short, recent, non-empty lines.
-- ctx.said = db.all(`SELECT text FROM stream_timeline_events WHERE user_id = ? AND kind = 'speech' AND LENGTH(text) BETWEEN 30 AND 140 ORDER BY created_at DESC LIMIT 12`, [userId]).map(r => r.text);
++ ctx.said = db.all(`SELECT text FROM stream_timeline_events WHERE user_id = ? AND kind = 'speech' AND LENGTH(text) BETWEEN 30 AND 140 ORDER BY created_at DESC LIMIT 24`, [userId]).map(r => r.text).filter(t => !isBannedText(t)).slice(0, 12);
} catch { ctx.said = []; }
return ctx;
}
@@@ -544,8 +578,10 @@ function quoteCandidates(userId, limit = 90) {
if (/\b(um+|uh+|like like)\b/.test(s)) n -= 1;
return n;
};
-- // Keep the spiciest, then fill with a spread across time so one stream cannot dominate.
-- const scored = rows.map(r => ({ ...r, score: score(r.text) })).sort((x, y) => y.score - x.score);
++ // Hard filter first — nothing on the banned list is ever a candidate, so the model
++ // cannot pick it. Then keep the spiciest, and fill with a spread across time so one
++ // stream cannot dominate.
++ const scored = rows.filter(r => !isBannedText(r.text)).map(r => ({ ...r, score: score(r.text) })).sort((x, y) => y.score - x.score);
const picked = scored.slice(0, Math.ceil(limit / 2));
const rest = scored.slice(Math.ceil(limit / 2));
const step = Math.max(1, Math.floor(rest.length / Math.max(1, limit - picked.length)));
@@@ -575,7 +611,8 @@ function fallbackQuotes(candidates) {
}
function materializeQuotes(candidates, sel) {
-- const pick = (i, why) => { const c = candidates[i]; return c ? { text: c.text, stream_id: c.stream_id, vod_id: c.vod_id, start_sec: Math.max(0, Math.floor(Number(c.start_sec) || 0) - 2), why } : null; };
++ // Belt and braces: candidates are pre-filtered, but never trust an index blindly.
++ const pick = (i, why) => { const c = candidates[i]; return c && !isBannedText(c.text) ? { text: c.text, stream_id: c.stream_id, vod_id: c.vod_id, start_sec: Math.max(0, Math.floor(Number(c.start_sec) || 0) - 2), why } : null; };
const picks = (sel.picks || []).map(p => pick(p.index, p.why)).filter(Boolean);
const walkout = pick(sel.walkout, 'walkout line') || picks[0] || null;
return { picks, walkout, voice_verdict: sel.voice_verdict || null, mic_style: sel.mic_style || null };
@@@ -1028,7 +1065,8 @@ function getLiveMatchups() {
// What they said most recently on this live stream — the arena "hot mic".
let lastLine = null;
try {
-- const r = db.get(`SELECT text, start_sec FROM stream_timeline_events WHERE stream_id = ? AND kind = 'speech' AND LENGTH(text) > 15 ORDER BY start_sec DESC LIMIT 1`, [s.id]);
++ const r = db.all(`SELECT text, start_sec FROM stream_timeline_events WHERE stream_id = ? AND kind = 'speech' AND LENGTH(text) > 15 ORDER BY start_sec DESC LIMIT 5`, [s.id])
++ .find(row => !isBannedText(row.text));
if (r) lastLine = { text: r.text, start_sec: Math.floor(r.start_sec) };
} catch { /* */ }
return {
@@@ -1100,4 +1138,5 @@ module.exports = {
_fallbackPersona: fallbackPersona,
_voiceStatsFor: voiceStatsFor,
_quoteCandidates: quoteCandidates,
++ _isBannedText: isBannedText,
};
ddiff --git a/test/arena.test.js b/test/arena.test.js
iindex 4885981e..78a98e36 100644
---- a/test/arena.test.js
++++ b/test/arena.test.js
@@@ -85,9 +85,23 @@ for (let i = 0; i < 40; i++) {
db.addTimelineEvents(lines);
db.addTimelineEvents([{ stream_id: s1, user_id: u1, vod_id: 901, kind: 'sound', start_sec: 30, end_sec: 33, text: null, label: 'Laughter', confidence: 0.8 }, { stream_id: s1, user_id: u1, vod_id: 901, kind: 'sound', start_sec: 90, end_sec: 93, text: null, label: 'Rock music', confidence: 0.8 }]);
++// Lines with slurs / hate / hard profanity must never become quotes — not even as candidates.
++db.addTimelineEvents([
++ { stream_id: s1, user_id: u1, vod_id: 901, kind: 'speech', start_sec: 5000, end_sec: 5010, text: "Let's go chat, you absolute f4ggots, that was insane!", label: null, confidence: 0.9 },
++ { stream_id: s1, user_id: u1, vod_id: 901, kind: 'speech', start_sec: 5100, end_sec: 5110, text: 'No way, chat, that guy is such a retard!', label: null, confidence: 0.9 },
++ { stream_id: s1, user_id: u1, vod_id: 901, kind: 'speech', start_sec: 5200, end_sec: 5210, text: 'Holy, that was insane, kill yourself if you disagree!', label: null, confidence: 0.9 },
++]);
++for (const bad of ["what up my n1ggas", "stop being a faggot", "he's a retard", "kys", "kill yourself", "such a whore", "pussy move"]) assert.ok(arena._isBannedText(bad), `banned: ${bad}`);
++for (const ok of ["the shell script is hell to debug", "we dug a dyke in the sand? no — a dike, the engineering kind"]) { /* documented: broad filter may catch "dyke" — acceptable */ }
++assert.ok(!arena._isBannedText("the shell script is hell to debug and I hate it"), 'ordinary sentences pass');
++const cands = arena._quoteCandidates(u1);
++assert.ok(cands.length >= 20, 'has candidates');
++assert.ok(cands.every(c => !arena._isBannedText(c.text)), 'no banned line is ever a candidate');
++console.log('✅ slurs/hate never surface as quotes');
++
const voice = arena._voiceStatsFor(u1, '-90 days');
assert.strictEqual(voice.has_data, true);
--assert.strictEqual(voice.lines, 40);
++assert.strictEqual(voice.lines, 43);
assert.ok(voice.talk_ratio_pct > 0 && voice.talk_ratio_pct <= 100, `talk ratio: ${voice.talk_ratio_pct}`);
assert.ok(voice.hype_hits >= 10, `hype hits counted: ${voice.hype_hits}`);
assert.strictEqual(voice.laughs, 1);