openvibers/openvibe.live Commits

Boot: the AI-text looks repair scans ~20k rows once per version of the scrub rules (a hash of their source), not on every boot (~0.8 s of database time each) (#222)

OP OpenVibers committed 27 days ago · ee541e10ca · 2 files, +22 −3

Changes

ddiff --git a/server/db/database.js b/server/db/database.js
iindex 7b7801f1..63c5810d 100644
---- a/server/db/database.js
++++ b/server/db/database.js
@@@ -665,7 +665,14 @@ function _extractDescFromMaybeJson(text) {
if (dm) { try { return JSON.parse(`"${dm[1]}"`); } catch { return dm[1]; } }
return text;
}
--async function cleanupMalformedAiText() {
++// The looks repair below scans ~20k rows with a regex (~0.8 s of database time on every boot). New rows are scrubbed when
++// written, so the scan only needs to run again when the scrub rules change: this hash of their source says when.
++let _looksRulesVersion = null;
++function looksRulesVersion() {
++ if (!_looksRulesVersion) _looksRulesVersion = require('crypto').createHash('sha1').update([_noLooks, _noClothing, _LOOKS, _PEOPLE, _WORN, _WORN_ANY, _FACE].map(String).join('|')).digest('hex').slice(0, 12);
++ return _looksRulesVersion;
++}
++async function cleanupMalformedAiText({ force = false } = {}) {
// Only Live-owned tables — the moved vods/clips/pastes tables are frozen for
// the OpenVibe.Media migration and must never be written.
const jobs = [
@@@ -706,10 +713,13 @@ async function cleanupMalformedAiText() {
} catch { /* */ }
// Frame notes and overviews that describe how a person looks (stored before the
// prompts forbade it): the person becomes "the streamer" and the card line follows.
++ let looksDone = false;
++ try { looksDone = !force && (await getState('ai_looks_repair_rules')) === looksRulesVersion(); } catch { looksDone = false; }
const looks = "~* '(shirtless|topless|bare-chested|young|teen|little|bearded|bald|blond|haired|tattooed|muscular|heavyset|older|elderly|middle-aged|skinny|slim|chubby|attractive|pretty|handsome|cute|boy|girl|kid|child|youth)|\\m(wear|dressed|sporting|shirt|t-shirt|tee|tank top|hoodie|sweat|jacket|coat|vest|beanie|cap|hat|helmet|headset|headphones|sunglasses|glasses|outfit|dress|jersey|suit|robe|bandana|scar|mask|attire|apparel|beard|mustache|moustache|goatee|stubble|tattoo|hair|ponytail|braids|dreadlocks|piercing|freckles|makeup|[a-z-]+-wearing)'";
for (const [table, key, col, short] of [['stream_memories', 'id', 'description', null], ['streams', 'id', 'ai_overview', 'ai_overview_short'],
['vod_ai_state', 'vod_id', 'ai_overview', 'ai_overview_short'], ['clip_ai_state', 'clip_id', 'ai_overview', 'ai_overview_short'],
['streamer_overviews', 'user_id', 'overview', 'overview_short']]) {
++ if (looksDone) break;
try {
const rows = await all(`SELECT ${key} AS k, ${col} AS v FROM ${table} WHERE ${col} ${looks}`);
for (const r of rows) {
@@@ -719,8 +729,10 @@ async function cleanupMalformedAiText() {
else await run(`UPDATE ${table} SET ${col} = ? WHERE ${key} = ?`, [clean, r.k]);
fixed++;
}
-- } catch (e) { console.warn(`[AI] looks repair (${table}):`, e.message); }
++ } catch (e) { console.warn(`[AI] looks repair (${table}):`, e.message); looksDone = null; }
}
++ // Recorded only after a clean pass, so a failed table is tried again at the next boot.
++ if (looksDone === false) { try { await setState('ai_looks_repair_rules', looksRulesVersion()); } catch { /* next boot tries again */ } }
if (fixed) console.log(`[AI] Cleaned ${fixed} malformed JSON AI text value(s)`);
return fixed;
}
ddiff --git a/test/ai-text-no-looks.test.js b/test/ai-text-no-looks.test.js
iindex 7bca0bae..d742ebf4 100644
---- a/test/ai-text-no-looks.test.js
++++ b/test/ai-text-no-looks.test.js
@@@ -61,8 +61,15 @@ const db = require('../server/db/database');
await db.run('UPDATE streams SET ai_overview = ?, ai_overview_short = ? WHERE id = ?', ['A young boy plays Fortnite.', 'A young boy plays Fortnite.', sid]);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9060, 'An older version of the game loads.']);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9120, 'A man wearing headphones sits at a desk.']);
-- await db.cleanupMalformedAiText();
++ await db.cleanupMalformedAiText({ force: true });
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9120')).description, 'A man sits at a desk.');
++ // The scan runs once per version of the rules: the next boot skips it (rows are scrubbed when written).
++ await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9180, 'A man wearing a hat waves.']);
++ await db.cleanupMalformedAiText();
++ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man wearing a hat waves.', 'skipped: the rules have not changed');
++ await db.setState('ai_looks_repair_rules', 'an older rule set');
++ await db.cleanupMalformedAiText();
++ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man waves.', 'new rules: scanned again');
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9000')).description, 'The streamer rests his chin on his hand.');
assert.deepStrictEqual(Object.values(await db.get('SELECT ai_overview, ai_overview_short FROM streams WHERE id = ?', [sid])), ['The streamer plays Fortnite.', 'The streamer plays Fortnite.']);
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9060')).description, 'An older version of the game loads.', 'looks words about things stay');