openvibers/openvibe.live Commits
Boot: the AI-text looks repair scans ~20k rows once per version of the scrub rules (a hash of their source), not on every boot (~0.8 s of database time each) (#222)
Changes
@@@ -665,7 +665,14 @@ function _extractDescFromMaybeJson(text) {if (dm) { try { return JSON.parse(`"${dm[1]}"`); } catch { return dm[1]; } }return text;}--async function cleanupMalformedAiText() {++// The looks repair below scans ~20k rows with a regex (~0.8 s of database time on every boot). New rows are scrubbed when++// written, so the scan only needs to run again when the scrub rules change: this hash of their source says when.++let _looksRulesVersion = null;++function looksRulesVersion() {++ if (!_looksRulesVersion) _looksRulesVersion = require('crypto').createHash('sha1').update([_noLooks, _noClothing, _LOOKS, _PEOPLE, _WORN, _WORN_ANY, _FACE].map(String).join('|')).digest('hex').slice(0, 12);++ return _looksRulesVersion;++}++async function cleanupMalformedAiText({ force = false } = {}) {// Only Live-owned tables — the moved vods/clips/pastes tables are frozen for// the OpenVibe.Media migration and must never be written.const jobs = [@@@ -706,10 +713,13 @@ async function cleanupMalformedAiText() {} catch { /* */ }// Frame notes and overviews that describe how a person looks (stored before the// prompts forbade it): the person becomes "the streamer" and the card line follows.++ let looksDone = false;++ try { looksDone = !force && (await getState('ai_looks_repair_rules')) === looksRulesVersion(); } catch { looksDone = false; }const looks = "~* '(shirtless|topless|bare-chested|young|teen|little|bearded|bald|blond|haired|tattooed|muscular|heavyset|older|elderly|middle-aged|skinny|slim|chubby|attractive|pretty|handsome|cute|boy|girl|kid|child|youth)|\\m(wear|dressed|sporting|shirt|t-shirt|tee|tank top|hoodie|sweat|jacket|coat|vest|beanie|cap|hat|helmet|headset|headphones|sunglasses|glasses|outfit|dress|jersey|suit|robe|bandana|scar|mask|attire|apparel|beard|mustache|moustache|goatee|stubble|tattoo|hair|ponytail|braids|dreadlocks|piercing|freckles|makeup|[a-z-]+-wearing)'";for (const [table, key, col, short] of [['stream_memories', 'id', 'description', null], ['streams', 'id', 'ai_overview', 'ai_overview_short'],['vod_ai_state', 'vod_id', 'ai_overview', 'ai_overview_short'], ['clip_ai_state', 'clip_id', 'ai_overview', 'ai_overview_short'],['streamer_overviews', 'user_id', 'overview', 'overview_short']]) {++ if (looksDone) break;try {const rows = await all(`SELECT ${key} AS k, ${col} AS v FROM ${table} WHERE ${col} ${looks}`);for (const r of rows) {@@@ -719,8 +729,10 @@ async function cleanupMalformedAiText() {else await run(`UPDATE ${table} SET ${col} = ? WHERE ${key} = ?`, [clean, r.k]);fixed++;}-- } catch (e) { console.warn(`[AI] looks repair (${table}):`, e.message); }++ } catch (e) { console.warn(`[AI] looks repair (${table}):`, e.message); looksDone = null; }}++ // Recorded only after a clean pass, so a failed table is tried again at the next boot.++ if (looksDone === false) { try { await setState('ai_looks_repair_rules', looksRulesVersion()); } catch { /* next boot tries again */ } }if (fixed) console.log(`[AI] Cleaned ${fixed} malformed JSON AI text value(s)`);return fixed;}@@@ -61,8 +61,15 @@ const db = require('../server/db/database');await db.run('UPDATE streams SET ai_overview = ?, ai_overview_short = ? WHERE id = ?', ['A young boy plays Fortnite.', 'A young boy plays Fortnite.', sid]);await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9060, 'An older version of the game loads.']);await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9120, 'A man wearing headphones sits at a desk.']);-- await db.cleanupMalformedAiText();++ await db.cleanupMalformedAiText({ force: true });assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9120')).description, 'A man sits at a desk.');++ // The scan runs once per version of the rules: the next boot skips it (rows are scrubbed when written).++ await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9180, 'A man wearing a hat waves.']);++ await db.cleanupMalformedAiText();++ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man wearing a hat waves.', 'skipped: the rules have not changed');++ await db.setState('ai_looks_repair_rules', 'an older rule set');++ await db.cleanupMalformedAiText();++ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man waves.', 'new rules: scanned again');assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9000')).description, 'The streamer rests his chin on his hand.');assert.deepStrictEqual(Object.values(await db.get('SELECT ai_overview, ai_overview_short FROM streams WHERE id = ?', [sid])), ['The streamer plays Fortnite.', 'The streamer plays Fortnite.']);assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9060')).description, 'An older version of the game loads.', 'looks words about things stay');
@@ -665,7 +665,14 @@ function _extractDescFromMaybeJson(text) {
@@ -665,7 +665,14 @@ function _extractDescFromMaybeJson(text) {
if (dm) { try { return JSON.parse(`"${dm[1]}"`); } catch { return dm[1]; } }
if (dm) { try { return JSON.parse(`"${dm[1]}"`); } catch { return dm[1]; } }
return text;
return text;
}
}
-async function cleanupMalformedAiText() {
+// The looks repair below scans ~20k rows with a regex (~0.8 s of database time on every boot). New rows are scrubbed when
+// written, so the scan only needs to run again when the scrub rules change: this hash of their source says when.
+let _looksRulesVersion = null;
+function looksRulesVersion() {
+ if (!_looksRulesVersion) _looksRulesVersion = require('crypto').createHash('sha1').update([_noLooks, _noClothing, _LOOKS, _PEOPLE, _WORN, _WORN_ANY, _FACE].map(String).join('|')).digest('hex').slice(0, 12);
+ return _looksRulesVersion;
+}
+async function cleanupMalformedAiText({ force = false } = {}) {
// Only Live-owned tables — the moved vods/clips/pastes tables are frozen for
// Only Live-owned tables — the moved vods/clips/pastes tables are frozen for
// the OpenVibe.Media migration and must never be written.
// the OpenVibe.Media migration and must never be written.
const jobs = [
const jobs = [
@@ -706,10 +713,13 @@ async function cleanupMalformedAiText() {
@@ -706,10 +713,13 @@ async function cleanupMalformedAiText() {
} catch { /* */ }
} catch { /* */ }
// Frame notes and overviews that describe how a person looks (stored before the
// Frame notes and overviews that describe how a person looks (stored before the
// prompts forbade it): the person becomes "the streamer" and the card line follows.
// prompts forbade it): the person becomes "the streamer" and the card line follows.
+ let looksDone = false;
+ try { looksDone = !force && (await getState('ai_looks_repair_rules')) === looksRulesVersion(); } catch { looksDone = false; }
const looks = "~* '(shirtless|topless|bare-chested|young|teen|little|bearded|bald|blond|haired|tattooed|muscular|heavyset|older|elderly|middle-aged|skinny|slim|chubby|attractive|pretty|handsome|cute|boy|girl|kid|child|youth)|\\m(wear|dressed|sporting|shirt|t-shirt|tee|tank top|hoodie|sweat|jacket|coat|vest|beanie|cap|hat|helmet|headset|headphones|sunglasses|glasses|outfit|dress|jersey|suit|robe|bandana|scar|mask|attire|apparel|beard|mustache|moustache|goatee|stubble|tattoo|hair|ponytail|braids|dreadlocks|piercing|freckles|makeup|[a-z-]+-wearing)'";
const looks = "~* '(shirtless|topless|bare-chested|young|teen|little|bearded|bald|blond|haired|tattooed|muscular|heavyset|older|elderly|middle-aged|skinny|slim|chubby|attractive|pretty|handsome|cute|boy|girl|kid|child|youth)|\\m(wear|dressed|sporting|shirt|t-shirt|tee|tank top|hoodie|sweat|jacket|coat|vest|beanie|cap|hat|helmet|headset|headphones|sunglasses|glasses|outfit|dress|jersey|suit|robe|bandana|scar|mask|attire|apparel|beard|mustache|moustache|goatee|stubble|tattoo|hair|ponytail|braids|dreadlocks|piercing|freckles|makeup|[a-z-]+-wearing)'";
for (const [table, key, col, short] of [['stream_memories', 'id', 'description', null], ['streams', 'id', 'ai_overview', 'ai_overview_short'],
for (const [table, key, col, short] of [['stream_memories', 'id', 'description', null], ['streams', 'id', 'ai_overview', 'ai_overview_short'],
['vod_ai_state', 'vod_id', 'ai_overview', 'ai_overview_short'], ['clip_ai_state', 'clip_id', 'ai_overview', 'ai_overview_short'],
['vod_ai_state', 'vod_id', 'ai_overview', 'ai_overview_short'], ['clip_ai_state', 'clip_id', 'ai_overview', 'ai_overview_short'],
['streamer_overviews', 'user_id', 'overview', 'overview_short']]) {
['streamer_overviews', 'user_id', 'overview', 'overview_short']]) {
+ if (looksDone) break;
try {
try {
const rows = await all(`SELECT ${key} AS k, ${col} AS v FROM ${table} WHERE ${col} ${looks}`);
const rows = await all(`SELECT ${key} AS k, ${col} AS v FROM ${table} WHERE ${col} ${looks}`);
for (const r of rows) {
for (const r of rows) {
@@ -719,8 +729,10 @@ async function cleanupMalformedAiText() {
@@ -719,8 +729,10 @@ async function cleanupMalformedAiText() {
else await run(`UPDATE ${table} SET ${col} = ? WHERE ${key} = ?`, [clean, r.k]);
else await run(`UPDATE ${table} SET ${col} = ? WHERE ${key} = ?`, [clean, r.k]);
fixed++;
fixed++;
}
}
- } catch (e) { console.warn(`[AI] looks repair (${table}):`, e.message); }
+ } catch (e) { console.warn(`[AI] looks repair (${table}):`, e.message); looksDone = null; }
}
}
+ // Recorded only after a clean pass, so a failed table is tried again at the next boot.
+ if (looksDone === false) { try { await setState('ai_looks_repair_rules', looksRulesVersion()); } catch { /* next boot tries again */ } }
if (fixed) console.log(`[AI] Cleaned ${fixed} malformed JSON AI text value(s)`);
if (fixed) console.log(`[AI] Cleaned ${fixed} malformed JSON AI text value(s)`);
return fixed;
return fixed;
}
}
@@ -61,8 +61,15 @@ const db = require('../server/db/database');
@@ -61,8 +61,15 @@ const db = require('../server/db/database');
await db.run('UPDATE streams SET ai_overview = ?, ai_overview_short = ? WHERE id = ?', ['A young boy plays Fortnite.', 'A young boy plays Fortnite.', sid]);
await db.run('UPDATE streams SET ai_overview = ?, ai_overview_short = ? WHERE id = ?', ['A young boy plays Fortnite.', 'A young boy plays Fortnite.', sid]);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9060, 'An older version of the game loads.']);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9060, 'An older version of the game loads.']);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9120, 'A man wearing headphones sits at a desk.']);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9120, 'A man wearing headphones sits at a desk.']);
- await db.cleanupMalformedAiText();
+ await db.cleanupMalformedAiText({ force: true });
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9120')).description, 'A man sits at a desk.');
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9120')).description, 'A man sits at a desk.');
+ // The scan runs once per version of the rules: the next boot skips it (rows are scrubbed when written).
+ await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9180, 'A man wearing a hat waves.']);
+ await db.cleanupMalformedAiText();
+ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man wearing a hat waves.', 'skipped: the rules have not changed');
+ await db.setState('ai_looks_repair_rules', 'an older rule set');
+ await db.cleanupMalformedAiText();
+ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man waves.', 'new rules: scanned again');
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9000')).description, 'The streamer rests his chin on his hand.');
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9000')).description, 'The streamer rests his chin on his hand.');
assert.deepStrictEqual(Object.values(await db.get('SELECT ai_overview, ai_overview_short FROM streams WHERE id = ?', [sid])), ['The streamer plays Fortnite.', 'The streamer plays Fortnite.']);
assert.deepStrictEqual(Object.values(await db.get('SELECT ai_overview, ai_overview_short FROM streams WHERE id = ?', [sid])), ['The streamer plays Fortnite.', 'The streamer plays Fortnite.']);
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9060')).description, 'An older version of the game loads.', 'looks words about things stay');
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9060')).description, 'An older version of the game loads.', 'looks words about things stay');