openvibers/openvibe.live Commits
Public AI text drops a guessed gender too ("a man" is "a person", "a male or female host" is "a host"), reads past slashed words before clothing, and drops sentences about someone's hair or clothes; the boot repair rewrites the 20 clip summaries that described people (#298)
Changes
@@@ -409,8 +409,11 @@ const _WITH_FACE = new RegExp(`\\s+with\\s+${_ART}(?:[\\w'-]+\\s+){0,2}?\\b(?:${const _WEARS_AND = new RegExp(`(?<!-)\\bwears\\s+${_WORN_NP}(?:\\s*(?:,|and|or|with|over|under)\\s+${_WORN_NP})*\\s+and\\s+(?=\\w)`, 'gi');// "…, while the other wears a dusty t-shirt;" goes as a whole clause.const _WEARS_CLAUSE = new RegExp(`,?\\s+(?:while|and|as|whereas)\\s+(?:the other|one|another|someone|he|she|they|the streamer|(?:a|the) (?:man|woman|person|guy))\\s+wears\\s+${_WORN_NP}(?:\\s*(?:,|and|or|with|over|under)\\s+${_WORN_NP})*(?=\\s*(?:[;,.!?]|$))`, 'gi');++// "a plaid/checkered shirt": a slashed pair of words before what is worn reads as one word, so the rules below see it.++const _SLASHED_WORN = new RegExp(`\\b([\\w'-]+)/[\\w'-]+(?=\\s+(?:[\\w'-]+\\s+){0,2}?(?:${_WORN_ANY})\\b)`, 'gi');function _noClothing(text) {return text.split(/("[^"]*"?|“[^”]*”?)/).map((part, i) => (i % 2 ? part : part++ .replace(_SLASHED_WORN, '$1').replace(/\b[\w-]+-wearing\s+/gi, '').replace(_WEARS_AND, '').replace(_WEARS_CLAUSE, '')@@@ -435,8 +438,33 @@ function _noLooksTags(tags) {}// Body language is the person too ("reacting with a thoughtful pose", "in a relaxed posture"): the phrase goes.const _POSE = /,?\s*(?:(?:appearing|seeming|seemingly)\s+(?:to\s+be\s+)?)?(?:(?:reacting|sitting|standing|leaning|posing|looking|resting)\s+)?(?:with|in)\s+(?:an?|their|his|her)\s+(?:[\w-]+\s+){0,2}(?:pose|posture|expression|facial expression|stance)\b/gi;++// A guessed gender is a description of the person too (owner's rule: no looks, body, clothing, hair, age or guessed++// gender): "a man in a kitchen" is "a person in a kitchen", "two men" are "two people", "a male or female host" is "a++// host". Only after an article or a count, so names (Iron Man, Pac-Man, Spice Girls) and quoted speech stay.++const _GENDERED = 'man|woman|guy|gal|lady|gentleman';++const _GENDERED_ONE = new RegExp(`\\b(a|an|the|one|another|this|that|each|every)\\s+(?:${_GENDERED})\\b(?!-)`, 'gi');++const _GENDERED_MANY = new RegExp(`\\b(two|three|four|five|several|some|the|both|many|other|few|these|those|\\d+)\\s+(?:men|women|guys|gals|ladies|gentlemen)\\b`, 'gi');++const _GENDER_ADJ = /\b(?:(?:a|an)\s+)?(?:male|female)\s+(?:or|and|\/)\s+(?:male|female)\s+|\b(?:male|female)\s+(?=(?:hosts?|streamers?|players?|speakers?|presenters?|gamers?|voices?|narrators?|commentators?|figures?|characters?|broadcasters?|viewers?|individuals?|persons?|people)\b)/gi;++// What is left of a sentence about someone's hair, clothes or body ("a person adjusts their hair or clothing near the++// counter") cannot be repaired word by word: the sentence goes.++const _GROOMING = /\b(?:their|his|her)\s+(?:own\s+)?(?:hair|beard|face|makeup|clothing|clothes|outfit|appearance|body|figure|looks)\b/i;++function _noGender(text) {++ return text.split(/("[^"]*"?|“[^”]*”?)/).map((part, i) => (i % 2 ? part : part++ .replace(_GENDER_ADJ, (m) => (/^an?\s/i.test(m) ? m.slice(0, m.indexOf(' ') + 1) : ''))++ .replace(_GENDERED_ONE, (m, art) => _cap(m, `${/^an$/i.test(art) ? 'a' : art.toLowerCase()} person`))++ .replace(_GENDERED_MANY, (m, n) => _cap(m, `${n.toLowerCase()} people`))++ .replace(/\ba\s+(?=[aeiou]\w*\s+(?:host|streamer|player|speaker|presenter|gamer|figure|character|broadcaster|person)\b)/gi, (m) => _cap(m, 'an '))++ .replace(/ {2,}/g, ' '))).join('');++}++function _noGrooming(text) {++ const sentences = text.match(/[^.!?]+(?:[.!?]+(?:["”])?|$)\s*/g) || [text];++ return sentences.filter((s) => !_GROOMING.test(s.replace(/"[^"]*"?|“[^”]*”?/g, ''))).join('').trim();++}function _noLooks(text) {if (!text || typeof text !== 'string') return text;++ return _noGrooming(_noGender(_noLooksWords(text)));++}++function _noLooksWords(text) {return _noClothing(text).replace(_POSE, '').replace(/\s+([,.;:!?])/g, '$1').replace(/ {2,}/g, ' ')// Looks named: the person on camera. Only an age ("a child sits nearby"): someone, who may not be the streamer..replace(_LOOKED_AT, (m, art, looks, noun) => (looks ? _cap(m, 'the streamer') : _AGE_NOUN.test(noun) ? _cap(m, /^the$/i.test(art) ? 'the person' : 'someone') : m))@@@ -708,7 +736,7 @@ function _extractDescFromMaybeJson(text) {// written, so the scan only needs to run again when the scrub rules change: this hash of their source says when.let _looksRulesVersion = null;function looksRulesVersion() {-- if (!_looksRulesVersion) _looksRulesVersion = require('crypto').createHash('sha1').update([_noLooks, _noClothing, _LOOKS, _PEOPLE, _WORN, _WORN_ANY, _FACE].map(String).join('|')).digest('hex').slice(0, 12);++ if (!_looksRulesVersion) _looksRulesVersion = require('crypto').createHash('sha1').update([_noLooks, _noLooksWords, _noGender, _noGrooming, _noClothing, _LOOKS, _PEOPLE, _WORN, _WORN_ANY, _FACE, _GENDER_ADJ, _GROOMING].map(String).join('|')).digest('hex').slice(0, 12);return _looksRulesVersion;}async function cleanupMalformedAiText({ force = false } = {}) {@@@ -754,7 +782,7 @@ async function cleanupMalformedAiText({ force = false } = {}) {// prompts forbade it): the person becomes "the streamer" and the card line follows.let looksDone = false;try { looksDone = !force && (await getState('ai_looks_repair_rules')) === looksRulesVersion(); } catch { looksDone = false; }-- const looks = "~* '(shirtless|topless|bare-chested|young|teen|little|bearded|bald|blond|haired|tattooed|muscular|heavyset|older|elderly|middle-aged|skinny|slim|chubby|attractive|pretty|handsome|cute|boy|girl|kid|child|youth)|\\m(wear|dressed|sporting|shirt|t-shirt|tee|tank top|hoodie|sweat|jacket|coat|vest|beanie|cap|hat|helmet|headset|headphones|sunglasses|glasses|outfit|dress|jersey|suit|robe|bandana|scar|mask|attire|apparel|beard|mustache|moustache|goatee|stubble|tattoo|hair|ponytail|braids|dreadlocks|piercing|freckles|makeup|[a-z-]+-wearing)'";++ const looks = "~* '(shirtless|topless|bare-chested|young|teen|little|bearded|bald|blond|haired|tattooed|muscular|heavyset|older|elderly|middle-aged|skinny|slim|chubby|attractive|pretty|handsome|cute|boy|girl|kid|child|youth)|\\m(wear|dressed|sporting|shirt|t-shirt|tee|tank top|hoodie|sweat|jacket|coat|vest|beanie|cap|hat|helmet|headset|headphones|sunglasses|glasses|outfit|dress|jersey|suit|robe|bandana|scar|mask|attire|apparel|beard|mustache|moustache|goatee|stubble|tattoo|hair|ponytail|braids|dreadlocks|piercing|freckles|makeup|[a-z-]+-wearing)|\\m(man|woman|men|women|guy|guys|gal|lady|ladies|gentleman|gentlemen|male|female)\\M'";for (const [table, key, col, short] of [['stream_memories', 'id', 'description', null], ['streams', 'id', 'ai_overview', 'ai_overview_short'],['vod_ai_state', 'vod_id', 'ai_overview', 'ai_overview_short'], ['clip_ai_state', 'clip_id', 'ai_overview', 'ai_overview_short'],['streamer_overviews', 'user_id', 'overview', 'overview_short']]) {@@@ -1,7 +1,8 @@'use strict';// Public AI text never describes how a person looks: a frame note or an overview that names looks or an age--// ("A shirtless man sits…", "a young boy at a desk") is stored with that person as "the streamer", a plain "a man"--// or "a black chair" stays as written, and rows stored before the rule are repaired by the AI cleanup pass.++// ("A shirtless man sits…", "a young boy at a desk") is stored with that person as "the streamer"; a guessed gender++// ("a man", "two men", "a male or female host") becomes a person; "a black chair" stays as written; and rows stored++// before a rule changed are repaired by the AI cleanup pass.const assert = require('assert');console.warn = () => {};@@@ -17,37 +18,44 @@ const db = require('../server/db/database');['A livestream frame showing a shirtless streamer in a small inset while playing a MOBA.', 'A livestream frame showing the streamer in a small inset while playing a MOBA.'],['Chat watches a young boy at a desk build a PC.', 'Chat watches the streamer at a desk build a PC.'],['Young, bearded man plays chess.', 'Streamer plays chess.'],-- ['A man plays chess next to a black chair.', 'A man plays chess next to a black chair.'],++ ['A man plays chess next to a black chair.', 'A person plays chess next to a black chair.'],['The streamer, topless, laughs at chat.', 'The streamer laughs at chat.'],-- ['A man is shirtless and appears to be talking to chat.', 'A man appears to be talking to chat.'],++ ['A man is shirtless and appears to be talking to chat.', 'A person appears to be talking to chat.'],['A person watches TV while a child sits nearby.', 'A person watches TV while someone sits nearby.'],['The girl waves at the camera.', 'The person waves at the camera.'],// Clothing, hair and face go; the rest of the sentence stays.['The streamer wearing a red beanie speaks to the camera outdoors, seemingly addressing chat.', 'The streamer speaks to the camera outdoors, seemingly addressing chat.'],['The view is close to the camera while wearing a black beanie and hoodie, with trees around.', 'The view is close to the camera, with trees around.'],-- ['A man in a gray tank top sits against a plain light background.', 'A man sits against a plain light background.'],-- ['A man with a long beard and plaid shirt examines a woven item.', 'A man examines a woven item.'],++ ['A man in a gray tank top sits against a plain light background.', 'A person sits against a plain light background.'],++ ['A man with a long beard and plaid shirt examines a woven item.', 'A person examines a woven item.'],// Pose and expression are the person too (2026-10-17, the home page's "Right now").['A live stream frame shows the streamer in a small window bottom-left, appearing to be reacting with a thoughtful pose while the main screen is black.','A live stream frame shows the streamer in a small window bottom-left while the main screen is black.'],['The streamer sits in a relaxed posture at the desk.', 'The streamer sits at the desk.'],['A kitchen studio with a headset-wearing host, microphone, and on-screen graphics.', 'A kitchen studio with a host, microphone, and on-screen graphics.'],['Panelists discuss while wearing headsets and microphones, with a trophy nearby.', 'Panelists discuss, with a trophy nearby.'],-- ['A man wearing sunglasses on his head looks down at the camera.', 'A man looks down at the camera.'],-- ['A collage of a man in a white shirt and tie wearing dark sunglasses in the left panel.', 'A collage of a man in the left panel.'],++ ['A man wearing sunglasses on his head looks down at the camera.', 'A person looks down at the camera.'],++ ['A collage of a man in a white shirt and tie wearing dark sunglasses in the left panel.', 'A collage of a person in the left panel.'],['Behind him a green mat; he wears a blue jacket and is leaning to the left.', 'Behind him a green mat; he is leaning to the left.'],-- ['Two men lift weights, one shirtless and sweating.', 'Two men lift weights, one sweating.'],-- ['Two men stand in an alley, one and drinking from a bottle while the other wears a dusty t-shirt; both look tired.', 'Two men stand in an alley, one drinking from a bottle; both look tired.'],-- ['A man wearing a green bucket hat with frog-like eyes sits indoors.', 'A man sits indoors.'],-- ['A man in sunglasses reaches toward the camera inside a car.', 'A man reaches toward the camera inside a car.'],++ ['Two men lift weights, one shirtless and sweating.', 'Two people lift weights, one sweating.'],++ ['Two men stand in an alley, one and drinking from a bottle while the other wears a dusty t-shirt; both look tired.', 'Two people stand in an alley, one drinking from a bottle; both look tired.'],++ ['A man wearing a green bucket hat with frog-like eyes sits indoors.', 'A person sits indoors.'],++ ['A man in sunglasses reaches toward the camera inside a car.', 'A person reaches toward the camera inside a car.'],['A live stream window shows a broadcaster with a headset on the left, a chat panel on the right.', 'A live stream window shows a broadcaster on the left, a chat panel on the right.'],-- ['A man with a red shirt sits in a room; a desk with a headset and keyboard.', 'A man sits in a room; a desk with a headset and keyboard.'],++ ['A man with a red shirt sits in a room; a desk with a headset and keyboard.', 'A person sits in a room; a desk with a headset and keyboard.'],['A gamer faces a casually dressed NPC in a grim interior.', 'A gamer faces a NPC in a grim interior.'],['A cartoon streaming setup: wearing a headset with a microphone, rugged jacket, backpack with a mat.', 'A cartoon streaming setup: backpack with a mat.'],['A partial view of a person wearing a gray zip-up jacket with a white zipper, seated indoors.', 'A partial view of a person, seated indoors.'],['Close-up of a person wearing a yellow vest over a gray knit top, with a backpack visible.', 'Close-up of a person, with a backpack visible.'],['A Minecraft player in a top-down view builds a house; the streamer reviews new headphones.', 'A Minecraft player in a top-down view builds a house; the streamer reviews new headphones.'],['The streamer laughs — heard: "Put your headset on. I like your hair."', 'The streamer laughs — heard: "Put your headset on. I like your hair."'],++ // Clip overviews found public on 2026-10-22: a guessed gender, a slashed pattern before a shirt, a grooming sentence.++ ['The video features a man in a plaid/checkered shirt delivering a live-streamed explanation or discussion.', 'The video features a person delivering a live-streamed explanation or discussion.'],++ ['The video shows a man in a kitchen setting, appearing to stream live from a couch or kitchen area.', 'The video shows a person in a kitchen setting, appearing to stream live from a couch or kitchen area.'],++ ['The video centers on a live stream featuring a male or female host wearing headphones, visible in a small webcam window.', 'The video centers on a live stream featuring a host, visible in a small webcam window.'],++ ['A home kitchen where a person adjusts their hair or clothing near the counter. A pot of soup simmers on the stove.', 'A pot of soup simmers on the stove.'],++ ['A female streamer and two guys race karts; the other men cheer.', 'A streamer and two people race karts; the other people cheer.'],++ ['An Iron Man skin fights Pac-Man in a man-made arena while chat says "that man is cracked".', 'An Iron Man skin fights Pac-Man in a man-made arena while chat says "that man is cracked".'],];let off = 0;// Public memory text is the observation alone: the raw transcript clause stays for the AI's context, never shown.@@@ -69,14 +77,19 @@ const db = require('../server/db/database');await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9060, 'An older version of the game loads.']);await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9120, 'A man wearing headphones sits at a desk.']);await db.cleanupMalformedAiText({ force: true });-- assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9120')).description, 'A man sits at a desk.');++ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9120')).description, 'A person sits at a desk.');// The scan runs once per version of the rules: the next boot skips it (rows are scrubbed when written).await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9180, 'A man wearing a hat waves.']);await db.cleanupMalformedAiText();assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man wearing a hat waves.', 'skipped: the rules have not changed');await db.setState('ai_looks_repair_rules', 'an older rule set');await db.cleanupMalformedAiText();-- assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man waves.', 'new rules: scanned again');++ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A person waves.', 'new rules: scanned again');++ // A plain guessed gender stored before this rule is repaired too (the scan's pattern names gendered words).++ await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9240, 'A woman cooks pasta.']);++ await db.setState('ai_looks_repair_rules', 'an older rule set');++ await db.cleanupMalformedAiText();++ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9240')).description, 'A person cooks pasta.');assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9000')).description, 'The streamer rests his chin on his hand.');assert.deepStrictEqual(Object.values(await db.get('SELECT ai_overview, ai_overview_short FROM streams WHERE id = ?', [sid])), ['The streamer plays Fortnite.', 'The streamer plays Fortnite.']);assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9060')).description, 'An older version of the game loads.', 'looks words about things stay');@@@ -87,7 +100,7 @@ const db = require('../server/db/database');// The chat popover and the profile's AI tab: every AI string in the answer, however nested.const { noLooksDeep } = require('../server/ai/chat-ai-routes')._internals;assert.deepStrictEqual(noLooksDeep({ overview_alltime: 'A shirtless host chats.', n: 3, jokes: ['the streamer wearing a hat dances'], sessions: [{ moments: [{ description: 'A man in a red shirt cooks.' }] }] }),-- { overview_alltime: 'A host chats.', n: 3, jokes: ['the streamer dances'], sessions: [{ moments: [{ description: 'A man cooks.' }] }] });++ { overview_alltime: 'A host chats.', n: 3, jokes: ['the streamer dances'], sessions: [{ moments: [{ description: 'A person cooks.' }] }] });// A session's AI title about looks is not used: the stream's own title is stored instead.await db.setStreamAiTitle(sid, 'Orange Shirt Kitchen Chat');
@@ -409,8 +409,11 @@ const _WITH_FACE = new RegExp(`\\s+with\\s+${_ART}(?:[\\w'-]+\\s+){0,2}?\\b(?:${
@@ -409,8 +409,11 @@ const _WITH_FACE = new RegExp(`\\s+with\\s+${_ART}(?:[\\w'-]+\\s+){0,2}?\\b(?:${
const _WEARS_AND = new RegExp(`(?<!-)\\bwears\\s+${_WORN_NP}(?:\\s*(?:,|and|or|with|over|under)\\s+${_WORN_NP})*\\s+and\\s+(?=\\w)`, 'gi');
const _WEARS_AND = new RegExp(`(?<!-)\\bwears\\s+${_WORN_NP}(?:\\s*(?:,|and|or|with|over|under)\\s+${_WORN_NP})*\\s+and\\s+(?=\\w)`, 'gi');
// "…, while the other wears a dusty t-shirt;" goes as a whole clause.
// "…, while the other wears a dusty t-shirt;" goes as a whole clause.
const _WEARS_CLAUSE = new RegExp(`,?\\s+(?:while|and|as|whereas)\\s+(?:the other|one|another|someone|he|she|they|the streamer|(?:a|the) (?:man|woman|person|guy))\\s+wears\\s+${_WORN_NP}(?:\\s*(?:,|and|or|with|over|under)\\s+${_WORN_NP})*(?=\\s*(?:[;,.!?]|$))`, 'gi');
const _WEARS_CLAUSE = new RegExp(`,?\\s+(?:while|and|as|whereas)\\s+(?:the other|one|another|someone|he|she|they|the streamer|(?:a|the) (?:man|woman|person|guy))\\s+wears\\s+${_WORN_NP}(?:\\s*(?:,|and|or|with|over|under)\\s+${_WORN_NP})*(?=\\s*(?:[;,.!?]|$))`, 'gi');
+// "a plaid/checkered shirt": a slashed pair of words before what is worn reads as one word, so the rules below see it.
+const _SLASHED_WORN = new RegExp(`\\b([\\w'-]+)/[\\w'-]+(?=\\s+(?:[\\w'-]+\\s+){0,2}?(?:${_WORN_ANY})\\b)`, 'gi');
function _noClothing(text) {
function _noClothing(text) {
return text.split(/("[^"]*"?|“[^”]*”?)/).map((part, i) => (i % 2 ? part : part
return text.split(/("[^"]*"?|“[^”]*”?)/).map((part, i) => (i % 2 ? part : part
+ .replace(_SLASHED_WORN, '$1')
.replace(/\b[\w-]+-wearing\s+/gi, '')
.replace(/\b[\w-]+-wearing\s+/gi, '')
.replace(_WEARS_AND, '')
.replace(_WEARS_AND, '')
.replace(_WEARS_CLAUSE, '')
.replace(_WEARS_CLAUSE, '')
@@ -435,8 +438,33 @@ function _noLooksTags(tags) {
@@ -435,8 +438,33 @@ function _noLooksTags(tags) {
}
}
// Body language is the person too ("reacting with a thoughtful pose", "in a relaxed posture"): the phrase goes.
// Body language is the person too ("reacting with a thoughtful pose", "in a relaxed posture"): the phrase goes.
const _POSE = /,?\s*(?:(?:appearing|seeming|seemingly)\s+(?:to\s+be\s+)?)?(?:(?:reacting|sitting|standing|leaning|posing|looking|resting)\s+)?(?:with|in)\s+(?:an?|their|his|her)\s+(?:[\w-]+\s+){0,2}(?:pose|posture|expression|facial expression|stance)\b/gi;
const _POSE = /,?\s*(?:(?:appearing|seeming|seemingly)\s+(?:to\s+be\s+)?)?(?:(?:reacting|sitting|standing|leaning|posing|looking|resting)\s+)?(?:with|in)\s+(?:an?|their|his|her)\s+(?:[\w-]+\s+){0,2}(?:pose|posture|expression|facial expression|stance)\b/gi;
+// A guessed gender is a description of the person too (owner's rule: no looks, body, clothing, hair, age or guessed
+// gender): "a man in a kitchen" is "a person in a kitchen", "two men" are "two people", "a male or female host" is "a
+// host". Only after an article or a count, so names (Iron Man, Pac-Man, Spice Girls) and quoted speech stay.
+const _GENDERED = 'man|woman|guy|gal|lady|gentleman';
+const _GENDERED_ONE = new RegExp(`\\b(a|an|the|one|another|this|that|each|every)\\s+(?:${_GENDERED})\\b(?!-)`, 'gi');
+const _GENDERED_MANY = new RegExp(`\\b(two|three|four|five|several|some|the|both|many|other|few|these|those|\\d+)\\s+(?:men|women|guys|gals|ladies|gentlemen)\\b`, 'gi');
+const _GENDER_ADJ = /\b(?:(?:a|an)\s+)?(?:male|female)\s+(?:or|and|\/)\s+(?:male|female)\s+|\b(?:male|female)\s+(?=(?:hosts?|streamers?|players?|speakers?|presenters?|gamers?|voices?|narrators?|commentators?|figures?|characters?|broadcasters?|viewers?|individuals?|persons?|people)\b)/gi;
+// What is left of a sentence about someone's hair, clothes or body ("a person adjusts their hair or clothing near the
+// counter") cannot be repaired word by word: the sentence goes.
+const _GROOMING = /\b(?:their|his|her)\s+(?:own\s+)?(?:hair|beard|face|makeup|clothing|clothes|outfit|appearance|body|figure|looks)\b/i;
+function _noGender(text) {
+ return text.split(/("[^"]*"?|“[^”]*”?)/).map((part, i) => (i % 2 ? part : part
+ .replace(_GENDER_ADJ, (m) => (/^an?\s/i.test(m) ? m.slice(0, m.indexOf(' ') + 1) : ''))
+ .replace(_GENDERED_ONE, (m, art) => _cap(m, `${/^an$/i.test(art) ? 'a' : art.toLowerCase()} person`))
+ .replace(_GENDERED_MANY, (m, n) => _cap(m, `${n.toLowerCase()} people`))
+ .replace(/\ba\s+(?=[aeiou]\w*\s+(?:host|streamer|player|speaker|presenter|gamer|figure|character|broadcaster|person)\b)/gi, (m) => _cap(m, 'an '))
+ .replace(/ {2,}/g, ' '))).join('');
+}
+function _noGrooming(text) {
+ const sentences = text.match(/[^.!?]+(?:[.!?]+(?:["”])?|$)\s*/g) || [text];
+ return sentences.filter((s) => !_GROOMING.test(s.replace(/"[^"]*"?|“[^”]*”?/g, ''))).join('').trim();
+}
function _noLooks(text) {
function _noLooks(text) {
if (!text || typeof text !== 'string') return text;
if (!text || typeof text !== 'string') return text;
+ return _noGrooming(_noGender(_noLooksWords(text)));
+}
+function _noLooksWords(text) {
return _noClothing(text).replace(_POSE, '').replace(/\s+([,.;:!?])/g, '$1').replace(/ {2,}/g, ' ')
return _noClothing(text).replace(_POSE, '').replace(/\s+([,.;:!?])/g, '$1').replace(/ {2,}/g, ' ')
// Looks named: the person on camera. Only an age ("a child sits nearby"): someone, who may not be the streamer.
// Looks named: the person on camera. Only an age ("a child sits nearby"): someone, who may not be the streamer.
.replace(_LOOKED_AT, (m, art, looks, noun) => (looks ? _cap(m, 'the streamer') : _AGE_NOUN.test(noun) ? _cap(m, /^the$/i.test(art) ? 'the person' : 'someone') : m))
.replace(_LOOKED_AT, (m, art, looks, noun) => (looks ? _cap(m, 'the streamer') : _AGE_NOUN.test(noun) ? _cap(m, /^the$/i.test(art) ? 'the person' : 'someone') : m))
@@ -708,7 +736,7 @@ function _extractDescFromMaybeJson(text) {
@@ -708,7 +736,7 @@ function _extractDescFromMaybeJson(text) {
// written, so the scan only needs to run again when the scrub rules change: this hash of their source says when.
// written, so the scan only needs to run again when the scrub rules change: this hash of their source says when.
let _looksRulesVersion = null;
let _looksRulesVersion = null;
function looksRulesVersion() {
function looksRulesVersion() {
- if (!_looksRulesVersion) _looksRulesVersion = require('crypto').createHash('sha1').update([_noLooks, _noClothing, _LOOKS, _PEOPLE, _WORN, _WORN_ANY, _FACE].map(String).join('|')).digest('hex').slice(0, 12);
+ if (!_looksRulesVersion) _looksRulesVersion = require('crypto').createHash('sha1').update([_noLooks, _noLooksWords, _noGender, _noGrooming, _noClothing, _LOOKS, _PEOPLE, _WORN, _WORN_ANY, _FACE, _GENDER_ADJ, _GROOMING].map(String).join('|')).digest('hex').slice(0, 12);
return _looksRulesVersion;
return _looksRulesVersion;
}
}
async function cleanupMalformedAiText({ force = false } = {}) {
async function cleanupMalformedAiText({ force = false } = {}) {
@@ -754,7 +782,7 @@ async function cleanupMalformedAiText({ force = false } = {}) {
@@ -754,7 +782,7 @@ async function cleanupMalformedAiText({ force = false } = {}) {
// prompts forbade it): the person becomes "the streamer" and the card line follows.
// prompts forbade it): the person becomes "the streamer" and the card line follows.
let looksDone = false;
let looksDone = false;
try { looksDone = !force && (await getState('ai_looks_repair_rules')) === looksRulesVersion(); } catch { looksDone = false; }
try { looksDone = !force && (await getState('ai_looks_repair_rules')) === looksRulesVersion(); } catch { looksDone = false; }
- const looks = "~* '(shirtless|topless|bare-chested|young|teen|little|bearded|bald|blond|haired|tattooed|muscular|heavyset|older|elderly|middle-aged|skinny|slim|chubby|attractive|pretty|handsome|cute|boy|girl|kid|child|youth)|\\m(wear|dressed|sporting|shirt|t-shirt|tee|tank top|hoodie|sweat|jacket|coat|vest|beanie|cap|hat|helmet|headset|headphones|sunglasses|glasses|outfit|dress|jersey|suit|robe|bandana|scar|mask|attire|apparel|beard|mustache|moustache|goatee|stubble|tattoo|hair|ponytail|braids|dreadlocks|piercing|freckles|makeup|[a-z-]+-wearing)'";
+ const looks = "~* '(shirtless|topless|bare-chested|young|teen|little|bearded|bald|blond|haired|tattooed|muscular|heavyset|older|elderly|middle-aged|skinny|slim|chubby|attractive|pretty|handsome|cute|boy|girl|kid|child|youth)|\\m(wear|dressed|sporting|shirt|t-shirt|tee|tank top|hoodie|sweat|jacket|coat|vest|beanie|cap|hat|helmet|headset|headphones|sunglasses|glasses|outfit|dress|jersey|suit|robe|bandana|scar|mask|attire|apparel|beard|mustache|moustache|goatee|stubble|tattoo|hair|ponytail|braids|dreadlocks|piercing|freckles|makeup|[a-z-]+-wearing)|\\m(man|woman|men|women|guy|guys|gal|lady|ladies|gentleman|gentlemen|male|female)\\M'";
for (const [table, key, col, short] of [['stream_memories', 'id', 'description', null], ['streams', 'id', 'ai_overview', 'ai_overview_short'],
for (const [table, key, col, short] of [['stream_memories', 'id', 'description', null], ['streams', 'id', 'ai_overview', 'ai_overview_short'],
['vod_ai_state', 'vod_id', 'ai_overview', 'ai_overview_short'], ['clip_ai_state', 'clip_id', 'ai_overview', 'ai_overview_short'],
['vod_ai_state', 'vod_id', 'ai_overview', 'ai_overview_short'], ['clip_ai_state', 'clip_id', 'ai_overview', 'ai_overview_short'],
['streamer_overviews', 'user_id', 'overview', 'overview_short']]) {
['streamer_overviews', 'user_id', 'overview', 'overview_short']]) {
@@ -1,7 +1,8 @@
@@ -1,7 +1,8 @@
'use strict';
'use strict';
// Public AI text never describes how a person looks: a frame note or an overview that names looks or an age
// Public AI text never describes how a person looks: a frame note or an overview that names looks or an age
-// ("A shirtless man sits…", "a young boy at a desk") is stored with that person as "the streamer", a plain "a man"
-// or "a black chair" stays as written, and rows stored before the rule are repaired by the AI cleanup pass.
+// ("A shirtless man sits…", "a young boy at a desk") is stored with that person as "the streamer"; a guessed gender
+// ("a man", "two men", "a male or female host") becomes a person; "a black chair" stays as written; and rows stored
+// before a rule changed are repaired by the AI cleanup pass.
const assert = require('assert');
const assert = require('assert');
console.warn = () => {};
console.warn = () => {};
@@ -17,37 +18,44 @@ const db = require('../server/db/database');
@@ -17,37 +18,44 @@ const db = require('../server/db/database');
['A livestream frame showing a shirtless streamer in a small inset while playing a MOBA.', 'A livestream frame showing the streamer in a small inset while playing a MOBA.'],
['A livestream frame showing a shirtless streamer in a small inset while playing a MOBA.', 'A livestream frame showing the streamer in a small inset while playing a MOBA.'],
['Chat watches a young boy at a desk build a PC.', 'Chat watches the streamer at a desk build a PC.'],
['Chat watches a young boy at a desk build a PC.', 'Chat watches the streamer at a desk build a PC.'],
['Young, bearded man plays chess.', 'Streamer plays chess.'],
['Young, bearded man plays chess.', 'Streamer plays chess.'],
- ['A man plays chess next to a black chair.', 'A man plays chess next to a black chair.'],
+ ['A man plays chess next to a black chair.', 'A person plays chess next to a black chair.'],
['The streamer, topless, laughs at chat.', 'The streamer laughs at chat.'],
['The streamer, topless, laughs at chat.', 'The streamer laughs at chat.'],
- ['A man is shirtless and appears to be talking to chat.', 'A man appears to be talking to chat.'],
+ ['A man is shirtless and appears to be talking to chat.', 'A person appears to be talking to chat.'],
['A person watches TV while a child sits nearby.', 'A person watches TV while someone sits nearby.'],
['A person watches TV while a child sits nearby.', 'A person watches TV while someone sits nearby.'],
['The girl waves at the camera.', 'The person waves at the camera.'],
['The girl waves at the camera.', 'The person waves at the camera.'],
// Clothing, hair and face go; the rest of the sentence stays.
// Clothing, hair and face go; the rest of the sentence stays.
['The streamer wearing a red beanie speaks to the camera outdoors, seemingly addressing chat.', 'The streamer speaks to the camera outdoors, seemingly addressing chat.'],
['The streamer wearing a red beanie speaks to the camera outdoors, seemingly addressing chat.', 'The streamer speaks to the camera outdoors, seemingly addressing chat.'],
['The view is close to the camera while wearing a black beanie and hoodie, with trees around.', 'The view is close to the camera, with trees around.'],
['The view is close to the camera while wearing a black beanie and hoodie, with trees around.', 'The view is close to the camera, with trees around.'],
- ['A man in a gray tank top sits against a plain light background.', 'A man sits against a plain light background.'],
- ['A man with a long beard and plaid shirt examines a woven item.', 'A man examines a woven item.'],
+ ['A man in a gray tank top sits against a plain light background.', 'A person sits against a plain light background.'],
+ ['A man with a long beard and plaid shirt examines a woven item.', 'A person examines a woven item.'],
// Pose and expression are the person too (2026-10-17, the home page's "Right now").
// Pose and expression are the person too (2026-10-17, the home page's "Right now").
['A live stream frame shows the streamer in a small window bottom-left, appearing to be reacting with a thoughtful pose while the main screen is black.',
['A live stream frame shows the streamer in a small window bottom-left, appearing to be reacting with a thoughtful pose while the main screen is black.',
'A live stream frame shows the streamer in a small window bottom-left while the main screen is black.'],
'A live stream frame shows the streamer in a small window bottom-left while the main screen is black.'],
['The streamer sits in a relaxed posture at the desk.', 'The streamer sits at the desk.'],
['The streamer sits in a relaxed posture at the desk.', 'The streamer sits at the desk.'],
['A kitchen studio with a headset-wearing host, microphone, and on-screen graphics.', 'A kitchen studio with a host, microphone, and on-screen graphics.'],
['A kitchen studio with a headset-wearing host, microphone, and on-screen graphics.', 'A kitchen studio with a host, microphone, and on-screen graphics.'],
['Panelists discuss while wearing headsets and microphones, with a trophy nearby.', 'Panelists discuss, with a trophy nearby.'],
['Panelists discuss while wearing headsets and microphones, with a trophy nearby.', 'Panelists discuss, with a trophy nearby.'],
- ['A man wearing sunglasses on his head looks down at the camera.', 'A man looks down at the camera.'],
- ['A collage of a man in a white shirt and tie wearing dark sunglasses in the left panel.', 'A collage of a man in the left panel.'],
+ ['A man wearing sunglasses on his head looks down at the camera.', 'A person looks down at the camera.'],
+ ['A collage of a man in a white shirt and tie wearing dark sunglasses in the left panel.', 'A collage of a person in the left panel.'],
['Behind him a green mat; he wears a blue jacket and is leaning to the left.', 'Behind him a green mat; he is leaning to the left.'],
['Behind him a green mat; he wears a blue jacket and is leaning to the left.', 'Behind him a green mat; he is leaning to the left.'],
- ['Two men lift weights, one shirtless and sweating.', 'Two men lift weights, one sweating.'],
- ['Two men stand in an alley, one and drinking from a bottle while the other wears a dusty t-shirt; both look tired.', 'Two men stand in an alley, one drinking from a bottle; both look tired.'],
- ['A man wearing a green bucket hat with frog-like eyes sits indoors.', 'A man sits indoors.'],
- ['A man in sunglasses reaches toward the camera inside a car.', 'A man reaches toward the camera inside a car.'],
+ ['Two men lift weights, one shirtless and sweating.', 'Two people lift weights, one sweating.'],
+ ['Two men stand in an alley, one and drinking from a bottle while the other wears a dusty t-shirt; both look tired.', 'Two people stand in an alley, one drinking from a bottle; both look tired.'],
+ ['A man wearing a green bucket hat with frog-like eyes sits indoors.', 'A person sits indoors.'],
+ ['A man in sunglasses reaches toward the camera inside a car.', 'A person reaches toward the camera inside a car.'],
['A live stream window shows a broadcaster with a headset on the left, a chat panel on the right.', 'A live stream window shows a broadcaster on the left, a chat panel on the right.'],
['A live stream window shows a broadcaster with a headset on the left, a chat panel on the right.', 'A live stream window shows a broadcaster on the left, a chat panel on the right.'],
- ['A man with a red shirt sits in a room; a desk with a headset and keyboard.', 'A man sits in a room; a desk with a headset and keyboard.'],
+ ['A man with a red shirt sits in a room; a desk with a headset and keyboard.', 'A person sits in a room; a desk with a headset and keyboard.'],
['A gamer faces a casually dressed NPC in a grim interior.', 'A gamer faces a NPC in a grim interior.'],
['A gamer faces a casually dressed NPC in a grim interior.', 'A gamer faces a NPC in a grim interior.'],
['A cartoon streaming setup: wearing a headset with a microphone, rugged jacket, backpack with a mat.', 'A cartoon streaming setup: backpack with a mat.'],
['A cartoon streaming setup: wearing a headset with a microphone, rugged jacket, backpack with a mat.', 'A cartoon streaming setup: backpack with a mat.'],
['A partial view of a person wearing a gray zip-up jacket with a white zipper, seated indoors.', 'A partial view of a person, seated indoors.'],
['A partial view of a person wearing a gray zip-up jacket with a white zipper, seated indoors.', 'A partial view of a person, seated indoors.'],
['Close-up of a person wearing a yellow vest over a gray knit top, with a backpack visible.', 'Close-up of a person, with a backpack visible.'],
['Close-up of a person wearing a yellow vest over a gray knit top, with a backpack visible.', 'Close-up of a person, with a backpack visible.'],
['A Minecraft player in a top-down view builds a house; the streamer reviews new headphones.', 'A Minecraft player in a top-down view builds a house; the streamer reviews new headphones.'],
['A Minecraft player in a top-down view builds a house; the streamer reviews new headphones.', 'A Minecraft player in a top-down view builds a house; the streamer reviews new headphones.'],
['The streamer laughs — heard: "Put your headset on. I like your hair."', 'The streamer laughs — heard: "Put your headset on. I like your hair."'],
['The streamer laughs — heard: "Put your headset on. I like your hair."', 'The streamer laughs — heard: "Put your headset on. I like your hair."'],
+ // Clip overviews found public on 2026-10-22: a guessed gender, a slashed pattern before a shirt, a grooming sentence.
+ ['The video features a man in a plaid/checkered shirt delivering a live-streamed explanation or discussion.', 'The video features a person delivering a live-streamed explanation or discussion.'],
+ ['The video shows a man in a kitchen setting, appearing to stream live from a couch or kitchen area.', 'The video shows a person in a kitchen setting, appearing to stream live from a couch or kitchen area.'],
+ ['The video centers on a live stream featuring a male or female host wearing headphones, visible in a small webcam window.', 'The video centers on a live stream featuring a host, visible in a small webcam window.'],
+ ['A home kitchen where a person adjusts their hair or clothing near the counter. A pot of soup simmers on the stove.', 'A pot of soup simmers on the stove.'],
+ ['A female streamer and two guys race karts; the other men cheer.', 'A streamer and two people race karts; the other people cheer.'],
+ ['An Iron Man skin fights Pac-Man in a man-made arena while chat says "that man is cracked".', 'An Iron Man skin fights Pac-Man in a man-made arena while chat says "that man is cracked".'],
];
];
let off = 0;
let off = 0;
// Public memory text is the observation alone: the raw transcript clause stays for the AI's context, never shown.
// Public memory text is the observation alone: the raw transcript clause stays for the AI's context, never shown.
@@ -69,14 +77,19 @@ const db = require('../server/db/database');
@@ -69,14 +77,19 @@ const db = require('../server/db/database');
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9060, 'An older version of the game loads.']);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9060, 'An older version of the game loads.']);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9120, 'A man wearing headphones sits at a desk.']);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9120, 'A man wearing headphones sits at a desk.']);
await db.cleanupMalformedAiText({ force: true });
await db.cleanupMalformedAiText({ force: true });
- assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9120')).description, 'A man sits at a desk.');
+ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9120')).description, 'A person sits at a desk.');
// The scan runs once per version of the rules: the next boot skips it (rows are scrubbed when written).
// The scan runs once per version of the rules: the next boot skips it (rows are scrubbed when written).
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9180, 'A man wearing a hat waves.']);
await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9180, 'A man wearing a hat waves.']);
await db.cleanupMalformedAiText();
await db.cleanupMalformedAiText();
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man wearing a hat waves.', 'skipped: the rules have not changed');
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man wearing a hat waves.', 'skipped: the rules have not changed');
await db.setState('ai_looks_repair_rules', 'an older rule set');
await db.setState('ai_looks_repair_rules', 'an older rule set');
await db.cleanupMalformedAiText();
await db.cleanupMalformedAiText();
- assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A man waves.', 'new rules: scanned again');
+ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9180')).description, 'A person waves.', 'new rules: scanned again');
+ // A plain guessed gender stored before this rule is repaired too (the scan's pattern names gendered words).
+ await db.run('INSERT INTO stream_memories (stream_id, user_id, offset_seconds, description) VALUES (?, ?, ?, ?)', [sid, uid, 9240, 'A woman cooks pasta.']);
+ await db.setState('ai_looks_repair_rules', 'an older rule set');
+ await db.cleanupMalformedAiText();
+ assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9240')).description, 'A person cooks pasta.');
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9000')).description, 'The streamer rests his chin on his hand.');
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9000')).description, 'The streamer rests his chin on his hand.');
assert.deepStrictEqual(Object.values(await db.get('SELECT ai_overview, ai_overview_short FROM streams WHERE id = ?', [sid])), ['The streamer plays Fortnite.', 'The streamer plays Fortnite.']);
assert.deepStrictEqual(Object.values(await db.get('SELECT ai_overview, ai_overview_short FROM streams WHERE id = ?', [sid])), ['The streamer plays Fortnite.', 'The streamer plays Fortnite.']);
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9060')).description, 'An older version of the game loads.', 'looks words about things stay');
assert.strictEqual((await db.get('SELECT description FROM stream_memories WHERE offset_seconds = 9060')).description, 'An older version of the game loads.', 'looks words about things stay');
@@ -87,7 +100,7 @@ const db = require('../server/db/database');
@@ -87,7 +100,7 @@ const db = require('../server/db/database');
// The chat popover and the profile's AI tab: every AI string in the answer, however nested.
// The chat popover and the profile's AI tab: every AI string in the answer, however nested.
const { noLooksDeep } = require('../server/ai/chat-ai-routes')._internals;
const { noLooksDeep } = require('../server/ai/chat-ai-routes')._internals;
assert.deepStrictEqual(noLooksDeep({ overview_alltime: 'A shirtless host chats.', n: 3, jokes: ['the streamer wearing a hat dances'], sessions: [{ moments: [{ description: 'A man in a red shirt cooks.' }] }] }),
assert.deepStrictEqual(noLooksDeep({ overview_alltime: 'A shirtless host chats.', n: 3, jokes: ['the streamer wearing a hat dances'], sessions: [{ moments: [{ description: 'A man in a red shirt cooks.' }] }] }),
- { overview_alltime: 'A host chats.', n: 3, jokes: ['the streamer dances'], sessions: [{ moments: [{ description: 'A man cooks.' }] }] });
+ { overview_alltime: 'A host chats.', n: 3, jokes: ['the streamer dances'], sessions: [{ moments: [{ description: 'A person cooks.' }] }] });
// A session's AI title about looks is not used: the stream's own title is stored instead.
// A session's AI title about looks is not used: the stream's own title is stored instead.
await db.setStreamAiTitle(sid, 'Orange Shirt Kitchen Chat');
await db.setStreamAiTitle(sid, 'Orange Shirt Kitchen Chat');