openvibers/openvibe.live Commits

Chat translation that knows which lines need it and never asks twice.

OP OpenVibers committed 29 Sept 2026 · 3e82fa978d · 8 files, +348 −54

public/js/lang-detect.js is one detector for the browser and the server (server/i18n/translate.js requires it): scripts decide Japanese, Korean, Chinese, Russian and the rest; Latin-script text is scored by common words and letters particular to a language (ñ, ã, ß, ł, ğ, ơ …), so Spanish, Portuguese, French, German, Italian, Dutch, Polish, Turkish, Indonesian and Vietnamese are told apart from English. Slang, emote names, commands, links and one short word stay unknown, so they are never offered or sent.

The translate button shows only on lines confidently in another language than the viewer's, and not on lines the channel's auto-translation already put into it (it used to sit on every line with two letters). Translations are kept for the session (sessionStorage), so a re-render, coming back to a room or a reload never asks again. Auto-translate is opt-in per viewer: a "Translate chat" switch appears in the tools row once a line in another language has been seen (a one-language room never shows it); when on, the room's foreign lines go out in batches (one request per ~0.8 s, up to 20, the newest 40 of a history load) to POST /api/i18n/translate-batch, which answers same-language and cached lines itself and sends the rest to the model in ONE call (translateMany), caching each line so the next viewer costs nothing. The rate limit counts lines.

Server: Spanish (and the other Latin-script languages) is detected as itself, so the translate endpoint no longer answers "already in English" for it. Translating foreign lines for everyone (translateChatMessage) happens only when detection is sure (confidence 0.85+); a lone 草 or a two-word line is left to each viewer's button.

test/lang-detect.test.js: detection, the button rule, one model call per batch, the cache, the sure-only rule. npm test 151/152, 1 skipped.

Changes

ddiff --git a/public/css/i18n-star.css b/public/css/i18n-star.css
iindex 30f0808b..f967eb18 100644
---- a/public/css/i18n-star.css
++++ b/public/css/i18n-star.css
@@@ -202,3 +202,14 @@
@media (prefers-reduced-motion: reduce) {
.star-expand, .star-expand i, .star-card.is-compact .star-avatar { transition: none; }
}
++
++/* ── Chat: "Translate chat" switch (shows once a line in another language arrives; chat.js _autoTrChip) ── */
++.chat-autotr {
++ display: inline-flex; align-items: center; gap: 5px; margin-left: auto; padding: 3px 10px; border-radius: 999px; cursor: pointer;
++ font: 600 0.72rem/1.2 inherit; white-space: nowrap; color: var(--text-secondary);
++ background: transparent; border: 1px solid var(--border); transition: color .15s, border-color .15s, background-color .15s;
++}
++.chat-autotr:hover { color: var(--text-primary); border-color: color-mix(in srgb, var(--accent) 50%, var(--border)); }
++.chat-autotr.is-on { color: var(--accent-light, var(--accent)); border-color: color-mix(in srgb, var(--accent) 45%, transparent); background: color-mix(in srgb, var(--accent) 12%, transparent); }
++.chat-autotr:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
++.chat-translate-hint ~ .chat-autotr { margin-left: 6px; }
ddiff --git a/public/index.html b/public/index.html
iindex 59bbacd3..0f7b68ad 100644
---- a/public/index.html
++++ b/public/index.html
@@@ -978,6 +978,7 @@
<script src="/js/app.js" defer></script>
<script src="/js/ov-navbar-live.js" defer></script>
<script src="/js/chat-shared.js" defer></script>
++<script src="/js/lang-detect.js" defer></script>
<script src="/js/chat.js" defer></script>
<!-- (js/admin.js removed — 130KB of unreachable code; /admin redirects off-site.) -->
<!-- ── Floating Global Chat Widget (visible on non-chat pages) ── -->
ddiff --git a/public/js/chat.js b/public/js/chat.js
iindex a5863bf0..670935be 100644
---- a/public/js/chat.js
++++ b/public/js/chat.js
@@@ -3024,21 +3024,88 @@ function _applyChatTranslation(evt) {
}
// The viewer's own language (browser setting, 2-letter), used by the on-demand translate button.
function _viewerLang() { try { return String(navigator.language || 'en').slice(0, 2).toLowerCase() || 'en'; } catch { return 'en'; } }
++/* ── Translating for this viewer ────────────────────────────────
++ Which lines are in another language is decided here, without a network call (public/js/lang-detect.js, the
++ same rules the server uses): only those get a translate button, and only they are ever sent. Translations are
++ kept for the session (sessionStorage), so a re-render, a room switch back or a reload never asks twice; the
++ server keeps its own cache too, shared by every viewer. Auto-translate (opt-in, remembered) sends the room's
++ foreign lines in batches: one request per ~0.8 s for up to 20 lines, answered from the cache where it can and
++ with one model call for the rest. */
++const _TR_KEY = 'ov_tr_cache_v1';
++const _trCache = (() => { try { return new Map(JSON.parse(sessionStorage.getItem(_TR_KEY) || '[]')); } catch { return new Map(); } })();
++let _trSaveT = 0;
++function _trRemember(to, text, val) {
++ _trCache.set(to + '\u0001' + text, val);
++ if (_trCache.size > 400) _trCache.delete(_trCache.keys().next().value);
++ clearTimeout(_trSaveT);
++ _trSaveT = setTimeout(() => { try { sessionStorage.setItem(_TR_KEY, JSON.stringify([..._trCache].slice(-400))); } catch { /* full */ } }, 800);
++}
++function _trShow(el, d, to) {
++ if (!d || !d.text || el.querySelector(`.chat-translation[data-to="${CSS.escape(to)}"]`)) return;
++ const node = _translationEl({ from: d.from, to, text: d.text });
++ node.dataset.to = to;
++ el.appendChild(node);
++}
async function _translateForMe(el, text, btn) {
const to = _viewerLang();
const existing = el.querySelector(`.chat-translation[data-to="${CSS.escape(to)}"]`);
if (existing) { existing.hidden = !existing.hidden; return; }
++ const hit = _trCache.get(to + '\u0001' + text);
++ if (hit) { _trShow(el, hit, to); return; }
if (btn) { btn.disabled = true; btn.innerHTML = '<i class="fa-solid fa-circle-notch fa-spin"></i>'; }
try {
const d = await api('/i18n/translate', { method: 'POST', body: { text, to } });
if (d && d.same) { toast(`Already in ${_LANG_LABEL[to] || to}`, 'info'); return; }
if (!d || !d.text) { toast('Translation unavailable right now', 'error'); return; }
-- const node = _translationEl({ from: d.from, to: d.to, text: d.text });
-- node.dataset.to = to;
-- el.appendChild(node);
++ _trRemember(to, text, { from: d.from, text: d.text });
++ _trShow(el, d, to);
} catch (err) { toast(err?.message || 'Translation failed', 'error'); }
finally { if (btn) { btn.disabled = false; btn.innerHTML = '<i class="fa-solid fa-language"></i>'; } }
}
++const _autoTr = {
++ on: (() => { try { return localStorage.getItem('ov_chat_autotr') === '1'; } catch { return false; } })(),
++ seen: 0, queue: [], timer: 0,
++ add(el, text) {
++ this.seen++;
++ if (this.seen === 1) _autoTrChip();
++ if (!this.on) return;
++ const to = _viewerLang(), hit = _trCache.get(to + '\u0001' + text);
++ if (hit) { if (hit.text) _trShow(el, hit, to); return; }
++ this.queue.push([el, text]);
++ if (this.queue.length > 40) this.queue.splice(0, this.queue.length - 40); // a history load: the newest lines
++ if (!this.timer) this.timer = setTimeout(() => this.flush(), 800);
++ },
++ async flush() {
++ this.timer = 0;
++ const batch = this.queue.splice(0, 20).filter(([el]) => el.isConnected);
++ if (this.queue.length) this.timer = setTimeout(() => this.flush(), 1500);
++ if (!batch.length) return;
++ const to = _viewerLang(), texts = [...new Set(batch.map(([, t]) => t))];
++ try {
++ const d = await api('/i18n/translate-batch', { method: 'POST', body: { texts, to } });
++ (d && d.results || []).forEach((r, i) => { if (r) _trRemember(to, texts[i], r.text ? { from: r.from, text: r.text } : { same: true }); });
++ for (const [el, t] of batch) { const r = _trCache.get(to + '\u0001' + t); if (r && r.text) _trShow(el, r, to); }
++ } catch { /* off or busy: the button still works */ }
++ },
++ toggle() {
++ this.on = !this.on;
++ try { localStorage.setItem('ov_chat_autotr', this.on ? '1' : '0'); } catch { /* */ }
++ _autoTrChip();
++ if (this.on) document.querySelectorAll('.chat-msg[data-lang]').forEach((el) => { if (el._ovRaw) this.add(el, el._ovRaw); });
++ },
++};
++// The switch shows in the tools row once a line in another language has been seen (a room of one language never shows it).
++function _autoTrChip() {
++ if (!_autoTr.seen) return;
++ document.querySelectorAll('.chat-tools-row').forEach((row) => {
++ let b = row.querySelector('.chat-autotr');
++ if (!b) { b = document.createElement('button'); b.type = 'button'; b.className = 'chat-autotr'; b.addEventListener('click', () => _autoTr.toggle()); row.appendChild(b); }
++ b.classList.toggle('is-on', _autoTr.on);
++ b.setAttribute('aria-pressed', String(_autoTr.on));
++ b.title = _autoTr.on ? `Lines in other languages are translated to ${_LANG_LABEL[_viewerLang()] || 'your language'} as they arrive. Click to stop.` : `Translate lines in other languages to ${_LANG_LABEL[_viewerLang()] || 'your language'} automatically`;
++ b.innerHTML = `<i class="fa-solid fa-language"></i> <span>${_autoTr.on ? 'Translating' : 'Translate chat'}</span>`;
++ });
++}
// "Auto-translated ↔ Japanese" pill in every chat tools row while in a non-English channel.
function _updateTranslateHint(lang) {
_chatChannelLang = lang || 'en';
@@@ -3724,14 +3791,18 @@ function buildChatMessageEl(msg, opts = {}) {
replyBtn.title = 'Reply';
replyBtn.innerHTML = '<i class="fa-solid fa-reply"></i>';
el.appendChild(replyBtn);
-- // "Translate to my language" — for every viewer, whatever the channel's language is
-- // (the automatic translations only cover English ↔ the streamer's language).
-- if (/\p{L}{2,}/u.test(rawText || '')) {
++ // "Translate to my language", only on lines confidently in another language than the viewer's (decided
++ // locally, public/js/lang-detect.js) and not already translated into it by the channel's auto-translation.
++ const _me = _viewerLang();
++ const _foreign = window.OVLang ? OVLang.foreignFor(displayRaw || '', _me) : null;
++ if (_foreign && !(_tr && _tr.to === _me)) {
++ el.dataset.lang = _foreign;
const trBtn = document.createElement('button');
trBtn.className = 'chat-translate-btn';
-- trBtn.title = `Translate to ${_LANG_LABEL[_viewerLang()] || 'my language'}`;
++ trBtn.title = `Translate from ${_LANG_LABEL[_foreign] || _foreign} to ${_LANG_LABEL[_me] || 'my language'}`;
trBtn.innerHTML = '<i class="fa-solid fa-language"></i>';
el.appendChild(trBtn);
++ _autoTr.add(el, displayRaw);
}
el.classList.add('chat-msg-hoverable');
}
ddiff --git a/public/js/lang-detect.js b/public/js/lang-detect.js
nnew file mode 100644
iindex 00000000..70c734e1
---- /dev/null
++++ b/public/js/lang-detect.js
@@@ -0,0 +1,91 @@
++/*
++ OpenVibe.Live — which language a short text is in, without a model call. One file for the browser
++ (window.OVLang, chat's translate button and auto-translate) and the server (require, server/i18n/translate.js),
++ so both sides agree on what counts as foreign.
++
++ detect(text) → { lang: 'es' | 'ja' | … | null, confidence: 0–1 }
++ foreignFor(text, me) → the language code when text is confidently not in `me`, else null
++
++ Non-Latin scripts decide by their characters (kana → Japanese, hangul → Korean, …). Latin-script text is scored
++ by common short words and letters particular to a language (ñ, ã, ß, ł, ğ, ơ, …); chat slang and emote names
++ carry no score, so "gg", "pog" and "lol" stay unknown rather than guessed. Unknown means "don't offer or spend
++ a translation".
++*/
++(function (root, factory) {
++ const api = factory();
++ if (typeof module === 'object' && module.exports) module.exports = api;
++ else root.OVLang = api;
++})(typeof self !== 'undefined' ? self : this, function () {
++ 'use strict';
++ const SCRIPTS = [
++ ['kana', /[぀-ヿヲ-゚]/g], ['hangul', /[가-힯ᄀ-ᇿ㄰-㆏]/g],
++ ['han', /[一-鿿㐀-䶿]/g], ['cyr', /[Ѐ-ӿ]/g], ['ar', /[؀-ۿ]/g],
++ ['th', /[฀-๿]/g], ['he', /[֐-׿]/g], ['el', /[Ͱ-Ͽ]/g], ['hi', /[ऀ-ॿ]/g],
++ ];
++ const W = {
++ en: 'the and you that this with for are was have not but what all were when your can there how which their will about would them been has its who did get just like know dont doesnt cant im ive youre thats yeah yes hello thanks thank good really think because some here want going people very much where they them then than also only',
++ es: 'que el los las por para con una como pero más mas muy está esta estoy esto eso también tambien porque cuando donde hola gracias bueno qué cómo tengo tiene hay nada todo soy eres del al pues vamos quiero puedo sabes ahora aquí aqui bien',
++ pt: 'que não nao uma com para por mais como mas você voce está estou isso também muito obrigado olá ola tudo então entao quando onde tem sou são aqui agora da do das dos na no eu ele ela vou pra',
++ fr: 'le les des une est pas que qui dans pour avec sur mais vous nous je tu il elle très tres bien merci bonjour oui non cest cette aussi quoi comment ça ca suis mon ton son',
++ de: 'der die das und ist nicht ich du wir ihr sie ein eine mit auf für fur auch aber wie was sehr gut danke hallo ja nein schon noch mal bin bist hast habe den dem zu',
++ it: 'il lo gli che di non una per con sono come ma anche molto bene grazie ciao questo quello cosa perché perche io tu lui lei sei è della',
++ nl: 'het een en van ik je niet dat is op te zijn met voor maar ook wat hoe goed dank hallo ja nee heb heeft wel dit',
++ pl: 'nie się sie jest to że ze na jak ale co tak czy już juz jestem mam dzięki dzieki cześć czesc bardzo dobrze tylko może moze też tez',
++ tr: 've bir bu için icin ile ama çok cok değil degil ne var yok ben sen evet hayır hayir merhaba teşekkürler nasıl nasil iyi',
++ id: 'yang dan di ini itu dengan untuk tidak ada saya kamu aku apa juga sudah bisa terima kasih halo baik',
++ vi: 'và của là không có được cho này những một người tôi bạn cảm ơn xin chào rất',
++ };
++ const SETS = {};
++ for (const k of Object.keys(W)) SETS[k] = new Set(W[k].split(' '));
++ const MARKS = {
++ es: /[ñ¿¡]/g, pt: /[ãõ]/g, fr: /[œæ]|[çèêëîïûù]/g, de: /ß|[äöü]/g, pl: /[ąćęłńśźż]/g, tr: /[ğış]/g,
++ vi: /[ăđơư]|[ạảấầẩẫậắằẳẵặẹẻẽếềểễệỉịọỏốồổỗộớờởỡợụủứừửữựỳỵỷỹ]/g,
++ };
++ const count = (s, re) => (s.match(re) || []).length;
++ const clean = (t) => String(t || '')
++ .replace(/https?:\/\/\S+/gi, ' ').replace(/:[a-z0-9_]+:/gi, ' ').replace(/(^|\s)[!/@#]\S+/g, ' ');
++
++ const memo = new Map();
++ function detect(text) {
++ const s = clean(text);
++ if (memo.has(s)) return memo.get(s);
++ const out = compute(s);
++ if (memo.size > 3000) memo.delete(memo.keys().next().value);
++ memo.set(s, out);
++ return out;
++ }
++ function compute(s) {
++ const n = {}; let nonLatin = 0;
++ for (const [k, re] of SCRIPTS) { n[k] = count(s, re); nonLatin += n[k]; }
++ const latin = count(s, /[A-Za-zÀ-ɏḀ-ỿ]/g);
++ if (!nonLatin && !latin) return { lang: null, confidence: 0 };
++ if (nonLatin && nonLatin / (nonLatin + latin) >= 0.34) {
++ const lang = n.kana ? 'ja' : n.hangul ? 'ko' : n.han ? 'zh' : n.cyr ? (/[іїєґ]/i.test(s) ? 'uk' : 'ru')
++ : n.ar ? 'ar' : n.th ? 'th' : n.he ? 'he' : n.el ? 'el' : 'hi';
++ return { lang, confidence: nonLatin >= 2 ? 0.95 : 0.6 };
++ }
++ const words = (s.toLowerCase().replace(/['’]/g, '').match(/[\p{L}]+/gu) || []);
++ if (!words.length) return { lang: null, confidence: 0 };
++ const score = {};
++ for (const k of Object.keys(SETS)) {
++ // A word of two letters or fewer (no, de, te, je) is shared by too many languages to count fully.
++ let v = 0; for (const w of words) if (SETS[k].has(w)) v += w.length <= 2 ? 0.5 : 1;
++ if (MARKS[k]) v += Math.min(3, count(s.toLowerCase(), MARKS[k])) * 0.75;
++ score[k] = v;
++ }
++ const ranked = Object.keys(score).sort((a, b) => score[b] - score[a]);
++ const best = ranked[0], a = score[best], b = score[ranked[1]];
++ if (a < 1) return { lang: null, confidence: 0 };
++ let confidence = 0.4;
++ if (a >= 2 && a >= b * 2) confidence = 0.9;
++ else if (a >= 1 && b === 0 && words.length <= 4) confidence = 0.65;
++ else if (a > b) confidence = 0.55;
++ return { lang: best, confidence };
++ }
++ /** The language of `text` when it is confidently not `me` (a 2-letter code), else null. */
++ function foreignFor(text, me, min = 0.6) {
++ const d = detect(text);
++ return d.lang && d.confidence >= min && d.lang !== String(me || 'en').slice(0, 2) ? d.lang : null;
++ }
++ return { detect, foreignFor };
++});
ddiff --git a/public/popout-chat.html b/public/popout-chat.html
iindex b7de9424..fe7d5839 100644
---- a/public/popout-chat.html
++++ b/public/popout-chat.html
@@@ -234,6 +234,7 @@
<script src="/js/vibes.js"></script>
<script src="/js/opencoins.js"></script>
<script src="/js/cosmetics-ui.js"></script>
++ <script src="/js/lang-detect.js"></script>
<script src="/js/chat.js"></script>
<script src="/js/chat-autocomplete.js"></script>
<script>
ddiff --git a/server/i18n/routes.js b/server/i18n/routes.js
iindex 2d7cc0ad..2bb862fa 100644
---- a/server/i18n/routes.js
++++ b/server/i18n/routes.js
@@@ -19,11 +19,12 @@ const i18n = require('./translate');
const router = express.Router();
const MAX_CHARS = 600;
const _hits = new Map(); // ip → [timestamps]
--function allow(ip, signedIn) {
++function allow(ip, signedIn, weight = 1) {
const now = Date.now(), win = 60_000, max = signedIn ? 40 : 15;
const list = (_hits.get(ip) || []).filter(t => now - t < win);
-- if (list.length >= max) { _hits.set(ip, list); return false; }
-- list.push(now); _hits.set(ip, list);
++ if (list.length + weight > max) { _hits.set(ip, list); return false; }
++ for (let i = 0; i < weight; i++) list.push(now);
++ _hits.set(ip, list);
if (_hits.size > 5000) for (const [k, v] of _hits) if (!v.length || now - v[v.length - 1] > win) _hits.delete(k);
return true;
}
@@@ -50,4 +51,22 @@ router.post('/translate', optionalAuth, async (req, res) => {
}
});
++// Auto-translate in chat: up to 20 lines per request, one model call for the ones no one has asked for before
++// (translateMany). The limit counts lines, a quarter each (most come from the cache), so a busy chat cannot
++// out-spend the one-at-a-time button.
++router.post('/translate-batch', optionalAuth, async (req, res) => {
++ try {
++ if (!i18n.available()) return res.status(503).json({ error: 'Translation is off right now' });
++ const to = String(req.body?.to || 'en').trim().toLowerCase();
++ if (!i18n.isAllowedLang(to) || to === 'auto') return res.status(400).json({ error: 'Unknown target language' });
++ const texts = (Array.isArray(req.body?.texts) ? req.body.texts : []).slice(0, 20).map((t) => String(t || '').trim().slice(0, MAX_CHARS));
++ if (!texts.length) return res.status(400).json({ error: 'Nothing to translate' });
++ if (!allow(String(req.ip || ''), !!req.user, Math.ceil(texts.length / 4))) return res.status(429).json({ error: 'Slow down — too many translations' });
++ const results = await i18n.translateMany(texts, { to, context: 'chat' });
++ res.json({ to, to_name: i18n.langName(to), results });
++ } catch {
++ res.status(500).json({ error: 'Translation failed' });
++ }
++});
++
module.exports = router;
ddiff --git a/server/i18n/translate.js b/server/i18n/translate.js
iindex 88b7221c..22e0b311 100644
---- a/server/i18n/translate.js
++++ b/server/i18n/translate.js
@@@ -39,36 +39,16 @@ function langName(code) { return LANG_NAMES[code] || code || 'English'; }
function langFlag(code) { return LANG_FLAGS[code] || '🌐'; }
function isAllowedLang(code) { return ALLOWED.has(String(code || '')); }
--// ── Detection (script-based; no model call) ─────────────────────────────────────
--// Latin script is reported as 'en' — we cannot tell English from Spanish without a model,
--// and for direction-picking "latin vs the channel's script" is what matters.
++// ── Detection (no model call) ──────────────────────────────────────────────────────
++// public/js/lang-detect.js, the same file chat uses in the browser: scripts decide non-Latin text, common words and
++// letters decide Latin-script languages (Spanish, French, German, …). Latin text it cannot place (slang, names, one
++// word) is reported as 'en', as before, so direction-picking stays "Latin vs the channel's script" for it.
++const OVLang = require('../../public/js/lang-detect');
function detectLang(text) {
-- const s = String(text || '');
-- const count = (re) => (s.match(re) || []).length;
-- const kana = count(/[\u3040-\u30ff\uff66-\uff9f]/g);
-- const han = count(/[\u4e00-\u9fff\u3400-\u4dbf]/g);
-- const hangul = count(/[\uac00-\ud7af\u1100-\u11ff\u3130-\u318f]/g);
-- const cyr = count(/[\u0400-\u04ff]/g);
-- const arab = count(/[\u0600-\u06ff]/g);
-- const thai = count(/[\u0e00-\u0e7f]/g);
-- const heb = count(/[\u0590-\u05ff]/g);
-- const greek = count(/[\u0370-\u03ff]/g);
-- const deva = count(/[\u0900-\u097f]/g);
-- const latin = count(/[A-Za-z\u00c0-\u024f]/g);
-- const nonLatin = kana + han + hangul + cyr + arab + thai + heb + greek + deva;
-- if (!nonLatin && !latin) return null;
-- if (!nonLatin) return 'en';
-- if (nonLatin / (nonLatin + latin) < 0.34) return 'en'; // "lol 草" — mostly English
-- if (kana) return 'ja';
-- if (hangul) return 'ko';
-- if (han) return 'zh';
-- if (cyr) return 'ru';
-- if (arab) return 'ar';
-- if (thai) return 'th';
-- if (heb) return 'he';
-- if (greek) return 'el';
-- if (deva) return 'hi';
-- return 'en';
++ const d = OVLang.detect(text);
++ if (d.lang && d.confidence >= 0.6) return d.lang;
++ if (/[A-Za-z\u00c0-\u024f]/.test(String(text || ''))) return 'en';
++ return d.lang || null;
}
/**
@@@ -159,6 +139,23 @@ const MEM_MAX = 1500;
function _memGet(k) { const v = _mem.get(k); if (v !== undefined) { _mem.delete(k); _mem.set(k, v); } return v; }
function _memSet(k, v) { _mem.set(k, v); if (_mem.size > MEM_MAX) _mem.delete(_mem.keys().next().value); }
function cacheKey(text, from, to) { return crypto.createHash('sha1').update(`${from}|${to}|${String(text).trim()}`).digest('hex'); }
++/** A cached translation: the text, '' when the model said it needs none, undefined when not cached. */
++function cached(text, from, to) {
++ const key = cacheKey(text, from, to);
++ const hit = _memGet(key);
++ if (hit !== undefined) return hit;
++ _ensureTable();
++ try {
++ const row = db.get('SELECT text FROM translations WHERE key = ?', [key]);
++ if (row && row.text) { _memSet(key, row.text); return row.text; }
++ } catch { /* no table yet */ }
++ return undefined;
++}
++function remember(text, from, to, out) {
++ const key = cacheKey(text, from, to);
++ _memSet(key, out || '');
++ if (out) { try { db.run('INSERT OR REPLACE INTO translations (key, src, dst, text) VALUES (?, ?, ?, ?)', [key, from, to, out]); } catch { /* */ } }
++}
let _inflight = 0;
const MAX_INFLIGHT = 3;
@@@ -182,25 +179,18 @@ function _release() {
async function translate(text, { from = 'auto', to = 'en', context = 'chat', maxTokens } = {}) {
if (!translatable(text)) return null;
if (from === to) return null;
-- const key = cacheKey(text, from, to);
-- const hit = _memGet(key);
-- if (hit !== undefined) return hit;
-- _ensureTable();
-- try {
-- const row = db.get('SELECT text FROM translations WHERE key = ?', [key]);
-- if (row && row.text) { _memSet(key, row.text); return row.text; }
-- } catch { /* no table yet */ }
++ const hit = cached(text, from, to);
++ if (hit !== undefined) return hit || null;
if (!available()) return null;
const ok = await _slot();
if (!ok) return null;
try {
// The prompt lives in OpenVibe.AI (workflow live.translate).
const o = await aiService.structured('live.translate', { text: String(text).trim(), from, to, context, max_tokens: maxTokens || undefined });
-- if (o && o.unchanged) { _memSet(key, ''); return null; }
++ if (o && o.unchanged) { remember(text, from, to, ''); return null; }
const out = o && typeof o.text === 'string' ? o.text.trim() : '';
if (!out) return null;
-- _memSet(key, out);
-- try { db.run('INSERT OR REPLACE INTO translations (key, src, dst, text) VALUES (?, ?, ?, ?)', [key, from, to, out]); } catch { /* */ }
++ remember(text, from, to, out);
return out;
} catch { return null; }
finally { _release(); }
@@@ -214,6 +204,9 @@ async function translateChatMessage(message, channelUserId) {
if (!available() || !translatable(message)) return null;
const from = detectLang(message);
if (!from) return null;
++ // Translating foreign text for everyone spends a model call on every such line, so only when detection is sure
++ // (a lone 草 or a two-word line stays as it is; each viewer's translate button covers those).
++ if (from !== 'en' && OVLang.detect(message).confidence < 0.85) return null;
const chan = channelLanguage(channelUserId);
let to = null;
if (from !== 'en') to = 'en'; // foreign → English, always
@@@ -227,22 +220,52 @@ async function translateChatMessage(message, channelUserId) {
* Translate an array of speech lines (one LLM call), returning an array aligned to the
* input (null where a line could not be translated).
*/
--async function translateLines(lines, { from, to = 'en' } = {}) {
++async function translateLines(lines, { from, to = 'en', context = 'speech' } = {}) {
const src = (lines || []).map(l => String(l || '').trim());
const out = new Array(src.length).fill(null);
const idx = src.map((l, i) => translatable(l) ? i : -1).filter(i => i >= 0);
if (!idx.length || !available()) return out;
const joined = idx.map(i => src[i].replace(/\s*\n\s*/g, ' ')).join('\n');
-- const t = await translate(joined, { from, to, context: 'speech', maxTokens: 1200 });
++ const t = await translate(joined, { from, to, context, maxTokens: 1200 });
if (!t) return out;
const parts = t.split('\n').map(x => x.trim()).filter(Boolean);
if (parts.length === idx.length) { idx.forEach((i, k) => { out[i] = parts[k]; }); return out; }
// Line count drifted — fall back to one call per line (bounded).
-- for (const i of idx.slice(0, 8)) out[i] = await translate(src[i], { from, to, context: 'speech' });
++ for (const i of idx.slice(0, 8)) out[i] = await translate(src[i], { from, to, context });
++ return out;
++}
++
++/**
++ * Many chat lines to one language, for a viewer (the batch endpoint behind auto-translate). Each line is checked
++ * before any model call: nothing to translate → null; already in `to` → { same: true }; cached → the cached text.
++ * What is left goes to the model in ONE call (translateLines), and each line is cached on its own, so the next
++ * viewer asking for the same line costs nothing.
++ * @returns {Promise<Array<{from:string,text:string}|{same:true,from:string}|null>>} aligned to `texts`
++ */
++async function translateMany(texts, { to = 'en', context = 'chat' } = {}) {
++ const out = texts.map(() => null);
++ const need = [];
++ texts.forEach((t, i) => {
++ if (!translatable(t)) return;
++ const d = OVLang.detect(t);
++ if (!d.lang) return; // slang, emote names, one short word: no language to translate from, no call
++ const from = d.confidence >= 0.6 ? d.lang : 'auto';
++ if (from === to) { out[i] = { same: true, from }; return; }
++ const hit = cached(t, from, to);
++ if (hit !== undefined) { out[i] = hit ? { from, text: hit } : { same: true, from }; return; }
++ need.push({ i, t, from });
++ });
++ if (!need.length || !available()) return out;
++ const lines = await translateLines(need.map((n) => n.t), { from: 'auto', to, context });
++ need.forEach((n, k) => {
++ if (!lines[k]) return;
++ out[n.i] = { from: n.from, text: lines[k] };
++ remember(n.t, n.from, to, lines[k]);
++ });
return out;
}
module.exports = {
-- detectLang, detectForeignInText, translatable, channelLanguage, channelMeta, invalidateChannel, translate, translateChatMessage, translateLines,
++ detectLang, detectForeignInText, translatable, channelLanguage, channelMeta, invalidateChannel, translate, translateChatMessage, translateLines, translateMany,
langName, langFlag, isAllowedLang, available, siteEnabled, LANG_NAMES, LANG_FLAGS,
};
ddiff --git a/test/lang-detect.test.js b/test/lang-detect.test.js
nnew file mode 100644
iindex 00000000..a2c4aec7
---- /dev/null
++++ b/test/lang-detect.test.js
@@@ -0,0 +1,77 @@
++'use strict';
++/**
++ * Chat translation without waste: public/js/lang-detect.js (shared by the browser and server/i18n/translate.js) says
++ * which lines are in another language without a model call, and translateMany (the batch behind auto-translate)
++ * answers same-language and cached lines itself and sends the rest to the model in one call.
++ */
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++
++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-live-lang-'));
++process.env.DATA_DIR = tmp;
++process.env.DB_PATH = path.join(tmp, 'live.db');
++
++const L = require('../public/js/lang-detect');
++
++let passed = 0;
++async function check(name, fn) { await fn(); passed++; console.log(` ✓ ${name}`); }
++
++(async () => {
++ await check('detection: scripts, Latin-script languages by their words and letters; slang and one short word stay unknown', async () => {
++ const lang = (s) => L.detect(s).lang;
++ assert.strictEqual(lang('こんにちは、元気ですか'), 'ja');
++ assert.strictEqual(lang('안녕하세요'), 'ko');
++ assert.strictEqual(lang('Привет как дела'), 'ru');
++ assert.strictEqual(lang('Hola, ¿cómo estás? Muy bien gracias'), 'es');
++ assert.strictEqual(lang('Olá, tudo bem com você?'), 'pt');
++ assert.strictEqual(lang('Bonjour, comment ça va ?'), 'fr');
++ assert.strictEqual(lang('Guten Tag, wie geht es dir? Danke'), 'de');
++ assert.strictEqual(lang('hello everyone how are you doing'), 'en');
++ for (const s of ['gg', 'lol', 'pog', 'no', 'te amo', 'ok', ':Kappa:', 'https://example.com/x', '!sr song']) assert.strictEqual(lang(s), null, s);
++ });
++
++ await check('foreignFor: only lines confidently in another language than the viewer\'s get a button', async () => {
++ assert.strictEqual(L.foreignFor('Hola, ¿cómo estás? Muy bien gracias', 'en'), 'es');
++ assert.strictEqual(L.foreignFor('Hola, ¿cómo estás? Muy bien gracias', 'es'), null);
++ assert.strictEqual(L.foreignFor('this stream is so good', 'en'), null);
++ assert.strictEqual(L.foreignFor('this stream is so good', 'ja'), 'en');
++ assert.strictEqual(L.foreignFor('gg', 'ja'), null, 'nothing to translate');
++ });
++
++ const llm = require('../server/ai/llm');
++ const ai = require('../server/ai/ai-service');
++ const i18n = require('../server/i18n/translate');
++ llm.isEnabled = () => true;
++ const calls = [];
++ ai.structured = async (wf, input) => { calls.push(input); return { text: String(input.text).split('\n').map((l) => `[${input.to}] ${l}`).join('\n') }; };
++
++ await check('server detection agrees: Spanish is Spanish now (it used to read as English and could not be translated)', async () => {
++ assert.strictEqual(i18n.detectLang('Hola, ¿cómo estás? Muy bien gracias'), 'es');
++ assert.strictEqual(i18n.detectLang('gg'), 'en', 'Latin text it cannot place stays English, as before');
++ assert.strictEqual(i18n.detectLang('日本語'), 'zh');
++ });
++
++ await check('translateMany: same-language lines cost nothing, the rest go in ONE model call, and the next ask is cached', async () => {
++ const texts = ['Hola, ¿cómo estás? Muy bien gracias', 'hello everyone how are you doing', 'Guten Tag, wie geht es dir? Danke', 'gg'];
++ const r1 = await i18n.translateMany(texts, { to: 'en' });
++ assert.strictEqual(calls.length, 1, 'one call for both foreign lines');
++ assert.deepStrictEqual(r1.map((r) => r && (r.same ? 'same' : r.text)), ['[en] Hola, ¿cómo estás? Muy bien gracias', 'same', '[en] Guten Tag, wie geht es dir? Danke', null]);
++ const r2 = await i18n.translateMany(texts.slice(0, 3), { to: 'en' });
++ assert.strictEqual(calls.length, 1, 'answered from the cache');
++ assert.strictEqual(r2[2].text, '[en] Guten Tag, wie geht es dir? Danke');
++ });
++
++ await check('auto-translation for everyone only when detection is sure', async () => {
++ const n = calls.length;
++ assert.strictEqual(await i18n.translateChatMessage('de nada', null), null, 'two words: left to the viewer\'s button');
++ assert.strictEqual(calls.length, n);
++ const t = await i18n.translateChatMessage('Hola, ¿cómo estás? Muy bien gracias amigo', null);
++ assert.strictEqual(t.from, 'es'); assert.strictEqual(t.to, 'en');
++ });
++
++ console.log(`lang-detect: ${passed} checks passed`);
++ fs.rmSync(tmp, { recursive: true, force: true });
++ process.exit(0);
++})().catch((e) => { console.error(e); process.exit(1); });