openvibers/openvibe.chat Commits

Chat: POST /api/vibe/listen, speech-to-text for the Dock's voice (#288)

OP OpenVibers committed 13 days ago · 4407ca7004 · 2 files, +173 −4

Changes

ddiff --git a/server/rooms/vibe-api.js b/server/rooms/vibe-api.js
iindex ff56519..beadaa3 100644
---- a/server/rooms/vibe-api.js
++++ b/server/rooms/vibe-api.js
@@@ -12,12 +12,18 @@
* post into that chat, else the latest one if it moved in the last 6 hours, else
* a new chat (fresh: always new). Vibe answers in the background, as on Chat.
* → { chat, message }
++ * POST /api/vibe/listen?lang=en the body is a short recording (Content-Type audio/webm, audio/ogg or
++ * audio/mp4, at most 1 MB) → { text }: Vibe's speech-to-text, for browsers
++ * without their own (plan L48 V2). The person's own Network token, never an API
++ * or Dock token; 20 a minute and 200 an hour each. The bytes stay in memory,
++ * go to OpenVibe.AI inline with Chat's service token, and are never stored.
*/
const express = require('express');
const { requireAuth } = require('../auth/auth');
const { limits } = require('../net/actor-limits');
const rooms = require('./rooms');
const vibe = require('./vibe');
++const aiClient = require('../ai/client');
const router = express.Router();
router.use(express.json({ limit: '32kb' }));
@@@ -25,6 +31,11 @@ router.use(requireAuth);
const SIX_HOURS = 6 * 3600 * 1000;
const asked = limits('chat.vibe.ask', { minute: 12, hour: 300 });
++const listened = limits('chat.vibe.listen', { minute: 20, hour: 200 });
++const AUDIO_MAX = 1024 * 1024;
++const AUDIO_TYPES = ['audio/webm', 'audio/ogg', 'audio/mp4'];
++/** Vibe speaks for nobody but the person signed in: API tokens get the rooms API instead. */
++const personOnly = (req, res, next) => (req.authSource === 'api_token' || req.authSource === 'dock' ? res.status(403).json({ error: 'Vibe is for the person themself', code: 'vibe.person_only' }) : next());
const zone = (tz) => { try { new Intl.DateTimeFormat('en-US', { timeZone: String(tz) }); return String(tz).slice(0, 64); } catch { return null; } };
const at = (ts) => { const d = new Date(String(ts || '').includes('T') ? ts : `${String(ts || '').replace(' ', 'T')}Z`); return Number.isNaN(d.getTime()) ? null : d.toISOString(); };
const wire = (m) => ({ id: Number(m.id), from: m.ai_member_id ? 'vibe' : 'you', text: String(m.message || ''), at: at(m.created_at) });
@@@ -36,8 +47,7 @@ async function latest(user) {
return first ? await vibe.ownChat(first.slug, user) : null;
}
--router.get('/chat', async (req, res) => {
-- if (req.authSource === 'api_token' || req.authSource === 'dock') return res.status(403).json({ error: 'Vibe is for the person themself' });
++router.get('/chat', personOnly, async (req, res) => {
const room = req.query.slug ? await vibe.ownChat(String(req.query.slug), req.user) : await latest(req.user);
if (!room) return res.set('Cache-Control', 'private, no-store').json({ chat: null, messages: [] });
const after = Number(req.query.after) || null;
@@@ -45,8 +55,47 @@ router.get('/chat', async (req, res) => {
return res.set('Cache-Control', 'private, no-store').json({ chat: chatOf(room), messages: rows.filter((m) => !m.is_deleted).map(wire) });
});
--router.post('/ask', asked, async (req, res) => {
-- if (req.authSource === 'api_token' || req.authSource === 'dock') return res.status(403).json({ error: 'Vibe is for the person themself' });
++
++const readAudio = express.raw({ type: AUDIO_TYPES, limit: AUDIO_MAX + 1 });
++const audioBody = (req, res, next) => readAudio(req, res, (err) => {
++ if (err && err.type === 'entity.too.large') return res.status(413).json({ error: 'That recording is too long: keep it under 30 seconds', code: 'vibe.listen_too_large' });
++ return err ? next(err) : next();
++});
++
++/** The recording's type, without MediaRecorder's `;codecs=opus`. */
++const audioType = (req) => String(req.get('content-type') || '').split(';')[0].trim().toLowerCase();
++
++/**
++ * A short recording → { text }. The bytes live in this request only: they are handed to OpenVibe.AI inline
++ * (data URL, chat.vibe.listen → its live.stt route) and are never written to disk or logged here. AI retains no
++ * input for that workflow at all (only its hash), and deletes its temp file in the same call. `force` is the
++ * person waiting, not a background job: it runs even when CHAT_AI_ENABLED (the insight jobs' switch) is off.
++ */
++router.post('/listen', personOnly, listened, audioBody, async (req, res) => {
++ const type = audioType(req);
++ if (!AUDIO_TYPES.includes(type)) return res.status(415).json({ error: 'Send the recording as audio/webm, audio/ogg or audio/mp4', code: 'vibe.listen_type' });
++ const bytes = Buffer.isBuffer(req.body) ? req.body : null;
++ if (!bytes || !bytes.length) return res.status(422).json({ error: 'The recording is empty', code: 'vibe.listen_empty' });
++ if (bytes.length > AUDIO_MAX) return res.status(413).json({ error: 'That recording is too long: keep it under 30 seconds', code: 'vibe.listen_too_large' });
++ const lang = req.query.lang == null || req.query.lang === '' ? null : String(req.query.lang).toLowerCase();
++ if (lang && !/^[a-z]{2}$/.test(lang)) return res.status(422).json({ error: 'lang is a two-letter language code, like en', code: 'vibe.listen_lang' });
++ const dataUrl = `data:${type};base64,${bytes.toString('base64')}`;
++ const subject = req.user.subject_id ? String(req.user.subject_id) : `user:${req.user.id}`;
++ let out = null;
++ try {
++ out = await aiClient.structured('chat.vibe.listen', {
++ audio: { data_url: dataUrl, mime: type },
++ ...(lang ? { language: lang } : {}),
++ }, { force: true, attribution: { service: 'chat', type: 'user', id: subject } });
++ } finally {
++ bytes.fill(0); // the request's copy goes as soon as AI has its own; nothing is kept after the answer
++ }
++ const text = String((out && out.text) || '').replace(/\s+/g, ' ').trim();
++ if (!text) return res.status(503).json({ error: 'I could not hear that just now. Try again.', code: 'vibe.listen_unavailable' });
++ return res.set('Cache-Control', 'private, no-store').json({ text });
++});
++
++router.post('/ask', personOnly, asked, async (req, res) => {
const b = req.body && typeof req.body === 'object' ? req.body : {};
const message = String(b.message || '').replace(/\r\n/g, '\n').trim();
if (!message) return res.status(422).json({ error: 'Write something first' });
ddiff --git a/test/vibe-listen.test.js b/test/vibe-listen.test.js
nnew file mode 100644
iindex 0000000..2778b9c
---- /dev/null
++++ b/test/vibe-listen.test.js
@@@ -0,0 +1,120 @@
++'use strict';
++/**
++ * Vibe by voice for browsers without their own speech recognition (plan L48 V2; server/rooms/vibe-api.js
++ * POST /api/vibe/listen): a short recording goes to OpenVibe.AI inline (chat.vibe.listen → live.stt) and comes
++ * back as { text }. Only the person themself, 20 a minute / 200 an hour, the audio never written down or logged.
++ */
++const assert = require('assert');
++const crypto = require('crypto');
++const fs = require('fs');
++const path = require('path');
++const { ids } = require('openvibe-contracts');
++const { boot, suite, b64url } = require('./helpers');
++
++const t = suite('vibe listen');
++const now = () => Math.floor(Date.now() / 1000);
++const jwt = (h, claims) => {
++ const head = `${b64url(JSON.stringify({ alg: 'RS256', typ: 'JWT' }))}.${b64url(JSON.stringify(claims))}`;
++ return `${head}.${b64url(crypto.sign('RSA-SHA256', Buffer.from(head), h.keys.privateKey))}`;
++};
++const dock = (h, u) => jwt(h, { iss: `${h.ISS}/dock`, sub: String(u.id), id: u.id, subject_id: u.subject_id, username: u.username, client_id: 'companion', scope: 'chat.read', fam: `fam_${'a'.repeat(24)}`, iat: now(), exp: now() + 3600 });
++const OGG = Buffer.concat([Buffer.from('OggS'), Buffer.from('vibe-listen-test-audio')]);
++const MARKER = 'OV-LISTEN-AUDIO-MARKER';
++
++let h, ann, apiUser;
++let seen = null;
++let fail = false;
++
++const listen = (token, body, { type = 'audio/ogg', query = '', headers = {} } = {}) =>
++ h.http('POST', `/api/vibe/listen${query}`, { token, raw: body, headers: { 'Content-Type': type, ...headers } });
++
++function filesUnder(dir) {
++ return fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => e.isDirectory() ? filesUnder(path.join(dir, e.name)) : [path.join(dir, e.name)]);
++}
++
++t('boot with a stand-in OpenVibe.AI', async () => {
++ h = await boot({ env: { CHAT_WEB_URL: 'https://openvibe.chat' } });
++ const aiClient = require('../server/ai/client');
++ aiClient._setClient({
++ runs: {
++ create: async (workflow, input, opts) => {
++ if (fail) throw new Error('AI is down');
++ seen = { workflow, input, opts };
++ return { run: { status: 'succeeded', output: { text: 'Remind me to stretch.', language: 'en', segments: [{ start: 0, end: 2, text: 'Remind me to stretch.' }] } } };
++ },
++ },
++ });
++ ann = h.addUser('ann', { subject: ids.newId('user') });
++ apiUser = h.addUser('bot', { subject: ids.newId('user'), apiScopes: ['chat'] });
++ await h.ctx.upsertUser(h.live.users.get(ann.id));
++ await h.ctx.upsertUser(h.live.users.get(apiUser.id));
++});
++
++t('a recording comes back as text, sent to AI inline', async () => {
++ const r = await listen(ann.token, OGG, { query: '?lang=en' });
++ assert.strictEqual(r.status, 200, r.text);
++ assert.deepStrictEqual(r.body, { text: 'Remind me to stretch.' });
++ assert.strictEqual(seen.workflow, 'chat.vibe.listen');
++ assert.strictEqual(seen.input.audio.mime, 'audio/ogg');
++ assert.strictEqual(seen.input.language, 'en');
++ assert.deepStrictEqual(seen.opts.attribution, { service: 'chat', type: 'user', id: ann.subject_id }, 'the run is attributed to the person who spoke');
++ assert.ok(seen, 'AI was called although CHAT_AI_ENABLED is off: the person is waiting, not a background job');
++ const b64 = /^data:audio\/ogg;base64,(.+)$/.exec(seen.input.audio.data_url);
++ assert.ok(b64, seen.input.audio.data_url.slice(0, 40));
++ assert.deepStrictEqual(Buffer.from(b64[1], 'base64'), OGG, 'the same bytes AI received');
++ assert.strictEqual(r.headers.get('cache-control'), 'private, no-store');
++});
++
++t('a MediaRecorder content type with codecs is accepted', async () => {
++ const r = await listen(ann.token, OGG, { type: 'audio/webm;codecs=opus' });
++ assert.strictEqual(r.status, 200, r.text);
++ assert.strictEqual(seen.input.audio.mime, 'audio/webm');
++ assert.ok(seen.input.audio.data_url.startsWith('data:audio/webm;base64,'));
++});
++
++t('only the person themself: signed out, an API token and the Dock token are refused', async () => {
++ assert.strictEqual((await listen(null, OGG)).status, 401);
++ assert.strictEqual((await listen(apiUser.token, OGG)).status, 403);
++ assert.strictEqual((await listen(dock(h, ann), OGG)).status, 401, 'a Dock token is a read-only DM token and never authenticates a write');
++});
++
++t('a wrong type, an empty body and a bad lang are refused', async () => {
++ let r = await listen(ann.token, Buffer.from('{"a":1}'), { type: 'application/json' });
++ assert.strictEqual(r.status, 415, r.text);
++ r = await listen(ann.token, Buffer.alloc(0));
++ assert.strictEqual(r.status, 422, r.text);
++ r = await listen(ann.token, OGG, { query: '?lang=english' });
++ assert.strictEqual(r.status, 422, r.text);
++});
++
++t('more than 1 MB is refused before AI is called', async () => {
++ seen = null;
++ const r = await listen(ann.token, Buffer.alloc(1024 * 1024 + 2, 1));
++ assert.strictEqual(r.status, 413, r.text);
++ assert.strictEqual(seen, null, 'nothing reached AI');
++});
++
++t('20 a minute per person', async () => {
++ const rita = h.addUser('rita', { subject: ids.newId('user') });
++ await h.ctx.upsertUser(h.live.users.get(rita.id));
++ for (let i = 0; i < 20; i++) assert.strictEqual((await listen(rita.token, OGG)).status, 200, `request ${i + 1}`);
++ assert.strictEqual((await listen(rita.token, OGG)).status, 429);
++});
++
++t('AI down answers a problem, and the recording is never written down', async () => {
++ fail = true;
++ try {
++ const r = await listen(ann.token, Buffer.concat([OGG, Buffer.from(MARKER)]));
++ assert.strictEqual(r.status, 503, r.text);
++ assert.strictEqual(r.body.code, 'vibe.listen_unavailable');
++ } finally { fail = false; }
++ const leaked = filesUnder(h.tmp).filter((f) => fs.readFileSync(f).includes(MARKER));
++ assert.deepStrictEqual(leaked, [], 'no file under Chat\'s data directory holds the audio');
++});
++
++t('done', async () => {
++ require('../server/ai/client')._setClient(null);
++ await h.close();
++});
++
++t.run();