openvibers/openvibe.ai Commits

AI: chat.vibe.listen, speech-to-text for short inline recordings (#235)

OP OpenVibers committed 12 days ago · b1c9f39a40 · 5 files, +96 −4

* AI: chat.vibe.listen transcribes inline short audio on live.stt

* Inline audio's temp file: an unguessable name, created exclusively

Changes

ddiff --git a/README.md b/README.md
iindex d4c3aa0..8e241ab 100644
---- a/README.md
++++ b/README.md
@@@ -531,6 +531,10 @@ the service boots without Valkey; with it on, production refuses to boot without
the tools Chat runs) passed through like `ai.complete` (the prompt is not kept) on route `chat.member`, so Chat's grant
and quota apply and a person's own key reaches only their provider; the choices come back with any tool calls, and
Chat sends the results in its next turn.
++ `chat.vibe.listen` is the Dock's voice on browsers without speech recognition: Chat sends a short recording
++ (audio/webm, audio/ogg or audio/mp4, at most 1 MB) inline as a data URL, the existing `transcribe` step runs it on
++ route `live.stt` (whisper.cpp/Parakeet), and the text comes back. The input is never retained (`retain_input:
++ false`); the engine writes it to a private temp file and deletes it in the same call.
- **Space's spaces** ([server/workflows/space.js](server/workflows/space.js), namespace `space`): `space.app.build`
is the AI builder of OpenVibe.Space. From what a person asks for (and a space's current files when changing one),
it returns a working space: name, description, the instructions its AI follows, the SDK features it uses, and up to
ddiff --git a/server/workflows/chat.js b/server/workflows/chat.js
iindex ab58d9c..4983a10 100644
---- a/server/workflows/chat.js
++++ b/server/workflows/chat.js
@@@ -64,6 +64,27 @@ const workflows = [
steps: [{ kind: 'llm', template: 'chat.member.reply', operation: 'chat', output: 'text', params: { max_tokens: 600, temperature: 0.7, timeout_ms: 25000 } }],
default_route: 'chat.member', cache_mode: 'none', metadata: { seed: true },
},
++ {
++ key: 'chat.vibe.listen', name: 'Chat: transcribe a short spoken message', namespace: 'chat',
++ description: 'Vibe by voice in the Dock where the browser has no speech recognition: a short recording (audio/webm, audio/ogg or audio/mp4) arrives inline and whisper.cpp/Parakeet on this host (route live.stt) returns the text. The audio is never stored.',
++ input_schema: OBJ({
++ audio: {
++ type: 'object', additionalProperties: false, required: ['data_url'],
++ properties: {
++ data_url: { type: 'string', maxLength: 1500000 },
++ mime: { enum: ['audio/webm', 'audio/ogg', 'audio/mp4'] },
++ },
++ },
++ language: { type: 'string', pattern: '^[a-z]{2}$' },
++ }, ['audio']),
++ output_schema: OBJ({
++ text: { type: 'string' },
++ language: STR(8),
++ segments: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' } } } },
++ }, ['text']),
++ steps: [{ kind: 'transcribe', route: 'live.stt' }],
++ cache_mode: 'none', metadata: { seed: true, retain_input: false },
++ },
{
key: 'chat.member.turn', name: 'Chat: an AI member\'s turn, with tools', namespace: 'chat',
description: 'One turn of a room\'s AI member as a Chat Completions request with the tools Chat runs (network search, the time, dice, a household\'s pantry); the choices come back with any tool calls. The prompt is not kept.',
ddiff --git a/server/workflows/engine.js b/server/workflows/engine.js
iindex 7db50eb..48d34d1 100644
---- a/server/workflows/engine.js
++++ b/server/workflows/engine.js
@@@ -8,7 +8,8 @@
* answer (structured JSON, or prose repaired like Live did), postprocess hook
* passthrough the caller supplies system/messages (Live features whose prompt Live still renders);
* routed by role (route_prefix.role) or a fixed route
-- * transcribe fetch the media (allow-listed hosts only) and run speech-to-text
++ * transcribe fetch the media (allow-listed hosts only), or take inline audio (a data URL, written to a
++ * private temp file deleted in the same call), and run speech-to-text
* embed embeddings
* media_analysis local-first media analysis: FFmpeg signals, scenes, local speech-to-text, highlights and
* an overview from a local model (or extractive; paid only with a budget). See media-analysis.js.
@@@ -25,6 +26,8 @@
* no output ever reads as verified when it is not.
*/
const fs = require('fs');
++const os = require('os');
++const path = require('path');
const { AiError, sha256, parseJsonLoose } = require('../util');
const { render } = require('../templates');
const { preferenceLines } = require('../user-modules');
@@@ -32,6 +35,24 @@ const schemas = require('../schemas');
const { PREPARE, POSTPROCESS } = require('./hooks');
const ROLE_TIMEOUT = { chat: 20000, vision: 30000, director: 25000, summary: 30000, legacy: 30000 };
++const INLINE_AUDIO_MAX = 1024 * 1024;
++
++/**
++ * Inline audio ({ audio: { data_url } }) -> a private temp file the transcribe step deletes. Chat's short spoken
++ * messages come this way, so the bytes never need a URL (and never touch storage beyond this call).
++ */
++function inlineAudioFile(audio) {
++ const raw = audio && audio.data_url;
++ const mm = /^data:(audio\/[a-z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+)$/i.exec(String(raw || ''));
++ if (!mm) throw new AiError(422, 'input.invalid', 'audio.data_url must be a base64 data:audio/* URL');
++ const bytes = Buffer.from(mm[2], 'base64');
++ if (!bytes.length) throw new AiError(422, 'input.invalid', 'the inline audio is empty');
++ if (bytes.length > INLINE_AUDIO_MAX) throw new AiError(413, 'source.too_large', 'inline audio is too large');
++ // An unguessable name, created only if nothing is there yet (never through a planted file or link).
++ const file = path.join(os.tmpdir(), `openvibe-ai-audio-${require('crypto').randomBytes(12).toString('hex')}`);
++ fs.writeFileSync(file, bytes, { mode: 0o600, flag: 'wx' });
++ return file;
++}
/** Every string in every `gaps` array of an output, in order, without repeats. */
function collectGaps(v, out = []) {
@@@ -241,9 +262,11 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
const route = await routeFor(step.route || 'live.stt');
// Read where it lies (the loopback reader), one window at a time: start_sec seeks with a range request, and
// timestamps are offset by start_sec unless offset_sec says otherwise. AI_MEDIA_STREAM=0 downloads first.
-- const streamed = !config.mediaAnalysis || config.mediaAnalysis.stream !== false;
++ // Inline audio (a short spoken message) needs no fetch: it becomes a private temp file, deleted below.
++ const inline = input.audio && input.audio.data_url ? inlineAudioFile(input.audio) : null;
++ const streamed = !inline && (!config.mediaAnalysis || config.mediaAnalysis.stream !== false);
const handle = streamed ? await reader().register(fetcher.mediaUrlOf(input)) : null;
-- const file = handle ? handle.url : (await fetcher.loadMediaToFile(input, { signal: ctx.signal })).file;
++ const file = inline || (handle ? handle.url : (await fetcher.loadMediaToFile(input, { signal: ctx.signal })).file);
try {
const startSec = Number(input.start_sec) || 0;
const offsetSec = input.offset_sec != null ? Number(input.offset_sec) : startSec;
ddiff --git a/test/transcribe-inline.test.js b/test/transcribe-inline.test.js
nnew file mode 100644
iindex 0000000..3f997c5
---- /dev/null
++++ b/test/transcribe-inline.test.js
@@@ -0,0 +1,43 @@
++'use strict';
++// chat.vibe.listen (Chat's Dock voice): inline audio is written to a private temp file for the existing
++// transcribe step, deleted in the same call, and the run record keeps only the data URL's digest.
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const { boot, request, token, suite, ALL } = require('./helpers');
++
++const t = suite('transcribe-inline');
++const tok = token('chat', ALL);
++const audioFiles = () => fs.readdirSync(os.tmpdir()).filter((f) => f.startsWith('openvibe-ai-audio-')).sort();
++let h;
++
++t.test('boot', async () => { h = await boot(); });
++
++t.test('a short recording is transcribed, and neither the run record nor a file keeps it', async () => {
++ const marker = 'OV-INLINE-AUDIO-MARKER';
++ const bytes = Buffer.concat([Buffer.from('OggS'), Buffer.from(marker)]);
++ const dataUrl = `data:audio/ogg;base64,${bytes.toString('base64')}`;
++ const before = audioFiles();
++ const r = await request(h.base, 'POST', '/api/v1/runs?wait=10000', { tok, body: { workflow: 'chat.vibe.listen', input: { audio: { data_url: dataUrl, mime: 'audio/ogg' }, language: 'en' } } });
++ assert.strictEqual(r.status, 201, r.text);
++ assert.strictEqual(r.body.run.status, 'succeeded', JSON.stringify(r.body.run.error));
++ assert.ok(r.body.run.output.text, 'the transcribe step answered');
++ const row = await h.db.prepare('SELECT input FROM runs WHERE id = ?').get(r.body.run.id);
++ assert.ok(!row.input.includes(marker), 'the audio bytes are not on the run record');
++ assert.ok(row.input.includes('not_retained'), 'the audio input is not retained at all, only its hash');
++ assert.deepStrictEqual(audioFiles(), before, 'the temp file was deleted in the same call');
++});
++
++t.test('Chat\'s chat.* grant reaches chat.vibe.listen, and nothing outside it', async () => {
++ const chatTok = token('chat', ALL, { ns: ['chat.*'] });
++ const ok = await request(h.base, 'POST', '/api/v1/runs?wait=10000', { tok: chatTok, body: { workflow: 'chat.vibe.listen', input: { audio: { data_url: 'data:audio/ogg;base64,T2dnUw==', mime: 'audio/ogg' } } } });
++ assert.strictEqual(ok.status, 201, ok.text);
++ assert.strictEqual(ok.body.run.status, 'succeeded', JSON.stringify(ok.body.run.error));
++ const no = await request(h.base, 'POST', '/api/v1/runs?wait=5000', { tok: chatTok, body: { workflow: 'live.media.transcribe', input: { media_url: 'https://openvibe.media/o/med_01J8Z3Q4R5S6T7V8W9X0Y1Z2A3' } } });
++ assert.strictEqual(no.status, 403, no.text);
++ assert.strictEqual(no.body.code, 'capability.namespace_denied');
++});
++
++t.test('done', async () => { await h.close(); });
++
++t.run();
ddiff --git a/test/workflows.test.js b/test/workflows.test.js
iindex 5ab29d0..88474be 100644
---- a/test/workflows.test.js
++++ b/test/workflows.test.js
@@@ -52,6 +52,7 @@ const EXAMPLES = {
'moderation.classify': { text: 'you absolute potato, gg', context: 'arena chat' },
'rent.screen_listing': { listing: { title: 'Bright room near the river', kind: 'room', price: 650, currency: 'EUR', period: 'month', city: 'Porto', country: 'PT', description: 'I am abroad for work; send the deposit by wire and I will mail the keys.' } },
'food.suggest_recipes': { pantry: ['rice', 'onions', 'canned tomatoes'], people: 2, sources: [{ source_type: 'food.food', source_id: 'lentils', title: 'Lentils, dried', content: '{"id":"lentils","unit":"kg"}' }] },
++ 'chat.vibe.listen': { audio: { data_url: 'data:audio/ogg;base64,T2dnUw==', mime: 'audio/ogg' }, language: 'en' },
'chat.member.reply': { ai_name: 'Pantry', persona: 'A cheerful cook who keeps it cheap.', room: 'Kim and Lee\'s kitchen', memory: 'Lee is vegetarian.', context: 'Pantry: rice, onions, canned tomatoes, lentils.', context_source: 'OpenVibe.Food household', transcript: 'kim: home late tonight\nlee: same', asked_by: 'kim', message: '@Pantry what can we cook in 20 minutes?' },
'food.read_pantry_photo': { image: { data_url: PNG }, kind: 'receipt', foods: ['Large Eggs (1 dozen)', 'Whole Milk (1 gallon)'] },
'pics.describe_image': { image: { data_url: PNG }, title: 'The coast' },
@@@ -101,7 +102,7 @@ t.test('boot seeds every required workflow key', async () => {
h = await boot({ env: { OV_MEDIA_INTERNAL_URL: mediaBase, AI_IMAGE_MODEL: 'gpt-image-1' } });
const keys = (await h.registry.listWorkflows()).map(w => w.key);
for (const k of ['wiki.generate_space', 'wiki.generate_page', 'blog.draft_post', 'blog.write_digest', 'blog.write_satire', 'news.summarize_story', 'news.compare_perspectives', 'reviews.summarize_entity', 'deals.enrich_deal',
-- 'coupons.extract_coupon', 'trade.summarize_market_context', 'codes.generate_docs', 'games.generate_lore', 'moderation.classify', 'tools.describe', 'food.suggest_recipes', 'food.read_pantry_photo', 'pics.describe_image', 'rent.screen_listing', 'chat.member.reply', 'chat.member.turn',
++ 'coupons.extract_coupon', 'trade.summarize_market_context', 'codes.generate_docs', 'games.generate_lore', 'moderation.classify', 'tools.describe', 'food.suggest_recipes', 'food.read_pantry_photo', 'pics.describe_image', 'rent.screen_listing', 'chat.member.reply', 'chat.member.turn', 'chat.vibe.listen',
'live.translate', 'live.stream.describe_frame', 'live.stream.summarize', 'live.viewers.reply', 'live.chat.insight', 'live.hero.slogans', 'live.stream.recap', 'network.site_copy']) {
assert.ok(keys.includes(k), `missing ${k}`);
}