openvibers/openvibe.ai Commits

Speech-to-text keeps word timing when asked (item 151, C12) (#274)

OP OpenVibers committed 10 days ago · 3eb1d6e3de · 6 files, +139 −16

Segments say when a whole sentence was spoken; a clip that should start and end on a word, or a moment whose words
must be found again, needs each word's time. live.media.transcribe takes words: true: whisper runs with -ojf and
its tokens become words (a leading space starts one, punctuation joins without moving the end, control tokens and
fillers are not words, Japanese gets one unit per token), Parakeet's token lines do the same, and the words ride
in their segment (dropped with it when it is noise). Without the flag nothing changes.

Changes

ddiff --git a/README.md b/README.md
iindex 76c7b75..3790e6c 100644
---- a/README.md
++++ b/README.md
@@@ -654,6 +654,7 @@ mix vectors from different providers (re-embed when the provider changes).
| The OpenAI Responses adapter (`kind: responses`) with a stubbed fetch: request shape, output and usage parsing, `text.format`, error mapping, the key never in an error | `test/provider-responses.test.js` |
| Ported adapters against fake OpenAI/Anthropic servers, stub determinism, whisper filter, templates | `test/providers.test.js` |
| Parakeet: engine by language, 30 s pieces past 45 s, fallback to whisper, `ASR_ENGINE=whisper` | `test/parakeet.test.js` |
++| Word timing (`live.media.transcribe` with `words: true`): whisper's `-ojf` tokens and Parakeet's token lines become each segment's words with their times; punctuation joins a word without moving its end; one unit per token in Japanese; fillers and control tokens are not words | `test/word-timing.test.js` |
| `media.analyze` on a real 28 s fixture: signals, scenes and loud moments measured, highlights with evidence, the local model's overview from the facts only; the local model down → extractive and said; paid only with `allow_paid` and a `media.paid` budget, never past it; too little disk refused before downloading | `test/media-analysis.test.js` |
| Import from a Live snapshot: dry run, holds, idempotent re-run | `test/import.test.js` |
| Capability proposals are valid contracts documents matching what is enforced | `test/proposals.test.js` |
ddiff --git a/server/providers/stub.js b/server/providers/stub.js
iindex 7cfacf5..90fcb7a 100644
---- a/server/providers/stub.js
++++ b/server/providers/stub.js
@@@ -135,7 +135,12 @@ function createStubProvider(record = { key: 'stub', metadata: {} }, { stats = nu
if (op === 'transcribe') {
const n = 2 + Math.floor(rnd() * 3);
const at = Number(req.offsetSec) || 0; // a window of a longer recording keeps the recording's times
-- const segments = Array.from({ length: n }, (_, i) => ({ start: at + i * 4, end: at + i * 4 + 3.5, text: `(synthetic) ${phrase(rnd, 5)}` }));
++ const segments = Array.from({ length: n }, (_, i) => {
++ const seg = { start: at + i * 4, end: at + i * 4 + 3.5, text: `(synthetic) ${phrase(rnd, 5)}` };
++ // words: true → each word with an even share of its segment's time (the shape whisper gives).
++ if (req.words === true) { const ws = seg.text.split(/\s+/); seg.words = ws.map((text, k) => ({ start: Math.round((seg.start + (3.5 * k) / ws.length) * 100) / 100, end: Math.round((seg.start + (3.5 * (k + 1)) / ws.length) * 100) / 100, text })); }
++ return seg;
++ });
return { text: segments.map(s => s.text).join(' '), segments, model, synthetic: true, usage: { input: 0, output: 0, cached: 0, estimated: true } };
}
let json = null;
ddiff --git a/server/providers/whisper.js b/server/providers/whisper.js
iindex d3601d4..9866be1 100644
---- a/server/providers/whisper.js
++++ b/server/providers/whisper.js
@@@ -49,6 +49,7 @@ function isNoise(t, hasVad) {
return false;
}
++const r2 = (v) => Math.round(v * 100) / 100;
function cleanSegments(segsRaw, hasVad) {
const out = [];
let last = '';
@@@ -58,24 +59,67 @@ function cleanSegments(segsRaw, hasVad) {
const norm = text.toLowerCase();
if (norm === last) continue;
last = norm;
-- out.push({ start: Math.round(seg.start * 100) / 100, end: Math.round(seg.end * 100) / 100, text });
++ const row = { start: r2(seg.start), end: r2(seg.end), text };
++ // Word timing, when it was asked for (item 151, C12): it goes with its segment, or not at all.
++ if (Array.isArray(seg.words)) row.words = seg.words.map((wd) => ({ start: r2(wd.start), end: r2(wd.end), text: wd.text }));
++ out.push(row);
}
return out;
}
++// ── Word timing (item 151, C12) ──
++// Clips start and end on words, and a moment's words can be found again, only if the times of the words survive: the
++// segments alone say when a whole sentence was spoken. whisper.cpp's -ojf names every token with its offsets (ms):
++// a token that starts with a space starts a word (" comp" + "iles"), punctuation joins the word before it without
++// moving its end, and control tokens ([_BEG_], [_TT_315], <|…|>) are not speech. A language written without spaces
++// (Japanese, Chinese) has no word breaks to read, so each of its tokens is a unit of its own.
++const FILLER_WORD = /^(?:uh+|um+|uhm|erm|mm+)[,.!?]*$/i;
++const SPACELESS = /[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff\uac00-\ud7af]/;
++const PUNCT = /^[\p{P}\p{S}\s]+$/u;
++function wordsFromTokens(tokens, offsetSec = 0) {
++ const out = [];
++ let cur = null;
++ for (const tk of Array.isArray(tokens) ? tokens : []) {
++ const raw = String((tk && tk.text) || '');
++ if (!raw.trim() || /^\[_.*\]$/.test(raw.trim()) || /^<\|.*\|>$/.test(raw.trim())) continue;
++ const o = (tk && tk.offsets) || {};
++ const from = Number(o.from), to = Number(o.to);
++ if (!Number.isFinite(from) || !Number.isFinite(to) || to < from) continue;
++ if (PUNCT.test(raw)) { if (cur) cur.text += raw.trim(); continue; }
++ const starts = !cur || /^\s/.test(raw) || SPACELESS.test(raw) || SPACELESS.test(cur.text);
++ if (starts) {
++ if (cur) out.push(cur);
++ cur = { start: offsetSec + from / 1000, end: offsetSec + to / 1000, text: raw.trim() };
++ } else {
++ cur.text += raw;
++ cur.end = Math.max(cur.end, offsetSec + to / 1000);
++ }
++ }
++ if (cur) out.push(cur);
++ return out.filter((wd) => wd.text && !FILLER_WORD.test(wd.text));
++}
++
// ── Parakeet output → timed sentences (Live's parseParakeet) ──
// `parakeet-cli -ps` prints every token on stderr with its start and end in centiseconds and whether it starts a word
// ("▁For"). A sentence ends at . ? or !, at a pause of 1.5 s, or after 20 s. Fillers ("uh", "um") are dropped.
const PARAKEET_LANGS = new Set(['bg', 'hr', 'cs', 'da', 'nl', 'en', 'et', 'fi', 'fr', 'de', 'el', 'hu', 'it', 'lv', 'lt', 'mt', 'pl', 'pt', 'ro', 'sk', 'sl', 'es', 'sv', 'ru', 'uk']);
const TOKEN_LINE = /t0=\s*(-?\d+)\s+t1=\s*(-?\d+)\s+word_start=(true|false)\s+"(.*)"\s*$/;
const FILLER_RE = /(^|[\s,])(?:uh+|um+|uhm|erm|mm+)(?=[\s,.!?]|$)[,]?/gi;
--function parseParakeet(stderr, offsetSec = 0) {
++function parseParakeet(stderr, offsetSec = 0, { words = false } = {}) {
const out = [];
let cur = null;
++ let word = null;
++ const closeWord = () => {
++ if (!word || !cur) { word = null; return; }
++ const text = word.text.trim();
++ if (text && !FILLER_WORD.test(text)) cur.words.push({ start: offsetSec + word.t0 / 100, end: offsetSec + word.t1 / 100, text });
++ word = null;
++ };
const close = () => {
if (!cur) return;
++ closeWord();
const text = cur.text.replace(FILLER_RE, '$1').replace(/\s+([,.!?])/g, '$1').replace(/\s+/g, ' ').replace(/^[,\s]+/, '').trim();
-- if (text) out.push({ start: offsetSec + cur.t0 / 100, end: offsetSec + cur.t1 / 100, text });
++ if (text) out.push({ start: offsetSec + cur.t0 / 100, end: offsetSec + cur.t1 / 100, text, ...(words ? { words: cur.words } : {}) });
cur = null;
};
for (const line of String(stderr || '').split('\n')) {
@@@ -87,9 +131,13 @@ function parseParakeet(stderr, offsetSec = 0) {
if (/^<.*>$/.test(raw)) continue; // <unk>, <|…|> control tokens
const piece = raw.replace(/\u2581/g, ' ');
if (cur && wordStart && (t0 - cur.t1 > 150 || t0 - cur.t0 > 2000)) close();
-- if (!cur) cur = { t0, t1, text: '' };
++ if (!cur) cur = { t0, t1, text: '', words: [] };
cur.text += wordStart && !piece.startsWith(' ') ? ` ${piece}` : piece;
cur.t1 = Math.max(cur.t1, t1);
++ // A word is the tokens from one word_start to the next; punctuation joins it without moving its end.
++ if (wordStart || !word) { closeWord(); word = { t0, t1, text: piece.trim() }; }
++ else if (PUNCT.test(piece)) word.text += piece.trim();
++ else { word.text += piece; word.t1 = Math.max(word.t1, t1); }
if (/[.!?]["')\]]?$/.test(piece.trim())) close();
}
close();
@@@ -162,7 +210,7 @@ function createWhisperProvider(record, { config }) {
});
}
-- function decode(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, signal, timeoutMs }) {
++ function decode(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, words = false, signal, timeoutMs }) {
return new Promise((resolve, reject) => {
const lang = language && language !== 'en' && multi() ? String(language) : 'en';
const model = lang !== 'en' ? multi() : (live && exists(MODEL_LIVE) ? MODEL_LIVE : MODEL);
@@@ -170,7 +218,8 @@ function createWhisperProvider(record, { config }) {
const threads = live || lowPower ? Math.max(1, Math.min(2, w.threads)) : w.threads;
const outBase = `${wavPath}.out`;
const jsonPath = `${outBase}.json`;
-- const args = ['-m', model, '-f', wavPath, '-oj', '-of', outBase, '-t', String(threads), '-l', lang];
++ // -ojf adds every token with its offsets (word timing, C12); only when asked, since it is a far larger file.
++ const args = ['-m', model, '-f', wavPath, words ? '-ojf' : '-oj', '-of', outBase, '-t', String(threads), '-l', lang];
if (w.beam > 1) args.push('-bs', String(w.beam));
const vm = vadModel();
if (vm) args.push('--vad', '-vm', vm);
@@@ -190,6 +239,7 @@ function createWhisperProvider(record, { config }) {
start: offsetSec + ((it.offsets && it.offsets.from) || 0) / 1000,
end: offsetSec + ((it.offsets && it.offsets.to) || 0) / 1000,
text: it.text || '',
++ ...(words ? { words: wordsFromTokens(it.tokens, offsetSec) } : {}),
})), Boolean(vm));
// With -l auto, whisper.cpp names the language it detected in result.language.
const detected = parsed && parsed.result && /^[a-z]{2,3}$/.test(String(parsed.result.language || '')) ? parsed.result.language : null;
@@@ -200,7 +250,7 @@ function createWhisperProvider(record, { config }) {
}
// One parakeet-cli run on a short WAV: the tokens on stderr become timed sentences.
-- function parakeetOnce(wavPath, { threads, offsetSec, signal, deadline }) {
++ function parakeetOnce(wavPath, { threads, offsetSec, words = false, signal, deadline }) {
return new Promise((resolve, reject) => {
let ch;
try { ch = spawnTracked('nice', ['-n', '15', parakeetBin(), '-t', String(threads), '-ps', '-np', '-m', w.parakeetModel, '-f', wavPath], { stderr: true }); }
@@@ -214,13 +264,13 @@ function createWhisperProvider(record, { config }) {
clearTimeout(timer);
if (signal && signal.aborted) return reject(new ProviderError('cancelled', { code: 'provider.cancelled' }));
if (code !== 0) return reject(new ProviderError(`parakeet exited ${code}`, { code: 'provider.error' }));
-- resolve(parseParakeet(Buffer.concat(chunks).toString('utf8'), offsetSec));
++ resolve(parseParakeet(Buffer.concat(chunks).toString('utf8'), offsetSec, { words }));
});
ch.on('error', (e) => { clearTimeout(timer); reject(new ProviderError(e.message, { code: 'provider.unavailable' })); });
});
}
// Parakeet on a whole WAV: in 30 s pieces past 45 s (PCM cut exactly, so a piece's times shift only by its start).
-- async function decodeParakeet(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, signal, timeoutMs }) {
++ async function decodeParakeet(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, words = false, signal, timeoutMs }) {
const threads = live || lowPower ? Math.max(1, Math.min(2, w.threads)) : w.threads;
const deadline = Date.now() + timeoutMs;
let pieces = [wavPath], dir = null;
@@@ -237,7 +287,7 @@ function createWhisperProvider(record, { config }) {
const raw = [];
for (let i = 0; i < pieces.length; i++) {
if (Date.now() >= deadline) throw new ProviderError('parakeet timed out', { code: 'provider.error' });
-- raw.push(...await parakeetOnce(pieces[i], { threads, offsetSec: offsetSec + i * CHUNK_SEC, signal, deadline }));
++ raw.push(...await parakeetOnce(pieces[i], { threads, offsetSec: offsetSec + i * CHUNK_SEC, words, signal, deadline }));
}
const segments = cleanSegments(raw, true);
return { text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: String(language || 'en'), model: path.basename(w.parakeetModel) };
@@@ -254,7 +304,7 @@ function createWhisperProvider(record, { config }) {
try {
await acquire(lane);
try {
-- const opts = { language: req.language, live: req.live, lowPower: req.lowPower === true, offsetSec: req.offsetSec || 0, signal: req.signal, timeoutMs };
++ const opts = { language: req.language, live: req.live, lowPower: req.lowPower === true, offsetSec: req.offsetSec || 0, words: req.words === true, signal: req.signal, timeoutMs };
let r = null;
if (engineFor(req.language) === 'parakeet') {
try { r = await decodeParakeet(wav, opts); }
@@@ -280,4 +330,4 @@ function createWhisperProvider(record, { config }) {
};
}
--module.exports = { createWhisperProvider, cleanSegments, isNoise, parseParakeet };
++module.exports = { createWhisperProvider, cleanSegments, isNoise, parseParakeet, wordsFromTokens };
ddiff --git a/server/workflows/engine.js b/server/workflows/engine.js
iindex 48d34d1..198eccb 100644
---- a/server/workflows/engine.js
++++ b/server/workflows/engine.js
@@@ -270,7 +270,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
try {
const startSec = Number(input.start_sec) || 0;
const offsetSec = input.offset_sec != null ? Number(input.offset_sec) : startSec;
-- const exec = await pool.execute(route, 'transcribe', { filePath: file, language: input.language || 'en', seconds: input.seconds || 0, startSec, offsetSec, lowPower: input.low_power === true, timeoutMs: 3600000 }, { ...ctx, routeKey: route.key, routeVersion: route.version, promptHash: null });
++ const exec = await pool.execute(route, 'transcribe', { filePath: file, language: input.language || 'en', seconds: input.seconds || 0, startSec, offsetSec, lowPower: input.low_power === true, words: input.words === true, timeoutMs: 3600000 }, { ...ctx, routeKey: route.key, routeVersion: route.version, promptHash: null });
const r = exec.result;
return { output: { text: r.text || '', language: r.language || input.language || 'en', segments: r.segments || [] }, exec, route };
} finally {
ddiff --git a/server/workflows/live.js b/server/workflows/live.js
iindex d2a798b..5bfd7c8 100644
---- a/server/workflows/live.js
++++ b/server/workflows/live.js
@@@ -498,8 +498,9 @@ const structured = [
{
key: 'live.media.transcribe', name: 'Speech-to-text for a VOD, clip or audio file', namespace: 'live',
description: 'whisper.cpp on this host (Live\'s transcribe.js), media fetched only from allow-listed OpenVibe hosts.',
-- input_schema: TEXT_OUT({ media: MEDIA_REF, media_url: STR(2048), language: LANG, seconds: { type: 'integer', minimum: 0, maximum: 36000 }, start_sec: { type: 'number', minimum: 0 }, offset_sec: { type: 'number', minimum: 0 }, low_power: { type: 'boolean' } }, []),
-- output_schema: TEXT_OUT({ text: { type: 'string' }, language: STR(8), segments: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' } } } } }, ['text', 'segments']),
++ input_schema: TEXT_OUT({ media: MEDIA_REF, media_url: STR(2048), language: LANG, seconds: { type: 'integer', minimum: 0, maximum: 36000 }, start_sec: { type: 'number', minimum: 0 }, offset_sec: { type: 'number', minimum: 0 }, low_power: { type: 'boolean' }, words: { type: 'boolean' } }, []),
++ // words: true adds each segment's words with their times (item 151, C12), so a clip can start and end on a word.
++ output_schema: TEXT_OUT({ text: { type: 'string' }, language: STR(8), segments: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' }, words: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' } } } } } } } }, ['text', 'segments']),
steps: [{ kind: 'transcribe', route: 'live.stt' }],
cache_mode: 'private',
},
ddiff --git a/test/word-timing.test.js b/test/word-timing.test.js
nnew file mode 100644
iindex 0000000..819108d
---- /dev/null
++++ b/test/word-timing.test.js
@@@ -0,0 +1,66 @@
++'use strict';
++// Word timing (item 151, C12): with `words: true`, live.media.transcribe keeps every word's start and end inside its
++// segment, so a clip can start and end on a word and a moment's words can be found again. whisper.cpp's -ojf tokens
++// (the fixture is what the host's whisper-cli large-v3-turbo printed for a synthetic voice on 2026-11-03) and
++// Parakeet's token lines both become words; punctuation joins its word without moving the word's end; control tokens
++// and fillers are not words; a language written without spaces gets one unit per token; times shift by the window's
++// offset; words go with their segment when a noise segment is dropped; nothing changes when words are not asked for.
++const assert = require('assert');
++const { wordsFromTokens, parseParakeet, cleanSegments } = require('../server/providers/whisper');
++const live = require('../server/workflows/live');
++
++// ── whisper.cpp -ojf ──
++const tok = (text, from, to) => ({ text, offsets: { from, to } });
++const host = [
++ tok('[_BEG_]', 0, 0), tok(' it', 0, 100), tok(' comp', 110, 340), tok('iles', 340, 570), tok('.', 570, 760),
++ tok(' Finally', 760, 1360), tok(',', 1360, 1540), tok(' after', 1540, 2020), tok(' three', 2020, 2300),
++ tok(' hours', 2300, 2520), tok(' of', 2520, 2610), tok(' trying', 2740, 2990), tok(',', 3000, 3090),
++ tok(' Chat', 4570, 4660), tok(',', 5010, 5040), tok(' did', 5040, 5270), tok(' you', 5270, 5500), tok(' see', 5500, 5720),
++ tok(' that', 5730, 6030), tok('?', 6040, 6280), tok('[_TT_315]', 6300, 6300),
++];
++const words = wordsFromTokens(host, 0);
++assert.deepStrictEqual(words.map((w) => w.text), ['it', 'compiles.', 'Finally,', 'after', 'three', 'hours', 'of', 'trying,', 'Chat,', 'did', 'you', 'see', 'that?']);
++const compiles = words[1];
++assert.strictEqual(compiles.start, 0.11);
++assert.strictEqual(compiles.end, 0.57, 'the full stop joins the word without stretching it to 0.76');
++assert.strictEqual(words[words.length - 1].end, 6.03);
++assert.strictEqual(wordsFromTokens(host, 120)[1].start, 120.11, 'a window of a longer recording keeps the recording\'s times');
++
++// Japanese: no spaces to break on, so each token is a unit; 「。」 joins the unit before it. Fillers and control tokens go.
++assert.deepStrictEqual(wordsFromTokens([tok(' 今日', 0, 300), tok('は', 300, 420), tok('ね', 420, 600), tok('。', 600, 700), tok('<|endoftext|>', 700, 700)], 0).map((w) => w.text), ['今日', 'は', 'ね。']);
++assert.deepStrictEqual(wordsFromTokens([tok(' so', 0, 100), tok(' uh', 100, 300), tok(' yes', 300, 500), tok(' broken', 600, 500), { text: ' x' }], 0).map((w) => w.text), ['so', 'yes'],
++ 'fillers, backwards and untimed tokens are not words');
++assert.deepStrictEqual(wordsFromTokens(null), []);
++
++// ── Parakeet's token lines ──
++const pk = [
++ 'tok t0= 10 t1= 40 word_start=true "▁IT"',
++ 'tok t0= 41 t1= 60 word_start=true "▁COM"',
++ 'tok t0= 60 t1= 90 word_start=false "PILES"',
++ 'tok t0= 90 t1= 95 word_start=false "."',
++ 'tok t0= 300 t1= 320 word_start=true "▁um"',
++ 'tok t0= 330 t1= 360 word_start=true "▁Wow"',
++ 'tok t0= 360 t1= 370 word_start=false "!"',
++].join('\n');
++const sentences = parseParakeet(pk, 100, { words: true });
++assert.deepStrictEqual(sentences.map((s) => s.text), ['IT COMPILES.', 'Wow!']);
++assert.deepStrictEqual(sentences[0].words, [{ start: 100.1, end: 100.4, text: 'IT' }, { start: 100.41, end: 100.9, text: 'COMPILES.' }]);
++assert.deepStrictEqual(sentences[1].words.map((w) => w.text), ['Wow!'], 'the filler is not a word');
++assert.ok(!('words' in parseParakeet(pk, 0)[0]), 'no words unless asked');
++
++// ── cleanSegments keeps words with their segment, and drops them with it ──
++const cleaned = cleanSegments([
++ { start: 1, end: 2, text: 'IT COMPILES.', words: [{ start: 1.004, end: 1.5, text: 'IT' }, { start: 1.5, end: 1.996, text: 'COMPILES.' }] },
++ { start: 3, end: 4, text: 'Thank you for watching!', words: [{ start: 3, end: 4, text: 'Thank' }] },
++], false);
++assert.strictEqual(cleaned.length, 1, 'whisper\'s invented sign-off is dropped, its words with it');
++assert.deepStrictEqual(cleaned[0].words, [{ start: 1, end: 1.5, text: 'IT' }, { start: 1.5, end: 2, text: 'COMPILES.' }]);
++assert.ok(!('words' in cleanSegments([{ start: 0, end: 1, text: 'plain' }], true)[0]));
++
++// ── The workflow asks for them and its output allows them ──
++const wf = (live.workflows || []).find((x) => x.key === 'live.media.transcribe');
++assert.ok(wf, 'live.media.transcribe exists');
++assert.deepStrictEqual(wf.input_schema.properties.words, { type: 'boolean' });
++assert.ok(wf.output_schema.properties.segments.items.properties.words, 'segments may carry words');
++
++console.log('word timing: all checks passed');