openvibers/openvibe.ai Commits
Speech-to-text keeps word timing when asked (item 151, C12) (#274)
Changes
@@@ -654,6 +654,7 @@ mix vectors from different providers (re-embed when the provider changes).| The OpenAI Responses adapter (`kind: responses`) with a stubbed fetch: request shape, output and usage parsing, `text.format`, error mapping, the key never in an error | `test/provider-responses.test.js` || Ported adapters against fake OpenAI/Anthropic servers, stub determinism, whisper filter, templates | `test/providers.test.js` || Parakeet: engine by language, 30 s pieces past 45 s, fallback to whisper, `ASR_ENGINE=whisper` | `test/parakeet.test.js` |++| Word timing (`live.media.transcribe` with `words: true`): whisper's `-ojf` tokens and Parakeet's token lines become each segment's words with their times; punctuation joins a word without moving its end; one unit per token in Japanese; fillers and control tokens are not words | `test/word-timing.test.js` || `media.analyze` on a real 28 s fixture: signals, scenes and loud moments measured, highlights with evidence, the local model's overview from the facts only; the local model down → extractive and said; paid only with `allow_paid` and a `media.paid` budget, never past it; too little disk refused before downloading | `test/media-analysis.test.js` || Import from a Live snapshot: dry run, holds, idempotent re-run | `test/import.test.js` || Capability proposals are valid contracts documents matching what is enforced | `test/proposals.test.js` |@@@ -135,7 +135,12 @@ function createStubProvider(record = { key: 'stub', metadata: {} }, { stats = nuif (op === 'transcribe') {const n = 2 + Math.floor(rnd() * 3);const at = Number(req.offsetSec) || 0; // a window of a longer recording keeps the recording's times-- const segments = Array.from({ length: n }, (_, i) => ({ start: at + i * 4, end: at + i * 4 + 3.5, text: `(synthetic) ${phrase(rnd, 5)}` }));++ const segments = Array.from({ length: n }, (_, i) => {++ const seg = { start: at + i * 4, end: at + i * 4 + 3.5, text: `(synthetic) ${phrase(rnd, 5)}` };++ // words: true → each word with an even share of its segment's time (the shape whisper gives).++ if (req.words === true) { const ws = seg.text.split(/\s+/); seg.words = ws.map((text, k) => ({ start: Math.round((seg.start + (3.5 * k) / ws.length) * 100) / 100, end: Math.round((seg.start + (3.5 * (k + 1)) / ws.length) * 100) / 100, text })); }++ return seg;++ });return { text: segments.map(s => s.text).join(' '), segments, model, synthetic: true, usage: { input: 0, output: 0, cached: 0, estimated: true } };}let json = null;@@@ -49,6 +49,7 @@ function isNoise(t, hasVad) {return false;}++const r2 = (v) => Math.round(v * 100) / 100;function cleanSegments(segsRaw, hasVad) {const out = [];let last = '';@@@ -58,24 +59,67 @@ function cleanSegments(segsRaw, hasVad) {const norm = text.toLowerCase();if (norm === last) continue;last = norm;-- out.push({ start: Math.round(seg.start * 100) / 100, end: Math.round(seg.end * 100) / 100, text });++ const row = { start: r2(seg.start), end: r2(seg.end), text };++ // Word timing, when it was asked for (item 151, C12): it goes with its segment, or not at all.++ if (Array.isArray(seg.words)) row.words = seg.words.map((wd) => ({ start: r2(wd.start), end: r2(wd.end), text: wd.text }));++ out.push(row);}return out;}++// ── Word timing (item 151, C12) ──++// Clips start and end on words, and a moment's words can be found again, only if the times of the words survive: the++// segments alone say when a whole sentence was spoken. whisper.cpp's -ojf names every token with its offsets (ms):++// a token that starts with a space starts a word (" comp" + "iles"), punctuation joins the word before it without++// moving its end, and control tokens ([_BEG_], [_TT_315], <|…|>) are not speech. A language written without spaces++// (Japanese, Chinese) has no word breaks to read, so each of its tokens is a unit of its own.++const FILLER_WORD = /^(?:uh+|um+|uhm|erm|mm+)[,.!?]*$/i;++const SPACELESS = /[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff\uac00-\ud7af]/;++const PUNCT = /^[\p{P}\p{S}\s]+$/u;++function wordsFromTokens(tokens, offsetSec = 0) {++ const out = [];++ let cur = null;++ for (const tk of Array.isArray(tokens) ? tokens : []) {++ const raw = String((tk && tk.text) || '');++ if (!raw.trim() || /^\[_.*\]$/.test(raw.trim()) || /^<\|.*\|>$/.test(raw.trim())) continue;++ const o = (tk && tk.offsets) || {};++ const from = Number(o.from), to = Number(o.to);++ if (!Number.isFinite(from) || !Number.isFinite(to) || to < from) continue;++ if (PUNCT.test(raw)) { if (cur) cur.text += raw.trim(); continue; }++ const starts = !cur || /^\s/.test(raw) || SPACELESS.test(raw) || SPACELESS.test(cur.text);++ if (starts) {++ if (cur) out.push(cur);++ cur = { start: offsetSec + from / 1000, end: offsetSec + to / 1000, text: raw.trim() };++ } else {++ cur.text += raw;++ cur.end = Math.max(cur.end, offsetSec + to / 1000);++ }++ }++ if (cur) out.push(cur);++ return out.filter((wd) => wd.text && !FILLER_WORD.test(wd.text));++}++// ── Parakeet output → timed sentences (Live's parseParakeet) ──// `parakeet-cli -ps` prints every token on stderr with its start and end in centiseconds and whether it starts a word// ("▁For"). A sentence ends at . ? or !, at a pause of 1.5 s, or after 20 s. Fillers ("uh", "um") are dropped.const PARAKEET_LANGS = new Set(['bg', 'hr', 'cs', 'da', 'nl', 'en', 'et', 'fi', 'fr', 'de', 'el', 'hu', 'it', 'lv', 'lt', 'mt', 'pl', 'pt', 'ro', 'sk', 'sl', 'es', 'sv', 'ru', 'uk']);const TOKEN_LINE = /t0=\s*(-?\d+)\s+t1=\s*(-?\d+)\s+word_start=(true|false)\s+"(.*)"\s*$/;const FILLER_RE = /(^|[\s,])(?:uh+|um+|uhm|erm|mm+)(?=[\s,.!?]|$)[,]?/gi;--function parseParakeet(stderr, offsetSec = 0) {++function parseParakeet(stderr, offsetSec = 0, { words = false } = {}) {const out = [];let cur = null;++ let word = null;++ const closeWord = () => {++ if (!word || !cur) { word = null; return; }++ const text = word.text.trim();++ if (text && !FILLER_WORD.test(text)) cur.words.push({ start: offsetSec + word.t0 / 100, end: offsetSec + word.t1 / 100, text });++ word = null;++ };const close = () => {if (!cur) return;++ closeWord();const text = cur.text.replace(FILLER_RE, '$1').replace(/\s+([,.!?])/g, '$1').replace(/\s+/g, ' ').replace(/^[,\s]+/, '').trim();-- if (text) out.push({ start: offsetSec + cur.t0 / 100, end: offsetSec + cur.t1 / 100, text });++ if (text) out.push({ start: offsetSec + cur.t0 / 100, end: offsetSec + cur.t1 / 100, text, ...(words ? { words: cur.words } : {}) });cur = null;};for (const line of String(stderr || '').split('\n')) {@@@ -87,9 +131,13 @@ function parseParakeet(stderr, offsetSec = 0) {if (/^<.*>$/.test(raw)) continue; // <unk>, <|…|> control tokensconst piece = raw.replace(/\u2581/g, ' ');if (cur && wordStart && (t0 - cur.t1 > 150 || t0 - cur.t0 > 2000)) close();-- if (!cur) cur = { t0, t1, text: '' };++ if (!cur) cur = { t0, t1, text: '', words: [] };cur.text += wordStart && !piece.startsWith(' ') ? ` ${piece}` : piece;cur.t1 = Math.max(cur.t1, t1);++ // A word is the tokens from one word_start to the next; punctuation joins it without moving its end.++ if (wordStart || !word) { closeWord(); word = { t0, t1, text: piece.trim() }; }++ else if (PUNCT.test(piece)) word.text += piece.trim();++ else { word.text += piece; word.t1 = Math.max(word.t1, t1); }if (/[.!?]["')\]]?$/.test(piece.trim())) close();}close();@@@ -162,7 +210,7 @@ function createWhisperProvider(record, { config }) {});}-- function decode(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, signal, timeoutMs }) {++ function decode(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, words = false, signal, timeoutMs }) {return new Promise((resolve, reject) => {const lang = language && language !== 'en' && multi() ? String(language) : 'en';const model = lang !== 'en' ? multi() : (live && exists(MODEL_LIVE) ? MODEL_LIVE : MODEL);@@@ -170,7 +218,8 @@ function createWhisperProvider(record, { config }) {const threads = live || lowPower ? Math.max(1, Math.min(2, w.threads)) : w.threads;const outBase = `${wavPath}.out`;const jsonPath = `${outBase}.json`;-- const args = ['-m', model, '-f', wavPath, '-oj', '-of', outBase, '-t', String(threads), '-l', lang];++ // -ojf adds every token with its offsets (word timing, C12); only when asked, since it is a far larger file.++ const args = ['-m', model, '-f', wavPath, words ? '-ojf' : '-oj', '-of', outBase, '-t', String(threads), '-l', lang];if (w.beam > 1) args.push('-bs', String(w.beam));const vm = vadModel();if (vm) args.push('--vad', '-vm', vm);@@@ -190,6 +239,7 @@ function createWhisperProvider(record, { config }) {start: offsetSec + ((it.offsets && it.offsets.from) || 0) / 1000,end: offsetSec + ((it.offsets && it.offsets.to) || 0) / 1000,text: it.text || '',++ ...(words ? { words: wordsFromTokens(it.tokens, offsetSec) } : {}),})), Boolean(vm));// With -l auto, whisper.cpp names the language it detected in result.language.const detected = parsed && parsed.result && /^[a-z]{2,3}$/.test(String(parsed.result.language || '')) ? parsed.result.language : null;@@@ -200,7 +250,7 @@ function createWhisperProvider(record, { config }) {}// One parakeet-cli run on a short WAV: the tokens on stderr become timed sentences.-- function parakeetOnce(wavPath, { threads, offsetSec, signal, deadline }) {++ function parakeetOnce(wavPath, { threads, offsetSec, words = false, signal, deadline }) {return new Promise((resolve, reject) => {let ch;try { ch = spawnTracked('nice', ['-n', '15', parakeetBin(), '-t', String(threads), '-ps', '-np', '-m', w.parakeetModel, '-f', wavPath], { stderr: true }); }@@@ -214,13 +264,13 @@ function createWhisperProvider(record, { config }) {clearTimeout(timer);if (signal && signal.aborted) return reject(new ProviderError('cancelled', { code: 'provider.cancelled' }));if (code !== 0) return reject(new ProviderError(`parakeet exited ${code}`, { code: 'provider.error' }));-- resolve(parseParakeet(Buffer.concat(chunks).toString('utf8'), offsetSec));++ resolve(parseParakeet(Buffer.concat(chunks).toString('utf8'), offsetSec, { words }));});ch.on('error', (e) => { clearTimeout(timer); reject(new ProviderError(e.message, { code: 'provider.unavailable' })); });});}// Parakeet on a whole WAV: in 30 s pieces past 45 s (PCM cut exactly, so a piece's times shift only by its start).-- async function decodeParakeet(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, signal, timeoutMs }) {++ async function decodeParakeet(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, words = false, signal, timeoutMs }) {const threads = live || lowPower ? Math.max(1, Math.min(2, w.threads)) : w.threads;const deadline = Date.now() + timeoutMs;let pieces = [wavPath], dir = null;@@@ -237,7 +287,7 @@ function createWhisperProvider(record, { config }) {const raw = [];for (let i = 0; i < pieces.length; i++) {if (Date.now() >= deadline) throw new ProviderError('parakeet timed out', { code: 'provider.error' });-- raw.push(...await parakeetOnce(pieces[i], { threads, offsetSec: offsetSec + i * CHUNK_SEC, signal, deadline }));++ raw.push(...await parakeetOnce(pieces[i], { threads, offsetSec: offsetSec + i * CHUNK_SEC, words, signal, deadline }));}const segments = cleanSegments(raw, true);return { text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: String(language || 'en'), model: path.basename(w.parakeetModel) };@@@ -254,7 +304,7 @@ function createWhisperProvider(record, { config }) {try {await acquire(lane);try {-- const opts = { language: req.language, live: req.live, lowPower: req.lowPower === true, offsetSec: req.offsetSec || 0, signal: req.signal, timeoutMs };++ const opts = { language: req.language, live: req.live, lowPower: req.lowPower === true, offsetSec: req.offsetSec || 0, words: req.words === true, signal: req.signal, timeoutMs };let r = null;if (engineFor(req.language) === 'parakeet') {try { r = await decodeParakeet(wav, opts); }@@@ -280,4 +330,4 @@ function createWhisperProvider(record, { config }) {};}--module.exports = { createWhisperProvider, cleanSegments, isNoise, parseParakeet };++module.exports = { createWhisperProvider, cleanSegments, isNoise, parseParakeet, wordsFromTokens };@@@ -270,7 +270,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spatry {const startSec = Number(input.start_sec) || 0;const offsetSec = input.offset_sec != null ? Number(input.offset_sec) : startSec;-- const exec = await pool.execute(route, 'transcribe', { filePath: file, language: input.language || 'en', seconds: input.seconds || 0, startSec, offsetSec, lowPower: input.low_power === true, timeoutMs: 3600000 }, { ...ctx, routeKey: route.key, routeVersion: route.version, promptHash: null });++ const exec = await pool.execute(route, 'transcribe', { filePath: file, language: input.language || 'en', seconds: input.seconds || 0, startSec, offsetSec, lowPower: input.low_power === true, words: input.words === true, timeoutMs: 3600000 }, { ...ctx, routeKey: route.key, routeVersion: route.version, promptHash: null });const r = exec.result;return { output: { text: r.text || '', language: r.language || input.language || 'en', segments: r.segments || [] }, exec, route };} finally {@@@ -498,8 +498,9 @@ const structured = [{key: 'live.media.transcribe', name: 'Speech-to-text for a VOD, clip or audio file', namespace: 'live',description: 'whisper.cpp on this host (Live\'s transcribe.js), media fetched only from allow-listed OpenVibe hosts.',-- input_schema: TEXT_OUT({ media: MEDIA_REF, media_url: STR(2048), language: LANG, seconds: { type: 'integer', minimum: 0, maximum: 36000 }, start_sec: { type: 'number', minimum: 0 }, offset_sec: { type: 'number', minimum: 0 }, low_power: { type: 'boolean' } }, []),-- output_schema: TEXT_OUT({ text: { type: 'string' }, language: STR(8), segments: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' } } } } }, ['text', 'segments']),++ input_schema: TEXT_OUT({ media: MEDIA_REF, media_url: STR(2048), language: LANG, seconds: { type: 'integer', minimum: 0, maximum: 36000 }, start_sec: { type: 'number', minimum: 0 }, offset_sec: { type: 'number', minimum: 0 }, low_power: { type: 'boolean' }, words: { type: 'boolean' } }, []),++ // words: true adds each segment's words with their times (item 151, C12), so a clip can start and end on a word.++ output_schema: TEXT_OUT({ text: { type: 'string' }, language: STR(8), segments: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' }, words: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' } } } } } } } }, ['text', 'segments']),steps: [{ kind: 'transcribe', route: 'live.stt' }],cache_mode: 'private',},nnew file mode 100644@@@ -0,0 +1,66 @@++'use strict';++// Word timing (item 151, C12): with `words: true`, live.media.transcribe keeps every word's start and end inside its++// segment, so a clip can start and end on a word and a moment's words can be found again. whisper.cpp's -ojf tokens++// (the fixture is what the host's whisper-cli large-v3-turbo printed for a synthetic voice on 2026-11-03) and++// Parakeet's token lines both become words; punctuation joins its word without moving the word's end; control tokens++// and fillers are not words; a language written without spaces gets one unit per token; times shift by the window's++// offset; words go with their segment when a noise segment is dropped; nothing changes when words are not asked for.++const assert = require('assert');++const { wordsFromTokens, parseParakeet, cleanSegments } = require('../server/providers/whisper');++const live = require('../server/workflows/live');++++// ── whisper.cpp -ojf ──++const tok = (text, from, to) => ({ text, offsets: { from, to } });++const host = [++ tok('[_BEG_]', 0, 0), tok(' it', 0, 100), tok(' comp', 110, 340), tok('iles', 340, 570), tok('.', 570, 760),++ tok(' Finally', 760, 1360), tok(',', 1360, 1540), tok(' after', 1540, 2020), tok(' three', 2020, 2300),++ tok(' hours', 2300, 2520), tok(' of', 2520, 2610), tok(' trying', 2740, 2990), tok(',', 3000, 3090),++ tok(' Chat', 4570, 4660), tok(',', 5010, 5040), tok(' did', 5040, 5270), tok(' you', 5270, 5500), tok(' see', 5500, 5720),++ tok(' that', 5730, 6030), tok('?', 6040, 6280), tok('[_TT_315]', 6300, 6300),++];++const words = wordsFromTokens(host, 0);++assert.deepStrictEqual(words.map((w) => w.text), ['it', 'compiles.', 'Finally,', 'after', 'three', 'hours', 'of', 'trying,', 'Chat,', 'did', 'you', 'see', 'that?']);++const compiles = words[1];++assert.strictEqual(compiles.start, 0.11);++assert.strictEqual(compiles.end, 0.57, 'the full stop joins the word without stretching it to 0.76');++assert.strictEqual(words[words.length - 1].end, 6.03);++assert.strictEqual(wordsFromTokens(host, 120)[1].start, 120.11, 'a window of a longer recording keeps the recording\'s times');++++// Japanese: no spaces to break on, so each token is a unit; 「。」 joins the unit before it. Fillers and control tokens go.++assert.deepStrictEqual(wordsFromTokens([tok(' 今日', 0, 300), tok('は', 300, 420), tok('ね', 420, 600), tok('。', 600, 700), tok('<|endoftext|>', 700, 700)], 0).map((w) => w.text), ['今日', 'は', 'ね。']);++assert.deepStrictEqual(wordsFromTokens([tok(' so', 0, 100), tok(' uh', 100, 300), tok(' yes', 300, 500), tok(' broken', 600, 500), { text: ' x' }], 0).map((w) => w.text), ['so', 'yes'],++ 'fillers, backwards and untimed tokens are not words');++assert.deepStrictEqual(wordsFromTokens(null), []);++++// ── Parakeet's token lines ──++const pk = [++ 'tok t0= 10 t1= 40 word_start=true "▁IT"',++ 'tok t0= 41 t1= 60 word_start=true "▁COM"',++ 'tok t0= 60 t1= 90 word_start=false "PILES"',++ 'tok t0= 90 t1= 95 word_start=false "."',++ 'tok t0= 300 t1= 320 word_start=true "▁um"',++ 'tok t0= 330 t1= 360 word_start=true "▁Wow"',++ 'tok t0= 360 t1= 370 word_start=false "!"',++].join('\n');++const sentences = parseParakeet(pk, 100, { words: true });++assert.deepStrictEqual(sentences.map((s) => s.text), ['IT COMPILES.', 'Wow!']);++assert.deepStrictEqual(sentences[0].words, [{ start: 100.1, end: 100.4, text: 'IT' }, { start: 100.41, end: 100.9, text: 'COMPILES.' }]);++assert.deepStrictEqual(sentences[1].words.map((w) => w.text), ['Wow!'], 'the filler is not a word');++assert.ok(!('words' in parseParakeet(pk, 0)[0]), 'no words unless asked');++++// ── cleanSegments keeps words with their segment, and drops them with it ──++const cleaned = cleanSegments([++ { start: 1, end: 2, text: 'IT COMPILES.', words: [{ start: 1.004, end: 1.5, text: 'IT' }, { start: 1.5, end: 1.996, text: 'COMPILES.' }] },++ { start: 3, end: 4, text: 'Thank you for watching!', words: [{ start: 3, end: 4, text: 'Thank' }] },++], false);++assert.strictEqual(cleaned.length, 1, 'whisper\'s invented sign-off is dropped, its words with it');++assert.deepStrictEqual(cleaned[0].words, [{ start: 1, end: 1.5, text: 'IT' }, { start: 1.5, end: 2, text: 'COMPILES.' }]);++assert.ok(!('words' in cleanSegments([{ start: 0, end: 1, text: 'plain' }], true)[0]));++++// ── The workflow asks for them and its output allows them ──++const wf = (live.workflows || []).find((x) => x.key === 'live.media.transcribe');++assert.ok(wf, 'live.media.transcribe exists');++assert.deepStrictEqual(wf.input_schema.properties.words, { type: 'boolean' });++assert.ok(wf.output_schema.properties.segments.items.properties.words, 'segments may carry words');++++console.log('word timing: all checks passed');
@@ -654,6 +654,7 @@ mix vectors from different providers (re-embed when the provider changes).
@@ -654,6 +654,7 @@ mix vectors from different providers (re-embed when the provider changes).
| The OpenAI Responses adapter (`kind: responses`) with a stubbed fetch: request shape, output and usage parsing, `text.format`, error mapping, the key never in an error | `test/provider-responses.test.js` |
| The OpenAI Responses adapter (`kind: responses`) with a stubbed fetch: request shape, output and usage parsing, `text.format`, error mapping, the key never in an error | `test/provider-responses.test.js` |
| Ported adapters against fake OpenAI/Anthropic servers, stub determinism, whisper filter, templates | `test/providers.test.js` |
| Ported adapters against fake OpenAI/Anthropic servers, stub determinism, whisper filter, templates | `test/providers.test.js` |
| Parakeet: engine by language, 30 s pieces past 45 s, fallback to whisper, `ASR_ENGINE=whisper` | `test/parakeet.test.js` |
| Parakeet: engine by language, 30 s pieces past 45 s, fallback to whisper, `ASR_ENGINE=whisper` | `test/parakeet.test.js` |
+| Word timing (`live.media.transcribe` with `words: true`): whisper's `-ojf` tokens and Parakeet's token lines become each segment's words with their times; punctuation joins a word without moving its end; one unit per token in Japanese; fillers and control tokens are not words | `test/word-timing.test.js` |
| `media.analyze` on a real 28 s fixture: signals, scenes and loud moments measured, highlights with evidence, the local model's overview from the facts only; the local model down → extractive and said; paid only with `allow_paid` and a `media.paid` budget, never past it; too little disk refused before downloading | `test/media-analysis.test.js` |
| `media.analyze` on a real 28 s fixture: signals, scenes and loud moments measured, highlights with evidence, the local model's overview from the facts only; the local model down → extractive and said; paid only with `allow_paid` and a `media.paid` budget, never past it; too little disk refused before downloading | `test/media-analysis.test.js` |
| Import from a Live snapshot: dry run, holds, idempotent re-run | `test/import.test.js` |
| Import from a Live snapshot: dry run, holds, idempotent re-run | `test/import.test.js` |
| Capability proposals are valid contracts documents matching what is enforced | `test/proposals.test.js` |
| Capability proposals are valid contracts documents matching what is enforced | `test/proposals.test.js` |
@@ -135,7 +135,12 @@ function createStubProvider(record = { key: 'stub', metadata: {} }, { stats = nu
@@ -135,7 +135,12 @@ function createStubProvider(record = { key: 'stub', metadata: {} }, { stats = nu
if (op === 'transcribe') {
if (op === 'transcribe') {
const n = 2 + Math.floor(rnd() * 3);
const n = 2 + Math.floor(rnd() * 3);
const at = Number(req.offsetSec) || 0; // a window of a longer recording keeps the recording's times
const at = Number(req.offsetSec) || 0; // a window of a longer recording keeps the recording's times
- const segments = Array.from({ length: n }, (_, i) => ({ start: at + i * 4, end: at + i * 4 + 3.5, text: `(synthetic) ${phrase(rnd, 5)}` }));
+ const segments = Array.from({ length: n }, (_, i) => {
+ const seg = { start: at + i * 4, end: at + i * 4 + 3.5, text: `(synthetic) ${phrase(rnd, 5)}` };
+ // words: true → each word with an even share of its segment's time (the shape whisper gives).
+ if (req.words === true) { const ws = seg.text.split(/\s+/); seg.words = ws.map((text, k) => ({ start: Math.round((seg.start + (3.5 * k) / ws.length) * 100) / 100, end: Math.round((seg.start + (3.5 * (k + 1)) / ws.length) * 100) / 100, text })); }
+ return seg;
+ });
return { text: segments.map(s => s.text).join(' '), segments, model, synthetic: true, usage: { input: 0, output: 0, cached: 0, estimated: true } };
return { text: segments.map(s => s.text).join(' '), segments, model, synthetic: true, usage: { input: 0, output: 0, cached: 0, estimated: true } };
}
}
let json = null;
let json = null;
@@ -49,6 +49,7 @@ function isNoise(t, hasVad) {
@@ -49,6 +49,7 @@ function isNoise(t, hasVad) {
return false;
return false;
}
}
+const r2 = (v) => Math.round(v * 100) / 100;
function cleanSegments(segsRaw, hasVad) {
function cleanSegments(segsRaw, hasVad) {
const out = [];
const out = [];
let last = '';
let last = '';
@@ -58,24 +59,67 @@ function cleanSegments(segsRaw, hasVad) {
@@ -58,24 +59,67 @@ function cleanSegments(segsRaw, hasVad) {
const norm = text.toLowerCase();
const norm = text.toLowerCase();
if (norm === last) continue;
if (norm === last) continue;
last = norm;
last = norm;
- out.push({ start: Math.round(seg.start * 100) / 100, end: Math.round(seg.end * 100) / 100, text });
+ const row = { start: r2(seg.start), end: r2(seg.end), text };
+ // Word timing, when it was asked for (item 151, C12): it goes with its segment, or not at all.
+ if (Array.isArray(seg.words)) row.words = seg.words.map((wd) => ({ start: r2(wd.start), end: r2(wd.end), text: wd.text }));
+ out.push(row);
}
}
return out;
return out;
}
}
+// ── Word timing (item 151, C12) ──
+// Clips start and end on words, and a moment's words can be found again, only if the times of the words survive: the
+// segments alone say when a whole sentence was spoken. whisper.cpp's -ojf names every token with its offsets (ms):
+// a token that starts with a space starts a word (" comp" + "iles"), punctuation joins the word before it without
+// moving its end, and control tokens ([_BEG_], [_TT_315], <|…|>) are not speech. A language written without spaces
+// (Japanese, Chinese) has no word breaks to read, so each of its tokens is a unit of its own.
+const FILLER_WORD = /^(?:uh+|um+|uhm|erm|mm+)[,.!?]*$/i;
+const SPACELESS = /[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff\uac00-\ud7af]/;
+const PUNCT = /^[\p{P}\p{S}\s]+$/u;
+function wordsFromTokens(tokens, offsetSec = 0) {
+ const out = [];
+ let cur = null;
+ for (const tk of Array.isArray(tokens) ? tokens : []) {
+ const raw = String((tk && tk.text) || '');
+ if (!raw.trim() || /^\[_.*\]$/.test(raw.trim()) || /^<\|.*\|>$/.test(raw.trim())) continue;
+ const o = (tk && tk.offsets) || {};
+ const from = Number(o.from), to = Number(o.to);
+ if (!Number.isFinite(from) || !Number.isFinite(to) || to < from) continue;
+ if (PUNCT.test(raw)) { if (cur) cur.text += raw.trim(); continue; }
+ const starts = !cur || /^\s/.test(raw) || SPACELESS.test(raw) || SPACELESS.test(cur.text);
+ if (starts) {
+ if (cur) out.push(cur);
+ cur = { start: offsetSec + from / 1000, end: offsetSec + to / 1000, text: raw.trim() };
+ } else {
+ cur.text += raw;
+ cur.end = Math.max(cur.end, offsetSec + to / 1000);
+ }
+ }
+ if (cur) out.push(cur);
+ return out.filter((wd) => wd.text && !FILLER_WORD.test(wd.text));
+}
+
// ── Parakeet output → timed sentences (Live's parseParakeet) ──
// ── Parakeet output → timed sentences (Live's parseParakeet) ──
// `parakeet-cli -ps` prints every token on stderr with its start and end in centiseconds and whether it starts a word
// `parakeet-cli -ps` prints every token on stderr with its start and end in centiseconds and whether it starts a word
// ("▁For"). A sentence ends at . ? or !, at a pause of 1.5 s, or after 20 s. Fillers ("uh", "um") are dropped.
// ("▁For"). A sentence ends at . ? or !, at a pause of 1.5 s, or after 20 s. Fillers ("uh", "um") are dropped.
const PARAKEET_LANGS = new Set(['bg', 'hr', 'cs', 'da', 'nl', 'en', 'et', 'fi', 'fr', 'de', 'el', 'hu', 'it', 'lv', 'lt', 'mt', 'pl', 'pt', 'ro', 'sk', 'sl', 'es', 'sv', 'ru', 'uk']);
const PARAKEET_LANGS = new Set(['bg', 'hr', 'cs', 'da', 'nl', 'en', 'et', 'fi', 'fr', 'de', 'el', 'hu', 'it', 'lv', 'lt', 'mt', 'pl', 'pt', 'ro', 'sk', 'sl', 'es', 'sv', 'ru', 'uk']);
const TOKEN_LINE = /t0=\s*(-?\d+)\s+t1=\s*(-?\d+)\s+word_start=(true|false)\s+"(.*)"\s*$/;
const TOKEN_LINE = /t0=\s*(-?\d+)\s+t1=\s*(-?\d+)\s+word_start=(true|false)\s+"(.*)"\s*$/;
const FILLER_RE = /(^|[\s,])(?:uh+|um+|uhm|erm|mm+)(?=[\s,.!?]|$)[,]?/gi;
const FILLER_RE = /(^|[\s,])(?:uh+|um+|uhm|erm|mm+)(?=[\s,.!?]|$)[,]?/gi;
-function parseParakeet(stderr, offsetSec = 0) {
+function parseParakeet(stderr, offsetSec = 0, { words = false } = {}) {
const out = [];
const out = [];
let cur = null;
let cur = null;
+ let word = null;
+ const closeWord = () => {
+ if (!word || !cur) { word = null; return; }
+ const text = word.text.trim();
+ if (text && !FILLER_WORD.test(text)) cur.words.push({ start: offsetSec + word.t0 / 100, end: offsetSec + word.t1 / 100, text });
+ word = null;
+ };
const close = () => {
const close = () => {
if (!cur) return;
if (!cur) return;
+ closeWord();
const text = cur.text.replace(FILLER_RE, '$1').replace(/\s+([,.!?])/g, '$1').replace(/\s+/g, ' ').replace(/^[,\s]+/, '').trim();
const text = cur.text.replace(FILLER_RE, '$1').replace(/\s+([,.!?])/g, '$1').replace(/\s+/g, ' ').replace(/^[,\s]+/, '').trim();
- if (text) out.push({ start: offsetSec + cur.t0 / 100, end: offsetSec + cur.t1 / 100, text });
+ if (text) out.push({ start: offsetSec + cur.t0 / 100, end: offsetSec + cur.t1 / 100, text, ...(words ? { words: cur.words } : {}) });
cur = null;
cur = null;
};
};
for (const line of String(stderr || '').split('\n')) {
for (const line of String(stderr || '').split('\n')) {
@@ -87,9 +131,13 @@ function parseParakeet(stderr, offsetSec = 0) {
@@ -87,9 +131,13 @@ function parseParakeet(stderr, offsetSec = 0) {
if (/^<.*>$/.test(raw)) continue; // <unk>, <|…|> control tokens
if (/^<.*>$/.test(raw)) continue; // <unk>, <|…|> control tokens
const piece = raw.replace(/\u2581/g, ' ');
const piece = raw.replace(/\u2581/g, ' ');
if (cur && wordStart && (t0 - cur.t1 > 150 || t0 - cur.t0 > 2000)) close();
if (cur && wordStart && (t0 - cur.t1 > 150 || t0 - cur.t0 > 2000)) close();
- if (!cur) cur = { t0, t1, text: '' };
+ if (!cur) cur = { t0, t1, text: '', words: [] };
cur.text += wordStart && !piece.startsWith(' ') ? ` ${piece}` : piece;
cur.text += wordStart && !piece.startsWith(' ') ? ` ${piece}` : piece;
cur.t1 = Math.max(cur.t1, t1);
cur.t1 = Math.max(cur.t1, t1);
+ // A word is the tokens from one word_start to the next; punctuation joins it without moving its end.
+ if (wordStart || !word) { closeWord(); word = { t0, t1, text: piece.trim() }; }
+ else if (PUNCT.test(piece)) word.text += piece.trim();
+ else { word.text += piece; word.t1 = Math.max(word.t1, t1); }
if (/[.!?]["')\]]?$/.test(piece.trim())) close();
if (/[.!?]["')\]]?$/.test(piece.trim())) close();
}
}
close();
close();
@@ -162,7 +210,7 @@ function createWhisperProvider(record, { config }) {
@@ -162,7 +210,7 @@ function createWhisperProvider(record, { config }) {
});
});
}
}
- function decode(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, signal, timeoutMs }) {
+ function decode(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, words = false, signal, timeoutMs }) {
return new Promise((resolve, reject) => {
return new Promise((resolve, reject) => {
const lang = language && language !== 'en' && multi() ? String(language) : 'en';
const lang = language && language !== 'en' && multi() ? String(language) : 'en';
const model = lang !== 'en' ? multi() : (live && exists(MODEL_LIVE) ? MODEL_LIVE : MODEL);
const model = lang !== 'en' ? multi() : (live && exists(MODEL_LIVE) ? MODEL_LIVE : MODEL);
@@ -170,7 +218,8 @@ function createWhisperProvider(record, { config }) {
@@ -170,7 +218,8 @@ function createWhisperProvider(record, { config }) {
const threads = live || lowPower ? Math.max(1, Math.min(2, w.threads)) : w.threads;
const threads = live || lowPower ? Math.max(1, Math.min(2, w.threads)) : w.threads;
const outBase = `${wavPath}.out`;
const outBase = `${wavPath}.out`;
const jsonPath = `${outBase}.json`;
const jsonPath = `${outBase}.json`;
- const args = ['-m', model, '-f', wavPath, '-oj', '-of', outBase, '-t', String(threads), '-l', lang];
+ // -ojf adds every token with its offsets (word timing, C12); only when asked, since it is a far larger file.
+ const args = ['-m', model, '-f', wavPath, words ? '-ojf' : '-oj', '-of', outBase, '-t', String(threads), '-l', lang];
if (w.beam > 1) args.push('-bs', String(w.beam));
if (w.beam > 1) args.push('-bs', String(w.beam));
const vm = vadModel();
const vm = vadModel();
if (vm) args.push('--vad', '-vm', vm);
if (vm) args.push('--vad', '-vm', vm);
@@ -190,6 +239,7 @@ function createWhisperProvider(record, { config }) {
@@ -190,6 +239,7 @@ function createWhisperProvider(record, { config }) {
start: offsetSec + ((it.offsets && it.offsets.from) || 0) / 1000,
start: offsetSec + ((it.offsets && it.offsets.from) || 0) / 1000,
end: offsetSec + ((it.offsets && it.offsets.to) || 0) / 1000,
end: offsetSec + ((it.offsets && it.offsets.to) || 0) / 1000,
text: it.text || '',
text: it.text || '',
+ ...(words ? { words: wordsFromTokens(it.tokens, offsetSec) } : {}),
})), Boolean(vm));
})), Boolean(vm));
// With -l auto, whisper.cpp names the language it detected in result.language.
// With -l auto, whisper.cpp names the language it detected in result.language.
const detected = parsed && parsed.result && /^[a-z]{2,3}$/.test(String(parsed.result.language || '')) ? parsed.result.language : null;
const detected = parsed && parsed.result && /^[a-z]{2,3}$/.test(String(parsed.result.language || '')) ? parsed.result.language : null;
@@ -200,7 +250,7 @@ function createWhisperProvider(record, { config }) {
@@ -200,7 +250,7 @@ function createWhisperProvider(record, { config }) {
}
}
// One parakeet-cli run on a short WAV: the tokens on stderr become timed sentences.
// One parakeet-cli run on a short WAV: the tokens on stderr become timed sentences.
- function parakeetOnce(wavPath, { threads, offsetSec, signal, deadline }) {
+ function parakeetOnce(wavPath, { threads, offsetSec, words = false, signal, deadline }) {
return new Promise((resolve, reject) => {
return new Promise((resolve, reject) => {
let ch;
let ch;
try { ch = spawnTracked('nice', ['-n', '15', parakeetBin(), '-t', String(threads), '-ps', '-np', '-m', w.parakeetModel, '-f', wavPath], { stderr: true }); }
try { ch = spawnTracked('nice', ['-n', '15', parakeetBin(), '-t', String(threads), '-ps', '-np', '-m', w.parakeetModel, '-f', wavPath], { stderr: true }); }
@@ -214,13 +264,13 @@ function createWhisperProvider(record, { config }) {
@@ -214,13 +264,13 @@ function createWhisperProvider(record, { config }) {
clearTimeout(timer);
clearTimeout(timer);
if (signal && signal.aborted) return reject(new ProviderError('cancelled', { code: 'provider.cancelled' }));
if (signal && signal.aborted) return reject(new ProviderError('cancelled', { code: 'provider.cancelled' }));
if (code !== 0) return reject(new ProviderError(`parakeet exited ${code}`, { code: 'provider.error' }));
if (code !== 0) return reject(new ProviderError(`parakeet exited ${code}`, { code: 'provider.error' }));
- resolve(parseParakeet(Buffer.concat(chunks).toString('utf8'), offsetSec));
+ resolve(parseParakeet(Buffer.concat(chunks).toString('utf8'), offsetSec, { words }));
});
});
ch.on('error', (e) => { clearTimeout(timer); reject(new ProviderError(e.message, { code: 'provider.unavailable' })); });
ch.on('error', (e) => { clearTimeout(timer); reject(new ProviderError(e.message, { code: 'provider.unavailable' })); });
});
});
}
}
// Parakeet on a whole WAV: in 30 s pieces past 45 s (PCM cut exactly, so a piece's times shift only by its start).
// Parakeet on a whole WAV: in 30 s pieces past 45 s (PCM cut exactly, so a piece's times shift only by its start).
- async function decodeParakeet(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, signal, timeoutMs }) {
+ async function decodeParakeet(wavPath, { language = 'en', live = false, lowPower = false, offsetSec = 0, words = false, signal, timeoutMs }) {
const threads = live || lowPower ? Math.max(1, Math.min(2, w.threads)) : w.threads;
const threads = live || lowPower ? Math.max(1, Math.min(2, w.threads)) : w.threads;
const deadline = Date.now() + timeoutMs;
const deadline = Date.now() + timeoutMs;
let pieces = [wavPath], dir = null;
let pieces = [wavPath], dir = null;
@@ -237,7 +287,7 @@ function createWhisperProvider(record, { config }) {
@@ -237,7 +287,7 @@ function createWhisperProvider(record, { config }) {
const raw = [];
const raw = [];
for (let i = 0; i < pieces.length; i++) {
for (let i = 0; i < pieces.length; i++) {
if (Date.now() >= deadline) throw new ProviderError('parakeet timed out', { code: 'provider.error' });
if (Date.now() >= deadline) throw new ProviderError('parakeet timed out', { code: 'provider.error' });
- raw.push(...await parakeetOnce(pieces[i], { threads, offsetSec: offsetSec + i * CHUNK_SEC, signal, deadline }));
+ raw.push(...await parakeetOnce(pieces[i], { threads, offsetSec: offsetSec + i * CHUNK_SEC, words, signal, deadline }));
}
}
const segments = cleanSegments(raw, true);
const segments = cleanSegments(raw, true);
return { text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: String(language || 'en'), model: path.basename(w.parakeetModel) };
return { text: segments.map(s => s.text).join(' ').replace(/\s+/g, ' ').trim(), segments, language: String(language || 'en'), model: path.basename(w.parakeetModel) };
@@ -254,7 +304,7 @@ function createWhisperProvider(record, { config }) {
@@ -254,7 +304,7 @@ function createWhisperProvider(record, { config }) {
try {
try {
await acquire(lane);
await acquire(lane);
try {
try {
- const opts = { language: req.language, live: req.live, lowPower: req.lowPower === true, offsetSec: req.offsetSec || 0, signal: req.signal, timeoutMs };
+ const opts = { language: req.language, live: req.live, lowPower: req.lowPower === true, offsetSec: req.offsetSec || 0, words: req.words === true, signal: req.signal, timeoutMs };
let r = null;
let r = null;
if (engineFor(req.language) === 'parakeet') {
if (engineFor(req.language) === 'parakeet') {
try { r = await decodeParakeet(wav, opts); }
try { r = await decodeParakeet(wav, opts); }
@@ -280,4 +330,4 @@ function createWhisperProvider(record, { config }) {
@@ -280,4 +330,4 @@ function createWhisperProvider(record, { config }) {
};
};
}
}
-module.exports = { createWhisperProvider, cleanSegments, isNoise, parseParakeet };
+module.exports = { createWhisperProvider, cleanSegments, isNoise, parseParakeet, wordsFromTokens };
@@ -270,7 +270,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
@@ -270,7 +270,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
try {
try {
const startSec = Number(input.start_sec) || 0;
const startSec = Number(input.start_sec) || 0;
const offsetSec = input.offset_sec != null ? Number(input.offset_sec) : startSec;
const offsetSec = input.offset_sec != null ? Number(input.offset_sec) : startSec;
- const exec = await pool.execute(route, 'transcribe', { filePath: file, language: input.language || 'en', seconds: input.seconds || 0, startSec, offsetSec, lowPower: input.low_power === true, timeoutMs: 3600000 }, { ...ctx, routeKey: route.key, routeVersion: route.version, promptHash: null });
+ const exec = await pool.execute(route, 'transcribe', { filePath: file, language: input.language || 'en', seconds: input.seconds || 0, startSec, offsetSec, lowPower: input.low_power === true, words: input.words === true, timeoutMs: 3600000 }, { ...ctx, routeKey: route.key, routeVersion: route.version, promptHash: null });
const r = exec.result;
const r = exec.result;
return { output: { text: r.text || '', language: r.language || input.language || 'en', segments: r.segments || [] }, exec, route };
return { output: { text: r.text || '', language: r.language || input.language || 'en', segments: r.segments || [] }, exec, route };
} finally {
} finally {
@@ -498,8 +498,9 @@ const structured = [
@@ -498,8 +498,9 @@ const structured = [
{
{
key: 'live.media.transcribe', name: 'Speech-to-text for a VOD, clip or audio file', namespace: 'live',
key: 'live.media.transcribe', name: 'Speech-to-text for a VOD, clip or audio file', namespace: 'live',
description: 'whisper.cpp on this host (Live\'s transcribe.js), media fetched only from allow-listed OpenVibe hosts.',
description: 'whisper.cpp on this host (Live\'s transcribe.js), media fetched only from allow-listed OpenVibe hosts.',
- input_schema: TEXT_OUT({ media: MEDIA_REF, media_url: STR(2048), language: LANG, seconds: { type: 'integer', minimum: 0, maximum: 36000 }, start_sec: { type: 'number', minimum: 0 }, offset_sec: { type: 'number', minimum: 0 }, low_power: { type: 'boolean' } }, []),
- output_schema: TEXT_OUT({ text: { type: 'string' }, language: STR(8), segments: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' } } } } }, ['text', 'segments']),
+ input_schema: TEXT_OUT({ media: MEDIA_REF, media_url: STR(2048), language: LANG, seconds: { type: 'integer', minimum: 0, maximum: 36000 }, start_sec: { type: 'number', minimum: 0 }, offset_sec: { type: 'number', minimum: 0 }, low_power: { type: 'boolean' }, words: { type: 'boolean' } }, []),
+ // words: true adds each segment's words with their times (item 151, C12), so a clip can start and end on a word.
+ output_schema: TEXT_OUT({ text: { type: 'string' }, language: STR(8), segments: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' }, words: { type: 'array', items: { type: 'object', required: ['start', 'end', 'text'], properties: { start: { type: 'number' }, end: { type: 'number' }, text: { type: 'string' } } } } } } } }, ['text', 'segments']),
steps: [{ kind: 'transcribe', route: 'live.stt' }],
steps: [{ kind: 'transcribe', route: 'live.stt' }],
cache_mode: 'private',
cache_mode: 'private',
},
},
new file mode 100644
new file mode 100644
@@ -0,0 +1,66 @@
@@ -0,0 +1,66 @@
+'use strict';
+// Word timing (item 151, C12): with `words: true`, live.media.transcribe keeps every word's start and end inside its
+// segment, so a clip can start and end on a word and a moment's words can be found again. whisper.cpp's -ojf tokens
+// (the fixture is what the host's whisper-cli large-v3-turbo printed for a synthetic voice on 2026-11-03) and
+// Parakeet's token lines both become words; punctuation joins its word without moving the word's end; control tokens
+// and fillers are not words; a language written without spaces gets one unit per token; times shift by the window's
+// offset; words go with their segment when a noise segment is dropped; nothing changes when words are not asked for.
+const assert = require('assert');
+const { wordsFromTokens, parseParakeet, cleanSegments } = require('../server/providers/whisper');
+const live = require('../server/workflows/live');
+
+// ── whisper.cpp -ojf ──
+const tok = (text, from, to) => ({ text, offsets: { from, to } });
+const host = [
+ tok('[_BEG_]', 0, 0), tok(' it', 0, 100), tok(' comp', 110, 340), tok('iles', 340, 570), tok('.', 570, 760),
+ tok(' Finally', 760, 1360), tok(',', 1360, 1540), tok(' after', 1540, 2020), tok(' three', 2020, 2300),
+ tok(' hours', 2300, 2520), tok(' of', 2520, 2610), tok(' trying', 2740, 2990), tok(',', 3000, 3090),
+ tok(' Chat', 4570, 4660), tok(',', 5010, 5040), tok(' did', 5040, 5270), tok(' you', 5270, 5500), tok(' see', 5500, 5720),
+ tok(' that', 5730, 6030), tok('?', 6040, 6280), tok('[_TT_315]', 6300, 6300),
+];
+const words = wordsFromTokens(host, 0);
+assert.deepStrictEqual(words.map((w) => w.text), ['it', 'compiles.', 'Finally,', 'after', 'three', 'hours', 'of', 'trying,', 'Chat,', 'did', 'you', 'see', 'that?']);
+const compiles = words[1];
+assert.strictEqual(compiles.start, 0.11);
+assert.strictEqual(compiles.end, 0.57, 'the full stop joins the word without stretching it to 0.76');
+assert.strictEqual(words[words.length - 1].end, 6.03);
+assert.strictEqual(wordsFromTokens(host, 120)[1].start, 120.11, 'a window of a longer recording keeps the recording\'s times');
+
+// Japanese: no spaces to break on, so each token is a unit; 「。」 joins the unit before it. Fillers and control tokens go.
+assert.deepStrictEqual(wordsFromTokens([tok(' 今日', 0, 300), tok('は', 300, 420), tok('ね', 420, 600), tok('。', 600, 700), tok('<|endoftext|>', 700, 700)], 0).map((w) => w.text), ['今日', 'は', 'ね。']);
+assert.deepStrictEqual(wordsFromTokens([tok(' so', 0, 100), tok(' uh', 100, 300), tok(' yes', 300, 500), tok(' broken', 600, 500), { text: ' x' }], 0).map((w) => w.text), ['so', 'yes'],
+ 'fillers, backwards and untimed tokens are not words');
+assert.deepStrictEqual(wordsFromTokens(null), []);
+
+// ── Parakeet's token lines ──
+const pk = [
+ 'tok t0= 10 t1= 40 word_start=true "▁IT"',
+ 'tok t0= 41 t1= 60 word_start=true "▁COM"',
+ 'tok t0= 60 t1= 90 word_start=false "PILES"',
+ 'tok t0= 90 t1= 95 word_start=false "."',
+ 'tok t0= 300 t1= 320 word_start=true "▁um"',
+ 'tok t0= 330 t1= 360 word_start=true "▁Wow"',
+ 'tok t0= 360 t1= 370 word_start=false "!"',
+].join('\n');
+const sentences = parseParakeet(pk, 100, { words: true });
+assert.deepStrictEqual(sentences.map((s) => s.text), ['IT COMPILES.', 'Wow!']);
+assert.deepStrictEqual(sentences[0].words, [{ start: 100.1, end: 100.4, text: 'IT' }, { start: 100.41, end: 100.9, text: 'COMPILES.' }]);
+assert.deepStrictEqual(sentences[1].words.map((w) => w.text), ['Wow!'], 'the filler is not a word');
+assert.ok(!('words' in parseParakeet(pk, 0)[0]), 'no words unless asked');
+
+// ── cleanSegments keeps words with their segment, and drops them with it ──
+const cleaned = cleanSegments([
+ { start: 1, end: 2, text: 'IT COMPILES.', words: [{ start: 1.004, end: 1.5, text: 'IT' }, { start: 1.5, end: 1.996, text: 'COMPILES.' }] },
+ { start: 3, end: 4, text: 'Thank you for watching!', words: [{ start: 3, end: 4, text: 'Thank' }] },
+], false);
+assert.strictEqual(cleaned.length, 1, 'whisper\'s invented sign-off is dropped, its words with it');
+assert.deepStrictEqual(cleaned[0].words, [{ start: 1, end: 1.5, text: 'IT' }, { start: 1.5, end: 2, text: 'COMPILES.' }]);
+assert.ok(!('words' in cleanSegments([{ start: 0, end: 1, text: 'plain' }], true)[0]));
+
+// ── The workflow asks for them and its output allows them ──
+const wf = (live.workflows || []).find((x) => x.key === 'live.media.transcribe');
+assert.ok(wf, 'live.media.transcribe exists');
+assert.deepStrictEqual(wf.input_schema.properties.words, { type: 'boolean' });
+assert.ok(wf.output_schema.properties.segments.items.properties.words, 'segments may carry words');
+
+console.log('word timing: all checks passed');