openvibers/openvibe.ai Commits
Read archives and audio, return private file artifacts (#474)
Changes
@@@ -38,7 +38,7 @@ const { EXACT_ID } = modelCatalog;const { resolveRequest } = require('../exchange/presets');const { compact, receiptId } = require('../exchange/receipts');--const FIELDS = ['system', 'tools', 'tool_choice', 'stop_sequences', 'temperature', 'top_p', 'top_k', 'metadata', 'files'];++const FIELDS = ['system', 'tools', 'tool_choice', 'stop_sequences', 'temperature', 'top_p', 'top_k', 'metadata', 'files', 'output_file'];const MAX_TOKENS = 64000;// Anthropic's error types by HTTP status (docs.anthropic.com/en/api/errors).@@@ -291,6 +291,7 @@ function messagesRouter({ runs, auth, config, registry, presets = null, receipts...(out.usage && out.usage.cache_creation_input_tokens != null ? { cache_creation_input_tokens: out.usage.cache_creation_input_tokens } : {}),},};++ if (Array.isArray(out.artifacts) && out.artifacts.length) message.artifacts = out.artifacts;if (b.usage?.include === true) Object.assign(message.usage, require('./compat-extra').accounting(run));const disclosure = await models.disclosureOf({ registry, config, run });const community = (disclosure && disclosure.community) || (run.explain && run.explain.community) || null;@@@ -302,7 +303,8 @@ function messagesRouter({ runs, auth, config, registry, presets = null, receiptscost_usd: run.usage.cost_usd, synthetic: Boolean(run.synthetic),...(community ? { community } : {}),...(run.output.label ? { policy: run.output.policy || 'uncensored', label: run.output.label } : {}),-- privacy: receipt && receipt.privacy ? receipt.privacy.label : null, receipt_id: receipt ? receipt.id : null };++ privacy: receipt && receipt.privacy ? receipt.privacy.label : null, receipt_id: receipt ? receipt.id : null,++ ...(out.artifacts?.length ? { artifacts: out.artifacts } : {}) };if (run.output.label) {message.label = run.output.label;message.policy = run.output.policy || 'uncensored';@@@ -52,7 +52,7 @@ const { EXACT_ID } = modelCatalog;const { resolveRequest } = require('../exchange/presets');const { compact, receiptId } = require('../exchange/receipts');--const FIELDS = ['messages', 'tools', 'tool_choice', 'parallel_tool_calls', 'response_format', 'stop', 'seed', 'top_p', 'presence_penalty', 'frequency_penalty', 'temperature', 'files'];++const FIELDS = ['messages', 'tools', 'tool_choice', 'parallel_tool_calls', 'response_format', 'stop', 'seed', 'top_p', 'presence_penalty', 'frequency_penalty', 'temperature', 'files', 'output_file'];// Cloudflare answers a 502 or 504 from the origin with its own HTML page, so an OpenAI client would get no JSON error:// an upstream that failed or could not be reached is 503 (OpenAI's own "unavailable, retry"), the body kept. An invalid@@@ -300,6 +300,7 @@ function openaiRouter({ runs, auth, config, registry, presets = null, receipts =...(run.output.label ? { policy: run.output.policy || 'uncensored', label: run.output.label } : {}),privacy: receipt && receipt.privacy ? receipt.privacy.label : null,receipt_id: receipt ? receipt.id : null,++ ...(run.output.artifacts?.length ? { artifacts: run.output.artifacts } : {}),};if (run.output.label) res.setHeader('X-OpenVibe-Content-Policy', run.output.policy || 'uncensored');if (!live) res.setHeader('X-OpenVibe-Run-Id', run.id); // a stream sent it with its headersnnew file mode 100644@@@ -0,0 +1,95 @@++'use strict';++/** Bounded, read-only extraction of ordinary ZIP and TAR archives attached to a turn. */++const zlib = require('node:zlib');++++const MAX_ENTRIES = 100;++const MAX_EXTRACTED = 2 * 1024 * 1024;++const MAX_ENTRY = 512 * 1024;++const TEXT_NAME = /\.(txt|text|md|markdown|csv|tsv|json|jsonl|log|xml|yaml|yml|toml|ini|cfg|conf|html?|css|scss|less|m?js|cjs|jsx|ts|tsx|py|rb|go|rs|java|kt|kts|c|h|cc|cpp|hpp|cs|php|sh|bash|zsh|sql|graphql|gql|proto|swift|lua|pl|r|m|vue|svelte|astro|tex|pdf)$/i;++++const safeName = (name) => {++ const value = String(name || '').replace(/\\/g, '/');++ return value && !value.startsWith('/') && !value.split('/').includes('..') && !/[\u0000-\u001f\u007f]/.test(value) ? value.slice(0, 255) : null;++};++const readable = (name) => Boolean(safeName(name) && TEXT_NAME.test(name));++++function zipEntries(buffer) {++ const start = Math.max(0, buffer.length - 65557);++ let end = -1;++ for (let i = buffer.length - 22; i >= start; i--) {++ if (buffer.readUInt32LE(i) === 0x06054b50) { end = i; break; }++ }++ if (end < 0) return { entries: [], reason: 'the ZIP directory is missing' };++ const count = buffer.readUInt16LE(end + 10);++ const size = buffer.readUInt32LE(end + 12);++ let offset = buffer.readUInt32LE(end + 16);++ if (count > MAX_ENTRIES || offset + size > buffer.length || buffer.readUInt16LE(end + 4) !== 0 || buffer.readUInt16LE(end + 6) !== 0) {++ return { entries: [], reason: 'the ZIP is too large or uses multiple disks' };++ }++ const entries = [];++ let total = 0;++ let skipped = 0;++ for (let i = 0; i < count; i++) {++ if (offset + 46 > buffer.length || buffer.readUInt32LE(offset) !== 0x02014b50) return { entries: [], reason: 'the ZIP directory is damaged' };++ const flags = buffer.readUInt16LE(offset + 8);++ const method = buffer.readUInt16LE(offset + 10);++ const packed = buffer.readUInt32LE(offset + 20);++ const length = buffer.readUInt32LE(offset + 24);++ const nameLength = buffer.readUInt16LE(offset + 28);++ const extraLength = buffer.readUInt16LE(offset + 30);++ const commentLength = buffer.readUInt16LE(offset + 32);++ const local = buffer.readUInt32LE(offset + 42);++ const next = offset + 46 + nameLength + extraLength + commentLength;++ if (next > buffer.length) return { entries: [], reason: 'the ZIP directory is damaged' };++ const name = buffer.subarray(offset + 46, offset + 46 + nameLength).toString('utf8');++ offset = next;++ if (!readable(name) || (flags & 1) || ![0, 8].includes(method) || length > MAX_ENTRY || total + length > MAX_EXTRACTED) { skipped++; continue; }++ if (local + 30 > buffer.length || buffer.readUInt32LE(local) !== 0x04034b50) { skipped++; continue; }++ const dataAt = local + 30 + buffer.readUInt16LE(local + 26) + buffer.readUInt16LE(local + 28);++ if (dataAt + packed > buffer.length) { skipped++; continue; }++ try {++ const compressed = buffer.subarray(dataAt, dataAt + packed);++ const bytes = method === 0 ? compressed : zlib.inflateRawSync(compressed, { maxOutputLength: MAX_ENTRY });++ if (bytes.length !== length) { skipped++; continue; }++ entries.push({ name: safeName(name), bytes });++ total += bytes.length;++ } catch { skipped++; }++ }++ return { entries, skipped };++}++++function tarEntries(buffer) {++ const entries = [];++ let total = 0;++ let skipped = 0;++ let offset = 0;++ let count = 0;++ while (offset + 512 <= buffer.length && buffer[offset] !== 0) {++ if (++count > MAX_ENTRIES) return { entries: [], reason: 'the TAR has too many entries' };++ const head = buffer.subarray(offset, offset + 512);++ const name = head.subarray(0, 100).toString('utf8').replace(/\u0000.*$/, '');++ const rawSize = head.subarray(124, 136).toString('ascii').replace(/\u0000.*$/, '').trim();++ if (!/^[0-7]+$/.test(rawSize)) return { entries: [], reason: 'the TAR header is damaged' };++ const size = parseInt(rawSize, 8);++ const type = head[156];++ const dataAt = offset + 512;++ if (!Number.isSafeInteger(size) || dataAt + size > buffer.length) return { entries: [], reason: 'the TAR entry is incomplete' };++ if ((type === 0 || type === 48) && readable(name) && size <= MAX_ENTRY && total + size <= MAX_EXTRACTED) {++ entries.push({ name: safeName(name), bytes: buffer.subarray(dataAt, dataAt + size) });++ total += size;++ } else skipped++;++ offset = dataAt + Math.ceil(size / 512) * 512;++ }++ return { entries, skipped };++}++++function extractArchive(buffer, type, name) {++ if (type === 'application/zip' || /\.zip$/i.test(name)) return zipEntries(buffer);++ if (/\.(?:tar\.gz|tgz)$/i.test(name) || type === 'application/gzip') {++ try { return tarEntries(zlib.gunzipSync(buffer, { maxOutputLength: MAX_EXTRACTED + MAX_ENTRIES * 512 })); }++ catch { return { entries: [], reason: 'the compressed TAR is damaged or too large' }; }++ }++ return tarEntries(buffer);++}++++module.exports = { extractArchive, MAX_ENTRIES, MAX_EXTRACTED };@@@ -4,14 +4,14 @@** A provider can answer with files beside its text: each entry names a Media object it already stored* (the reference shape), which is returned as it is. A provider that hands back the bytes themselves-- * (`data`/`b64`) needs a Media store in this service: `store.put({ name, media_type, bytes, subject })`-- * returns the stored ai.file@1. When no store is configured such a file is REFUSED with-- * files.store_unavailable, never dropped and never put in the run output as inline bytes. The store is-- * configured by the operator (a Media tenant AI may write to); nothing here pretends to have one.++ * (`data`/`b64`) goes to the existing Media uploader, private to the person on the turn.++ * When Media is unavailable, such a file is refused with files.store_unavailable and is never++ * put in the run output as inline bytes.*/const { AiError } = require('./util');const MAX_ARTIFACTS = 10;++const MAX_ARTIFACT_BYTES = 10 * 1024 * 1024;const MEDIA_ID = /^med_[0-9A-HJKMNP-TV-Z]{26}$/;function storedRef(file) {@@@ -29,10 +29,20 @@ function storedRef(file) {};}--/** The store a run may write to (config.media.artifactStore with put(), or null). */--function storeOf(config) {++/** The Media store for inline output bytes; a generated file belongs to the person on the turn. */++function storeOf(config, mediaStore = null) {const s = config && config.media && config.media.artifactStore;-- return s && typeof s.put === 'function' ? s : null;++ if (s && typeof s.put === 'function') return s;++ if (!mediaStore || typeof mediaStore.put !== 'function' || typeof mediaStore.configured !== 'function' || !mediaStore.configured()) return null;++ return { put: async ({ name, media_type: mediaType, bytes, subject }) => {++ if (!subject) throw new AiError(422, 'files.subject_required', 'a generated file needs a person on the turn so Media can keep it private to them');++ if (bytes.length > MAX_ARTIFACT_BYTES) throw new AiError(413, 'files.too_large', 'a generated file may be up to 10 MiB');++ const saved = await mediaStore.put({ bytes, mime: mediaType || 'application/octet-stream', filename: name || 'file', owner: subject,++ visibility: 'private', metadata: { origin: 'ai.turn.artifact' } });++ const ref = storedRef({ id: saved.media_id, name, media_type: mediaType, size: bytes.length, access: { readers: [subject] } });++ if (!ref) throw new AiError(502, 'files.store_invalid', 'Media did not return a valid file reference');++ return ref;++ } };}/**@@@ -51,10 +61,21 @@ async function collectArtifacts(message, { store = null, subject = null } = {}): typeof (file && (file.data || file.b64)) === 'string' ? Buffer.from(file.data || file.b64, file.encoding === 'utf8' ? 'utf8' : 'base64'): null;if (!bytes) throw new AiError(422, 'files.invalid', `a model returned a file without bytes or a Media reference: ${String((file && file.name) || 'unnamed').slice(0, 80)}`);++ if (bytes.length > MAX_ARTIFACT_BYTES) throw new AiError(413, 'files.too_large', 'a generated file may be up to 10 MiB');if (!store) throw new AiError(503, 'files.store_unavailable', 'a model produced a file but AI has no Media store configured to keep it');out.push(await store.put({ name: file.name, media_type: file.media_type || file.type, bytes, subject }));}return out;}--module.exports = { collectArtifacts, storeOf, MAX_ARTIFACTS };++/** Turn a requested text answer into a private Media file with the same text as the answer. */++async function collectTextArtifact(text, request, { store = null, subject = null } = {}) {++ if (!request) return [];++ const name = String(request.name || '');++ if (!name || name.length > 255 || /[/\\\u0000-\u001f\u007f]/.test(name)) throw new AiError(422, 'files.name_invalid', 'output_file.name must be a filename without a path');++ if (!['text/plain', 'text/markdown', 'text/csv', 'application/json'].includes(request.media_type)) throw new AiError(422, 'files.type_invalid', 'output_file.media_type must be a supported text type');++ if (typeof text !== 'string' || !text.length) throw new AiError(422, 'files.text_missing', 'the model returned no text to save as output_file');++ return collectArtifacts({ artifacts: [{ name, media_type: request.media_type, data: text, encoding: 'utf8' }] }, { store, subject });++}++++module.exports = { collectArtifacts, collectTextArtifact, storeOf, MAX_ARTIFACTS };@@@ -11,6 +11,8 @@* text, markdown, code, CSV and JSON a text block, cut at MAX_TEXT_CHARS with an honest "(truncated)" line* PDF its text, page by page, through the small extractor below (no dependency)* PNG, JPEG, WebP, GIF an image part, when the model can see (vision); otherwise a named note++ * audio a transcript from the host's local ASR, or an honest unavailable note++ * ZIP, TAR, TAR.GZ bounded text/code/PDF entries with their paths; other entries named as skipped* anything else a named note saying it could not be read, never silently dropped** Limits: MAX_FILES per turn, MAX_FILE_BYTES each and MAX_TOTAL_BYTES together. Per-person, per-window@@@ -23,6 +25,7 @@*/const zlib = require('node:zlib');const { AiError } = require('./util');++const { extractArchive } = require('./archive-inputs');const MAX_FILES = 10;const MAX_FILE_BYTES = 10 * 1024 * 1024;@@@ -34,6 +37,8 @@ const TEXT_EXT = /\.(txt|text|md|markdown|csv|tsv|json|jsonl|log|xml|yaml|yml|toconst CODE_TYPE = /^(text\/[a-z0-9.+-]+|application\/(json|jsonl|xml|javascript|ecmascript|x-javascript|x-httpd-php|x-sh|x-shellscript|x-python|x-ruby|x-yaml|x-yml|x-toml|x-tex|x-latex|x-c|x-c\+\+|x-java-source|x-php|x-perl|x-rust|x-go|x-typescript|x-www-form-urlencoded))$/i;const IMAGE_TYPE = /^image\/(png|jpe?g|webp|gif)$/i;const PDF_TYPE = /^application\/pdf$/i;++const ARCHIVE_TYPE = /^(application\/(zip|x-tar|gzip|x-gzip|x-gtar|x-compressed-tar))$/i;++const AUDIO_TYPE = /^audio\/(mpeg|mp3|mp4|x-m4a|wav|x-wav|wave|ogg|webm|flac|x-flac)$/i;/** The named virus-scan extension point; null means no scan is configured (and none is pretended). */let scanner = null;@@@ -57,6 +62,9 @@ function effectiveType(name, contentType, declared) {if (!type || type === 'application/octet-stream' || type === 'binary/octet-stream') type = typeOf(declared);if ((!type || type === 'application/octet-stream' || type === 'binary/octet-stream') && TEXT_EXT.test(String(name || ''))) type = 'text/plain';if ((!type || type === 'application/octet-stream') && extOf(name) === 'pdf') type = 'application/pdf';++ if ((!type || type === 'application/octet-stream') && /\.zip$/i.test(name)) type = 'application/zip';++ if ((!type || type === 'application/octet-stream') && /\.(?:tar|tar\.gz|tgz)$/i.test(name)) type = 'application/x-tar';++ if ((!type || type === 'application/octet-stream') && /\.(?:mp3|m4a|mp4|wav|ogg|webm|flac)$/i.test(name)) type = ({ mp3: 'audio/mpeg', m4a: 'audio/mp4', mp4: 'audio/mp4', wav: 'audio/wav', ogg: 'audio/ogg', webm: 'audio/webm', flac: 'audio/flac' })[extOf(name)];if ((!type || type === 'application/octet-stream') && /^(png|jpe?g|webp|gif)$/.test(extOf(name))) type = `image/${extOf(name).replace('jpg', 'jpeg')}`;return type || 'application/octet-stream';}@@@ -183,7 +191,7 @@ function referenceOf(file, mediaId, name, mediaType, size, read) {}/** Read one Media-backed file for the model. → { reference, part, note } */--async function readFile(file, fetcher, { signal, format = 'openai', vision = true, scan = null } = {}) {++async function readFile(file, fetcher, { signal, format = 'openai', vision = true, scan = null, transcribe = null } = {}) {const mediaId = mediaIdOf(file);const name = nameOf(file);const declaredSize = Number(file && file.size);@@@ -249,6 +257,36 @@ async function readFile(file, fetcher, { signal, format = 'openai', vision = truconst reason = 'this model cannot see images, so the picture itself was not passed';return { reference: referenceOf(file, mediaId, name, type, size, { method: 'not_readable', reason }), part: null, note: `[Attached image: ${name} (${type}); ${reason}.]` };}++ if (AUDIO_TYPE.test(type)) {++ let result = null;++ if (transcribe) {++ try { result = await transcribe({ bytes: buffer, media_type: type, name, signal }); }++ catch (err) { if (signal && signal.aborted) throw err; }++ }++ const transcript = result && typeof result.text === 'string' ? capText(result.text.trim()) : '';++ if (transcript) return { reference: referenceOf(file, mediaId, name, type, size, { method: 'text' }),++ part: textPart(format, `\n\n[Attached audio transcript: ${name} (${type})]\n${transcript}\n`), note: null };++ const reason = transcribe ? 'local speech recognition could not read this audio' : 'local speech recognition is unavailable';++ return { reference: referenceOf(file, mediaId, name, type, size, { method: 'not_readable', reason }), part: null,++ note: `[Attached audio: ${name} (${type}); ${reason}.]` };++ }++ if (ARCHIVE_TYPE.test(type) || /\.(?:zip|tar|tar\.gz|tgz)$/i.test(name)) {++ const archive = extractArchive(buffer, type, name);++ const parts = (archive.entries || []).map((entry) => {++ if (/\.pdf$/i.test(entry.name)) {++ const pdf = pdfText(entry.bytes);++ return pdf.pages.length ? `--- ${entry.name} (PDF) ---\n${pdf.pages.join('\n\n')}` : null;++ }++ return `--- ${entry.name} ---\n${entry.bytes.toString('utf8').replace(/\u0000/g, '')}`;++ }).filter(Boolean);++ if (parts.length) {++ const skipped = (archive.skipped || 0) + (archive.entries.length - parts.length);++ const text = capText(`${parts.join('\n\n')}${skipped ? `\n\n(${skipped} archive entries were not read)` : ''}`);++ return { reference: referenceOf(file, mediaId, name, type, size, { method: 'text' }), part: textPart(format, `\n\n[Attached archive: ${name} (${type})]\n${text}\n`), note: null };++ }++ const reason = archive.reason || 'the archive has no readable text, code or PDF entries';++ return { reference: referenceOf(file, mediaId, name, type, size, { method: 'not_readable', reason }), part: null, note: `[Attached archive: ${name} (${type}); ${reason}.]` };++ }if (CODE_TYPE.test(type) || TEXT_EXT.test(name) || /^text\//.test(type)) {const text = capText(buffer.toString('utf8').replace(/\u0000/g, ''));if (!text.trim()) {@@@ -322,7 +360,7 @@ function userContent(format, content) {* Returns a NEW input: the messages gain parts, the file references stay metadata-only (`read` says what* happened to each, which is the ai.file@1 shape). No bytes or extracted text are put into `input.files`.*/--async function attachFiles(input, fetcher, { signal, format = 'openai', vision = true, scan = null } = {}) {++async function attachFiles(input, fetcher, { signal, format = 'openai', vision = true, scan = null, transcribe = null } = {}) {const files = Array.isArray(input && input.files) ? input.files : [];if (!files.length) return input;checkLimits(files);@@@ -331,7 +369,7 @@ async function attachFiles(input, fetcher, { signal, format = 'openai', vision =let total = 0;let hasImage = false;for (const file of files) {-- const read = await readFile(file, fetcher, { signal, format, vision, scan });++ const read = await readFile(file, fetcher, { signal, format, vision, scan, transcribe });total += read.reference.size;if (total > MAX_TOTAL_BYTES) throw new AiError(413, 'files.too_large', `attached files may be up to ${MAX_TOTAL_BYTES / 1024 / 1024} MiB together`);references.push(read.reference);@@@ -58,7 +58,7 @@ function createMediaStore({ config, env = process.env, fetchImpl = globalThis.fe* pictures in (blog, wiki, or AI's own ai); absent, the AI tenant. → { media_id, url, width, height, mime }.* metadata keeps where it came from (source_url, license, author) or what made it (model, ai).*/-- async function put({ bytes, mime, filename = 'picture', owner = null, app: tenant = null, metadata = {}, timeoutMs = 30000 } = {}) {++ async function put({ bytes, mime, filename = 'picture', owner = null, app: tenant = null, metadata = {}, visibility = 'public', timeoutMs = 30000 } = {}) {if (!configured()) throw Object.assign(new Error('OpenVibe.Media is not configured (OV_MEDIA_INTERNAL_URL and the AI service secret are needed)'), { status: 503, code: 'media.unavailable' });if (!Buffer.isBuffer(bytes) || !bytes.length) throw Object.assign(new Error('the picture has no bytes'), { status: 422, code: 'media.empty' });const target = tenant && /^[a-z][a-z0-9-]{1,31}$/.test(String(tenant)) ? String(tenant) : app;@@@ -68,7 +68,7 @@ function createMediaStore({ config, env = process.env, fetchImpl = globalThis.feconst init = await call(base, {method: 'POST', headers: { ...auth, ...subject, 'Content-Type': 'application/json', Accept: 'application/json' },body: JSON.stringify({-- kind: 'asset', visibility: 'public', mime_type: mime, size_bytes: bytes.length,++ kind: 'asset', visibility, mime_type: mime, size_bytes: bytes.length,filename: String(filename || 'picture').slice(0, 120), content_hash: crypto.createHash('sha256').update(bytes).digest('hex'),metadata,}),@@@ -17,6 +17,7 @@ const FILE_REF = { type: 'object', additionalProperties: false, properties: {read: { type: 'object' }, role: { enum: ['input', 'artifact'] },} };const TURN_FILES = { type: 'array', maxItems: 10, items: FILE_REF };++const OUTPUT_FILE = OBJ({ name: STR(255), media_type: { enum: ['text/plain', 'text/markdown', 'text/csv', 'application/json'] } }, ['name', 'media_type']);const COMMON = { max_tokens: { type: 'integer', minimum: 1, maximum: 16000 }, temperature: { type: ['number', 'null'], minimum: 0, maximum: 2 },// AI Exchange F: a liability cap on the Balance hold for this request (optional; nothing when unset).max_cost_usd: { type: ['number', 'string', 'null'] } };@@@ -141,6 +142,7 @@ const workflows = [presence_penalty: { type: ['number', 'null'], minimum: -2, maximum: 2 },frequency_penalty: { type: ['number', 'null'], minimum: -2, maximum: 2 },files: TURN_FILES,++ output_file: OUTPUT_FILE,...COMMON,}, ['messages']),output_schema: OBJ({ choices: { type: 'array', minItems: 1, maxItems: 8, items: { type: 'object' } }, system_fingerprint: { type: ['string', 'null'] }, model: { type: ['string', 'null'] }, artifacts: { type: 'array', maxItems: 10, items: { type: 'object' } } }, ['choices']),@@@ -171,6 +173,7 @@ const workflows = [top_k: { type: ['integer', 'null'], minimum: 0 },metadata: { type: 'object' },files: TURN_FILES,++ output_file: OUTPUT_FILE,}, ['messages', 'max_tokens']),output_schema: OBJ({id: STR(200), type: STR(20), role: STR(20), model: { type: ['string', 'null'] },@@@ -168,6 +168,17 @@ function batchExec(b, result) {function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spawnImpl, log = console, research = null, db = null, clock = { now: () => Date.now() }, mediaStore = null }) {pool = require('../effort/pool').wrapPool(pool);++ // Turn attachments use only the host's local speech recognizer. Reading an audio file must never++ // select a paid transcription provider as a side effect of a chat request.++ let localAsr = null;++ async function transcribeAttachment({ bytes, signal }) {++ localAsr ||= require('../providers/whisper').createWhisperProvider({ key: 'whisper' }, { config });++ if (!localAsr.supports('transcribe')) throw new AiError(503, 'files.audio_unavailable', 'local speech recognition is unavailable');++ const file = path.join(os.tmpdir(), `openvibe-ai-attachment-${require('crypto').randomBytes(12).toString('hex')}`);++ fs.writeFileSync(file, bytes, { mode: 0o600, flag: 'wx' });++ try { return await localAsr.transcribe({ filePath: file, language: 'auto', signal, timeoutMs: 180000 }); }++ finally { try { fs.unlinkSync(file); } catch { /* already removed */ } }++ }// One loopback reader for everything that streams media (server/media-proxy.js): transcripts and media.analyze.let proxy = null;const reader = () => proxy || (proxy = require('../media-proxy').createMediaProxy({ fetcher, log }));@@@ -278,7 +289,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spaconst base = { ...input, messages: normMessages(input.messages, input.user) };// Files on the turn (ai.file@1) become text or image content for this call only.const turn = Array.isArray(input.files) && input.files.length-- ? await fileInputs.attachFiles(base, fetcher, { signal: ctx.signal, format: 'openai', vision: true })++ ? await fileInputs.attachFiles(base, fetcher, { signal: ctx.signal, format: 'openai', vision: true, transcribe: transcribeAttachment }): base;const messages = turn.messages;const image = input.image ? await fetcher.loadImage({ ...input.image, max_width: input.image.max_width || 1024 }, { signal: ctx.signal }) : null;@@@ -408,7 +419,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spaconst { route, selection } = await selectedRoute(await routeFor(step.route || 'default.chat'), input);// Media-backed files on the turn (ai.file@1) become text or image content for this call only.const turn = Array.isArray(input.files) && input.files.length-- ? await fileInputs.attachFiles(input, fetcher, { signal: ctx.signal, format: 'openai', vision: true })++ ? await fileInputs.attachFiles(input, fetcher, { signal: ctx.signal, format: 'openai', vision: true, transcribe: transcribeAttachment }): input;const openai = {};for (const k of ['messages', 'tools', 'tool_choice', 'parallel_tool_calls', 'response_format', 'stop', 'seed', 'top_p', 'presence_penalty', 'frequency_penalty']) {@@@ -431,9 +442,22 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spaif (!c || !Array.isArray(c.choices) || !c.choices.length) throw new AiError(502, 'output.empty', 'the provider answered, but with no choices');// Files the model produced beside its text: Media references pass through; inline bytes need the// configured Media store (server/artifacts.js), never the run's output.-- const subject = ctx.onBehalfOf && ctx.onBehalfOf.id ? ctx.onBehalfOf.id : null;-- const artifacts = await filesOut.collectArtifacts(c.choices[0] && c.choices[0].message, { store: filesOut.storeOf(config), subject });-- return { output: { choices: c.choices, system_fingerprint: c.system_fingerprint || null, model: exec.model, ...(artifacts.length ? { artifacts } : {}) }, exec, route };++ const subject = typeof ctx.onBehalfOf === 'string' ? ctx.onBehalfOf : ctx.onBehalfOf?.id || null;++ const artifacts = [];++ const choices = [];++ const artifactStore = filesOut.storeOf(config, mediaStore);++ for (const choice of c.choices) {++ const saved = await filesOut.collectArtifacts(choice && choice.message, { store: artifactStore, subject });++ if (input.output_file) saved.push(...await filesOut.collectTextArtifact(choice && choice.message && choice.message.content, input.output_file, { store: artifactStore, subject }));++ artifacts.push(...saved);++ if (artifacts.length > filesOut.MAX_ARTIFACTS) throw new AiError(413, 'files.too_many', `a model may return at most ${filesOut.MAX_ARTIFACTS} files`);++ const message = { ...(choice.message || {}) };++ delete message.files;++ delete message.artifacts;++ if (saved.length) message.artifacts = saved;++ choices.push({ ...choice, message });++ }++ return { output: { choices, system_fingerprint: c.system_fingerprint || null, model: exec.model, ...(artifacts.length ? { artifacts } : {}) }, exec, route };}/**@@@ -444,7 +468,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spaconst { route, selection } = await selectedRoute(await routeFor(step.route || 'default.messages'), input);// Media-backed files on the turn (ai.file@1): text parts, or base64 image sources for a vision model.const turn = Array.isArray(input.files) && input.files.length-- ? await fileInputs.attachFiles(input, fetcher, { signal: ctx.signal, format: 'anthropic', vision: true })++ ? await fileInputs.attachFiles(input, fetcher, { signal: ctx.signal, format: 'anthropic', vision: true, transcribe: transcribeAttachment }): input;const req = {messages: turn.messages,@@@ -460,9 +484,14 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spaconst exec = await pool.execute(route, 'messages', req, { ...ctx, ...selectionCtx(selection), routeKey: route.key, routeVersion: route.version, promptHash: sha256({ m: turn.messages, t: input.tools || null }) });const m = exec.result.message;if (!m || !Array.isArray(m.content)) throw new AiError(502, 'output.empty', 'the provider answered, but not with a message');-- const subject = ctx.onBehalfOf && ctx.onBehalfOf.id ? ctx.onBehalfOf.id : null;-- const artifacts = await filesOut.collectArtifacts(m, { store: filesOut.storeOf(config), subject });-- return { output: { ...m, model: m.model || exec.model, ...(artifacts.length ? { artifacts } : {}) }, exec, route };++ const subject = typeof ctx.onBehalfOf === 'string' ? ctx.onBehalfOf : ctx.onBehalfOf?.id || null;++ const artifactStore = filesOut.storeOf(config, mediaStore);++ const artifacts = await filesOut.collectArtifacts(m, { store: artifactStore, subject });++ if (input.output_file) artifacts.push(...await filesOut.collectTextArtifact(m.content.filter((part) => part && part.type === 'text').map((part) => part.text).join('\n'), input.output_file, { store: artifactStore, subject }));++ const message = { ...m };++ delete message.files;++ delete message.artifacts;++ return { output: { ...message, model: m.model || exec.model, ...(artifacts.length ? { artifacts } : {}) }, exec, route };}async function transcribeStep(step, input, wf, run, ctx) {@@@ -1,6 +1,6 @@'use strict';const assert = require('node:assert/strict');--const { collectArtifacts, storeOf } = require('../server/artifacts');++const { collectArtifacts, collectTextArtifact, storeOf } = require('../server/artifacts');const { suite } = require('./helpers');const t = suite('file-artifacts');@@@ -37,9 +37,34 @@ t.test('storeOf finds a configured store and nothing else', () => {assert.equal(storeOf({ media: { artifactStore: store } }), store);});++t.test('the production Media uploader stores generated bytes privately for the person', async () => {++ const puts = [];++ const media = { configured: () => true, put: async (what) => { puts.push(what); return { media_id: ID }; } };++ const store = storeOf({}, media);++ const subject = 'usr_01ARZ3NDEKTSV4RRFFQ69G5FAV';++ const out = await collectArtifacts({ artifacts: [{ name: 'answer.txt', media_type: 'text/plain', data: Buffer.from('private output bytes').toString('base64') }] }, { store, subject });++ assert.equal(puts[0].visibility, 'private');++ assert.equal(puts[0].owner, subject);++ assert.equal(puts[0].bytes.toString(), 'private output bytes');++ assert.equal(out[0].id, ID);++ assert.deepEqual(out[0].access.readers, [subject]);++ assert.ok(!JSON.stringify(out).includes('private output bytes'), 'the reference contains no inline bytes');++ await assert.rejects(() => collectArtifacts({ artifacts: [{ name: 'x.txt', data: 'eA==' }] }, { store }),++ (err) => err.code === 'files.subject_required');++});++t.test('more than ten produced files is a refusal', async () => {const artifacts = Array.from({ length: 11 }, () => ({ id: ID, name: 'a.txt', media_type: 'text/plain', size: 1 }));await assert.rejects(() => collectArtifacts({ artifacts }), (err) => err.status === 413 && err.code === 'files.too_many');});++t.test('a requested text output is saved as a file and a path is refused', async () => {++ const puts = [];++ const store = { put: async (what) => { puts.push(what); return { id: ID, name: what.name, role: 'artifact' }; } };++ const files = await collectTextArtifact('name,score\nAda,10\n', { name: 'scores.csv', media_type: 'text/csv' }, { store, subject: 'usr_01ARZ3NDEKTSV4RRFFQ69G5FAV' });++ assert.equal(files.length, 1);++ assert.equal(puts[0].bytes.toString(), 'name,score\nAda,10\n');++ await assert.rejects(() => collectTextArtifact('x', { name: '../x.csv', media_type: 'text/csv' }, { store }), (err) => err.code === 'files.name_invalid');++});++t.run();@@@ -1,5 +1,6 @@'use strict';const assert = require('node:assert/strict');++const zlib = require('node:zlib');const { attachFiles, readFile, pdfText, setScanner, MAX_FILES, MAX_FILE_BYTES } = require('../server/file-inputs');const { suite } = require('./helpers');@@@ -29,6 +30,41 @@ const pdfFile = { body: pdf('Quarterly report: revenue up 12%'), type: 'applicatconst png = { body: Buffer.from([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]), type: 'image/png' };const zip = { body: Buffer.from([0x50, 0x4b, 3, 4, 5]), type: 'application/zip' };++function zipOf(name, text) {++ const data = Buffer.from(text);++ const packed = zlib.deflateRawSync(data);++ const label = Buffer.from(name);++ const local = Buffer.alloc(30);++ local.writeUInt32LE(0x04034b50, 0);++ local.writeUInt16LE(8, 8);++ local.writeUInt32LE(packed.length, 18);++ local.writeUInt32LE(data.length, 22);++ local.writeUInt16LE(label.length, 26);++ const central = Buffer.alloc(46);++ central.writeUInt32LE(0x02014b50, 0);++ central.writeUInt16LE(8, 10);++ central.writeUInt32LE(packed.length, 20);++ central.writeUInt32LE(data.length, 24);++ central.writeUInt16LE(label.length, 28);++ const directory = Buffer.concat([central, label]);++ const end = Buffer.alloc(22);++ end.writeUInt32LE(0x06054b50, 0);++ end.writeUInt16LE(1, 8);++ end.writeUInt16LE(1, 10);++ end.writeUInt32LE(directory.length, 12);++ end.writeUInt32LE(local.length + label.length + packed.length, 16);++ return Buffer.concat([local, label, packed, directory, end]);++}++++function tarOf(name, text) {++ const data = Buffer.from(text);++ const head = Buffer.alloc(512);++ head.write(name, 0, 100);++ head.write(data.length.toString(8).padStart(11, '0') + '\0', 124, 12);++ head[156] = 48;++ return Buffer.concat([head, data, Buffer.alloc(Math.ceil(data.length / 512) * 512 - data.length + 1024)]);++}++t.test('CSV, PDF, image and unknown types become model content by type; references never carry bytes', async () => {const fetcher = fetcherOf({ [ID(1)]: csv, [ID(2)]: pdfFile, [ID(3)]: png, [ID(4)]: zip });const out = await attachFiles({@@@ -45,7 +81,7 @@ t.test('CSV, PDF, image and unknown types become model content by type; referencconst texts = parts.filter((p) => p.type === 'text').map((p) => p.text).join('\n');assert.match(texts, /name,score/);assert.match(texts, /Quarterly report: revenue up 12%/);-- assert.match(texts, /bundle\.zip \(application\/zip\); this file type cannot be read/);++ assert.match(texts, /bundle\.zip \(application\/zip\); the ZIP directory is missing/);assert.ok(parts.some((p) => p.type === 'image_url' && /^data:image\/png;base64,/.test(p.image_url.url)), 'the picture goes to a vision model');assert.equal(out.files.length, 4);assert.deepEqual(out.files.map((f) => f.read.method), ['text', 'pages', 'image', 'not_readable']);@@@ -57,6 +93,56 @@ t.test('CSV, PDF, image and unknown types become model content by type; referencassert.equal(out.hasImage, true);});++t.test('ZIP and compressed TAR expose bounded text entries with paths and keep bytes out of references', async () => {++ const bundle = zipOf('src/app.js', 'export const answer = 42;');++ const archive = zlib.gzipSync(tarOf('notes/readme.md', 'A useful note.'));++ const fetcher = fetcherOf({++ [ID(5)]: { body: bundle, type: 'application/zip' },++ [ID(6)]: { body: archive, type: 'application/gzip' },++ });++ const out = await attachFiles({ messages: [{ role: 'user', content: 'Summarize the files' }], files: [++ { id: ID(5), name: 'code.zip', size: bundle.length },++ { id: ID(6), name: 'docs.tar.gz', size: archive.length },++ ] }, fetcher);++ const text = out.messages[0].content.map((part) => part.text).join('\n');++ assert.match(text, /src\/app\.js[\s\S]*answer = 42/);++ assert.match(text, /notes\/readme\.md[\s\S]*A useful note/);++ assert.deepEqual(out.files.map((file) => file.read.method), ['text', 'text']);++ assert.ok(!JSON.stringify(out.files).includes('useful note'));++});++++t.test('unsafe archive paths and overlarge expanded entries are skipped', async () => {++ const unsafe = zipOf('../private.txt', 'do not show this');++ const large = zipOf('large.txt', 'A'.repeat(600000));++ const fetcher = fetcherOf({ [ID(7)]: { body: unsafe, type: 'application/zip' }, [ID(8)]: { body: large, type: 'application/zip' } });++ for (const [id, name, size] of [[ID(7), 'unsafe.zip', unsafe.length], [ID(8), 'large.zip', large.length]]) {++ const out = await attachFiles({ messages: [{ role: 'user', content: 'Read' }], files: [{ id, name, size }] }, fetcher);++ assert.equal(out.files[0].read.method, 'not_readable');++ assert.ok(!JSON.stringify(out.messages).includes('do not show this'));++ assert.ok(!JSON.stringify(out.messages).includes('AAAAAA'));++ }++});++++t.test('audio is transcribed for a text model with local ASR, with an honest note if unavailable', async () => {++ const bytes = Buffer.from('audio fixture bytes');++ const fetcher = fetcherOf({ [ID(9)]: { body: bytes, type: 'audio/ogg' } });++ const input = { messages: [{ role: 'user', content: 'What did I say?' }], files: [{ id: ID(9), name: 'voice.ogg', size: bytes.length }] };++ let called = 0;++ const heard = await attachFiles(input, fetcher, { transcribe: async ({ bytes: got, media_type: type }) => {++ called++;++ assert.equal(type, 'audio/ogg');++ assert.deepEqual(got, bytes);++ return { text: 'A spoken answer.' };++ } });++ assert.equal(called, 1);++ assert.match(heard.messages[0].content[1].text, /Attached audio transcript:[\s\S]*A spoken answer/);++ assert.equal(heard.files[0].read.method, 'text');++ assert.ok(!JSON.stringify(heard.files).includes('spoken answer'));++ const unavailable = await attachFiles(input, fetcher);++ assert.equal(unavailable.files[0].read.method, 'not_readable');++ assert.match(unavailable.messages[0].content[1].text, /local speech recognition is unavailable/);++});++t.test('a text-only model gets an honest note instead of a dropped picture', async () => {const fetcher = fetcherOf({ [ID(3)]: png });const out = await attachFiles({ messages: [{ role: 'user', content: 'What is this?' }], files: [{ id: ID(3), name: 'shot.png', size: 8 }] }, fetcher, { vision: false });@@@ -6,17 +6,45 @@ const { boot, request, token, suite, seamServer, ALL } = require('./helpers');const t = suite('file-turn');const ID = 'med_01ARZ3NDEKTSV4RRFFQ69G5FAV';--let h; let seam; let media; let service;++const ZIP_ID = 'med_01ARZ3NDEKTSV4RRFFQ69G5FAW';++const PERSON = 'usr_01ARZ3NDEKTSV4RRFFQ69G5FAV';++let h; let seam; let media; let service; let uploads;++++function codeZip() {++ const name = Buffer.from('src/answer.js');++ const data = Buffer.from('export const answer = 42;');++ const local = Buffer.alloc(30);++ local.writeUInt32LE(0x04034b50, 0);++ local.writeUInt32LE(data.length, 18);++ local.writeUInt32LE(data.length, 22);++ local.writeUInt16LE(name.length, 26);++ const central = Buffer.alloc(46);++ central.writeUInt32LE(0x02014b50, 0);++ central.writeUInt32LE(data.length, 20);++ central.writeUInt32LE(data.length, 24);++ central.writeUInt16LE(name.length, 28);++ const directory = Buffer.concat([central, name]);++ const end = Buffer.alloc(22);++ end.writeUInt32LE(0x06054b50, 0);++ end.writeUInt16LE(1, 8);++ end.writeUInt16LE(1, 10);++ end.writeUInt32LE(directory.length, 12);++ end.writeUInt32LE(local.length + name.length + data.length, 16);++ return Buffer.concat([local, name, data, directory, end]);++}t.test('a Media server and a counting OpenAI-shaped provider', async () => {media = nodeHttp.createServer((req, res) => {if (req.url === `/o/${ID}`) { res.writeHead(200, { 'Content-Type': 'text/csv' }); return res.end('name,score\nAda,10\n'); }++ if (req.url === `/o/${ZIP_ID}`) { res.writeHead(200, { 'Content-Type': 'application/zip' }); return res.end(codeZip()); }res.writeHead(404, { 'Content-Type': 'application/json' }); res.end('{"error":"Not found"}');});await new Promise((r) => media.listen(0, '127.0.0.1', r));const mediaUrl = `http://127.0.0.1:${media.address().port}`;seam = await seamServer(() => ({ choices: [{ index: 0, message: { role: 'assistant', content: 'It has two rows.' } }], usage: { prompt_tokens: 10, completion_tokens: 5 } }));-- h = await boot({ env: { AI_STUB_FALLBACK: 'false', OV_MEDIA_URL: mediaUrl, OV_MEDIA_INTERNAL_URL: mediaUrl } });++ uploads = [];++ const mediaStore = { configured: () => true, put: async (what) => { uploads.push(what); return { media_id: ZIP_ID }; } };++ h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_FREE_PROVIDERS: 'seam', OV_MEDIA_URL: mediaUrl, OV_MEDIA_INTERNAL_URL: mediaUrl }, mediaStore });service = token('chat', ALL);let r = await request(h.base, 'POST', '/api/v1/providers', { tok: service, body: { key: 'seam', kind: 'openai', base_url: `${seam.url}/v1`, auth_mode: 'none', capabilities: ['chat', 'complete'], default_model: 'm1' } });assert.equal(r.status, 201, r.text);@@@ -26,7 +54,57 @@ t.test('a Media server and a counting OpenAI-shaped provider', async () => {assert.equal(r.status, 201, r.text);});++t.test('POST /v1/chat/completions returns a generated file by private Media reference', async () => {++ const previous = seam.handler;++ seam.handler = () => ({ choices: [{ index: 0, message: { role: 'assistant', content: 'Download the result.', artifacts: [++ { name: 'answer.txt', media_type: 'text/plain', data: Buffer.from('private output bytes').toString('base64') },++ ] } }], usage: { prompt_tokens: 10, completion_tokens: 5 } });++ try {++ const acting = token('chat', ALL, { extra: { on_behalf_of: PERSON } });++ const r = await request(h.base, 'POST', '/v1/chat/completions', { tok: acting, body: { model: 'openvibe', messages: [{ role: 'user', content: 'Make a text file' }] } });++ assert.equal(r.status, 200, r.text);++ assert.equal(uploads.length, 1);++ assert.equal(uploads[0].visibility, 'private');++ assert.equal(uploads[0].owner, PERSON);++ assert.equal(uploads[0].bytes.toString(), 'private output bytes');++ const files = r.body.choices[0].message.artifacts;++ assert.equal(files[0].id, ZIP_ID);++ assert.equal(files[0].role, 'artifact');++ assert.ok(!r.text.includes('private output bytes'));++ assert.ok(!r.text.includes(Buffer.from('private output bytes').toString('base64')));++ const stored = await h.db.prepare('SELECT output FROM runs ORDER BY created_at DESC LIMIT 1').get();++ assert.ok(!JSON.stringify(stored.output).includes('private output bytes'));++ } finally { seam.handler = previous; }++});++++t.test('POST /v1/chat/completions saves a normal model answer when output_file is requested', async () => {++ const before = uploads.length;++ const acting = token('chat', ALL, { extra: { on_behalf_of: PERSON } });++ const r = await request(h.base, 'POST', '/v1/chat/completions', { tok: acting, body: {++ model: 'openvibe', messages: [{ role: 'user', content: 'Write a short text file' }],++ output_file: { name: 'answer.txt', media_type: 'text/plain' },++ } });++ assert.equal(r.status, 200, r.text);++ assert.equal(uploads.length, before + 1);++ assert.equal(uploads[before].bytes.toString(), 'It has two rows.');++ assert.equal(uploads[before].visibility, 'private');++ assert.equal(r.body.choices[0].message.artifacts[0].id, ZIP_ID);++ assert.equal(r.body.x_openvibe.artifacts[0].id, ZIP_ID);++});++++t.test('POST /v1/chat/completions reads a ZIP code file into the turn', async () => {++ const r = await request(h.base, 'POST', '/v1/chat/completions', {++ tok: service,++ body: { model: 'openvibe', messages: [{ role: 'user', content: 'What does this code return?' }], files: [{ id: ZIP_ID, name: 'code.zip', size: codeZip().length }] },++ });++ assert.equal(r.status, 200, r.text);++ assert.match(JSON.stringify(seam.last.messages), /src\/answer\.js[\s\S]*answer = 42/);++ const stored = await h.db.prepare('SELECT input FROM runs ORDER BY created_at DESC LIMIT 1').get();++ assert.ok(!JSON.stringify(stored.input).includes('export const answer'), 'archive content is never kept in the run');++});++t.test('POST /v1/chat/completions reads the file into the provider request, not the run', async () => {++ const before = seam.calls;const r = await request(h.base, 'POST', '/v1/chat/completions', {tok: service,body: {@@@ -38,7 +116,7 @@ t.test('POST /v1/chat/completions reads the file into the provider request, notconst sent = JSON.stringify(seam.last.messages);assert.match(sent, /Attached file: scores\.csv/);assert.match(sent, /name,score/);-- assert.equal(seam.calls, 1);++ assert.equal(seam.calls, before + 1);const stored = await h.db.prepare('SELECT input FROM runs ORDER BY created_at DESC LIMIT 1').get();const raw = typeof stored.input === 'string' ? stored.input : JSON.stringify(stored.input);assert.ok(!raw.includes('Ada,10'), 'the file bytes are not in the run input');@@@ -45,7 +45,7 @@ function tmpDir() {// A boot on a dir used before is a restart on the same database: its database stays open.const restartable = new Map();--async function boot({ searchFetch, env = {}, dir, clock, fetchImpl, credentialFetch, research, machineModels, log, variantEvidence, community, communityEndpoints, webhooks, memoryTransport, memoryLookup, ownerNotify } = {}) {++async function boot({ searchFetch, env = {}, dir, clock, fetchImpl, credentialFetch, research, machineModels, log, variantEvidence, community, communityEndpoints, webhooks, memoryTransport, memoryLookup, ownerNotify, mediaStore } = {}) {const reuse = dir !== undefined;if (dir === undefined) dir = tmpDir();const fullEnv = {@@@ -62,7 +62,7 @@ async function boot({ searchFetch, env = {}, dir, clock, fetchImpl, credentialFe// An injected endpoint store needs the migrated database, so a factory is called with it before the app boots.const endpointStore = typeof communityEndpoints === 'function' ? communityEndpoints(testdb.db) : communityEndpoints;const webhookStore = typeof webhooks === 'function' ? webhooks(testdb.db) : webhooks;-- const h = await start({ searchFetch, config, db: testdb.db, env: fullEnv, log: log || silent, clock, fetchImpl, credentialFetch, research: researcher, machineModels, variantEvidence, community, communityEndpoints: endpointStore, webhooks: webhookStore, memoryTransport, memoryLookup, ownerNotify });++ const h = await start({ searchFetch, config, db: testdb.db, env: fullEnv, log: log || silent, clock, fetchImpl, credentialFetch, research: researcher, machineModels, variantEvidence, community, communityEndpoints: endpointStore, webhooks: webhookStore, memoryTransport, memoryLookup, ownerNotify, mediaStore });const base = `http://127.0.0.1:${h.server.address().port}`;return { ...h, base, dir, env: fullEnv, testdb, stop: async () => { await h.close(); if (!reuse) await testdb.close(); } };}
@@ -38,7 +38,7 @@ const { EXACT_ID } = modelCatalog;
@@ -38,7 +38,7 @@ const { EXACT_ID } = modelCatalog;
const { resolveRequest } = require('../exchange/presets');
const { resolveRequest } = require('../exchange/presets');
const { compact, receiptId } = require('../exchange/receipts');
const { compact, receiptId } = require('../exchange/receipts');
-const FIELDS = ['system', 'tools', 'tool_choice', 'stop_sequences', 'temperature', 'top_p', 'top_k', 'metadata', 'files'];
+const FIELDS = ['system', 'tools', 'tool_choice', 'stop_sequences', 'temperature', 'top_p', 'top_k', 'metadata', 'files', 'output_file'];
const MAX_TOKENS = 64000;
const MAX_TOKENS = 64000;
// Anthropic's error types by HTTP status (docs.anthropic.com/en/api/errors).
// Anthropic's error types by HTTP status (docs.anthropic.com/en/api/errors).
@@ -291,6 +291,7 @@ function messagesRouter({ runs, auth, config, registry, presets = null, receipts
@@ -291,6 +291,7 @@ function messagesRouter({ runs, auth, config, registry, presets = null, receipts
...(out.usage && out.usage.cache_creation_input_tokens != null ? { cache_creation_input_tokens: out.usage.cache_creation_input_tokens } : {}),
...(out.usage && out.usage.cache_creation_input_tokens != null ? { cache_creation_input_tokens: out.usage.cache_creation_input_tokens } : {}),
},
},
};
};
+ if (Array.isArray(out.artifacts) && out.artifacts.length) message.artifacts = out.artifacts;
if (b.usage?.include === true) Object.assign(message.usage, require('./compat-extra').accounting(run));
if (b.usage?.include === true) Object.assign(message.usage, require('./compat-extra').accounting(run));
const disclosure = await models.disclosureOf({ registry, config, run });
const disclosure = await models.disclosureOf({ registry, config, run });
const community = (disclosure && disclosure.community) || (run.explain && run.explain.community) || null;
const community = (disclosure && disclosure.community) || (run.explain && run.explain.community) || null;
@@ -302,7 +303,8 @@ function messagesRouter({ runs, auth, config, registry, presets = null, receipts
@@ -302,7 +303,8 @@ function messagesRouter({ runs, auth, config, registry, presets = null, receipts
cost_usd: run.usage.cost_usd, synthetic: Boolean(run.synthetic),
cost_usd: run.usage.cost_usd, synthetic: Boolean(run.synthetic),
...(community ? { community } : {}),
...(community ? { community } : {}),
...(run.output.label ? { policy: run.output.policy || 'uncensored', label: run.output.label } : {}),
...(run.output.label ? { policy: run.output.policy || 'uncensored', label: run.output.label } : {}),
- privacy: receipt && receipt.privacy ? receipt.privacy.label : null, receipt_id: receipt ? receipt.id : null };
+ privacy: receipt && receipt.privacy ? receipt.privacy.label : null, receipt_id: receipt ? receipt.id : null,
+ ...(out.artifacts?.length ? { artifacts: out.artifacts } : {}) };
if (run.output.label) {
if (run.output.label) {
message.label = run.output.label;
message.label = run.output.label;
message.policy = run.output.policy || 'uncensored';
message.policy = run.output.policy || 'uncensored';
@@ -52,7 +52,7 @@ const { EXACT_ID } = modelCatalog;
@@ -52,7 +52,7 @@ const { EXACT_ID } = modelCatalog;
const { resolveRequest } = require('../exchange/presets');
const { resolveRequest } = require('../exchange/presets');
const { compact, receiptId } = require('../exchange/receipts');
const { compact, receiptId } = require('../exchange/receipts');
-const FIELDS = ['messages', 'tools', 'tool_choice', 'parallel_tool_calls', 'response_format', 'stop', 'seed', 'top_p', 'presence_penalty', 'frequency_penalty', 'temperature', 'files'];
+const FIELDS = ['messages', 'tools', 'tool_choice', 'parallel_tool_calls', 'response_format', 'stop', 'seed', 'top_p', 'presence_penalty', 'frequency_penalty', 'temperature', 'files', 'output_file'];
// Cloudflare answers a 502 or 504 from the origin with its own HTML page, so an OpenAI client would get no JSON error:
// Cloudflare answers a 502 or 504 from the origin with its own HTML page, so an OpenAI client would get no JSON error:
// an upstream that failed or could not be reached is 503 (OpenAI's own "unavailable, retry"), the body kept. An invalid
// an upstream that failed or could not be reached is 503 (OpenAI's own "unavailable, retry"), the body kept. An invalid
@@ -300,6 +300,7 @@ function openaiRouter({ runs, auth, config, registry, presets = null, receipts =
@@ -300,6 +300,7 @@ function openaiRouter({ runs, auth, config, registry, presets = null, receipts =
...(run.output.label ? { policy: run.output.policy || 'uncensored', label: run.output.label } : {}),
...(run.output.label ? { policy: run.output.policy || 'uncensored', label: run.output.label } : {}),
privacy: receipt && receipt.privacy ? receipt.privacy.label : null,
privacy: receipt && receipt.privacy ? receipt.privacy.label : null,
receipt_id: receipt ? receipt.id : null,
receipt_id: receipt ? receipt.id : null,
+ ...(run.output.artifacts?.length ? { artifacts: run.output.artifacts } : {}),
};
};
if (run.output.label) res.setHeader('X-OpenVibe-Content-Policy', run.output.policy || 'uncensored');
if (run.output.label) res.setHeader('X-OpenVibe-Content-Policy', run.output.policy || 'uncensored');
if (!live) res.setHeader('X-OpenVibe-Run-Id', run.id); // a stream sent it with its headers
if (!live) res.setHeader('X-OpenVibe-Run-Id', run.id); // a stream sent it with its headers
new file mode 100644
new file mode 100644
@@ -0,0 +1,95 @@
@@ -0,0 +1,95 @@
+'use strict';
+/** Bounded, read-only extraction of ordinary ZIP and TAR archives attached to a turn. */
+const zlib = require('node:zlib');
+
+const MAX_ENTRIES = 100;
+const MAX_EXTRACTED = 2 * 1024 * 1024;
+const MAX_ENTRY = 512 * 1024;
+const TEXT_NAME = /\.(txt|text|md|markdown|csv|tsv|json|jsonl|log|xml|yaml|yml|toml|ini|cfg|conf|html?|css|scss|less|m?js|cjs|jsx|ts|tsx|py|rb|go|rs|java|kt|kts|c|h|cc|cpp|hpp|cs|php|sh|bash|zsh|sql|graphql|gql|proto|swift|lua|pl|r|m|vue|svelte|astro|tex|pdf)$/i;
+
+const safeName = (name) => {
+ const value = String(name || '').replace(/\\/g, '/');
+ return value && !value.startsWith('/') && !value.split('/').includes('..') && !/[\u0000-\u001f\u007f]/.test(value) ? value.slice(0, 255) : null;
+};
+const readable = (name) => Boolean(safeName(name) && TEXT_NAME.test(name));
+
+function zipEntries(buffer) {
+ const start = Math.max(0, buffer.length - 65557);
+ let end = -1;
+ for (let i = buffer.length - 22; i >= start; i--) {
+ if (buffer.readUInt32LE(i) === 0x06054b50) { end = i; break; }
+ }
+ if (end < 0) return { entries: [], reason: 'the ZIP directory is missing' };
+ const count = buffer.readUInt16LE(end + 10);
+ const size = buffer.readUInt32LE(end + 12);
+ let offset = buffer.readUInt32LE(end + 16);
+ if (count > MAX_ENTRIES || offset + size > buffer.length || buffer.readUInt16LE(end + 4) !== 0 || buffer.readUInt16LE(end + 6) !== 0) {
+ return { entries: [], reason: 'the ZIP is too large or uses multiple disks' };
+ }
+ const entries = [];
+ let total = 0;
+ let skipped = 0;
+ for (let i = 0; i < count; i++) {
+ if (offset + 46 > buffer.length || buffer.readUInt32LE(offset) !== 0x02014b50) return { entries: [], reason: 'the ZIP directory is damaged' };
+ const flags = buffer.readUInt16LE(offset + 8);
+ const method = buffer.readUInt16LE(offset + 10);
+ const packed = buffer.readUInt32LE(offset + 20);
+ const length = buffer.readUInt32LE(offset + 24);
+ const nameLength = buffer.readUInt16LE(offset + 28);
+ const extraLength = buffer.readUInt16LE(offset + 30);
+ const commentLength = buffer.readUInt16LE(offset + 32);
+ const local = buffer.readUInt32LE(offset + 42);
+ const next = offset + 46 + nameLength + extraLength + commentLength;
+ if (next > buffer.length) return { entries: [], reason: 'the ZIP directory is damaged' };
+ const name = buffer.subarray(offset + 46, offset + 46 + nameLength).toString('utf8');
+ offset = next;
+ if (!readable(name) || (flags & 1) || ![0, 8].includes(method) || length > MAX_ENTRY || total + length > MAX_EXTRACTED) { skipped++; continue; }
+ if (local + 30 > buffer.length || buffer.readUInt32LE(local) !== 0x04034b50) { skipped++; continue; }
+ const dataAt = local + 30 + buffer.readUInt16LE(local + 26) + buffer.readUInt16LE(local + 28);
+ if (dataAt + packed > buffer.length) { skipped++; continue; }
+ try {
+ const compressed = buffer.subarray(dataAt, dataAt + packed);
+ const bytes = method === 0 ? compressed : zlib.inflateRawSync(compressed, { maxOutputLength: MAX_ENTRY });
+ if (bytes.length !== length) { skipped++; continue; }
+ entries.push({ name: safeName(name), bytes });
+ total += bytes.length;
+ } catch { skipped++; }
+ }
+ return { entries, skipped };
+}
+
+function tarEntries(buffer) {
+ const entries = [];
+ let total = 0;
+ let skipped = 0;
+ let offset = 0;
+ let count = 0;
+ while (offset + 512 <= buffer.length && buffer[offset] !== 0) {
+ if (++count > MAX_ENTRIES) return { entries: [], reason: 'the TAR has too many entries' };
+ const head = buffer.subarray(offset, offset + 512);
+ const name = head.subarray(0, 100).toString('utf8').replace(/\u0000.*$/, '');
+ const rawSize = head.subarray(124, 136).toString('ascii').replace(/\u0000.*$/, '').trim();
+ if (!/^[0-7]+$/.test(rawSize)) return { entries: [], reason: 'the TAR header is damaged' };
+ const size = parseInt(rawSize, 8);
+ const type = head[156];
+ const dataAt = offset + 512;
+ if (!Number.isSafeInteger(size) || dataAt + size > buffer.length) return { entries: [], reason: 'the TAR entry is incomplete' };
+ if ((type === 0 || type === 48) && readable(name) && size <= MAX_ENTRY && total + size <= MAX_EXTRACTED) {
+ entries.push({ name: safeName(name), bytes: buffer.subarray(dataAt, dataAt + size) });
+ total += size;
+ } else skipped++;
+ offset = dataAt + Math.ceil(size / 512) * 512;
+ }
+ return { entries, skipped };
+}
+
+function extractArchive(buffer, type, name) {
+ if (type === 'application/zip' || /\.zip$/i.test(name)) return zipEntries(buffer);
+ if (/\.(?:tar\.gz|tgz)$/i.test(name) || type === 'application/gzip') {
+ try { return tarEntries(zlib.gunzipSync(buffer, { maxOutputLength: MAX_EXTRACTED + MAX_ENTRIES * 512 })); }
+ catch { return { entries: [], reason: 'the compressed TAR is damaged or too large' }; }
+ }
+ return tarEntries(buffer);
+}
+
+module.exports = { extractArchive, MAX_ENTRIES, MAX_EXTRACTED };
@@ -4,14 +4,14 @@
@@ -4,14 +4,14 @@
*
*
* A provider can answer with files beside its text: each entry names a Media object it already stored
* A provider can answer with files beside its text: each entry names a Media object it already stored
* (the reference shape), which is returned as it is. A provider that hands back the bytes themselves
* (the reference shape), which is returned as it is. A provider that hands back the bytes themselves
- * (`data`/`b64`) needs a Media store in this service: `store.put({ name, media_type, bytes, subject })`
- * returns the stored ai.file@1. When no store is configured such a file is REFUSED with
- * files.store_unavailable, never dropped and never put in the run output as inline bytes. The store is
- * configured by the operator (a Media tenant AI may write to); nothing here pretends to have one.
+ * (`data`/`b64`) goes to the existing Media uploader, private to the person on the turn.
+ * When Media is unavailable, such a file is refused with files.store_unavailable and is never
+ * put in the run output as inline bytes.
*/
*/
const { AiError } = require('./util');
const { AiError } = require('./util');
const MAX_ARTIFACTS = 10;
const MAX_ARTIFACTS = 10;
+const MAX_ARTIFACT_BYTES = 10 * 1024 * 1024;
const MEDIA_ID = /^med_[0-9A-HJKMNP-TV-Z]{26}$/;
const MEDIA_ID = /^med_[0-9A-HJKMNP-TV-Z]{26}$/;
function storedRef(file) {
function storedRef(file) {
@@ -29,10 +29,20 @@ function storedRef(file) {
@@ -29,10 +29,20 @@ function storedRef(file) {
};
};
}
}
-/** The store a run may write to (config.media.artifactStore with put(), or null). */
-function storeOf(config) {
+/** The Media store for inline output bytes; a generated file belongs to the person on the turn. */
+function storeOf(config, mediaStore = null) {
const s = config && config.media && config.media.artifactStore;
const s = config && config.media && config.media.artifactStore;
- return s && typeof s.put === 'function' ? s : null;
+ if (s && typeof s.put === 'function') return s;
+ if (!mediaStore || typeof mediaStore.put !== 'function' || typeof mediaStore.configured !== 'function' || !mediaStore.configured()) return null;
+ return { put: async ({ name, media_type: mediaType, bytes, subject }) => {
+ if (!subject) throw new AiError(422, 'files.subject_required', 'a generated file needs a person on the turn so Media can keep it private to them');
+ if (bytes.length > MAX_ARTIFACT_BYTES) throw new AiError(413, 'files.too_large', 'a generated file may be up to 10 MiB');
+ const saved = await mediaStore.put({ bytes, mime: mediaType || 'application/octet-stream', filename: name || 'file', owner: subject,
+ visibility: 'private', metadata: { origin: 'ai.turn.artifact' } });
+ const ref = storedRef({ id: saved.media_id, name, media_type: mediaType, size: bytes.length, access: { readers: [subject] } });
+ if (!ref) throw new AiError(502, 'files.store_invalid', 'Media did not return a valid file reference');
+ return ref;
+ } };
}
}
/**
/**
@@ -51,10 +61,21 @@ async function collectArtifacts(message, { store = null, subject = null } = {})
@@ -51,10 +61,21 @@ async function collectArtifacts(message, { store = null, subject = null } = {})
: typeof (file && (file.data || file.b64)) === 'string' ? Buffer.from(file.data || file.b64, file.encoding === 'utf8' ? 'utf8' : 'base64')
: typeof (file && (file.data || file.b64)) === 'string' ? Buffer.from(file.data || file.b64, file.encoding === 'utf8' ? 'utf8' : 'base64')
: null;
: null;
if (!bytes) throw new AiError(422, 'files.invalid', `a model returned a file without bytes or a Media reference: ${String((file && file.name) || 'unnamed').slice(0, 80)}`);
if (!bytes) throw new AiError(422, 'files.invalid', `a model returned a file without bytes or a Media reference: ${String((file && file.name) || 'unnamed').slice(0, 80)}`);
+ if (bytes.length > MAX_ARTIFACT_BYTES) throw new AiError(413, 'files.too_large', 'a generated file may be up to 10 MiB');
if (!store) throw new AiError(503, 'files.store_unavailable', 'a model produced a file but AI has no Media store configured to keep it');
if (!store) throw new AiError(503, 'files.store_unavailable', 'a model produced a file but AI has no Media store configured to keep it');
out.push(await store.put({ name: file.name, media_type: file.media_type || file.type, bytes, subject }));
out.push(await store.put({ name: file.name, media_type: file.media_type || file.type, bytes, subject }));
}
}
return out;
return out;
}
}
-module.exports = { collectArtifacts, storeOf, MAX_ARTIFACTS };
+/** Turn a requested text answer into a private Media file with the same text as the answer. */
+async function collectTextArtifact(text, request, { store = null, subject = null } = {}) {
+ if (!request) return [];
+ const name = String(request.name || '');
+ if (!name || name.length > 255 || /[/\\\u0000-\u001f\u007f]/.test(name)) throw new AiError(422, 'files.name_invalid', 'output_file.name must be a filename without a path');
+ if (!['text/plain', 'text/markdown', 'text/csv', 'application/json'].includes(request.media_type)) throw new AiError(422, 'files.type_invalid', 'output_file.media_type must be a supported text type');
+ if (typeof text !== 'string' || !text.length) throw new AiError(422, 'files.text_missing', 'the model returned no text to save as output_file');
+ return collectArtifacts({ artifacts: [{ name, media_type: request.media_type, data: text, encoding: 'utf8' }] }, { store, subject });
+}
+
+module.exports = { collectArtifacts, collectTextArtifact, storeOf, MAX_ARTIFACTS };
@@ -11,6 +11,8 @@
@@ -11,6 +11,8 @@
* text, markdown, code, CSV and JSON a text block, cut at MAX_TEXT_CHARS with an honest "(truncated)" line
* text, markdown, code, CSV and JSON a text block, cut at MAX_TEXT_CHARS with an honest "(truncated)" line
* PDF its text, page by page, through the small extractor below (no dependency)
* PDF its text, page by page, through the small extractor below (no dependency)
* PNG, JPEG, WebP, GIF an image part, when the model can see (vision); otherwise a named note
* PNG, JPEG, WebP, GIF an image part, when the model can see (vision); otherwise a named note
+ * audio a transcript from the host's local ASR, or an honest unavailable note
+ * ZIP, TAR, TAR.GZ bounded text/code/PDF entries with their paths; other entries named as skipped
* anything else a named note saying it could not be read, never silently dropped
* anything else a named note saying it could not be read, never silently dropped
*
*
* Limits: MAX_FILES per turn, MAX_FILE_BYTES each and MAX_TOTAL_BYTES together. Per-person, per-window
* Limits: MAX_FILES per turn, MAX_FILE_BYTES each and MAX_TOTAL_BYTES together. Per-person, per-window
@@ -23,6 +25,7 @@
@@ -23,6 +25,7 @@
*/
*/
const zlib = require('node:zlib');
const zlib = require('node:zlib');
const { AiError } = require('./util');
const { AiError } = require('./util');
+const { extractArchive } = require('./archive-inputs');
const MAX_FILES = 10;
const MAX_FILES = 10;
const MAX_FILE_BYTES = 10 * 1024 * 1024;
const MAX_FILE_BYTES = 10 * 1024 * 1024;
@@ -34,6 +37,8 @@ const TEXT_EXT = /\.(txt|text|md|markdown|csv|tsv|json|jsonl|log|xml|yaml|yml|to
@@ -34,6 +37,8 @@ const TEXT_EXT = /\.(txt|text|md|markdown|csv|tsv|json|jsonl|log|xml|yaml|yml|to
const CODE_TYPE = /^(text\/[a-z0-9.+-]+|application\/(json|jsonl|xml|javascript|ecmascript|x-javascript|x-httpd-php|x-sh|x-shellscript|x-python|x-ruby|x-yaml|x-yml|x-toml|x-tex|x-latex|x-c|x-c\+\+|x-java-source|x-php|x-perl|x-rust|x-go|x-typescript|x-www-form-urlencoded))$/i;
const CODE_TYPE = /^(text\/[a-z0-9.+-]+|application\/(json|jsonl|xml|javascript|ecmascript|x-javascript|x-httpd-php|x-sh|x-shellscript|x-python|x-ruby|x-yaml|x-yml|x-toml|x-tex|x-latex|x-c|x-c\+\+|x-java-source|x-php|x-perl|x-rust|x-go|x-typescript|x-www-form-urlencoded))$/i;
const IMAGE_TYPE = /^image\/(png|jpe?g|webp|gif)$/i;
const IMAGE_TYPE = /^image\/(png|jpe?g|webp|gif)$/i;
const PDF_TYPE = /^application\/pdf$/i;
const PDF_TYPE = /^application\/pdf$/i;
+const ARCHIVE_TYPE = /^(application\/(zip|x-tar|gzip|x-gzip|x-gtar|x-compressed-tar))$/i;
+const AUDIO_TYPE = /^audio\/(mpeg|mp3|mp4|x-m4a|wav|x-wav|wave|ogg|webm|flac|x-flac)$/i;
/** The named virus-scan extension point; null means no scan is configured (and none is pretended). */
/** The named virus-scan extension point; null means no scan is configured (and none is pretended). */
let scanner = null;
let scanner = null;
@@ -57,6 +62,9 @@ function effectiveType(name, contentType, declared) {
@@ -57,6 +62,9 @@ function effectiveType(name, contentType, declared) {
if (!type || type === 'application/octet-stream' || type === 'binary/octet-stream') type = typeOf(declared);
if (!type || type === 'application/octet-stream' || type === 'binary/octet-stream') type = typeOf(declared);
if ((!type || type === 'application/octet-stream' || type === 'binary/octet-stream') && TEXT_EXT.test(String(name || ''))) type = 'text/plain';
if ((!type || type === 'application/octet-stream' || type === 'binary/octet-stream') && TEXT_EXT.test(String(name || ''))) type = 'text/plain';
if ((!type || type === 'application/octet-stream') && extOf(name) === 'pdf') type = 'application/pdf';
if ((!type || type === 'application/octet-stream') && extOf(name) === 'pdf') type = 'application/pdf';
+ if ((!type || type === 'application/octet-stream') && /\.zip$/i.test(name)) type = 'application/zip';
+ if ((!type || type === 'application/octet-stream') && /\.(?:tar|tar\.gz|tgz)$/i.test(name)) type = 'application/x-tar';
+ if ((!type || type === 'application/octet-stream') && /\.(?:mp3|m4a|mp4|wav|ogg|webm|flac)$/i.test(name)) type = ({ mp3: 'audio/mpeg', m4a: 'audio/mp4', mp4: 'audio/mp4', wav: 'audio/wav', ogg: 'audio/ogg', webm: 'audio/webm', flac: 'audio/flac' })[extOf(name)];
if ((!type || type === 'application/octet-stream') && /^(png|jpe?g|webp|gif)$/.test(extOf(name))) type = `image/${extOf(name).replace('jpg', 'jpeg')}`;
if ((!type || type === 'application/octet-stream') && /^(png|jpe?g|webp|gif)$/.test(extOf(name))) type = `image/${extOf(name).replace('jpg', 'jpeg')}`;
return type || 'application/octet-stream';
return type || 'application/octet-stream';
}
}
@@ -183,7 +191,7 @@ function referenceOf(file, mediaId, name, mediaType, size, read) {
@@ -183,7 +191,7 @@ function referenceOf(file, mediaId, name, mediaType, size, read) {
}
}
/** Read one Media-backed file for the model. → { reference, part, note } */
/** Read one Media-backed file for the model. → { reference, part, note } */
-async function readFile(file, fetcher, { signal, format = 'openai', vision = true, scan = null } = {}) {
+async function readFile(file, fetcher, { signal, format = 'openai', vision = true, scan = null, transcribe = null } = {}) {
const mediaId = mediaIdOf(file);
const mediaId = mediaIdOf(file);
const name = nameOf(file);
const name = nameOf(file);
const declaredSize = Number(file && file.size);
const declaredSize = Number(file && file.size);
@@ -249,6 +257,36 @@ async function readFile(file, fetcher, { signal, format = 'openai', vision = tru
@@ -249,6 +257,36 @@ async function readFile(file, fetcher, { signal, format = 'openai', vision = tru
const reason = 'this model cannot see images, so the picture itself was not passed';
const reason = 'this model cannot see images, so the picture itself was not passed';
return { reference: referenceOf(file, mediaId, name, type, size, { method: 'not_readable', reason }), part: null, note: `[Attached image: ${name} (${type}); ${reason}.]` };
return { reference: referenceOf(file, mediaId, name, type, size, { method: 'not_readable', reason }), part: null, note: `[Attached image: ${name} (${type}); ${reason}.]` };
}
}
+ if (AUDIO_TYPE.test(type)) {
+ let result = null;
+ if (transcribe) {
+ try { result = await transcribe({ bytes: buffer, media_type: type, name, signal }); }
+ catch (err) { if (signal && signal.aborted) throw err; }
+ }
+ const transcript = result && typeof result.text === 'string' ? capText(result.text.trim()) : '';
+ if (transcript) return { reference: referenceOf(file, mediaId, name, type, size, { method: 'text' }),
+ part: textPart(format, `\n\n[Attached audio transcript: ${name} (${type})]\n${transcript}\n`), note: null };
+ const reason = transcribe ? 'local speech recognition could not read this audio' : 'local speech recognition is unavailable';
+ return { reference: referenceOf(file, mediaId, name, type, size, { method: 'not_readable', reason }), part: null,
+ note: `[Attached audio: ${name} (${type}); ${reason}.]` };
+ }
+ if (ARCHIVE_TYPE.test(type) || /\.(?:zip|tar|tar\.gz|tgz)$/i.test(name)) {
+ const archive = extractArchive(buffer, type, name);
+ const parts = (archive.entries || []).map((entry) => {
+ if (/\.pdf$/i.test(entry.name)) {
+ const pdf = pdfText(entry.bytes);
+ return pdf.pages.length ? `--- ${entry.name} (PDF) ---\n${pdf.pages.join('\n\n')}` : null;
+ }
+ return `--- ${entry.name} ---\n${entry.bytes.toString('utf8').replace(/\u0000/g, '')}`;
+ }).filter(Boolean);
+ if (parts.length) {
+ const skipped = (archive.skipped || 0) + (archive.entries.length - parts.length);
+ const text = capText(`${parts.join('\n\n')}${skipped ? `\n\n(${skipped} archive entries were not read)` : ''}`);
+ return { reference: referenceOf(file, mediaId, name, type, size, { method: 'text' }), part: textPart(format, `\n\n[Attached archive: ${name} (${type})]\n${text}\n`), note: null };
+ }
+ const reason = archive.reason || 'the archive has no readable text, code or PDF entries';
+ return { reference: referenceOf(file, mediaId, name, type, size, { method: 'not_readable', reason }), part: null, note: `[Attached archive: ${name} (${type}); ${reason}.]` };
+ }
if (CODE_TYPE.test(type) || TEXT_EXT.test(name) || /^text\//.test(type)) {
if (CODE_TYPE.test(type) || TEXT_EXT.test(name) || /^text\//.test(type)) {
const text = capText(buffer.toString('utf8').replace(/\u0000/g, ''));
const text = capText(buffer.toString('utf8').replace(/\u0000/g, ''));
if (!text.trim()) {
if (!text.trim()) {
@@ -322,7 +360,7 @@ function userContent(format, content) {
@@ -322,7 +360,7 @@ function userContent(format, content) {
* Returns a NEW input: the messages gain parts, the file references stay metadata-only (`read` says what
* Returns a NEW input: the messages gain parts, the file references stay metadata-only (`read` says what
* happened to each, which is the ai.file@1 shape). No bytes or extracted text are put into `input.files`.
* happened to each, which is the ai.file@1 shape). No bytes or extracted text are put into `input.files`.
*/
*/
-async function attachFiles(input, fetcher, { signal, format = 'openai', vision = true, scan = null } = {}) {
+async function attachFiles(input, fetcher, { signal, format = 'openai', vision = true, scan = null, transcribe = null } = {}) {
const files = Array.isArray(input && input.files) ? input.files : [];
const files = Array.isArray(input && input.files) ? input.files : [];
if (!files.length) return input;
if (!files.length) return input;
checkLimits(files);
checkLimits(files);
@@ -331,7 +369,7 @@ async function attachFiles(input, fetcher, { signal, format = 'openai', vision =
@@ -331,7 +369,7 @@ async function attachFiles(input, fetcher, { signal, format = 'openai', vision =
let total = 0;
let total = 0;
let hasImage = false;
let hasImage = false;
for (const file of files) {
for (const file of files) {
- const read = await readFile(file, fetcher, { signal, format, vision, scan });
+ const read = await readFile(file, fetcher, { signal, format, vision, scan, transcribe });
total += read.reference.size;
total += read.reference.size;
if (total > MAX_TOTAL_BYTES) throw new AiError(413, 'files.too_large', `attached files may be up to ${MAX_TOTAL_BYTES / 1024 / 1024} MiB together`);
if (total > MAX_TOTAL_BYTES) throw new AiError(413, 'files.too_large', `attached files may be up to ${MAX_TOTAL_BYTES / 1024 / 1024} MiB together`);
references.push(read.reference);
references.push(read.reference);
@@ -58,7 +58,7 @@ function createMediaStore({ config, env = process.env, fetchImpl = globalThis.fe
@@ -58,7 +58,7 @@ function createMediaStore({ config, env = process.env, fetchImpl = globalThis.fe
* pictures in (blog, wiki, or AI's own ai); absent, the AI tenant. → { media_id, url, width, height, mime }.
* pictures in (blog, wiki, or AI's own ai); absent, the AI tenant. → { media_id, url, width, height, mime }.
* metadata keeps where it came from (source_url, license, author) or what made it (model, ai).
* metadata keeps where it came from (source_url, license, author) or what made it (model, ai).
*/
*/
- async function put({ bytes, mime, filename = 'picture', owner = null, app: tenant = null, metadata = {}, timeoutMs = 30000 } = {}) {
+ async function put({ bytes, mime, filename = 'picture', owner = null, app: tenant = null, metadata = {}, visibility = 'public', timeoutMs = 30000 } = {}) {
if (!configured()) throw Object.assign(new Error('OpenVibe.Media is not configured (OV_MEDIA_INTERNAL_URL and the AI service secret are needed)'), { status: 503, code: 'media.unavailable' });
if (!configured()) throw Object.assign(new Error('OpenVibe.Media is not configured (OV_MEDIA_INTERNAL_URL and the AI service secret are needed)'), { status: 503, code: 'media.unavailable' });
if (!Buffer.isBuffer(bytes) || !bytes.length) throw Object.assign(new Error('the picture has no bytes'), { status: 422, code: 'media.empty' });
if (!Buffer.isBuffer(bytes) || !bytes.length) throw Object.assign(new Error('the picture has no bytes'), { status: 422, code: 'media.empty' });
const target = tenant && /^[a-z][a-z0-9-]{1,31}$/.test(String(tenant)) ? String(tenant) : app;
const target = tenant && /^[a-z][a-z0-9-]{1,31}$/.test(String(tenant)) ? String(tenant) : app;
@@ -68,7 +68,7 @@ function createMediaStore({ config, env = process.env, fetchImpl = globalThis.fe
@@ -68,7 +68,7 @@ function createMediaStore({ config, env = process.env, fetchImpl = globalThis.fe
const init = await call(base, {
const init = await call(base, {
method: 'POST', headers: { ...auth, ...subject, 'Content-Type': 'application/json', Accept: 'application/json' },
method: 'POST', headers: { ...auth, ...subject, 'Content-Type': 'application/json', Accept: 'application/json' },
body: JSON.stringify({
body: JSON.stringify({
- kind: 'asset', visibility: 'public', mime_type: mime, size_bytes: bytes.length,
+ kind: 'asset', visibility, mime_type: mime, size_bytes: bytes.length,
filename: String(filename || 'picture').slice(0, 120), content_hash: crypto.createHash('sha256').update(bytes).digest('hex'),
filename: String(filename || 'picture').slice(0, 120), content_hash: crypto.createHash('sha256').update(bytes).digest('hex'),
metadata,
metadata,
}),
}),
@@ -17,6 +17,7 @@ const FILE_REF = { type: 'object', additionalProperties: false, properties: {
@@ -17,6 +17,7 @@ const FILE_REF = { type: 'object', additionalProperties: false, properties: {
read: { type: 'object' }, role: { enum: ['input', 'artifact'] },
read: { type: 'object' }, role: { enum: ['input', 'artifact'] },
} };
} };
const TURN_FILES = { type: 'array', maxItems: 10, items: FILE_REF };
const TURN_FILES = { type: 'array', maxItems: 10, items: FILE_REF };
+const OUTPUT_FILE = OBJ({ name: STR(255), media_type: { enum: ['text/plain', 'text/markdown', 'text/csv', 'application/json'] } }, ['name', 'media_type']);
const COMMON = { max_tokens: { type: 'integer', minimum: 1, maximum: 16000 }, temperature: { type: ['number', 'null'], minimum: 0, maximum: 2 },
const COMMON = { max_tokens: { type: 'integer', minimum: 1, maximum: 16000 }, temperature: { type: ['number', 'null'], minimum: 0, maximum: 2 },
// AI Exchange F: a liability cap on the Balance hold for this request (optional; nothing when unset).
// AI Exchange F: a liability cap on the Balance hold for this request (optional; nothing when unset).
max_cost_usd: { type: ['number', 'string', 'null'] } };
max_cost_usd: { type: ['number', 'string', 'null'] } };
@@ -141,6 +142,7 @@ const workflows = [
@@ -141,6 +142,7 @@ const workflows = [
presence_penalty: { type: ['number', 'null'], minimum: -2, maximum: 2 },
presence_penalty: { type: ['number', 'null'], minimum: -2, maximum: 2 },
frequency_penalty: { type: ['number', 'null'], minimum: -2, maximum: 2 },
frequency_penalty: { type: ['number', 'null'], minimum: -2, maximum: 2 },
files: TURN_FILES,
files: TURN_FILES,
+ output_file: OUTPUT_FILE,
...COMMON,
...COMMON,
}, ['messages']),
}, ['messages']),
output_schema: OBJ({ choices: { type: 'array', minItems: 1, maxItems: 8, items: { type: 'object' } }, system_fingerprint: { type: ['string', 'null'] }, model: { type: ['string', 'null'] }, artifacts: { type: 'array', maxItems: 10, items: { type: 'object' } } }, ['choices']),
output_schema: OBJ({ choices: { type: 'array', minItems: 1, maxItems: 8, items: { type: 'object' } }, system_fingerprint: { type: ['string', 'null'] }, model: { type: ['string', 'null'] }, artifacts: { type: 'array', maxItems: 10, items: { type: 'object' } } }, ['choices']),
@@ -171,6 +173,7 @@ const workflows = [
@@ -171,6 +173,7 @@ const workflows = [
top_k: { type: ['integer', 'null'], minimum: 0 },
top_k: { type: ['integer', 'null'], minimum: 0 },
metadata: { type: 'object' },
metadata: { type: 'object' },
files: TURN_FILES,
files: TURN_FILES,
+ output_file: OUTPUT_FILE,
}, ['messages', 'max_tokens']),
}, ['messages', 'max_tokens']),
output_schema: OBJ({
output_schema: OBJ({
id: STR(200), type: STR(20), role: STR(20), model: { type: ['string', 'null'] },
id: STR(200), type: STR(20), role: STR(20), model: { type: ['string', 'null'] },
@@ -168,6 +168,17 @@ function batchExec(b, result) {
@@ -168,6 +168,17 @@ function batchExec(b, result) {
function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spawnImpl, log = console, research = null, db = null, clock = { now: () => Date.now() }, mediaStore = null }) {
function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spawnImpl, log = console, research = null, db = null, clock = { now: () => Date.now() }, mediaStore = null }) {
pool = require('../effort/pool').wrapPool(pool);
pool = require('../effort/pool').wrapPool(pool);
+ // Turn attachments use only the host's local speech recognizer. Reading an audio file must never
+ // select a paid transcription provider as a side effect of a chat request.
+ let localAsr = null;
+ async function transcribeAttachment({ bytes, signal }) {
+ localAsr ||= require('../providers/whisper').createWhisperProvider({ key: 'whisper' }, { config });
+ if (!localAsr.supports('transcribe')) throw new AiError(503, 'files.audio_unavailable', 'local speech recognition is unavailable');
+ const file = path.join(os.tmpdir(), `openvibe-ai-attachment-${require('crypto').randomBytes(12).toString('hex')}`);
+ fs.writeFileSync(file, bytes, { mode: 0o600, flag: 'wx' });
+ try { return await localAsr.transcribe({ filePath: file, language: 'auto', signal, timeoutMs: 180000 }); }
+ finally { try { fs.unlinkSync(file); } catch { /* already removed */ } }
+ }
// One loopback reader for everything that streams media (server/media-proxy.js): transcripts and media.analyze.
// One loopback reader for everything that streams media (server/media-proxy.js): transcripts and media.analyze.
let proxy = null;
let proxy = null;
const reader = () => proxy || (proxy = require('../media-proxy').createMediaProxy({ fetcher, log }));
const reader = () => proxy || (proxy = require('../media-proxy').createMediaProxy({ fetcher, log }));
@@ -278,7 +289,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
@@ -278,7 +289,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
const base = { ...input, messages: normMessages(input.messages, input.user) };
const base = { ...input, messages: normMessages(input.messages, input.user) };
// Files on the turn (ai.file@1) become text or image content for this call only.
// Files on the turn (ai.file@1) become text or image content for this call only.
const turn = Array.isArray(input.files) && input.files.length
const turn = Array.isArray(input.files) && input.files.length
- ? await fileInputs.attachFiles(base, fetcher, { signal: ctx.signal, format: 'openai', vision: true })
+ ? await fileInputs.attachFiles(base, fetcher, { signal: ctx.signal, format: 'openai', vision: true, transcribe: transcribeAttachment })
: base;
: base;
const messages = turn.messages;
const messages = turn.messages;
const image = input.image ? await fetcher.loadImage({ ...input.image, max_width: input.image.max_width || 1024 }, { signal: ctx.signal }) : null;
const image = input.image ? await fetcher.loadImage({ ...input.image, max_width: input.image.max_width || 1024 }, { signal: ctx.signal }) : null;
@@ -408,7 +419,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
@@ -408,7 +419,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
const { route, selection } = await selectedRoute(await routeFor(step.route || 'default.chat'), input);
const { route, selection } = await selectedRoute(await routeFor(step.route || 'default.chat'), input);
// Media-backed files on the turn (ai.file@1) become text or image content for this call only.
// Media-backed files on the turn (ai.file@1) become text or image content for this call only.
const turn = Array.isArray(input.files) && input.files.length
const turn = Array.isArray(input.files) && input.files.length
- ? await fileInputs.attachFiles(input, fetcher, { signal: ctx.signal, format: 'openai', vision: true })
+ ? await fileInputs.attachFiles(input, fetcher, { signal: ctx.signal, format: 'openai', vision: true, transcribe: transcribeAttachment })
: input;
: input;
const openai = {};
const openai = {};
for (const k of ['messages', 'tools', 'tool_choice', 'parallel_tool_calls', 'response_format', 'stop', 'seed', 'top_p', 'presence_penalty', 'frequency_penalty']) {
for (const k of ['messages', 'tools', 'tool_choice', 'parallel_tool_calls', 'response_format', 'stop', 'seed', 'top_p', 'presence_penalty', 'frequency_penalty']) {
@@ -431,9 +442,22 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
@@ -431,9 +442,22 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
if (!c || !Array.isArray(c.choices) || !c.choices.length) throw new AiError(502, 'output.empty', 'the provider answered, but with no choices');
if (!c || !Array.isArray(c.choices) || !c.choices.length) throw new AiError(502, 'output.empty', 'the provider answered, but with no choices');
// Files the model produced beside its text: Media references pass through; inline bytes need the
// Files the model produced beside its text: Media references pass through; inline bytes need the
// configured Media store (server/artifacts.js), never the run's output.
// configured Media store (server/artifacts.js), never the run's output.
- const subject = ctx.onBehalfOf && ctx.onBehalfOf.id ? ctx.onBehalfOf.id : null;
- const artifacts = await filesOut.collectArtifacts(c.choices[0] && c.choices[0].message, { store: filesOut.storeOf(config), subject });
- return { output: { choices: c.choices, system_fingerprint: c.system_fingerprint || null, model: exec.model, ...(artifacts.length ? { artifacts } : {}) }, exec, route };
+ const subject = typeof ctx.onBehalfOf === 'string' ? ctx.onBehalfOf : ctx.onBehalfOf?.id || null;
+ const artifacts = [];
+ const choices = [];
+ const artifactStore = filesOut.storeOf(config, mediaStore);
+ for (const choice of c.choices) {
+ const saved = await filesOut.collectArtifacts(choice && choice.message, { store: artifactStore, subject });
+ if (input.output_file) saved.push(...await filesOut.collectTextArtifact(choice && choice.message && choice.message.content, input.output_file, { store: artifactStore, subject }));
+ artifacts.push(...saved);
+ if (artifacts.length > filesOut.MAX_ARTIFACTS) throw new AiError(413, 'files.too_many', `a model may return at most ${filesOut.MAX_ARTIFACTS} files`);
+ const message = { ...(choice.message || {}) };
+ delete message.files;
+ delete message.artifacts;
+ if (saved.length) message.artifacts = saved;
+ choices.push({ ...choice, message });
+ }
+ return { output: { choices, system_fingerprint: c.system_fingerprint || null, model: exec.model, ...(artifacts.length ? { artifacts } : {}) }, exec, route };
}
}
/**
/**
@@ -444,7 +468,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
@@ -444,7 +468,7 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
const { route, selection } = await selectedRoute(await routeFor(step.route || 'default.messages'), input);
const { route, selection } = await selectedRoute(await routeFor(step.route || 'default.messages'), input);
// Media-backed files on the turn (ai.file@1): text parts, or base64 image sources for a vision model.
// Media-backed files on the turn (ai.file@1): text parts, or base64 image sources for a vision model.
const turn = Array.isArray(input.files) && input.files.length
const turn = Array.isArray(input.files) && input.files.length
- ? await fileInputs.attachFiles(input, fetcher, { signal: ctx.signal, format: 'anthropic', vision: true })
+ ? await fileInputs.attachFiles(input, fetcher, { signal: ctx.signal, format: 'anthropic', vision: true, transcribe: transcribeAttachment })
: input;
: input;
const req = {
const req = {
messages: turn.messages,
messages: turn.messages,
@@ -460,9 +484,14 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
@@ -460,9 +484,14 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
const exec = await pool.execute(route, 'messages', req, { ...ctx, ...selectionCtx(selection), routeKey: route.key, routeVersion: route.version, promptHash: sha256({ m: turn.messages, t: input.tools || null }) });
const exec = await pool.execute(route, 'messages', req, { ...ctx, ...selectionCtx(selection), routeKey: route.key, routeVersion: route.version, promptHash: sha256({ m: turn.messages, t: input.tools || null }) });
const m = exec.result.message;
const m = exec.result.message;
if (!m || !Array.isArray(m.content)) throw new AiError(502, 'output.empty', 'the provider answered, but not with a message');
if (!m || !Array.isArray(m.content)) throw new AiError(502, 'output.empty', 'the provider answered, but not with a message');
- const subject = ctx.onBehalfOf && ctx.onBehalfOf.id ? ctx.onBehalfOf.id : null;
- const artifacts = await filesOut.collectArtifacts(m, { store: filesOut.storeOf(config), subject });
- return { output: { ...m, model: m.model || exec.model, ...(artifacts.length ? { artifacts } : {}) }, exec, route };
+ const subject = typeof ctx.onBehalfOf === 'string' ? ctx.onBehalfOf : ctx.onBehalfOf?.id || null;
+ const artifactStore = filesOut.storeOf(config, mediaStore);
+ const artifacts = await filesOut.collectArtifacts(m, { store: artifactStore, subject });
+ if (input.output_file) artifacts.push(...await filesOut.collectTextArtifact(m.content.filter((part) => part && part.type === 'text').map((part) => part.text).join('\n'), input.output_file, { store: artifactStore, subject }));
+ const message = { ...m };
+ delete message.files;
+ delete message.artifacts;
+ return { output: { ...message, model: m.model || exec.model, ...(artifacts.length ? { artifacts } : {}) }, exec, route };
}
}
async function transcribeStep(step, input, wf, run, ctx) {
async function transcribeStep(step, input, wf, run, ctx) {
@@ -1,6 +1,6 @@
@@ -1,6 +1,6 @@
'use strict';
'use strict';
const assert = require('node:assert/strict');
const assert = require('node:assert/strict');
-const { collectArtifacts, storeOf } = require('../server/artifacts');
+const { collectArtifacts, collectTextArtifact, storeOf } = require('../server/artifacts');
const { suite } = require('./helpers');
const { suite } = require('./helpers');
const t = suite('file-artifacts');
const t = suite('file-artifacts');
@@ -37,9 +37,34 @@ t.test('storeOf finds a configured store and nothing else', () => {
@@ -37,9 +37,34 @@ t.test('storeOf finds a configured store and nothing else', () => {
assert.equal(storeOf({ media: { artifactStore: store } }), store);
assert.equal(storeOf({ media: { artifactStore: store } }), store);
});
});
+t.test('the production Media uploader stores generated bytes privately for the person', async () => {
+ const puts = [];
+ const media = { configured: () => true, put: async (what) => { puts.push(what); return { media_id: ID }; } };
+ const store = storeOf({}, media);
+ const subject = 'usr_01ARZ3NDEKTSV4RRFFQ69G5FAV';
+ const out = await collectArtifacts({ artifacts: [{ name: 'answer.txt', media_type: 'text/plain', data: Buffer.from('private output bytes').toString('base64') }] }, { store, subject });
+ assert.equal(puts[0].visibility, 'private');
+ assert.equal(puts[0].owner, subject);
+ assert.equal(puts[0].bytes.toString(), 'private output bytes');
+ assert.equal(out[0].id, ID);
+ assert.deepEqual(out[0].access.readers, [subject]);
+ assert.ok(!JSON.stringify(out).includes('private output bytes'), 'the reference contains no inline bytes');
+ await assert.rejects(() => collectArtifacts({ artifacts: [{ name: 'x.txt', data: 'eA==' }] }, { store }),
+ (err) => err.code === 'files.subject_required');
+});
+
t.test('more than ten produced files is a refusal', async () => {
t.test('more than ten produced files is a refusal', async () => {
const artifacts = Array.from({ length: 11 }, () => ({ id: ID, name: 'a.txt', media_type: 'text/plain', size: 1 }));
const artifacts = Array.from({ length: 11 }, () => ({ id: ID, name: 'a.txt', media_type: 'text/plain', size: 1 }));
await assert.rejects(() => collectArtifacts({ artifacts }), (err) => err.status === 413 && err.code === 'files.too_many');
await assert.rejects(() => collectArtifacts({ artifacts }), (err) => err.status === 413 && err.code === 'files.too_many');
});
});
+t.test('a requested text output is saved as a file and a path is refused', async () => {
+ const puts = [];
+ const store = { put: async (what) => { puts.push(what); return { id: ID, name: what.name, role: 'artifact' }; } };
+ const files = await collectTextArtifact('name,score\nAda,10\n', { name: 'scores.csv', media_type: 'text/csv' }, { store, subject: 'usr_01ARZ3NDEKTSV4RRFFQ69G5FAV' });
+ assert.equal(files.length, 1);
+ assert.equal(puts[0].bytes.toString(), 'name,score\nAda,10\n');
+ await assert.rejects(() => collectTextArtifact('x', { name: '../x.csv', media_type: 'text/csv' }, { store }), (err) => err.code === 'files.name_invalid');
+});
+
t.run();
t.run();
@@ -1,5 +1,6 @@
@@ -1,5 +1,6 @@
'use strict';
'use strict';
const assert = require('node:assert/strict');
const assert = require('node:assert/strict');
+const zlib = require('node:zlib');
const { attachFiles, readFile, pdfText, setScanner, MAX_FILES, MAX_FILE_BYTES } = require('../server/file-inputs');
const { attachFiles, readFile, pdfText, setScanner, MAX_FILES, MAX_FILE_BYTES } = require('../server/file-inputs');
const { suite } = require('./helpers');
const { suite } = require('./helpers');
@@ -29,6 +30,41 @@ const pdfFile = { body: pdf('Quarterly report: revenue up 12%'), type: 'applicat
@@ -29,6 +30,41 @@ const pdfFile = { body: pdf('Quarterly report: revenue up 12%'), type: 'applicat
const png = { body: Buffer.from([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]), type: 'image/png' };
const png = { body: Buffer.from([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]), type: 'image/png' };
const zip = { body: Buffer.from([0x50, 0x4b, 3, 4, 5]), type: 'application/zip' };
const zip = { body: Buffer.from([0x50, 0x4b, 3, 4, 5]), type: 'application/zip' };
+function zipOf(name, text) {
+ const data = Buffer.from(text);
+ const packed = zlib.deflateRawSync(data);
+ const label = Buffer.from(name);
+ const local = Buffer.alloc(30);
+ local.writeUInt32LE(0x04034b50, 0);
+ local.writeUInt16LE(8, 8);
+ local.writeUInt32LE(packed.length, 18);
+ local.writeUInt32LE(data.length, 22);
+ local.writeUInt16LE(label.length, 26);
+ const central = Buffer.alloc(46);
+ central.writeUInt32LE(0x02014b50, 0);
+ central.writeUInt16LE(8, 10);
+ central.writeUInt32LE(packed.length, 20);
+ central.writeUInt32LE(data.length, 24);
+ central.writeUInt16LE(label.length, 28);
+ const directory = Buffer.concat([central, label]);
+ const end = Buffer.alloc(22);
+ end.writeUInt32LE(0x06054b50, 0);
+ end.writeUInt16LE(1, 8);
+ end.writeUInt16LE(1, 10);
+ end.writeUInt32LE(directory.length, 12);
+ end.writeUInt32LE(local.length + label.length + packed.length, 16);
+ return Buffer.concat([local, label, packed, directory, end]);
+}
+
+function tarOf(name, text) {
+ const data = Buffer.from(text);
+ const head = Buffer.alloc(512);
+ head.write(name, 0, 100);
+ head.write(data.length.toString(8).padStart(11, '0') + '\0', 124, 12);
+ head[156] = 48;
+ return Buffer.concat([head, data, Buffer.alloc(Math.ceil(data.length / 512) * 512 - data.length + 1024)]);
+}
+
t.test('CSV, PDF, image and unknown types become model content by type; references never carry bytes', async () => {
t.test('CSV, PDF, image and unknown types become model content by type; references never carry bytes', async () => {
const fetcher = fetcherOf({ [ID(1)]: csv, [ID(2)]: pdfFile, [ID(3)]: png, [ID(4)]: zip });
const fetcher = fetcherOf({ [ID(1)]: csv, [ID(2)]: pdfFile, [ID(3)]: png, [ID(4)]: zip });
const out = await attachFiles({
const out = await attachFiles({
@@ -45,7 +81,7 @@ t.test('CSV, PDF, image and unknown types become model content by type; referenc
@@ -45,7 +81,7 @@ t.test('CSV, PDF, image and unknown types become model content by type; referenc
const texts = parts.filter((p) => p.type === 'text').map((p) => p.text).join('\n');
const texts = parts.filter((p) => p.type === 'text').map((p) => p.text).join('\n');
assert.match(texts, /name,score/);
assert.match(texts, /name,score/);
assert.match(texts, /Quarterly report: revenue up 12%/);
assert.match(texts, /Quarterly report: revenue up 12%/);
- assert.match(texts, /bundle\.zip \(application\/zip\); this file type cannot be read/);
+ assert.match(texts, /bundle\.zip \(application\/zip\); the ZIP directory is missing/);
assert.ok(parts.some((p) => p.type === 'image_url' && /^data:image\/png;base64,/.test(p.image_url.url)), 'the picture goes to a vision model');
assert.ok(parts.some((p) => p.type === 'image_url' && /^data:image\/png;base64,/.test(p.image_url.url)), 'the picture goes to a vision model');
assert.equal(out.files.length, 4);
assert.equal(out.files.length, 4);
assert.deepEqual(out.files.map((f) => f.read.method), ['text', 'pages', 'image', 'not_readable']);
assert.deepEqual(out.files.map((f) => f.read.method), ['text', 'pages', 'image', 'not_readable']);
@@ -57,6 +93,56 @@ t.test('CSV, PDF, image and unknown types become model content by type; referenc
@@ -57,6 +93,56 @@ t.test('CSV, PDF, image and unknown types become model content by type; referenc
assert.equal(out.hasImage, true);
assert.equal(out.hasImage, true);
});
});
+t.test('ZIP and compressed TAR expose bounded text entries with paths and keep bytes out of references', async () => {
+ const bundle = zipOf('src/app.js', 'export const answer = 42;');
+ const archive = zlib.gzipSync(tarOf('notes/readme.md', 'A useful note.'));
+ const fetcher = fetcherOf({
+ [ID(5)]: { body: bundle, type: 'application/zip' },
+ [ID(6)]: { body: archive, type: 'application/gzip' },
+ });
+ const out = await attachFiles({ messages: [{ role: 'user', content: 'Summarize the files' }], files: [
+ { id: ID(5), name: 'code.zip', size: bundle.length },
+ { id: ID(6), name: 'docs.tar.gz', size: archive.length },
+ ] }, fetcher);
+ const text = out.messages[0].content.map((part) => part.text).join('\n');
+ assert.match(text, /src\/app\.js[\s\S]*answer = 42/);
+ assert.match(text, /notes\/readme\.md[\s\S]*A useful note/);
+ assert.deepEqual(out.files.map((file) => file.read.method), ['text', 'text']);
+ assert.ok(!JSON.stringify(out.files).includes('useful note'));
+});
+
+t.test('unsafe archive paths and overlarge expanded entries are skipped', async () => {
+ const unsafe = zipOf('../private.txt', 'do not show this');
+ const large = zipOf('large.txt', 'A'.repeat(600000));
+ const fetcher = fetcherOf({ [ID(7)]: { body: unsafe, type: 'application/zip' }, [ID(8)]: { body: large, type: 'application/zip' } });
+ for (const [id, name, size] of [[ID(7), 'unsafe.zip', unsafe.length], [ID(8), 'large.zip', large.length]]) {
+ const out = await attachFiles({ messages: [{ role: 'user', content: 'Read' }], files: [{ id, name, size }] }, fetcher);
+ assert.equal(out.files[0].read.method, 'not_readable');
+ assert.ok(!JSON.stringify(out.messages).includes('do not show this'));
+ assert.ok(!JSON.stringify(out.messages).includes('AAAAAA'));
+ }
+});
+
+t.test('audio is transcribed for a text model with local ASR, with an honest note if unavailable', async () => {
+ const bytes = Buffer.from('audio fixture bytes');
+ const fetcher = fetcherOf({ [ID(9)]: { body: bytes, type: 'audio/ogg' } });
+ const input = { messages: [{ role: 'user', content: 'What did I say?' }], files: [{ id: ID(9), name: 'voice.ogg', size: bytes.length }] };
+ let called = 0;
+ const heard = await attachFiles(input, fetcher, { transcribe: async ({ bytes: got, media_type: type }) => {
+ called++;
+ assert.equal(type, 'audio/ogg');
+ assert.deepEqual(got, bytes);
+ return { text: 'A spoken answer.' };
+ } });
+ assert.equal(called, 1);
+ assert.match(heard.messages[0].content[1].text, /Attached audio transcript:[\s\S]*A spoken answer/);
+ assert.equal(heard.files[0].read.method, 'text');
+ assert.ok(!JSON.stringify(heard.files).includes('spoken answer'));
+ const unavailable = await attachFiles(input, fetcher);
+ assert.equal(unavailable.files[0].read.method, 'not_readable');
+ assert.match(unavailable.messages[0].content[1].text, /local speech recognition is unavailable/);
+});
+
t.test('a text-only model gets an honest note instead of a dropped picture', async () => {
t.test('a text-only model gets an honest note instead of a dropped picture', async () => {
const fetcher = fetcherOf({ [ID(3)]: png });
const fetcher = fetcherOf({ [ID(3)]: png });
const out = await attachFiles({ messages: [{ role: 'user', content: 'What is this?' }], files: [{ id: ID(3), name: 'shot.png', size: 8 }] }, fetcher, { vision: false });
const out = await attachFiles({ messages: [{ role: 'user', content: 'What is this?' }], files: [{ id: ID(3), name: 'shot.png', size: 8 }] }, fetcher, { vision: false });
@@ -6,17 +6,45 @@ const { boot, request, token, suite, seamServer, ALL } = require('./helpers');
@@ -6,17 +6,45 @@ const { boot, request, token, suite, seamServer, ALL } = require('./helpers');
const t = suite('file-turn');
const t = suite('file-turn');
const ID = 'med_01ARZ3NDEKTSV4RRFFQ69G5FAV';
const ID = 'med_01ARZ3NDEKTSV4RRFFQ69G5FAV';
-let h; let seam; let media; let service;
+const ZIP_ID = 'med_01ARZ3NDEKTSV4RRFFQ69G5FAW';
+const PERSON = 'usr_01ARZ3NDEKTSV4RRFFQ69G5FAV';
+let h; let seam; let media; let service; let uploads;
+
+function codeZip() {
+ const name = Buffer.from('src/answer.js');
+ const data = Buffer.from('export const answer = 42;');
+ const local = Buffer.alloc(30);
+ local.writeUInt32LE(0x04034b50, 0);
+ local.writeUInt32LE(data.length, 18);
+ local.writeUInt32LE(data.length, 22);
+ local.writeUInt16LE(name.length, 26);
+ const central = Buffer.alloc(46);
+ central.writeUInt32LE(0x02014b50, 0);
+ central.writeUInt32LE(data.length, 20);
+ central.writeUInt32LE(data.length, 24);
+ central.writeUInt16LE(name.length, 28);
+ const directory = Buffer.concat([central, name]);
+ const end = Buffer.alloc(22);
+ end.writeUInt32LE(0x06054b50, 0);
+ end.writeUInt16LE(1, 8);
+ end.writeUInt16LE(1, 10);
+ end.writeUInt32LE(directory.length, 12);
+ end.writeUInt32LE(local.length + name.length + data.length, 16);
+ return Buffer.concat([local, name, data, directory, end]);
+}
t.test('a Media server and a counting OpenAI-shaped provider', async () => {
t.test('a Media server and a counting OpenAI-shaped provider', async () => {
media = nodeHttp.createServer((req, res) => {
media = nodeHttp.createServer((req, res) => {
if (req.url === `/o/${ID}`) { res.writeHead(200, { 'Content-Type': 'text/csv' }); return res.end('name,score\nAda,10\n'); }
if (req.url === `/o/${ID}`) { res.writeHead(200, { 'Content-Type': 'text/csv' }); return res.end('name,score\nAda,10\n'); }
+ if (req.url === `/o/${ZIP_ID}`) { res.writeHead(200, { 'Content-Type': 'application/zip' }); return res.end(codeZip()); }
res.writeHead(404, { 'Content-Type': 'application/json' }); res.end('{"error":"Not found"}');
res.writeHead(404, { 'Content-Type': 'application/json' }); res.end('{"error":"Not found"}');
});
});
await new Promise((r) => media.listen(0, '127.0.0.1', r));
await new Promise((r) => media.listen(0, '127.0.0.1', r));
const mediaUrl = `http://127.0.0.1:${media.address().port}`;
const mediaUrl = `http://127.0.0.1:${media.address().port}`;
seam = await seamServer(() => ({ choices: [{ index: 0, message: { role: 'assistant', content: 'It has two rows.' } }], usage: { prompt_tokens: 10, completion_tokens: 5 } }));
seam = await seamServer(() => ({ choices: [{ index: 0, message: { role: 'assistant', content: 'It has two rows.' } }], usage: { prompt_tokens: 10, completion_tokens: 5 } }));
- h = await boot({ env: { AI_STUB_FALLBACK: 'false', OV_MEDIA_URL: mediaUrl, OV_MEDIA_INTERNAL_URL: mediaUrl } });
+ uploads = [];
+ const mediaStore = { configured: () => true, put: async (what) => { uploads.push(what); return { media_id: ZIP_ID }; } };
+ h = await boot({ env: { AI_STUB_FALLBACK: 'false', AI_FREE_PROVIDERS: 'seam', OV_MEDIA_URL: mediaUrl, OV_MEDIA_INTERNAL_URL: mediaUrl }, mediaStore });
service = token('chat', ALL);
service = token('chat', ALL);
let r = await request(h.base, 'POST', '/api/v1/providers', { tok: service, body: { key: 'seam', kind: 'openai', base_url: `${seam.url}/v1`, auth_mode: 'none', capabilities: ['chat', 'complete'], default_model: 'm1' } });
let r = await request(h.base, 'POST', '/api/v1/providers', { tok: service, body: { key: 'seam', kind: 'openai', base_url: `${seam.url}/v1`, auth_mode: 'none', capabilities: ['chat', 'complete'], default_model: 'm1' } });
assert.equal(r.status, 201, r.text);
assert.equal(r.status, 201, r.text);
@@ -26,7 +54,57 @@ t.test('a Media server and a counting OpenAI-shaped provider', async () => {
@@ -26,7 +54,57 @@ t.test('a Media server and a counting OpenAI-shaped provider', async () => {
assert.equal(r.status, 201, r.text);
assert.equal(r.status, 201, r.text);
});
});
+t.test('POST /v1/chat/completions returns a generated file by private Media reference', async () => {
+ const previous = seam.handler;
+ seam.handler = () => ({ choices: [{ index: 0, message: { role: 'assistant', content: 'Download the result.', artifacts: [
+ { name: 'answer.txt', media_type: 'text/plain', data: Buffer.from('private output bytes').toString('base64') },
+ ] } }], usage: { prompt_tokens: 10, completion_tokens: 5 } });
+ try {
+ const acting = token('chat', ALL, { extra: { on_behalf_of: PERSON } });
+ const r = await request(h.base, 'POST', '/v1/chat/completions', { tok: acting, body: { model: 'openvibe', messages: [{ role: 'user', content: 'Make a text file' }] } });
+ assert.equal(r.status, 200, r.text);
+ assert.equal(uploads.length, 1);
+ assert.equal(uploads[0].visibility, 'private');
+ assert.equal(uploads[0].owner, PERSON);
+ assert.equal(uploads[0].bytes.toString(), 'private output bytes');
+ const files = r.body.choices[0].message.artifacts;
+ assert.equal(files[0].id, ZIP_ID);
+ assert.equal(files[0].role, 'artifact');
+ assert.ok(!r.text.includes('private output bytes'));
+ assert.ok(!r.text.includes(Buffer.from('private output bytes').toString('base64')));
+ const stored = await h.db.prepare('SELECT output FROM runs ORDER BY created_at DESC LIMIT 1').get();
+ assert.ok(!JSON.stringify(stored.output).includes('private output bytes'));
+ } finally { seam.handler = previous; }
+});
+
+t.test('POST /v1/chat/completions saves a normal model answer when output_file is requested', async () => {
+ const before = uploads.length;
+ const acting = token('chat', ALL, { extra: { on_behalf_of: PERSON } });
+ const r = await request(h.base, 'POST', '/v1/chat/completions', { tok: acting, body: {
+ model: 'openvibe', messages: [{ role: 'user', content: 'Write a short text file' }],
+ output_file: { name: 'answer.txt', media_type: 'text/plain' },
+ } });
+ assert.equal(r.status, 200, r.text);
+ assert.equal(uploads.length, before + 1);
+ assert.equal(uploads[before].bytes.toString(), 'It has two rows.');
+ assert.equal(uploads[before].visibility, 'private');
+ assert.equal(r.body.choices[0].message.artifacts[0].id, ZIP_ID);
+ assert.equal(r.body.x_openvibe.artifacts[0].id, ZIP_ID);
+});
+
+t.test('POST /v1/chat/completions reads a ZIP code file into the turn', async () => {
+ const r = await request(h.base, 'POST', '/v1/chat/completions', {
+ tok: service,
+ body: { model: 'openvibe', messages: [{ role: 'user', content: 'What does this code return?' }], files: [{ id: ZIP_ID, name: 'code.zip', size: codeZip().length }] },
+ });
+ assert.equal(r.status, 200, r.text);
+ assert.match(JSON.stringify(seam.last.messages), /src\/answer\.js[\s\S]*answer = 42/);
+ const stored = await h.db.prepare('SELECT input FROM runs ORDER BY created_at DESC LIMIT 1').get();
+ assert.ok(!JSON.stringify(stored.input).includes('export const answer'), 'archive content is never kept in the run');
+});
+
t.test('POST /v1/chat/completions reads the file into the provider request, not the run', async () => {
t.test('POST /v1/chat/completions reads the file into the provider request, not the run', async () => {
+ const before = seam.calls;
const r = await request(h.base, 'POST', '/v1/chat/completions', {
const r = await request(h.base, 'POST', '/v1/chat/completions', {
tok: service,
tok: service,
body: {
body: {
@@ -38,7 +116,7 @@ t.test('POST /v1/chat/completions reads the file into the provider request, not
@@ -38,7 +116,7 @@ t.test('POST /v1/chat/completions reads the file into the provider request, not
const sent = JSON.stringify(seam.last.messages);
const sent = JSON.stringify(seam.last.messages);
assert.match(sent, /Attached file: scores\.csv/);
assert.match(sent, /Attached file: scores\.csv/);
assert.match(sent, /name,score/);
assert.match(sent, /name,score/);
- assert.equal(seam.calls, 1);
+ assert.equal(seam.calls, before + 1);
const stored = await h.db.prepare('SELECT input FROM runs ORDER BY created_at DESC LIMIT 1').get();
const stored = await h.db.prepare('SELECT input FROM runs ORDER BY created_at DESC LIMIT 1').get();
const raw = typeof stored.input === 'string' ? stored.input : JSON.stringify(stored.input);
const raw = typeof stored.input === 'string' ? stored.input : JSON.stringify(stored.input);
assert.ok(!raw.includes('Ada,10'), 'the file bytes are not in the run input');
assert.ok(!raw.includes('Ada,10'), 'the file bytes are not in the run input');
@@ -45,7 +45,7 @@ function tmpDir() {
@@ -45,7 +45,7 @@ function tmpDir() {
// A boot on a dir used before is a restart on the same database: its database stays open.
// A boot on a dir used before is a restart on the same database: its database stays open.
const restartable = new Map();
const restartable = new Map();
-async function boot({ searchFetch, env = {}, dir, clock, fetchImpl, credentialFetch, research, machineModels, log, variantEvidence, community, communityEndpoints, webhooks, memoryTransport, memoryLookup, ownerNotify } = {}) {
+async function boot({ searchFetch, env = {}, dir, clock, fetchImpl, credentialFetch, research, machineModels, log, variantEvidence, community, communityEndpoints, webhooks, memoryTransport, memoryLookup, ownerNotify, mediaStore } = {}) {
const reuse = dir !== undefined;
const reuse = dir !== undefined;
if (dir === undefined) dir = tmpDir();
if (dir === undefined) dir = tmpDir();
const fullEnv = {
const fullEnv = {
@@ -62,7 +62,7 @@ async function boot({ searchFetch, env = {}, dir, clock, fetchImpl, credentialFe
@@ -62,7 +62,7 @@ async function boot({ searchFetch, env = {}, dir, clock, fetchImpl, credentialFe
// An injected endpoint store needs the migrated database, so a factory is called with it before the app boots.
// An injected endpoint store needs the migrated database, so a factory is called with it before the app boots.
const endpointStore = typeof communityEndpoints === 'function' ? communityEndpoints(testdb.db) : communityEndpoints;
const endpointStore = typeof communityEndpoints === 'function' ? communityEndpoints(testdb.db) : communityEndpoints;
const webhookStore = typeof webhooks === 'function' ? webhooks(testdb.db) : webhooks;
const webhookStore = typeof webhooks === 'function' ? webhooks(testdb.db) : webhooks;
- const h = await start({ searchFetch, config, db: testdb.db, env: fullEnv, log: log || silent, clock, fetchImpl, credentialFetch, research: researcher, machineModels, variantEvidence, community, communityEndpoints: endpointStore, webhooks: webhookStore, memoryTransport, memoryLookup, ownerNotify });
+ const h = await start({ searchFetch, config, db: testdb.db, env: fullEnv, log: log || silent, clock, fetchImpl, credentialFetch, research: researcher, machineModels, variantEvidence, community, communityEndpoints: endpointStore, webhooks: webhookStore, memoryTransport, memoryLookup, ownerNotify, mediaStore });
const base = `http://127.0.0.1:${h.server.address().port}`;
const base = `http://127.0.0.1:${h.server.address().port}`;
return { ...h, base, dir, env: fullEnv, testdb, stop: async () => { await h.close(); if (!reuse) await testdb.close(); } };
return { ...h, base, dir, env: fullEnv, testdb, stop: async () => { await h.close(); if (!reuse) await testdb.close(); } };
}
}