openvibers/openvibe.network Commits

Frame: a caption for a picture, in any composer (#885)

OP OpenVibers committed 13 h ago · 48fe3155f7 · 4 files, +171 −0

* Frame: a caption for a picture, in any composer

POST /api/frame/caption { image: { data_url }, title? } gives alt text, a caption and tags for one picture from AI's pics.describe_image, as the person (their token: free allowance, Balance or their own key pays), signed in only, at most 20 a minute, nothing kept. The composer's "Caption it" on a picture and a post's missing alt text ask here (the owner, 11-12: "it'll auto caption using OpenVibe.Exchange, using the AI features").

* SSRF review: the caption route's one call to AI

Changes

ddiff --git a/server/frame/caption.js b/server/frame/caption.js
nnew file mode 100644
iindex 00000000..b7456d7c
---- /dev/null
++++ b/server/frame/caption.js
@@@ -0,0 +1,95 @@
++'use strict';
++/**
++ * A caption for a picture, in any composer (the owner, 2026-11-12: "it'll auto caption using OpenVibe.Exchange, using
++ * the AI features … like some Instagram shit"):
++ *
++ * POST /api/frame/caption { image: { data_url }, title? } → 200 { alt, caption, tags }
++ *
++ * Signed in only, and the person's own token pays (AI's free allowance, their Balance, or their own key or computer),
++ * exactly as Ask does. The picture goes to AI's pics.describe_image workflow (alt text of at most 140 characters, a
++ * one or two sentence caption, up to six tags) and nothing is kept here: AI keeps only the image's hash. The composer
++ * sends a small JPEG (its long side 768 px, about 100-200 KB), well under the app's 1 MB JSON limit.
++ */
++const express = require('express');
++
++const TOKEN_RE = /^[A-Za-z0-9._~+/=-]{20,4096}$/;
++const DATA_URL_RE = /^data:image\/(jpeg|png|webp);base64,[A-Za-z0-9+/=]+$/;
++const MAX_DATA_URL = 950_000; // under the app's 1 MB JSON parser (server/index.js)
++
++const clean = (v, n) => String(v == null ? '' : v).replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, n);
++
++/** The request's body, or { error }: one picture as a data URL, and an optional title to steer the words. */
++function readCaption(body) {
++ const b = body && typeof body === 'object' ? body : {};
++ const url = b.image && typeof b.image === 'object' ? String(b.image.data_url || '') : '';
++ if (!url) return { error: 'Send the picture as image.data_url' };
++ if (url.length > MAX_DATA_URL) return { error: 'The picture is too large to describe: send it at most 768 px on its long side' };
++ if (!DATA_URL_RE.test(url)) return { error: 'The picture must be a JPEG, PNG or WebP data URL' };
++ return { dataUrl: url, title: clean(b.title, 120) };
++}
++
++function createFrameCaption({ aiUrl = 'http://127.0.0.1:4700', requireAuth = null, fetchImpl = globalThis.fetch, now = Date.now, perMinute = 20, timeoutMs = 45_000 } = {}) {
++ const aiBase = String(aiUrl || '').replace(/\/+$/, '');
++ const guard = typeof requireAuth === 'function' ? requireAuth : (_req, res) => res.status(401).json({ error: 'Sign in to caption a picture' });
++ const asks = new Map(); // subject -> [timestamps]
++
++ const subjectOf = (req) => {
++ const user = req.user || {};
++ const claims = req.tokenClaims || {};
++ const id = user.subject_id || user.subject || claims.subject_id || claims.sub || user.sub;
++ return /^usr_[0-9A-HJKMNP-TV-Z]{26}$/.test(String(id || '')) ? String(id) : null;
++ };
++ const bearerOf = (req) => {
++ const m = /^Bearer\s+(\S+)$/i.exec(String(req.get('authorization') || ''));
++ const t = m ? m[1] : (req.token || '');
++ return TOKEN_RE.test(String(t || '')) ? String(t) : null;
++ };
++ function allowed(subject) {
++ const at = now();
++ const list = (asks.get(subject) || []).filter((t) => at - t < 60_000);
++ if (list.length >= perMinute) { asks.set(subject, list); return false; }
++ list.push(at); asks.set(subject, list);
++ if (asks.size > 5000) asks.clear();
++ return true;
++ }
++
++ const caption = [guard, express.json({ limit: '1mb' }), async (req, res) => {
++ res.set('Cache-Control', 'private, no-store');
++ const subject = subjectOf(req);
++ const token = bearerOf(req);
++ if (!subject || !token) return res.status(401).json({ error: 'Sign in to caption a picture', code: 'frame.sign_in' });
++ const body = readCaption(req.body);
++ if (body.error) return res.status(422).json({ error: body.error, code: 'frame.caption_invalid' });
++ if (!allowed(subject)) { res.set('Retry-After', '60'); return res.status(429).json({ error: 'Too many captions just now: try again in a minute', code: 'frame.rate_limited' }); }
++ let r;
++ try {
++ r = await fetchImpl(`${aiBase}/api/v1/runs?wait=${timeoutMs - 5000}`, {
++ method: 'POST',
++ headers: { 'Content-Type': 'application/json', Accept: 'application/json', Authorization: `Bearer ${token}`, ...(req.ip ? { 'X-Forwarded-For': req.ip } : {}) },
++ body: JSON.stringify({ workflow: 'pics.describe_image', input: { image: { data_url: body.dataUrl }, ...(body.title ? { title: body.title } : {}) } }),
++ redirect: 'manual',
++ signal: AbortSignal.timeout(timeoutMs),
++ });
++ } catch {
++ return res.status(504).json({ error: 'The AI did not answer in time: try again', code: 'frame.caption_timeout' });
++ }
++ const out = await r.json().catch(() => null);
++ if (!r.ok) {
++ const status = r.status === 402 || r.status === 403 || r.status === 429 ? r.status : 502;
++ if (r.status === 429) res.set('Retry-After', String(r.headers.get('retry-after') || '60'));
++ return res.status(status).json({ error: (out && (out.detail || out.error)) || 'The AI could not describe the picture', code: (out && out.code) || 'frame.caption_failed' });
++ }
++ const run = out && out.run;
++ const o = run && run.output;
++ if (!o || typeof o !== 'object' || (!o.alt && !o.caption)) {
++ const pending = run && !['succeeded', 'failed', 'cancelled'].includes(String(run.status));
++ return res.status(pending ? 504 : 502).json({ error: pending ? 'The AI is still describing it: try again in a moment' : 'The AI could not describe the picture', code: 'frame.caption_failed' });
++ }
++ const tags = Array.isArray(o.tags) ? o.tags.map((t) => clean(t, 30)).filter(Boolean).slice(0, 6) : [];
++ return res.json({ alt: clean(o.alt, 160), caption: clean(o.caption, 400), tags });
++ }];
++
++ return { caption };
++}
++
++module.exports = { createFrameCaption, readCaption };
ddiff --git a/server/frame/service.js b/server/frame/service.js
iindex f26e50fd..e5f6115a 100644
---- a/server/frame/service.js
++++ b/server/frame/service.js
@@@ -34,6 +34,7 @@ const { SITES, LISTED, LISTED_SITES, siteForHost, GROUPS, groupOf, PRODUCTS, MER
const { createMenuCards } = require('./menu-cards');
const { createFrameChoices } = require('./choices');
const { createFrameAsk } = require('./ask');
++const { createFrameCaption } = require('./caption');
const { createFrameCodeSearch } = require('./code-search');
const { read: readCommands } = require('./commands');
const { createFrameMentions } = require('./mentions');
@@@ -181,6 +182,8 @@ async function createFrameService(db, config, analytics, { privateKey = null, is
});
// The composer's GitHub tool (Shared 4.225.0): Exchange's public code search, through the Frame.
const codeSearch = createFrameCodeSearch({ exchangeUrl: (services.exchange && services.exchange.internalUrl) || 'http://127.0.0.1:5150', now });
++ // A picture's caption and alt text in any composer (11-12): AI's pics.describe_image, as the person.
++ const caption = createFrameCaption({ aiUrl, now, requireAuth });
const ask = createFrameAsk({
aiUrl, now, requireAuth, sheet: choices,
searchUrl: (services.search && services.search.internalUrl) || 'http://127.0.0.1:4710',
@@@ -1114,6 +1117,10 @@ async function createFrameService(db, config, analytics, { privateKey = null, is
// /api/v1/runs/:id/events (mounted in server/index.js), replayable from Last-Event-ID.
router.post('/ask', ...ask.ask);
++ // POST /api/frame/caption — alt text, a caption and tags for one picture (AI's pics.describe_image), signed in only
++ // and on the person's own allowance; the composer's "Caption it" and a post's missing alt text ask here (11-12).
++ router.post('/caption', ...caption.caption);
++
// GET /api/frame/query/providers — the Sources popover's web ladder: Search's offers (provider, scopes, price,
// availability, today's use), asked through Network for the same connect-src reason as /query. Same CORS and
// caching: a person's token goes along and nothing is kept; signed out the offers are kept 60 s.
ddiff --git a/test/frame-caption.test.js b/test/frame-caption.test.js
nnew file mode 100644
iindex 00000000..05eba59f
---- /dev/null
++++ b/test/frame-caption.test.js
@@@ -0,0 +1,68 @@
++'use strict';
++// POST /api/frame/caption (11-12): a picture's alt text, caption and tags from AI's pics.describe_image, as the person
++// (their token goes to AI), signed in only, validated, rate-limited, and an AI refusal passed on with its words.
++const assert = require('assert');
++const http = require('http');
++const express = require('express');
++const { createFrameCaption, readCaption } = require('../server/frame/caption');
++
++const PNG = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
++const SUBJECT = 'usr_01KKT9AC60KM7CRTB3WN1Z8P56';
++const TOKEN = 'eyJhbGciOiJSUzI1NiJ9.person-token.signature';
++
++(async () => {
++ assert.deepStrictEqual(readCaption({}), { error: 'Send the picture as image.data_url' });
++ assert.match(readCaption({ image: { data_url: 'data:image/svg+xml;base64,PHN2Zz4=' } }).error, /JPEG, PNG or WebP/);
++ assert.match(readCaption({ image: { data_url: `data:image/jpeg;base64,${'A'.repeat(950_001)}` } }).error, /too large/);
++ assert.deepStrictEqual(readCaption({ image: { data_url: PNG }, title: ' My\ncat ' }), { dataUrl: PNG, title: 'My cat' });
++
++ const sent = [];
++ let answer = { status: 201, body: { run: { id: 'run_1', status: 'succeeded', output: { alt: 'A tabby cat asleep on a red blanket', caption: 'Nap time, again.', tags: ['cat', 'nap', ' '] } } } };
++ const fetchImpl = async (url, init) => {
++ sent.push({ url, init, body: JSON.parse(init.body) });
++ return { ok: answer.status < 400, status: answer.status, headers: { get: () => null }, json: async () => answer.body };
++ };
++ let t = 0;
++ const requireAuth = (req, res, next) => {
++ if (req.get('authorization') !== `Bearer ${TOKEN}`) return res.status(401).json({ error: 'sign in' });
++ req.user = { subject_id: SUBJECT };
++ return next();
++ };
++ const { caption } = createFrameCaption({ aiUrl: 'http://ai.test/', requireAuth, fetchImpl, now: () => t, perMinute: 2 });
++ const app = express();
++ app.post('/api/frame/caption', ...caption);
++ const server = http.createServer(app);
++ await new Promise((ok) => server.listen(0, '127.0.0.1', ok));
++ const base = `http://127.0.0.1:${server.address().port}/api/frame/caption`;
++ const post = (body, auth = true) => fetch(base, { method: 'POST', headers: { 'Content-Type': 'application/json', ...(auth ? { Authorization: `Bearer ${TOKEN}` } : {}) }, body: JSON.stringify(body) });
++ try {
++ let r = await post({ image: { data_url: PNG } }, false);
++ assert.strictEqual(r.status, 401, 'signed in only');
++ assert.strictEqual(sent.length, 0);
++
++ r = await post({ image: { data_url: PNG }, title: 'cat' });
++ assert.strictEqual(r.status, 200);
++ assert.deepStrictEqual(await r.json(), { alt: 'A tabby cat asleep on a red blanket', caption: 'Nap time, again.', tags: ['cat', 'nap'] });
++ assert.strictEqual(r.headers.get('cache-control'), 'private, no-store');
++ assert.ok(sent[0].url.startsWith('http://ai.test/api/v1/runs?wait='), 'one run on AI');
++ assert.strictEqual(sent[0].init.headers.Authorization, `Bearer ${TOKEN}`, 'as the person: their token pays');
++ assert.deepStrictEqual(sent[0].body, { workflow: 'pics.describe_image', input: { image: { data_url: PNG }, title: 'cat' } });
++
++ r = await post({ image: { data_url: 'data:text/html;base64,PGI+' } });
++ assert.strictEqual(r.status, 422);
++
++ answer = { status: 402, body: { code: 'quota.exhausted', detail: 'Your free AI allowance for today is used up' } };
++ r = await post({ image: { data_url: PNG } });
++ assert.deepStrictEqual([r.status, (await r.json()).error], [402, 'Your free AI allowance for today is used up'], 'AI\'s refusal, in its own words');
++
++ r = await post({ image: { data_url: PNG } });
++ assert.strictEqual(r.status, 429, 'two a minute in this test');
++ t += 61_000;
++ answer = { status: 202, body: { run: { id: 'run_2', status: 'running', output: null } } };
++ r = await post({ image: { data_url: PNG } });
++ assert.strictEqual(r.status, 504, 'a run still going is a timeout the composer can retry');
++ console.log('frame caption: as the person, validated, AI\'s words passed on, rate-limited');
++ } finally {
++ server.close();
++ }
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/test/security-ssrf.test.js b/test/security-ssrf.test.js
iindex 6f278c08..eab6bc08 100644
---- a/test/security-ssrf.test.js
++++ b/test/security-ssrf.test.js
@@@ -121,6 +121,7 @@ const REVIEWED = {
'server/frame/choices.js fetch': [1, 'the composer\'s Model sheet: AI (OV_AI_INTERNAL_URL), Exchange (OV_EXCHANGE_INTERNAL_URL / config.services.exchange) and Report (OV_REPORT_INTERNAL_URL) internal URLs from operator config, fixed /api/v1/choices, /api/v1/models and /api/v1/ai/leaderboard paths, the task percent-encoded and a checked person Bearer only; no request value names a host, redirects refused, 3-8 s timeouts'],
'server/frame/code-search.js fetch': [1, 'The composer\'s GitHub tool: only the operator-configured internal URL of Exchange (services.exchange.internalUrl), the fixed /api/v1/code/search path with the query encoded and the limit clamped to 10, the person\'s address and Bearer forwarded as every Frame relay does, redirects refused, a 9 s timeout; never a request-chosen host'],
'server/frame/runs.js fetch': [1, 'Frame Runs: only the operator-configured internal URLs of Chat, Codes and Actor (svcUrl), fixed /api/vibe and /api/v1 paths with encoded ids from a checked ref, the person\'s own Bearer, redirects refused, 4-15 s timeouts; never a request-chosen host'],
++ 'server/frame/caption.js fetch': [1, 'the composer\'s Caption it: AI (OV_AI_INTERNAL_URL, the operator-configured internal URL) at the fixed /api/v1/runs?wait= path, workflow pics.describe_image, with the person\'s Bearer and a checked JPEG/PNG/WebP data URL; no address from the request is fetched, redirect manual, 45 s timeout'],
'server/frame/ask.js fetch': [2, 'the composer\'s Ask: Search (OV_SEARCH_INTERNAL_URL) fixed /api/v1/query path with checked scopes and the person\'s Bearer, and AI (OV_AI_INTERNAL_URL) fixed /api/v1/runs?stream=1 path with the person\'s Bearer; both operator-configured internal URLs, no request value names a host, redirects refused, 30-150 s timeouts'],
'server/admin/analytics-routes.js fetch': [2, 'staff analytics: the services\' internal URLs from the registry (owner/admin set), and Insights\' Prometheus (PROMETHEUS_URL, loopback by default) with fixed queries; never a request value'],
'server/admin/events-ops.js fetch': [1, 'OV_EVENTS_INTERNAL_URL (operator config) for staff DLQ views'],