openvibers/openvibe.vision Commits

VIS-6: meals, surroundings, plants, landmarks, screenshot components; every task's answer saves (#81)

OP OpenVibers committed 10 days ago · 4a8b732580 · 8 files, +85 −8

Five prompted tasks on the Ask path (no question needed, always the routed model) and an MCP tool for each:
vision.meal, vision.describe_surroundings, vision.identify_plant, vision.identify_landmark,
vision.screenshot_components. The Ask page offers them.

Migration 0006 (expand): vision_answers' task check allowed only ask, describe, ocr and compare, so a translate,
identify or count answer (vision.translate_sign, three of the Ask page's own choices) failed its insert. It now
allows every task the service runs.

Changes

ddiff --git a/README.md b/README.md
iindex 5ba0273..8c41219 100644
---- a/README.md
++++ b/README.md
@@@ -134,7 +134,10 @@ client `vision`. No new AI capability, no provider key, no direct paid provider.
token responds with a `WWW-Authenticate` challenge pointing to that metadata. The tools are `vision.ask_image`,
`vision.read_text`, `vision.translate_sign`, `vision.find_similar`, `vision.search_my_media`,
`vision.describe_clip`, `vision.detect_objects`, `vision.blur_faces`, `vision.check_ui`,
--`vision.compare_models` and `vision.get_job`. Face enrolment, verification, cast matching and consent mutation
++`vision.compare_models` and `vision.get_job`, and five that run one prompted task each through the routed model
++(VIS-6; also `task=` on `POST /api/v1/ask`, no question needed): `vision.meal` (`meal`: foods, ingredients, portions,
++rough calories), `vision.describe_surroundings` (`surroundings`), `vision.identify_plant` (`plant`),
++`vision.identify_landmark` (`landmark`) and `vision.screenshot_components` (`components`). Face enrolment, verification, cast matching and consent mutation
are available only through their person or Live-specific API routes, never through MCP.
When configured with Events and a Network client secret, successful owner results publish `vision.result.ready`
ddiff --git a/migrations/0006_answer_tasks.sql b/migrations/0006_answer_tasks.sql
nnew file mode 100644
iindex 0000000..6c0602c
---- /dev/null
++++ b/migrations/0006_answer_tasks.sql
@@@ -0,0 +1,10 @@
++-- phase: expand
++-- VIS-6 (2026-11-03): an answer is saved under the task it ran as. 0001 allowed only ask, describe, ocr and compare,
++-- so a translate, identify or count answer (vision.translate_sign, the Ask page's own choices) failed its insert. This
++-- allows every task the service runs, the wave-2 ones included: meal, surroundings, plant, landmark and components.
++-- A superset of the old check: code from before this migration still writes only the old four.
++ALTER TABLE vision_answers DROP CONSTRAINT vision_answers_task;
++ALTER TABLE vision_answers ADD CONSTRAINT vision_answers_task CHECK (task IN (
++ 'ask', 'describe', 'ocr', 'compare', 'translate', 'identify', 'count',
++ 'meal', 'surroundings', 'plant', 'landmark', 'components'
++));
ddiff --git a/server/http/mcp.js b/server/http/mcp.js
iindex 77e78a3..aeb55af 100644
---- a/server/http/mcp.js
++++ b/server/http/mcp.js
@@@ -14,6 +14,11 @@ const TOOLS = Object.freeze([
{ name: 'vision.ask_image', description: 'Ask about an image. Anonymous requests use local OCR only.', inputSchema: input({ ...image, question: { type: 'string' }, mode: { type: 'string', enum: ['auto', 'local', 'ai'] } }) },
{ name: 'vision.read_text', description: 'Read visible text with local OCR.', inputSchema: input({ ...image, lang: { type: 'string' } }) },
{ name: 'vision.translate_sign', description: 'Translate visible sign text through a routed model.', inputSchema: input({ ...image, lang: { type: 'string' } }) },
++ { name: 'vision.meal', description: 'List the foods in a photo of a meal, with likely ingredients, estimated portions and rough calories.', inputSchema: input(image) },
++ { name: 'vision.describe_surroundings', description: 'Describe surroundings for someone who cannot see them: what is ahead, obstacles, signs. Never who people are.', inputSchema: input(image) },
++ { name: 'vision.identify_plant', description: 'Name a plant (common and scientific), with confidence and whether it is commonly poisonous.', inputSchema: input(image) },
++ { name: 'vision.identify_landmark', description: 'Name a landmark, building or place, with confidence. Never identifies a person.', inputSchema: input(image) },
++ { name: 'vision.screenshot_components', description: 'Break a screenshot of an app or page into its interface components, with the HTML element or ARIA role for each.', inputSchema: input(image) },
{ name: 'vision.find_similar', description: 'Search your private visual index by image.', inputSchema: input({ image: image.image }, ['image']) },
{ name: 'vision.search_my_media', description: 'Search your private visual index by words.', inputSchema: input({ query: { type: 'string' } }, ['query']) },
{ name: 'vision.describe_clip', description: 'Find scene changes and moments in your own frames.', inputSchema: input({ frames: { type: 'array', items: { type: 'object' } }, prompts: { type: 'array', items: { type: 'string' } } }, ['frames']) },
@@@ -27,6 +32,8 @@ const TOOLS = Object.freeze([
const ACCESS = Object.freeze({
'vision.ask_image': ['vision.image.ask', 'local'], 'vision.read_text': ['vision.text.read', 'anonymous'],
'vision.translate_sign': ['vision.image.ask'], 'vision.find_similar': ['vision.media.find', 'person'],
++ 'vision.meal': ['vision.image.ask'], 'vision.describe_surroundings': ['vision.image.ask'], 'vision.identify_plant': ['vision.image.ask'],
++ 'vision.identify_landmark': ['vision.image.ask'], 'vision.screenshot_components': ['vision.image.ask'],
'vision.search_my_media': ['vision.media.find', 'person'], 'vision.describe_clip': ['vision.video.moments', 'person'],
'vision.detect_objects': ['vision.camera.watch', 'person'], 'vision.blur_faces': ['vision.face.blur', 'anonymous'],
'vision.check_ui': ['vision.ui.check', 'anonymous'], 'vision.compare_models': ['vision.model.compare'],
@@@ -34,10 +41,17 @@ const ACCESS = Object.freeze({
});
const EVENT_PATH = Object.freeze({
'vision.ask_image': '/ask', 'vision.read_text': '/ocr', 'vision.translate_sign': '/ask',
++ 'vision.meal': '/ask', 'vision.describe_surroundings': '/ask', 'vision.identify_plant': '/ask', 'vision.identify_landmark': '/ask', 'vision.screenshot_components': '/ask',
'vision.find_similar': '/find', 'vision.search_my_media': '/find', 'vision.describe_clip': '/moments',
'vision.detect_objects': '/watch', 'vision.check_ui': '/ui-check', 'vision.compare_models': '/compare',
});
++// VIS-6 wave 2: each tool is one of the service's prompted tasks (server/vision/ai.js TASK_PROMPTS).
++const PROMPTED_TOOL = Object.freeze({
++ 'vision.meal': 'meal', 'vision.describe_surroundings': 'surroundings', 'vision.identify_plant': 'plant',
++ 'vision.identify_landmark': 'landmark', 'vision.screenshot_components': 'components',
++});
++
function rpcResult(id, value) { return { jsonrpc: '2.0', id, result: value }; }
function rpcError(id, code, message) { return { jsonrpc: '2.0', id: id === undefined ? null : id, error: { code, message } }; }
function toolResult(id, value) {
@@@ -81,6 +95,8 @@ async function invoke(name, args, req, ctx) {
}
case 'vision.read_text': return await vision.ocr({ ...request, image: await load(), lang: args.lang, fallback: false });
case 'vision.translate_sign': return { answer: await vision.ask({ ...request, image: await load(), task: 'translate', question: 'Translate the visible sign text.', mode: 'ai', lang: args.lang }) };
++ case 'vision.meal': case 'vision.describe_surroundings': case 'vision.identify_plant': case 'vision.identify_landmark': case 'vision.screenshot_components':
++ return { answer: await vision.ask({ ...request, image: await load(), task: PROMPTED_TOOL[name], question: '', mode: 'ai' }) };
case 'vision.find_similar': return { answer: await networkTools.find(subject, await loadPrivate()) };
case 'vision.search_my_media': {
if (typeof args.query !== 'string' || !args.query.trim() || args.query.length > 500) throw new Error('query must contain 1–500 characters');
ddiff --git a/server/views/index.js b/server/views/index.js
iindex 9c15b23..0017072 100644
---- a/server/views/index.js
++++ b/server/views/index.js
@@@ -139,7 +139,7 @@ function ask({ signedIn = false, ai = false, problem = null, values = {} } = {})
const v = { task: 'ask', mode: 'auto', ...values };
return `<div class="vi-wrap" id="vi-ask">
<header class="vi-page-h"><p class="vi-kicker">Ask a picture</p><h1>What does the image show?</h1>
--<p class="vi-lede">Describe, read, translate, identify or count. A read question goes to the local OCR reader first; the answer says which engine answered and why. Face matching has separate consent controls on <a href="/people">People</a>.</p></header>
++<p class="vi-lede">Describe, read, translate, identify or count; name a plant or a place, list what is in a meal, describe your surroundings, or break a screenshot into components. A read question goes to the local OCR reader first; the answer says which engine answered and why. Face matching has separate consent controls on <a href="/people">People</a>.</p></header>
${problem ? `<p class="vi-note is-bad" role="alert">${esc(problem)}</p>` : ''}
${signedIn ? '' : `<p class="vi-note">Ask saves the answer to an account. <a href="/auth/login?next=%2Fask">Sign in with OpenVibe</a> to ask; <a href="/ocr">OCR</a> and the <a href="/ui-check">UI check</a> work without one.</p>`}
${ai ? '' : '<p class="vi-note is-warm">OpenVibe.AI is not configured on this host. A read question still runs on the local OCR engine.</p>'}
@@@ -147,7 +147,8 @@ ${ai ? '' : '<p class="vi-note is-warm">OpenVibe.AI is not configured on this ho
${imageFields({ values: v })}
<div class="vi-field-grid">
<label class="vi-field"><span>Task</span><select name="task">
--${[['ask', 'Ask a question'], ['describe', 'Describe the image'], ['identify', 'Identify what it is'], ['count', 'Count something'], ['translate', 'Read and translate'], ['ocr', 'Read the text']].map(([k, label]) => `<option value="${k}"${v.task === k ? ' selected' : ''}>${label}</option>`).join('')}
++${[['ask', 'Ask a question'], ['describe', 'Describe the image'], ['identify', 'Identify what it is'], ['count', 'Count something'], ['translate', 'Read and translate'], ['ocr', 'Read the text'],
++ ['meal', 'What is in this meal'], ['surroundings', 'Describe my surroundings'], ['plant', 'Name this plant'], ['landmark', 'Name this place'], ['components', 'Screenshot to components']].map(([k, label]) => `<option value="${k}"${v.task === k ? ' selected' : ''}>${label}</option>`).join('')}
</select></label>
<label class="vi-field"><span>Mode</span><select name="mode">
${[['auto', 'Auto: cheapest engine first'], ['local', 'Local only: no model call'], ['ai', 'AI directly']].map(([k, label]) => `<option value="${k}"${v.mode === k ? ' selected' : ''}>${label}</option>`).join('')}
ddiff --git a/server/vision/ai.js b/server/vision/ai.js
iindex 66d0a3b..3033927 100644
---- a/server/vision/ai.js
++++ b/server/vision/ai.js
@@@ -30,6 +30,12 @@ const TASK_PROMPTS = {
translate: 'Read the visible text and translate it into the target language. Give the original text, then the translation.',
ocr: 'Read every visible piece of text in the image, verbatim, preserving line breaks and reading order. Do not summarize and do not add anything that is not written.',
compare: 'Answer the question about the image precisely, for a side-by-side comparison. Say when the image does not show the answer.',
++ // VIS-6 wave 2. Each answers without a question; a question narrows it.
++ meal: 'List each food and drink you can see. For each: its likely ingredients, an estimated portion (grams, or a household measure such as a cup) and a rough calorie estimate. Then give a total. Say plainly that these are estimates from a photo, and name what a photo cannot show (oil, butter, sugar, sauces, what is underneath). If there is no food, say so.',
++ surroundings: 'Describe these surroundings for someone who cannot see them, to help them find their way: what is straight ahead and to each side, steps, kerbs, doors, obstacles and hazards, people and moving things (what they are doing, never who they are), and any signs or text. Most important first, in short sentences. Say how sure you are when the picture is unclear.',
++ plant: 'Identify the plant: its common name and its scientific name, how sure you are, and the features you went by (leaves, flowers, fruit, bark, habit). When the picture supports it, say whether any part is commonly poisonous to people or pets. If the picture is not of a plant, say so.',
++ landmark: 'Identify the landmark, building or place: its name, city and country, how sure you are, and the features you went by. If it is not a recognisable landmark, say so and describe the kind of place instead. Never identify a person in the picture.',
++ components: 'Break this screenshot of an app or web page into its interface components, as a nested outline from the page down: each region (header, navigation, main content, sidebar, footer, dialog) and inside it each component (heading, button, link, text field, checkbox, list, card, tab, image, icon), with its visible label or text. After each component, the HTML element or ARIA role that fits it. End with anything that looks broken: overlapping, cut off, unreadable or unlabelled.',
};
const NO_IDENTITY = 'Never identify, name or recognise a person, and never use a face as identity.';
ddiff --git a/server/vision/service.js b/server/vision/service.js
iindex 607f132..19cec97 100644
---- a/server/vision/service.js
++++ b/server/vision/service.js
@@@ -22,14 +22,17 @@ class VisionError extends Error {
}
}
--const TASKS = new Set(['ask', 'describe', 'identify', 'count', 'translate', 'ocr', 'compare']);
++const TASKS = new Set(['ask', 'describe', 'identify', 'count', 'translate', 'ocr', 'compare', 'meal', 'surroundings', 'plant', 'landmark', 'components']);
++// VIS-6 wave 2: tasks whose prompt says everything (server/vision/ai.js TASK_PROMPTS), so they need no question. Each
++// always goes to the routed model: the local OCR reader cannot answer them.
++const PROMPTED = new Set(['meal', 'surroundings', 'plant', 'landmark', 'components']);
const READ_RE = /\b(read|say|says|said|text|word|words|label|labels|sign|signs|receipt|receipts|handwriting|ocr|written|print|printed|translate|translation|caption|menu|address|number)\b/i;
function isReadTask(task, question) {
if (task === 'ocr') return true;
// Local OCR can read the words, but it cannot translate them: a translate task (or a question that asks for a
// translation) goes to the model when auto mode is on. mode=local still returns the reading, and says so.
-- if (task === 'translate') return false;
++ if (task === 'translate' || PROMPTED.has(task)) return false;
if (/\btranslate|translation\b/i.test(String(question || ''))) return false;
return task === 'ask' && READ_RE.test(String(question || ''));
}
@@@ -87,11 +90,12 @@ function createVision({ config, s, images, ocr, ai, store, log = console }) {
*/
async function ask({ requester, subject = null, image, task = 'ask', question = '', mode = 'auto', lang = null, model = null }) {
const t = asTask(task);
-- if (!t || t === 'compare') throw new VisionError(400, 'vision.invalid_task', 'task must be ask, describe, identify, count, translate or ocr');
++ if (!t || t === 'compare') throw new VisionError(400, 'vision.invalid_task', 'task must be ask, describe, identify, count, translate, ocr, meal, surroundings, plant, landmark or components');
if (mode !== 'auto' && mode !== 'local' && mode !== 'ai') throw new VisionError(400, 'vision.invalid_mode', 'mode must be auto, local or ai');
const started = Date.now();
const q = String(question || '').trim().slice(0, 2000);
-- if (t !== 'describe' && t !== 'ocr' && !q && mode !== 'local') throw new VisionError(400, 'vision.question_required', 'ask, identify, count and translate need a question');
++ if (t !== 'describe' && t !== 'ocr' && !PROMPTED.has(t) && !q && mode !== 'local') throw new VisionError(400, 'vision.question_required', 'ask, identify, count and translate need a question');
++ if (PROMPTED.has(t) && mode === 'local') throw new VisionError(400, 'vision.needs_model', `${t} needs the routed model: the local OCR reader cannot answer it`);
let local = null;
if (mode !== 'ai' && (isReadTask(t, q) || mode === 'local')) {
ddiff --git a/test/api.test.js b/test/api.test.js
iindex f1623fc..8ec1b20 100644
---- a/test/api.test.js
++++ b/test/api.test.js
@@@ -45,6 +45,30 @@ const { boot, check, done } = require('./helpers/boot');
assert.ok(t.aiServer.requests.some((x) => x.path === '/v1/chat/completions' && JSON.stringify(x.body).includes('image_url')));
});
++ await check('VIS-6: every task is saved as itself, and the prompted tasks need no question', async () => {
++ // 0001 allowed only ask/describe/ocr/compare: a translate, identify or count answer failed its insert (0006).
++ for (const task of ['translate', 'identify', 'count']) {
++ const r = await t.get('/api/v1/ask', { as: user, multipart: { fields: { task, question: 'What is it?', mode: 'ai' }, image: png } });
++ assert.strictEqual(r.status, 201, `${task}: ${r.text}`);
++ assert.strictEqual(r.json().answer.task, task);
++ }
++ const prompts = {
++ meal: /estimated portion[\s\S]*rough calorie/, surroundings: /someone who cannot see them[\s\S]*never who they are/,
++ plant: /scientific name[\s\S]*poisonous/, landmark: /city and country[\s\S]*Never identify a person/, components: /ARIA role/,
++ };
++ for (const [task, re] of Object.entries(prompts)) {
++ const r = await t.get('/api/v1/ask', { as: user, multipart: { fields: { task }, image: png } });
++ assert.strictEqual(r.status, 201, `${task}: ${r.text}`);
++ assert.strictEqual(r.json().answer.task, task);
++ assert.strictEqual(r.json().answer.answer.engine, 'ai', `${task} goes to the model`);
++ const sent = t.aiServer.requests.filter((x) => x.path === '/v1/chat/completions').pop();
++ assert.match(JSON.stringify(sent.body.messages[0].content), re, `${task} carries its prompt`);
++ }
++ const local = await t.get('/api/v1/ask', { as: user, multipart: { fields: { task: 'meal', mode: 'local' }, image: png } });
++ assert.strictEqual(local.status, 400);
++ assert.strictEqual(local.json().code, 'vision.needs_model');
++ });
++
await check('POST /describe is the plan\'s operation name for the same ask, and a data URL works', async () => {
const r = await t.get('/api/v1/describe', { as: user, json: { question: 'What is this?', image: `data:image/png;base64,${png.toString('base64')}` } });
assert.strictEqual(r.status, 201, r.text);
ddiff --git a/test/mcp.test.js b/test/mcp.test.js
iindex dcecb2a..e238084 100644
---- a/test/mcp.test.js
++++ b/test/mcp.test.js
@@@ -18,7 +18,10 @@ const rpc = (method, params = {}, id = 1) => ({ jsonrpc: '2.0', id, method, para
assert.strictEqual(init.status, 200, init.text);
assert.strictEqual(init.json().result.serverInfo.name, 'openvibe-vision');
const list = await t.get('/mcp', { json: rpc('tools/list') });
-- assert.strictEqual(list.json().result.tools.length, 11);
++ assert.strictEqual(list.json().result.tools.length, 16);
++ for (const name of ['vision.meal', 'vision.describe_surroundings', 'vision.identify_plant', 'vision.identify_landmark', 'vision.screenshot_components']) {
++ assert.ok(list.json().result.tools.some((x) => x.name === name), `VIS-6 tool ${name}`);
++ }
assert.ok(list.json().result.tools.some((x) => x.name === 'vision.blur_faces'));
assert.ok(!list.json().result.tools.some((x) => /enrol|verify|cast/.test(x.name)));
});
@@@ -44,6 +47,16 @@ const rpc = (method, params = {}, id = 1) => ({ jsonrpc: '2.0', id, method, para
assert.match(r.json().result.content[0].text, /image data URL only/);
});
++ await check('VIS-6: a prompted tool runs its task through the routed model and saves it as that task', async () => {
++ const tiny = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
++ const r = await t.get('/mcp', { bearer: t.network.userToken(owner), json: rpc('tools/call', { name: 'vision.identify_plant', arguments: { image: tiny } }) });
++ assert.strictEqual(r.status, 200, r.text);
++ assert.strictEqual(r.json().result.isError, false, r.text);
++ assert.strictEqual(r.json().result.structuredContent.answer.task, 'plant');
++ const sent = t.aiServer.requests.filter((x) => x.path === '/v1/chat/completions').pop();
++ assert.match(JSON.stringify(sent.body.messages[0].content), /scientific name/);
++ });
++
await check('an owner may read their own UI check, while another owner cannot', async () => {
const saved = await t.get('/api/v1/ui-check', { as: owner, json: { html: '<div>hello</div>' } });
assert.strictEqual(saved.status, 201, saved.text);