openvibers/openvibe.vision Commits
VIS-6: meals, surroundings, plants, landmarks, screenshot components; every task's answer saves (#81)
Changes
@@@ -134,7 +134,10 @@ client `vision`. No new AI capability, no provider key, no direct paid provider.token responds with a `WWW-Authenticate` challenge pointing to that metadata. The tools are `vision.ask_image`,`vision.read_text`, `vision.translate_sign`, `vision.find_similar`, `vision.search_my_media`,`vision.describe_clip`, `vision.detect_objects`, `vision.blur_faces`, `vision.check_ui`,--`vision.compare_models` and `vision.get_job`. Face enrolment, verification, cast matching and consent mutation++`vision.compare_models` and `vision.get_job`, and five that run one prompted task each through the routed model++(VIS-6; also `task=` on `POST /api/v1/ask`, no question needed): `vision.meal` (`meal`: foods, ingredients, portions,++rough calories), `vision.describe_surroundings` (`surroundings`), `vision.identify_plant` (`plant`),++`vision.identify_landmark` (`landmark`) and `vision.screenshot_components` (`components`). Face enrolment, verification, cast matching and consent mutationare available only through their person or Live-specific API routes, never through MCP.When configured with Events and a Network client secret, successful owner results publish `vision.result.ready`nnew file mode 100644@@@ -0,0 +1,10 @@++-- phase: expand++-- VIS-6 (2026-11-03): an answer is saved under the task it ran as. 0001 allowed only ask, describe, ocr and compare,++-- so a translate, identify or count answer (vision.translate_sign, the Ask page's own choices) failed its insert. This++-- allows every task the service runs, the wave-2 ones included: meal, surroundings, plant, landmark and components.++-- A superset of the old check: code from before this migration still writes only the old four.++ALTER TABLE vision_answers DROP CONSTRAINT vision_answers_task;++ALTER TABLE vision_answers ADD CONSTRAINT vision_answers_task CHECK (task IN (++ 'ask', 'describe', 'ocr', 'compare', 'translate', 'identify', 'count',++ 'meal', 'surroundings', 'plant', 'landmark', 'components'++));@@@ -14,6 +14,11 @@ const TOOLS = Object.freeze([{ name: 'vision.ask_image', description: 'Ask about an image. Anonymous requests use local OCR only.', inputSchema: input({ ...image, question: { type: 'string' }, mode: { type: 'string', enum: ['auto', 'local', 'ai'] } }) },{ name: 'vision.read_text', description: 'Read visible text with local OCR.', inputSchema: input({ ...image, lang: { type: 'string' } }) },{ name: 'vision.translate_sign', description: 'Translate visible sign text through a routed model.', inputSchema: input({ ...image, lang: { type: 'string' } }) },++ { name: 'vision.meal', description: 'List the foods in a photo of a meal, with likely ingredients, estimated portions and rough calories.', inputSchema: input(image) },++ { name: 'vision.describe_surroundings', description: 'Describe surroundings for someone who cannot see them: what is ahead, obstacles, signs. Never who people are.', inputSchema: input(image) },++ { name: 'vision.identify_plant', description: 'Name a plant (common and scientific), with confidence and whether it is commonly poisonous.', inputSchema: input(image) },++ { name: 'vision.identify_landmark', description: 'Name a landmark, building or place, with confidence. Never identifies a person.', inputSchema: input(image) },++ { name: 'vision.screenshot_components', description: 'Break a screenshot of an app or page into its interface components, with the HTML element or ARIA role for each.', inputSchema: input(image) },{ name: 'vision.find_similar', description: 'Search your private visual index by image.', inputSchema: input({ image: image.image }, ['image']) },{ name: 'vision.search_my_media', description: 'Search your private visual index by words.', inputSchema: input({ query: { type: 'string' } }, ['query']) },{ name: 'vision.describe_clip', description: 'Find scene changes and moments in your own frames.', inputSchema: input({ frames: { type: 'array', items: { type: 'object' } }, prompts: { type: 'array', items: { type: 'string' } } }, ['frames']) },@@@ -27,6 +32,8 @@ const TOOLS = Object.freeze([const ACCESS = Object.freeze({'vision.ask_image': ['vision.image.ask', 'local'], 'vision.read_text': ['vision.text.read', 'anonymous'],'vision.translate_sign': ['vision.image.ask'], 'vision.find_similar': ['vision.media.find', 'person'],++ 'vision.meal': ['vision.image.ask'], 'vision.describe_surroundings': ['vision.image.ask'], 'vision.identify_plant': ['vision.image.ask'],++ 'vision.identify_landmark': ['vision.image.ask'], 'vision.screenshot_components': ['vision.image.ask'],'vision.search_my_media': ['vision.media.find', 'person'], 'vision.describe_clip': ['vision.video.moments', 'person'],'vision.detect_objects': ['vision.camera.watch', 'person'], 'vision.blur_faces': ['vision.face.blur', 'anonymous'],'vision.check_ui': ['vision.ui.check', 'anonymous'], 'vision.compare_models': ['vision.model.compare'],@@@ -34,10 +41,17 @@ const ACCESS = Object.freeze({});const EVENT_PATH = Object.freeze({'vision.ask_image': '/ask', 'vision.read_text': '/ocr', 'vision.translate_sign': '/ask',++ 'vision.meal': '/ask', 'vision.describe_surroundings': '/ask', 'vision.identify_plant': '/ask', 'vision.identify_landmark': '/ask', 'vision.screenshot_components': '/ask','vision.find_similar': '/find', 'vision.search_my_media': '/find', 'vision.describe_clip': '/moments','vision.detect_objects': '/watch', 'vision.check_ui': '/ui-check', 'vision.compare_models': '/compare',});++// VIS-6 wave 2: each tool is one of the service's prompted tasks (server/vision/ai.js TASK_PROMPTS).++const PROMPTED_TOOL = Object.freeze({++ 'vision.meal': 'meal', 'vision.describe_surroundings': 'surroundings', 'vision.identify_plant': 'plant',++ 'vision.identify_landmark': 'landmark', 'vision.screenshot_components': 'components',++});++function rpcResult(id, value) { return { jsonrpc: '2.0', id, result: value }; }function rpcError(id, code, message) { return { jsonrpc: '2.0', id: id === undefined ? null : id, error: { code, message } }; }function toolResult(id, value) {@@@ -81,6 +95,8 @@ async function invoke(name, args, req, ctx) {}case 'vision.read_text': return await vision.ocr({ ...request, image: await load(), lang: args.lang, fallback: false });case 'vision.translate_sign': return { answer: await vision.ask({ ...request, image: await load(), task: 'translate', question: 'Translate the visible sign text.', mode: 'ai', lang: args.lang }) };++ case 'vision.meal': case 'vision.describe_surroundings': case 'vision.identify_plant': case 'vision.identify_landmark': case 'vision.screenshot_components':++ return { answer: await vision.ask({ ...request, image: await load(), task: PROMPTED_TOOL[name], question: '', mode: 'ai' }) };case 'vision.find_similar': return { answer: await networkTools.find(subject, await loadPrivate()) };case 'vision.search_my_media': {if (typeof args.query !== 'string' || !args.query.trim() || args.query.length > 500) throw new Error('query must contain 1–500 characters');@@@ -139,7 +139,7 @@ function ask({ signedIn = false, ai = false, problem = null, values = {} } = {})const v = { task: 'ask', mode: 'auto', ...values };return `<div class="vi-wrap" id="vi-ask"><header class="vi-page-h"><p class="vi-kicker">Ask a picture</p><h1>What does the image show?</h1>--<p class="vi-lede">Describe, read, translate, identify or count. A read question goes to the local OCR reader first; the answer says which engine answered and why. Face matching has separate consent controls on <a href="/people">People</a>.</p></header>++<p class="vi-lede">Describe, read, translate, identify or count; name a plant or a place, list what is in a meal, describe your surroundings, or break a screenshot into components. A read question goes to the local OCR reader first; the answer says which engine answered and why. Face matching has separate consent controls on <a href="/people">People</a>.</p></header>${problem ? `<p class="vi-note is-bad" role="alert">${esc(problem)}</p>` : ''}${signedIn ? '' : `<p class="vi-note">Ask saves the answer to an account. <a href="/auth/login?next=%2Fask">Sign in with OpenVibe</a> to ask; <a href="/ocr">OCR</a> and the <a href="/ui-check">UI check</a> work without one.</p>`}${ai ? '' : '<p class="vi-note is-warm">OpenVibe.AI is not configured on this host. A read question still runs on the local OCR engine.</p>'}@@@ -147,7 +147,8 @@ ${ai ? '' : '<p class="vi-note is-warm">OpenVibe.AI is not configured on this ho${imageFields({ values: v })}<div class="vi-field-grid"><label class="vi-field"><span>Task</span><select name="task">--${[['ask', 'Ask a question'], ['describe', 'Describe the image'], ['identify', 'Identify what it is'], ['count', 'Count something'], ['translate', 'Read and translate'], ['ocr', 'Read the text']].map(([k, label]) => `<option value="${k}"${v.task === k ? ' selected' : ''}>${label}</option>`).join('')}++${[['ask', 'Ask a question'], ['describe', 'Describe the image'], ['identify', 'Identify what it is'], ['count', 'Count something'], ['translate', 'Read and translate'], ['ocr', 'Read the text'],++ ['meal', 'What is in this meal'], ['surroundings', 'Describe my surroundings'], ['plant', 'Name this plant'], ['landmark', 'Name this place'], ['components', 'Screenshot to components']].map(([k, label]) => `<option value="${k}"${v.task === k ? ' selected' : ''}>${label}</option>`).join('')}</select></label><label class="vi-field"><span>Mode</span><select name="mode">${[['auto', 'Auto: cheapest engine first'], ['local', 'Local only: no model call'], ['ai', 'AI directly']].map(([k, label]) => `<option value="${k}"${v.mode === k ? ' selected' : ''}>${label}</option>`).join('')}@@@ -30,6 +30,12 @@ const TASK_PROMPTS = {translate: 'Read the visible text and translate it into the target language. Give the original text, then the translation.',ocr: 'Read every visible piece of text in the image, verbatim, preserving line breaks and reading order. Do not summarize and do not add anything that is not written.',compare: 'Answer the question about the image precisely, for a side-by-side comparison. Say when the image does not show the answer.',++ // VIS-6 wave 2. Each answers without a question; a question narrows it.++ meal: 'List each food and drink you can see. For each: its likely ingredients, an estimated portion (grams, or a household measure such as a cup) and a rough calorie estimate. Then give a total. Say plainly that these are estimates from a photo, and name what a photo cannot show (oil, butter, sugar, sauces, what is underneath). If there is no food, say so.',++ surroundings: 'Describe these surroundings for someone who cannot see them, to help them find their way: what is straight ahead and to each side, steps, kerbs, doors, obstacles and hazards, people and moving things (what they are doing, never who they are), and any signs or text. Most important first, in short sentences. Say how sure you are when the picture is unclear.',++ plant: 'Identify the plant: its common name and its scientific name, how sure you are, and the features you went by (leaves, flowers, fruit, bark, habit). When the picture supports it, say whether any part is commonly poisonous to people or pets. If the picture is not of a plant, say so.',++ landmark: 'Identify the landmark, building or place: its name, city and country, how sure you are, and the features you went by. If it is not a recognisable landmark, say so and describe the kind of place instead. Never identify a person in the picture.',++ components: 'Break this screenshot of an app or web page into its interface components, as a nested outline from the page down: each region (header, navigation, main content, sidebar, footer, dialog) and inside it each component (heading, button, link, text field, checkbox, list, card, tab, image, icon), with its visible label or text. After each component, the HTML element or ARIA role that fits it. End with anything that looks broken: overlapping, cut off, unreadable or unlabelled.',};const NO_IDENTITY = 'Never identify, name or recognise a person, and never use a face as identity.';@@@ -22,14 +22,17 @@ class VisionError extends Error {}}--const TASKS = new Set(['ask', 'describe', 'identify', 'count', 'translate', 'ocr', 'compare']);++const TASKS = new Set(['ask', 'describe', 'identify', 'count', 'translate', 'ocr', 'compare', 'meal', 'surroundings', 'plant', 'landmark', 'components']);++// VIS-6 wave 2: tasks whose prompt says everything (server/vision/ai.js TASK_PROMPTS), so they need no question. Each++// always goes to the routed model: the local OCR reader cannot answer them.++const PROMPTED = new Set(['meal', 'surroundings', 'plant', 'landmark', 'components']);const READ_RE = /\b(read|say|says|said|text|word|words|label|labels|sign|signs|receipt|receipts|handwriting|ocr|written|print|printed|translate|translation|caption|menu|address|number)\b/i;function isReadTask(task, question) {if (task === 'ocr') return true;// Local OCR can read the words, but it cannot translate them: a translate task (or a question that asks for a// translation) goes to the model when auto mode is on. mode=local still returns the reading, and says so.-- if (task === 'translate') return false;++ if (task === 'translate' || PROMPTED.has(task)) return false;if (/\btranslate|translation\b/i.test(String(question || ''))) return false;return task === 'ask' && READ_RE.test(String(question || ''));}@@@ -87,11 +90,12 @@ function createVision({ config, s, images, ocr, ai, store, log = console }) {*/async function ask({ requester, subject = null, image, task = 'ask', question = '', mode = 'auto', lang = null, model = null }) {const t = asTask(task);-- if (!t || t === 'compare') throw new VisionError(400, 'vision.invalid_task', 'task must be ask, describe, identify, count, translate or ocr');++ if (!t || t === 'compare') throw new VisionError(400, 'vision.invalid_task', 'task must be ask, describe, identify, count, translate, ocr, meal, surroundings, plant, landmark or components');if (mode !== 'auto' && mode !== 'local' && mode !== 'ai') throw new VisionError(400, 'vision.invalid_mode', 'mode must be auto, local or ai');const started = Date.now();const q = String(question || '').trim().slice(0, 2000);-- if (t !== 'describe' && t !== 'ocr' && !q && mode !== 'local') throw new VisionError(400, 'vision.question_required', 'ask, identify, count and translate need a question');++ if (t !== 'describe' && t !== 'ocr' && !PROMPTED.has(t) && !q && mode !== 'local') throw new VisionError(400, 'vision.question_required', 'ask, identify, count and translate need a question');++ if (PROMPTED.has(t) && mode === 'local') throw new VisionError(400, 'vision.needs_model', `${t} needs the routed model: the local OCR reader cannot answer it`);let local = null;if (mode !== 'ai' && (isReadTask(t, q) || mode === 'local')) {@@@ -45,6 +45,30 @@ const { boot, check, done } = require('./helpers/boot');assert.ok(t.aiServer.requests.some((x) => x.path === '/v1/chat/completions' && JSON.stringify(x.body).includes('image_url')));});++ await check('VIS-6: every task is saved as itself, and the prompted tasks need no question', async () => {++ // 0001 allowed only ask/describe/ocr/compare: a translate, identify or count answer failed its insert (0006).++ for (const task of ['translate', 'identify', 'count']) {++ const r = await t.get('/api/v1/ask', { as: user, multipart: { fields: { task, question: 'What is it?', mode: 'ai' }, image: png } });++ assert.strictEqual(r.status, 201, `${task}: ${r.text}`);++ assert.strictEqual(r.json().answer.task, task);++ }++ const prompts = {++ meal: /estimated portion[\s\S]*rough calorie/, surroundings: /someone who cannot see them[\s\S]*never who they are/,++ plant: /scientific name[\s\S]*poisonous/, landmark: /city and country[\s\S]*Never identify a person/, components: /ARIA role/,++ };++ for (const [task, re] of Object.entries(prompts)) {++ const r = await t.get('/api/v1/ask', { as: user, multipart: { fields: { task }, image: png } });++ assert.strictEqual(r.status, 201, `${task}: ${r.text}`);++ assert.strictEqual(r.json().answer.task, task);++ assert.strictEqual(r.json().answer.answer.engine, 'ai', `${task} goes to the model`);++ const sent = t.aiServer.requests.filter((x) => x.path === '/v1/chat/completions').pop();++ assert.match(JSON.stringify(sent.body.messages[0].content), re, `${task} carries its prompt`);++ }++ const local = await t.get('/api/v1/ask', { as: user, multipart: { fields: { task: 'meal', mode: 'local' }, image: png } });++ assert.strictEqual(local.status, 400);++ assert.strictEqual(local.json().code, 'vision.needs_model');++ });++await check('POST /describe is the plan\'s operation name for the same ask, and a data URL works', async () => {const r = await t.get('/api/v1/describe', { as: user, json: { question: 'What is this?', image: `data:image/png;base64,${png.toString('base64')}` } });assert.strictEqual(r.status, 201, r.text);@@@ -18,7 +18,10 @@ const rpc = (method, params = {}, id = 1) => ({ jsonrpc: '2.0', id, method, paraassert.strictEqual(init.status, 200, init.text);assert.strictEqual(init.json().result.serverInfo.name, 'openvibe-vision');const list = await t.get('/mcp', { json: rpc('tools/list') });-- assert.strictEqual(list.json().result.tools.length, 11);++ assert.strictEqual(list.json().result.tools.length, 16);++ for (const name of ['vision.meal', 'vision.describe_surroundings', 'vision.identify_plant', 'vision.identify_landmark', 'vision.screenshot_components']) {++ assert.ok(list.json().result.tools.some((x) => x.name === name), `VIS-6 tool ${name}`);++ }assert.ok(list.json().result.tools.some((x) => x.name === 'vision.blur_faces'));assert.ok(!list.json().result.tools.some((x) => /enrol|verify|cast/.test(x.name)));});@@@ -44,6 +47,16 @@ const rpc = (method, params = {}, id = 1) => ({ jsonrpc: '2.0', id, method, paraassert.match(r.json().result.content[0].text, /image data URL only/);});++ await check('VIS-6: a prompted tool runs its task through the routed model and saves it as that task', async () => {++ const tiny = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';++ const r = await t.get('/mcp', { bearer: t.network.userToken(owner), json: rpc('tools/call', { name: 'vision.identify_plant', arguments: { image: tiny } }) });++ assert.strictEqual(r.status, 200, r.text);++ assert.strictEqual(r.json().result.isError, false, r.text);++ assert.strictEqual(r.json().result.structuredContent.answer.task, 'plant');++ const sent = t.aiServer.requests.filter((x) => x.path === '/v1/chat/completions').pop();++ assert.match(JSON.stringify(sent.body.messages[0].content), /scientific name/);++ });++await check('an owner may read their own UI check, while another owner cannot', async () => {const saved = await t.get('/api/v1/ui-check', { as: owner, json: { html: '<div>hello</div>' } });assert.strictEqual(saved.status, 201, saved.text);
@@ -134,7 +134,10 @@ client `vision`. No new AI capability, no provider key, no direct paid provider.
@@ -134,7 +134,10 @@ client `vision`. No new AI capability, no provider key, no direct paid provider.
token responds with a `WWW-Authenticate` challenge pointing to that metadata. The tools are `vision.ask_image`,
token responds with a `WWW-Authenticate` challenge pointing to that metadata. The tools are `vision.ask_image`,
`vision.read_text`, `vision.translate_sign`, `vision.find_similar`, `vision.search_my_media`,
`vision.read_text`, `vision.translate_sign`, `vision.find_similar`, `vision.search_my_media`,
`vision.describe_clip`, `vision.detect_objects`, `vision.blur_faces`, `vision.check_ui`,
`vision.describe_clip`, `vision.detect_objects`, `vision.blur_faces`, `vision.check_ui`,
-`vision.compare_models` and `vision.get_job`. Face enrolment, verification, cast matching and consent mutation
+`vision.compare_models` and `vision.get_job`, and five that run one prompted task each through the routed model
+(VIS-6; also `task=` on `POST /api/v1/ask`, no question needed): `vision.meal` (`meal`: foods, ingredients, portions,
+rough calories), `vision.describe_surroundings` (`surroundings`), `vision.identify_plant` (`plant`),
+`vision.identify_landmark` (`landmark`) and `vision.screenshot_components` (`components`). Face enrolment, verification, cast matching and consent mutation
are available only through their person or Live-specific API routes, never through MCP.
are available only through their person or Live-specific API routes, never through MCP.
When configured with Events and a Network client secret, successful owner results publish `vision.result.ready`
When configured with Events and a Network client secret, successful owner results publish `vision.result.ready`
new file mode 100644
new file mode 100644
@@ -0,0 +1,10 @@
@@ -0,0 +1,10 @@
+-- phase: expand
+-- VIS-6 (2026-11-03): an answer is saved under the task it ran as. 0001 allowed only ask, describe, ocr and compare,
+-- so a translate, identify or count answer (vision.translate_sign, the Ask page's own choices) failed its insert. This
+-- allows every task the service runs, the wave-2 ones included: meal, surroundings, plant, landmark and components.
+-- A superset of the old check: code from before this migration still writes only the old four.
+ALTER TABLE vision_answers DROP CONSTRAINT vision_answers_task;
+ALTER TABLE vision_answers ADD CONSTRAINT vision_answers_task CHECK (task IN (
+ 'ask', 'describe', 'ocr', 'compare', 'translate', 'identify', 'count',
+ 'meal', 'surroundings', 'plant', 'landmark', 'components'
+));
@@ -14,6 +14,11 @@ const TOOLS = Object.freeze([
@@ -14,6 +14,11 @@ const TOOLS = Object.freeze([
{ name: 'vision.ask_image', description: 'Ask about an image. Anonymous requests use local OCR only.', inputSchema: input({ ...image, question: { type: 'string' }, mode: { type: 'string', enum: ['auto', 'local', 'ai'] } }) },
{ name: 'vision.ask_image', description: 'Ask about an image. Anonymous requests use local OCR only.', inputSchema: input({ ...image, question: { type: 'string' }, mode: { type: 'string', enum: ['auto', 'local', 'ai'] } }) },
{ name: 'vision.read_text', description: 'Read visible text with local OCR.', inputSchema: input({ ...image, lang: { type: 'string' } }) },
{ name: 'vision.read_text', description: 'Read visible text with local OCR.', inputSchema: input({ ...image, lang: { type: 'string' } }) },
{ name: 'vision.translate_sign', description: 'Translate visible sign text through a routed model.', inputSchema: input({ ...image, lang: { type: 'string' } }) },
{ name: 'vision.translate_sign', description: 'Translate visible sign text through a routed model.', inputSchema: input({ ...image, lang: { type: 'string' } }) },
+ { name: 'vision.meal', description: 'List the foods in a photo of a meal, with likely ingredients, estimated portions and rough calories.', inputSchema: input(image) },
+ { name: 'vision.describe_surroundings', description: 'Describe surroundings for someone who cannot see them: what is ahead, obstacles, signs. Never who people are.', inputSchema: input(image) },
+ { name: 'vision.identify_plant', description: 'Name a plant (common and scientific), with confidence and whether it is commonly poisonous.', inputSchema: input(image) },
+ { name: 'vision.identify_landmark', description: 'Name a landmark, building or place, with confidence. Never identifies a person.', inputSchema: input(image) },
+ { name: 'vision.screenshot_components', description: 'Break a screenshot of an app or page into its interface components, with the HTML element or ARIA role for each.', inputSchema: input(image) },
{ name: 'vision.find_similar', description: 'Search your private visual index by image.', inputSchema: input({ image: image.image }, ['image']) },
{ name: 'vision.find_similar', description: 'Search your private visual index by image.', inputSchema: input({ image: image.image }, ['image']) },
{ name: 'vision.search_my_media', description: 'Search your private visual index by words.', inputSchema: input({ query: { type: 'string' } }, ['query']) },
{ name: 'vision.search_my_media', description: 'Search your private visual index by words.', inputSchema: input({ query: { type: 'string' } }, ['query']) },
{ name: 'vision.describe_clip', description: 'Find scene changes and moments in your own frames.', inputSchema: input({ frames: { type: 'array', items: { type: 'object' } }, prompts: { type: 'array', items: { type: 'string' } } }, ['frames']) },
{ name: 'vision.describe_clip', description: 'Find scene changes and moments in your own frames.', inputSchema: input({ frames: { type: 'array', items: { type: 'object' } }, prompts: { type: 'array', items: { type: 'string' } } }, ['frames']) },
@@ -27,6 +32,8 @@ const TOOLS = Object.freeze([
@@ -27,6 +32,8 @@ const TOOLS = Object.freeze([
const ACCESS = Object.freeze({
const ACCESS = Object.freeze({
'vision.ask_image': ['vision.image.ask', 'local'], 'vision.read_text': ['vision.text.read', 'anonymous'],
'vision.ask_image': ['vision.image.ask', 'local'], 'vision.read_text': ['vision.text.read', 'anonymous'],
'vision.translate_sign': ['vision.image.ask'], 'vision.find_similar': ['vision.media.find', 'person'],
'vision.translate_sign': ['vision.image.ask'], 'vision.find_similar': ['vision.media.find', 'person'],
+ 'vision.meal': ['vision.image.ask'], 'vision.describe_surroundings': ['vision.image.ask'], 'vision.identify_plant': ['vision.image.ask'],
+ 'vision.identify_landmark': ['vision.image.ask'], 'vision.screenshot_components': ['vision.image.ask'],
'vision.search_my_media': ['vision.media.find', 'person'], 'vision.describe_clip': ['vision.video.moments', 'person'],
'vision.search_my_media': ['vision.media.find', 'person'], 'vision.describe_clip': ['vision.video.moments', 'person'],
'vision.detect_objects': ['vision.camera.watch', 'person'], 'vision.blur_faces': ['vision.face.blur', 'anonymous'],
'vision.detect_objects': ['vision.camera.watch', 'person'], 'vision.blur_faces': ['vision.face.blur', 'anonymous'],
'vision.check_ui': ['vision.ui.check', 'anonymous'], 'vision.compare_models': ['vision.model.compare'],
'vision.check_ui': ['vision.ui.check', 'anonymous'], 'vision.compare_models': ['vision.model.compare'],
@@ -34,10 +41,17 @@ const ACCESS = Object.freeze({
@@ -34,10 +41,17 @@ const ACCESS = Object.freeze({
});
});
const EVENT_PATH = Object.freeze({
const EVENT_PATH = Object.freeze({
'vision.ask_image': '/ask', 'vision.read_text': '/ocr', 'vision.translate_sign': '/ask',
'vision.ask_image': '/ask', 'vision.read_text': '/ocr', 'vision.translate_sign': '/ask',
+ 'vision.meal': '/ask', 'vision.describe_surroundings': '/ask', 'vision.identify_plant': '/ask', 'vision.identify_landmark': '/ask', 'vision.screenshot_components': '/ask',
'vision.find_similar': '/find', 'vision.search_my_media': '/find', 'vision.describe_clip': '/moments',
'vision.find_similar': '/find', 'vision.search_my_media': '/find', 'vision.describe_clip': '/moments',
'vision.detect_objects': '/watch', 'vision.check_ui': '/ui-check', 'vision.compare_models': '/compare',
'vision.detect_objects': '/watch', 'vision.check_ui': '/ui-check', 'vision.compare_models': '/compare',
});
});
+// VIS-6 wave 2: each tool is one of the service's prompted tasks (server/vision/ai.js TASK_PROMPTS).
+const PROMPTED_TOOL = Object.freeze({
+ 'vision.meal': 'meal', 'vision.describe_surroundings': 'surroundings', 'vision.identify_plant': 'plant',
+ 'vision.identify_landmark': 'landmark', 'vision.screenshot_components': 'components',
+});
+
function rpcResult(id, value) { return { jsonrpc: '2.0', id, result: value }; }
function rpcResult(id, value) { return { jsonrpc: '2.0', id, result: value }; }
function rpcError(id, code, message) { return { jsonrpc: '2.0', id: id === undefined ? null : id, error: { code, message } }; }
function rpcError(id, code, message) { return { jsonrpc: '2.0', id: id === undefined ? null : id, error: { code, message } }; }
function toolResult(id, value) {
function toolResult(id, value) {
@@ -81,6 +95,8 @@ async function invoke(name, args, req, ctx) {
@@ -81,6 +95,8 @@ async function invoke(name, args, req, ctx) {
}
}
case 'vision.read_text': return await vision.ocr({ ...request, image: await load(), lang: args.lang, fallback: false });
case 'vision.read_text': return await vision.ocr({ ...request, image: await load(), lang: args.lang, fallback: false });
case 'vision.translate_sign': return { answer: await vision.ask({ ...request, image: await load(), task: 'translate', question: 'Translate the visible sign text.', mode: 'ai', lang: args.lang }) };
case 'vision.translate_sign': return { answer: await vision.ask({ ...request, image: await load(), task: 'translate', question: 'Translate the visible sign text.', mode: 'ai', lang: args.lang }) };
+ case 'vision.meal': case 'vision.describe_surroundings': case 'vision.identify_plant': case 'vision.identify_landmark': case 'vision.screenshot_components':
+ return { answer: await vision.ask({ ...request, image: await load(), task: PROMPTED_TOOL[name], question: '', mode: 'ai' }) };
case 'vision.find_similar': return { answer: await networkTools.find(subject, await loadPrivate()) };
case 'vision.find_similar': return { answer: await networkTools.find(subject, await loadPrivate()) };
case 'vision.search_my_media': {
case 'vision.search_my_media': {
if (typeof args.query !== 'string' || !args.query.trim() || args.query.length > 500) throw new Error('query must contain 1–500 characters');
if (typeof args.query !== 'string' || !args.query.trim() || args.query.length > 500) throw new Error('query must contain 1–500 characters');
@@ -139,7 +139,7 @@ function ask({ signedIn = false, ai = false, problem = null, values = {} } = {})
@@ -139,7 +139,7 @@ function ask({ signedIn = false, ai = false, problem = null, values = {} } = {})
const v = { task: 'ask', mode: 'auto', ...values };
const v = { task: 'ask', mode: 'auto', ...values };
return `<div class="vi-wrap" id="vi-ask">
return `<div class="vi-wrap" id="vi-ask">
<header class="vi-page-h"><p class="vi-kicker">Ask a picture</p><h1>What does the image show?</h1>
<header class="vi-page-h"><p class="vi-kicker">Ask a picture</p><h1>What does the image show?</h1>
-<p class="vi-lede">Describe, read, translate, identify or count. A read question goes to the local OCR reader first; the answer says which engine answered and why. Face matching has separate consent controls on <a href="/people">People</a>.</p></header>
+<p class="vi-lede">Describe, read, translate, identify or count; name a plant or a place, list what is in a meal, describe your surroundings, or break a screenshot into components. A read question goes to the local OCR reader first; the answer says which engine answered and why. Face matching has separate consent controls on <a href="/people">People</a>.</p></header>
${problem ? `<p class="vi-note is-bad" role="alert">${esc(problem)}</p>` : ''}
${problem ? `<p class="vi-note is-bad" role="alert">${esc(problem)}</p>` : ''}
${signedIn ? '' : `<p class="vi-note">Ask saves the answer to an account. <a href="/auth/login?next=%2Fask">Sign in with OpenVibe</a> to ask; <a href="/ocr">OCR</a> and the <a href="/ui-check">UI check</a> work without one.</p>`}
${signedIn ? '' : `<p class="vi-note">Ask saves the answer to an account. <a href="/auth/login?next=%2Fask">Sign in with OpenVibe</a> to ask; <a href="/ocr">OCR</a> and the <a href="/ui-check">UI check</a> work without one.</p>`}
${ai ? '' : '<p class="vi-note is-warm">OpenVibe.AI is not configured on this host. A read question still runs on the local OCR engine.</p>'}
${ai ? '' : '<p class="vi-note is-warm">OpenVibe.AI is not configured on this host. A read question still runs on the local OCR engine.</p>'}
@@ -147,7 +147,8 @@ ${ai ? '' : '<p class="vi-note is-warm">OpenVibe.AI is not configured on this ho
@@ -147,7 +147,8 @@ ${ai ? '' : '<p class="vi-note is-warm">OpenVibe.AI is not configured on this ho
${imageFields({ values: v })}
${imageFields({ values: v })}
<div class="vi-field-grid">
<div class="vi-field-grid">
<label class="vi-field"><span>Task</span><select name="task">
<label class="vi-field"><span>Task</span><select name="task">
-${[['ask', 'Ask a question'], ['describe', 'Describe the image'], ['identify', 'Identify what it is'], ['count', 'Count something'], ['translate', 'Read and translate'], ['ocr', 'Read the text']].map(([k, label]) => `<option value="${k}"${v.task === k ? ' selected' : ''}>${label}</option>`).join('')}
+${[['ask', 'Ask a question'], ['describe', 'Describe the image'], ['identify', 'Identify what it is'], ['count', 'Count something'], ['translate', 'Read and translate'], ['ocr', 'Read the text'],
+ ['meal', 'What is in this meal'], ['surroundings', 'Describe my surroundings'], ['plant', 'Name this plant'], ['landmark', 'Name this place'], ['components', 'Screenshot to components']].map(([k, label]) => `<option value="${k}"${v.task === k ? ' selected' : ''}>${label}</option>`).join('')}
</select></label>
</select></label>
<label class="vi-field"><span>Mode</span><select name="mode">
<label class="vi-field"><span>Mode</span><select name="mode">
${[['auto', 'Auto: cheapest engine first'], ['local', 'Local only: no model call'], ['ai', 'AI directly']].map(([k, label]) => `<option value="${k}"${v.mode === k ? ' selected' : ''}>${label}</option>`).join('')}
${[['auto', 'Auto: cheapest engine first'], ['local', 'Local only: no model call'], ['ai', 'AI directly']].map(([k, label]) => `<option value="${k}"${v.mode === k ? ' selected' : ''}>${label}</option>`).join('')}
@@ -30,6 +30,12 @@ const TASK_PROMPTS = {
@@ -30,6 +30,12 @@ const TASK_PROMPTS = {
translate: 'Read the visible text and translate it into the target language. Give the original text, then the translation.',
translate: 'Read the visible text and translate it into the target language. Give the original text, then the translation.',
ocr: 'Read every visible piece of text in the image, verbatim, preserving line breaks and reading order. Do not summarize and do not add anything that is not written.',
ocr: 'Read every visible piece of text in the image, verbatim, preserving line breaks and reading order. Do not summarize and do not add anything that is not written.',
compare: 'Answer the question about the image precisely, for a side-by-side comparison. Say when the image does not show the answer.',
compare: 'Answer the question about the image precisely, for a side-by-side comparison. Say when the image does not show the answer.',
+ // VIS-6 wave 2. Each answers without a question; a question narrows it.
+ meal: 'List each food and drink you can see. For each: its likely ingredients, an estimated portion (grams, or a household measure such as a cup) and a rough calorie estimate. Then give a total. Say plainly that these are estimates from a photo, and name what a photo cannot show (oil, butter, sugar, sauces, what is underneath). If there is no food, say so.',
+ surroundings: 'Describe these surroundings for someone who cannot see them, to help them find their way: what is straight ahead and to each side, steps, kerbs, doors, obstacles and hazards, people and moving things (what they are doing, never who they are), and any signs or text. Most important first, in short sentences. Say how sure you are when the picture is unclear.',
+ plant: 'Identify the plant: its common name and its scientific name, how sure you are, and the features you went by (leaves, flowers, fruit, bark, habit). When the picture supports it, say whether any part is commonly poisonous to people or pets. If the picture is not of a plant, say so.',
+ landmark: 'Identify the landmark, building or place: its name, city and country, how sure you are, and the features you went by. If it is not a recognisable landmark, say so and describe the kind of place instead. Never identify a person in the picture.',
+ components: 'Break this screenshot of an app or web page into its interface components, as a nested outline from the page down: each region (header, navigation, main content, sidebar, footer, dialog) and inside it each component (heading, button, link, text field, checkbox, list, card, tab, image, icon), with its visible label or text. After each component, the HTML element or ARIA role that fits it. End with anything that looks broken: overlapping, cut off, unreadable or unlabelled.',
};
};
const NO_IDENTITY = 'Never identify, name or recognise a person, and never use a face as identity.';
const NO_IDENTITY = 'Never identify, name or recognise a person, and never use a face as identity.';
@@ -22,14 +22,17 @@ class VisionError extends Error {
@@ -22,14 +22,17 @@ class VisionError extends Error {
}
}
}
}
-const TASKS = new Set(['ask', 'describe', 'identify', 'count', 'translate', 'ocr', 'compare']);
+const TASKS = new Set(['ask', 'describe', 'identify', 'count', 'translate', 'ocr', 'compare', 'meal', 'surroundings', 'plant', 'landmark', 'components']);
+// VIS-6 wave 2: tasks whose prompt says everything (server/vision/ai.js TASK_PROMPTS), so they need no question. Each
+// always goes to the routed model: the local OCR reader cannot answer them.
+const PROMPTED = new Set(['meal', 'surroundings', 'plant', 'landmark', 'components']);
const READ_RE = /\b(read|say|says|said|text|word|words|label|labels|sign|signs|receipt|receipts|handwriting|ocr|written|print|printed|translate|translation|caption|menu|address|number)\b/i;
const READ_RE = /\b(read|say|says|said|text|word|words|label|labels|sign|signs|receipt|receipts|handwriting|ocr|written|print|printed|translate|translation|caption|menu|address|number)\b/i;
function isReadTask(task, question) {
function isReadTask(task, question) {
if (task === 'ocr') return true;
if (task === 'ocr') return true;
// Local OCR can read the words, but it cannot translate them: a translate task (or a question that asks for a
// Local OCR can read the words, but it cannot translate them: a translate task (or a question that asks for a
// translation) goes to the model when auto mode is on. mode=local still returns the reading, and says so.
// translation) goes to the model when auto mode is on. mode=local still returns the reading, and says so.
- if (task === 'translate') return false;
+ if (task === 'translate' || PROMPTED.has(task)) return false;
if (/\btranslate|translation\b/i.test(String(question || ''))) return false;
if (/\btranslate|translation\b/i.test(String(question || ''))) return false;
return task === 'ask' && READ_RE.test(String(question || ''));
return task === 'ask' && READ_RE.test(String(question || ''));
}
}
@@ -87,11 +90,12 @@ function createVision({ config, s, images, ocr, ai, store, log = console }) {
@@ -87,11 +90,12 @@ function createVision({ config, s, images, ocr, ai, store, log = console }) {
*/
*/
async function ask({ requester, subject = null, image, task = 'ask', question = '', mode = 'auto', lang = null, model = null }) {
async function ask({ requester, subject = null, image, task = 'ask', question = '', mode = 'auto', lang = null, model = null }) {
const t = asTask(task);
const t = asTask(task);
- if (!t || t === 'compare') throw new VisionError(400, 'vision.invalid_task', 'task must be ask, describe, identify, count, translate or ocr');
+ if (!t || t === 'compare') throw new VisionError(400, 'vision.invalid_task', 'task must be ask, describe, identify, count, translate, ocr, meal, surroundings, plant, landmark or components');
if (mode !== 'auto' && mode !== 'local' && mode !== 'ai') throw new VisionError(400, 'vision.invalid_mode', 'mode must be auto, local or ai');
if (mode !== 'auto' && mode !== 'local' && mode !== 'ai') throw new VisionError(400, 'vision.invalid_mode', 'mode must be auto, local or ai');
const started = Date.now();
const started = Date.now();
const q = String(question || '').trim().slice(0, 2000);
const q = String(question || '').trim().slice(0, 2000);
- if (t !== 'describe' && t !== 'ocr' && !q && mode !== 'local') throw new VisionError(400, 'vision.question_required', 'ask, identify, count and translate need a question');
+ if (t !== 'describe' && t !== 'ocr' && !PROMPTED.has(t) && !q && mode !== 'local') throw new VisionError(400, 'vision.question_required', 'ask, identify, count and translate need a question');
+ if (PROMPTED.has(t) && mode === 'local') throw new VisionError(400, 'vision.needs_model', `${t} needs the routed model: the local OCR reader cannot answer it`);
let local = null;
let local = null;
if (mode !== 'ai' && (isReadTask(t, q) || mode === 'local')) {
if (mode !== 'ai' && (isReadTask(t, q) || mode === 'local')) {
@@ -45,6 +45,30 @@ const { boot, check, done } = require('./helpers/boot');
@@ -45,6 +45,30 @@ const { boot, check, done } = require('./helpers/boot');
assert.ok(t.aiServer.requests.some((x) => x.path === '/v1/chat/completions' && JSON.stringify(x.body).includes('image_url')));
assert.ok(t.aiServer.requests.some((x) => x.path === '/v1/chat/completions' && JSON.stringify(x.body).includes('image_url')));
});
});
+ await check('VIS-6: every task is saved as itself, and the prompted tasks need no question', async () => {
+ // 0001 allowed only ask/describe/ocr/compare: a translate, identify or count answer failed its insert (0006).
+ for (const task of ['translate', 'identify', 'count']) {
+ const r = await t.get('/api/v1/ask', { as: user, multipart: { fields: { task, question: 'What is it?', mode: 'ai' }, image: png } });
+ assert.strictEqual(r.status, 201, `${task}: ${r.text}`);
+ assert.strictEqual(r.json().answer.task, task);
+ }
+ const prompts = {
+ meal: /estimated portion[\s\S]*rough calorie/, surroundings: /someone who cannot see them[\s\S]*never who they are/,
+ plant: /scientific name[\s\S]*poisonous/, landmark: /city and country[\s\S]*Never identify a person/, components: /ARIA role/,
+ };
+ for (const [task, re] of Object.entries(prompts)) {
+ const r = await t.get('/api/v1/ask', { as: user, multipart: { fields: { task }, image: png } });
+ assert.strictEqual(r.status, 201, `${task}: ${r.text}`);
+ assert.strictEqual(r.json().answer.task, task);
+ assert.strictEqual(r.json().answer.answer.engine, 'ai', `${task} goes to the model`);
+ const sent = t.aiServer.requests.filter((x) => x.path === '/v1/chat/completions').pop();
+ assert.match(JSON.stringify(sent.body.messages[0].content), re, `${task} carries its prompt`);
+ }
+ const local = await t.get('/api/v1/ask', { as: user, multipart: { fields: { task: 'meal', mode: 'local' }, image: png } });
+ assert.strictEqual(local.status, 400);
+ assert.strictEqual(local.json().code, 'vision.needs_model');
+ });
+
await check('POST /describe is the plan\'s operation name for the same ask, and a data URL works', async () => {
await check('POST /describe is the plan\'s operation name for the same ask, and a data URL works', async () => {
const r = await t.get('/api/v1/describe', { as: user, json: { question: 'What is this?', image: `data:image/png;base64,${png.toString('base64')}` } });
const r = await t.get('/api/v1/describe', { as: user, json: { question: 'What is this?', image: `data:image/png;base64,${png.toString('base64')}` } });
assert.strictEqual(r.status, 201, r.text);
assert.strictEqual(r.status, 201, r.text);
@@ -18,7 +18,10 @@ const rpc = (method, params = {}, id = 1) => ({ jsonrpc: '2.0', id, method, para
@@ -18,7 +18,10 @@ const rpc = (method, params = {}, id = 1) => ({ jsonrpc: '2.0', id, method, para
assert.strictEqual(init.status, 200, init.text);
assert.strictEqual(init.status, 200, init.text);
assert.strictEqual(init.json().result.serverInfo.name, 'openvibe-vision');
assert.strictEqual(init.json().result.serverInfo.name, 'openvibe-vision');
const list = await t.get('/mcp', { json: rpc('tools/list') });
const list = await t.get('/mcp', { json: rpc('tools/list') });
- assert.strictEqual(list.json().result.tools.length, 11);
+ assert.strictEqual(list.json().result.tools.length, 16);
+ for (const name of ['vision.meal', 'vision.describe_surroundings', 'vision.identify_plant', 'vision.identify_landmark', 'vision.screenshot_components']) {
+ assert.ok(list.json().result.tools.some((x) => x.name === name), `VIS-6 tool ${name}`);
+ }
assert.ok(list.json().result.tools.some((x) => x.name === 'vision.blur_faces'));
assert.ok(list.json().result.tools.some((x) => x.name === 'vision.blur_faces'));
assert.ok(!list.json().result.tools.some((x) => /enrol|verify|cast/.test(x.name)));
assert.ok(!list.json().result.tools.some((x) => /enrol|verify|cast/.test(x.name)));
});
});
@@ -44,6 +47,16 @@ const rpc = (method, params = {}, id = 1) => ({ jsonrpc: '2.0', id, method, para
@@ -44,6 +47,16 @@ const rpc = (method, params = {}, id = 1) => ({ jsonrpc: '2.0', id, method, para
assert.match(r.json().result.content[0].text, /image data URL only/);
assert.match(r.json().result.content[0].text, /image data URL only/);
});
});
+ await check('VIS-6: a prompted tool runs its task through the routed model and saves it as that task', async () => {
+ const tiny = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
+ const r = await t.get('/mcp', { bearer: t.network.userToken(owner), json: rpc('tools/call', { name: 'vision.identify_plant', arguments: { image: tiny } }) });
+ assert.strictEqual(r.status, 200, r.text);
+ assert.strictEqual(r.json().result.isError, false, r.text);
+ assert.strictEqual(r.json().result.structuredContent.answer.task, 'plant');
+ const sent = t.aiServer.requests.filter((x) => x.path === '/v1/chat/completions').pop();
+ assert.match(JSON.stringify(sent.body.messages[0].content), /scientific name/);
+ });
+
await check('an owner may read their own UI check, while another owner cannot', async () => {
await check('an owner may read their own UI check, while another owner cannot', async () => {
const saved = await t.get('/api/v1/ui-check', { as: owner, json: { html: '<div>hello</div>' } });
const saved = await t.get('/api/v1/ui-check', { as: owner, json: { html: '<div>hello</div>' } });
assert.strictEqual(saved.status, 201, saved.text);
assert.strictEqual(saved.status, 201, saved.text);