openvibers/openvibe.ai Commits
AI: food.read_pantry_photo, which foods a cupboard photo or receipt shows (#92)
Changes
@@@ -268,6 +268,10 @@ const PREPARE = {sources(input) {return { vars: { ...input, sources_block: sourcesBlock(input.sources), source_count: (input.sources || []).length } };},++ 'food.read_pantry_photo'(input) {++ const foods = (Array.isArray(input.foods) ? input.foods : []).slice(0, 80);++ return { vars: { kind: input.kind === 'receipt' ? 'grocery receipt' : 'cupboard, fridge or shelf', foods_block: foods.map((f, i) => `${i}: ${clean(f, 120)}`).join('\n') } };++ },// ── Direct operations ──'ai.summarize'(input) {return { vars: { ...input, max_words: input.max_words || 120 } };@@@ -11,7 +11,7 @@* - Trade is informational only (fixed disclaimer, observation timestamps, stale sources flagged)* - moderation follows OpenVibe policy: profanity is allowed; threats, minors and doxxing are not*/--const { SOURCES, CITES, GAPS } = require('./common-schemas');++const { SOURCES, CITES, GAPS, IMAGE } = require('./common-schemas');const STR = (n) => ({ type: 'string', maxLength: n });const OBJ = (props, required) => ({ type: 'object', additionalProperties: false, required: required || Object.keys(props), properties: props });@@@ -210,4 +210,28 @@ const workflows = [metadata: { draft: true, seed: true },}));++// Food's cupboard by photo or receipt (OpenVibe.Food /pantry/photo): a picture, not sources, so it has its own step.++templates.push({++ key: 'food.read_pantry_photo', name: 'Food: read a cupboard photo or a grocery receipt',++ description: 'Which foods from OpenVibe.Food\'s list one photo clearly shows: a cupboard, fridge or shelf, or a grocery receipt.',++ input_schema: { type: 'object' }, output_schema: { type: 'object' },++ system_prompt: 'You read one photo for OpenVibe.Food: a cupboard, fridge or shelf, or a grocery receipt. Say which foods from the numbered list it clearly shows, by their numbers in `items` (each number once). On a receipt a line names a food when its words or a usual receipt abbreviation do ("BNLS CHKN BRST" is chicken breasts, "LG EGGS 12CT" is eggs). Never guess at something you cannot see or read. Put foods you do see that are not on the list in `unlisted` (short names). `seen` says what you saw for that item, in a few words ("a carton of 12 eggs", "receipt line: GV WHL MILK"). confidence: high when the label or the receipt line is plain, medium when it is likely, low otherwise. Never write out prices, totals, card numbers, store names or addresses, loyalty numbers, names or anything else personal on a receipt. Output only JSON matching the schema.',++ user_prompt: 'This photo is a {{kind}}.\n\nThe list (number: food):\n{{foods_block}}',++ default_route: 'food.vision', owner: 'food', metadata: { json_name: 'food_read_pantry_photo' },++});++workflows.push({++ key: 'food.read_pantry_photo', name: 'Read a cupboard photo or a receipt', namespace: 'food',++ input_schema: OBJ({ image: IMAGE, kind: { enum: ['shelf', 'receipt'] }, foods: ARR(STR(120), 80) }, ['image', 'foods']),++ output_schema: OBJ({++ items: ARR(OBJ({ food: { type: 'integer', minimum: 0, maximum: 79 }, seen: STR(160), confidence: { enum: ['high', 'medium', 'low'] } }), 60),++ unlisted: ARR(STR(120), 30),++ gaps: GAPS,++ }),++ steps: [{ kind: 'llm', template: 'food.read_pantry_photo', operation: 'generate', output: 'json', image: 'image', image_max_width: 1280, prepare: 'food.read_pantry_photo', params: { max_tokens: 1200, temperature: 0.1 } }],++ default_route: 'food.vision',++ // A person's photo: never shared between callers (the stored run keeps only the image's hash, server/runs.js).++ cache_mode: 'none',++ metadata: { seed: true },++});++module.exports = { templates, workflows };@@@ -122,6 +122,8 @@ async function seed({ registry, quotas, config, env = process.env, db }) {// Images (metered per image) only with AI_IMAGE_MODEL: paid providers' registered image models, the shared one first.if (imageModel) await registry.seedVersioned('route', 'default.image', pool(120000, { 'capability': 'image', constraints: { objective: 'correctness', provider_scope: 'paid', authority: `shared:${imageModel}` } }));for (const key of HISTORICAL_ROUTES) await registry.seedVersioned('route', key, { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: null, response_format: 'json', timeout_ms: null, alias_of: 'default.json' });++ // Food's cupboard photos and receipts (food.read_pantry_photo): JSON from an image, on the JSON pool (its models read images).++ await registry.seedVersioned('route', 'food.vision', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 1200, response_format: 'json', timeout_ms: 60000, alias_of: 'default.json' });// Chat's AI members (server/workflows/chat.js): conversation text, so an alias of the chat pool.await registry.seedVersioned('route', 'chat.member', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 600, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });@@@ -49,6 +49,7 @@ const EXAMPLES = {'rent.screen_listing': { listing: { title: 'Bright room near the river', kind: 'room', price: 650, currency: 'EUR', period: 'month', city: 'Porto', country: 'PT', description: 'I am abroad for work; send the deposit by wire and I will mail the keys.' } },'food.suggest_recipes': { pantry: ['rice', 'onions', 'canned tomatoes'], people: 2, sources: [{ source_type: 'food.food', source_id: 'lentils', title: 'Lentils, dried', content: '{"id":"lentils","unit":"kg"}' }] },'chat.member.reply': { ai_name: 'Pantry', persona: 'A cheerful cook who keeps it cheap.', room: 'Kim and Lee\'s kitchen', memory: 'Lee is vegetarian.', context: 'Pantry: rice, onions, canned tomatoes, lentils.', context_source: 'OpenVibe.Food household', transcript: 'kim: home late tonight\nlee: same', asked_by: 'kim', message: '@Pantry what can we cook in 20 minutes?' },++ 'food.read_pantry_photo': { image: { data_url: PNG }, kind: 'receipt', foods: ['Large Eggs (1 dozen)', 'Whole Milk (1 gallon)'] },'tools.describe': { tool: 'img.convert', sources: [{ source_type: 'tools.tool', title: 'img.convert descriptor', content: '{"id":"img.convert","inputs":["file","format"]}' }] },};for (const k of new Set(Object.values(KIND_TO_WORKFLOW).concat('live.complete'))) EXAMPLES[k] = PASS;@@@ -93,11 +94,11 @@ t.test('boot seeds every required workflow key', async () => {h = await boot({ env: { OV_MEDIA_INTERNAL_URL: mediaBase, AI_IMAGE_MODEL: 'gpt-image-1' } });const keys = (await h.registry.listWorkflows()).map(w => w.key);for (const k of ['wiki.generate_space', 'wiki.generate_page', 'blog.draft_post', 'news.summarize_story', 'news.compare_perspectives', 'reviews.summarize_entity', 'deals.enrich_deal',-- 'coupons.extract_coupon', 'trade.summarize_market_context', 'codes.generate_docs', 'games.generate_lore', 'moderation.classify', 'tools.describe', 'food.suggest_recipes', 'rent.screen_listing', 'chat.member.reply',++ 'coupons.extract_coupon', 'trade.summarize_market_context', 'codes.generate_docs', 'games.generate_lore', 'moderation.classify', 'tools.describe', 'food.suggest_recipes', 'food.read_pantry_photo', 'rent.screen_listing', 'chat.member.reply','live.translate', 'live.stream.describe_frame', 'live.stream.summarize', 'live.viewers.reply', 'live.chat.insight', 'live.hero.slogans', 'live.stream.recap', 'network.site_copy']) {assert.ok(keys.includes(k), `missing ${k}`);}-- for (const k of ['default.chat', 'default.json', 'default.embedding', 'default.image', 'wiki.generate', 'blog.draft', 'news.summarize', 'reviews.summarize', 'deals.enrich', 'coupons.extract', 'trade.summarize', 'codes.generate_docs', 'games.generate_lore', 'tools.describe', 'moderation.classify', 'food.suggest', 'rent.screen', 'chat.member']) {++ for (const k of ['default.chat', 'default.json', 'default.embedding', 'default.image', 'wiki.generate', 'blog.draft', 'news.summarize', 'reviews.summarize', 'deals.enrich', 'coupons.extract', 'trade.summarize', 'codes.generate_docs', 'games.generate_lore', 'tools.describe', 'moderation.classify', 'food.suggest', 'food.vision', 'rent.screen', 'chat.member']) {assert.ok(await h.registry.getRoute(k), `missing historical route ${k}`);}});
@@ -268,6 +268,10 @@ const PREPARE = {
@@ -268,6 +268,10 @@ const PREPARE = {
sources(input) {
sources(input) {
return { vars: { ...input, sources_block: sourcesBlock(input.sources), source_count: (input.sources || []).length } };
return { vars: { ...input, sources_block: sourcesBlock(input.sources), source_count: (input.sources || []).length } };
},
},
+ 'food.read_pantry_photo'(input) {
+ const foods = (Array.isArray(input.foods) ? input.foods : []).slice(0, 80);
+ return { vars: { kind: input.kind === 'receipt' ? 'grocery receipt' : 'cupboard, fridge or shelf', foods_block: foods.map((f, i) => `${i}: ${clean(f, 120)}`).join('\n') } };
+ },
// ── Direct operations ──
// ── Direct operations ──
'ai.summarize'(input) {
'ai.summarize'(input) {
return { vars: { ...input, max_words: input.max_words || 120 } };
return { vars: { ...input, max_words: input.max_words || 120 } };
@@ -11,7 +11,7 @@
@@ -11,7 +11,7 @@
* - Trade is informational only (fixed disclaimer, observation timestamps, stale sources flagged)
* - Trade is informational only (fixed disclaimer, observation timestamps, stale sources flagged)
* - moderation follows OpenVibe policy: profanity is allowed; threats, minors and doxxing are not
* - moderation follows OpenVibe policy: profanity is allowed; threats, minors and doxxing are not
*/
*/
-const { SOURCES, CITES, GAPS } = require('./common-schemas');
+const { SOURCES, CITES, GAPS, IMAGE } = require('./common-schemas');
const STR = (n) => ({ type: 'string', maxLength: n });
const STR = (n) => ({ type: 'string', maxLength: n });
const OBJ = (props, required) => ({ type: 'object', additionalProperties: false, required: required || Object.keys(props), properties: props });
const OBJ = (props, required) => ({ type: 'object', additionalProperties: false, required: required || Object.keys(props), properties: props });
@@ -210,4 +210,28 @@ const workflows = [
@@ -210,4 +210,28 @@ const workflows = [
metadata: { draft: true, seed: true },
metadata: { draft: true, seed: true },
}));
}));
+// Food's cupboard by photo or receipt (OpenVibe.Food /pantry/photo): a picture, not sources, so it has its own step.
+templates.push({
+ key: 'food.read_pantry_photo', name: 'Food: read a cupboard photo or a grocery receipt',
+ description: 'Which foods from OpenVibe.Food\'s list one photo clearly shows: a cupboard, fridge or shelf, or a grocery receipt.',
+ input_schema: { type: 'object' }, output_schema: { type: 'object' },
+ system_prompt: 'You read one photo for OpenVibe.Food: a cupboard, fridge or shelf, or a grocery receipt. Say which foods from the numbered list it clearly shows, by their numbers in `items` (each number once). On a receipt a line names a food when its words or a usual receipt abbreviation do ("BNLS CHKN BRST" is chicken breasts, "LG EGGS 12CT" is eggs). Never guess at something you cannot see or read. Put foods you do see that are not on the list in `unlisted` (short names). `seen` says what you saw for that item, in a few words ("a carton of 12 eggs", "receipt line: GV WHL MILK"). confidence: high when the label or the receipt line is plain, medium when it is likely, low otherwise. Never write out prices, totals, card numbers, store names or addresses, loyalty numbers, names or anything else personal on a receipt. Output only JSON matching the schema.',
+ user_prompt: 'This photo is a {{kind}}.\n\nThe list (number: food):\n{{foods_block}}',
+ default_route: 'food.vision', owner: 'food', metadata: { json_name: 'food_read_pantry_photo' },
+});
+workflows.push({
+ key: 'food.read_pantry_photo', name: 'Read a cupboard photo or a receipt', namespace: 'food',
+ input_schema: OBJ({ image: IMAGE, kind: { enum: ['shelf', 'receipt'] }, foods: ARR(STR(120), 80) }, ['image', 'foods']),
+ output_schema: OBJ({
+ items: ARR(OBJ({ food: { type: 'integer', minimum: 0, maximum: 79 }, seen: STR(160), confidence: { enum: ['high', 'medium', 'low'] } }), 60),
+ unlisted: ARR(STR(120), 30),
+ gaps: GAPS,
+ }),
+ steps: [{ kind: 'llm', template: 'food.read_pantry_photo', operation: 'generate', output: 'json', image: 'image', image_max_width: 1280, prepare: 'food.read_pantry_photo', params: { max_tokens: 1200, temperature: 0.1 } }],
+ default_route: 'food.vision',
+ // A person's photo: never shared between callers (the stored run keeps only the image's hash, server/runs.js).
+ cache_mode: 'none',
+ metadata: { seed: true },
+});
+
module.exports = { templates, workflows };
module.exports = { templates, workflows };
@@ -122,6 +122,8 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
@@ -122,6 +122,8 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
// Images (metered per image) only with AI_IMAGE_MODEL: paid providers' registered image models, the shared one first.
// Images (metered per image) only with AI_IMAGE_MODEL: paid providers' registered image models, the shared one first.
if (imageModel) await registry.seedVersioned('route', 'default.image', pool(120000, { 'capability': 'image', constraints: { objective: 'correctness', provider_scope: 'paid', authority: `shared:${imageModel}` } }));
if (imageModel) await registry.seedVersioned('route', 'default.image', pool(120000, { 'capability': 'image', constraints: { objective: 'correctness', provider_scope: 'paid', authority: `shared:${imageModel}` } }));
for (const key of HISTORICAL_ROUTES) await registry.seedVersioned('route', key, { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: null, response_format: 'json', timeout_ms: null, alias_of: 'default.json' });
for (const key of HISTORICAL_ROUTES) await registry.seedVersioned('route', key, { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: null, response_format: 'json', timeout_ms: null, alias_of: 'default.json' });
+ // Food's cupboard photos and receipts (food.read_pantry_photo): JSON from an image, on the JSON pool (its models read images).
+ await registry.seedVersioned('route', 'food.vision', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 1200, response_format: 'json', timeout_ms: 60000, alias_of: 'default.json' });
// Chat's AI members (server/workflows/chat.js): conversation text, so an alias of the chat pool.
// Chat's AI members (server/workflows/chat.js): conversation text, so an alias of the chat pool.
await registry.seedVersioned('route', 'chat.member', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 600, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });
await registry.seedVersioned('route', 'chat.member', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 600, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });
@@ -49,6 +49,7 @@ const EXAMPLES = {
@@ -49,6 +49,7 @@ const EXAMPLES = {
'rent.screen_listing': { listing: { title: 'Bright room near the river', kind: 'room', price: 650, currency: 'EUR', period: 'month', city: 'Porto', country: 'PT', description: 'I am abroad for work; send the deposit by wire and I will mail the keys.' } },
'rent.screen_listing': { listing: { title: 'Bright room near the river', kind: 'room', price: 650, currency: 'EUR', period: 'month', city: 'Porto', country: 'PT', description: 'I am abroad for work; send the deposit by wire and I will mail the keys.' } },
'food.suggest_recipes': { pantry: ['rice', 'onions', 'canned tomatoes'], people: 2, sources: [{ source_type: 'food.food', source_id: 'lentils', title: 'Lentils, dried', content: '{"id":"lentils","unit":"kg"}' }] },
'food.suggest_recipes': { pantry: ['rice', 'onions', 'canned tomatoes'], people: 2, sources: [{ source_type: 'food.food', source_id: 'lentils', title: 'Lentils, dried', content: '{"id":"lentils","unit":"kg"}' }] },
'chat.member.reply': { ai_name: 'Pantry', persona: 'A cheerful cook who keeps it cheap.', room: 'Kim and Lee\'s kitchen', memory: 'Lee is vegetarian.', context: 'Pantry: rice, onions, canned tomatoes, lentils.', context_source: 'OpenVibe.Food household', transcript: 'kim: home late tonight\nlee: same', asked_by: 'kim', message: '@Pantry what can we cook in 20 minutes?' },
'chat.member.reply': { ai_name: 'Pantry', persona: 'A cheerful cook who keeps it cheap.', room: 'Kim and Lee\'s kitchen', memory: 'Lee is vegetarian.', context: 'Pantry: rice, onions, canned tomatoes, lentils.', context_source: 'OpenVibe.Food household', transcript: 'kim: home late tonight\nlee: same', asked_by: 'kim', message: '@Pantry what can we cook in 20 minutes?' },
+ 'food.read_pantry_photo': { image: { data_url: PNG }, kind: 'receipt', foods: ['Large Eggs (1 dozen)', 'Whole Milk (1 gallon)'] },
'tools.describe': { tool: 'img.convert', sources: [{ source_type: 'tools.tool', title: 'img.convert descriptor', content: '{"id":"img.convert","inputs":["file","format"]}' }] },
'tools.describe': { tool: 'img.convert', sources: [{ source_type: 'tools.tool', title: 'img.convert descriptor', content: '{"id":"img.convert","inputs":["file","format"]}' }] },
};
};
for (const k of new Set(Object.values(KIND_TO_WORKFLOW).concat('live.complete'))) EXAMPLES[k] = PASS;
for (const k of new Set(Object.values(KIND_TO_WORKFLOW).concat('live.complete'))) EXAMPLES[k] = PASS;
@@ -93,11 +94,11 @@ t.test('boot seeds every required workflow key', async () => {
@@ -93,11 +94,11 @@ t.test('boot seeds every required workflow key', async () => {
h = await boot({ env: { OV_MEDIA_INTERNAL_URL: mediaBase, AI_IMAGE_MODEL: 'gpt-image-1' } });
h = await boot({ env: { OV_MEDIA_INTERNAL_URL: mediaBase, AI_IMAGE_MODEL: 'gpt-image-1' } });
const keys = (await h.registry.listWorkflows()).map(w => w.key);
const keys = (await h.registry.listWorkflows()).map(w => w.key);
for (const k of ['wiki.generate_space', 'wiki.generate_page', 'blog.draft_post', 'news.summarize_story', 'news.compare_perspectives', 'reviews.summarize_entity', 'deals.enrich_deal',
for (const k of ['wiki.generate_space', 'wiki.generate_page', 'blog.draft_post', 'news.summarize_story', 'news.compare_perspectives', 'reviews.summarize_entity', 'deals.enrich_deal',
- 'coupons.extract_coupon', 'trade.summarize_market_context', 'codes.generate_docs', 'games.generate_lore', 'moderation.classify', 'tools.describe', 'food.suggest_recipes', 'rent.screen_listing', 'chat.member.reply',
+ 'coupons.extract_coupon', 'trade.summarize_market_context', 'codes.generate_docs', 'games.generate_lore', 'moderation.classify', 'tools.describe', 'food.suggest_recipes', 'food.read_pantry_photo', 'rent.screen_listing', 'chat.member.reply',
'live.translate', 'live.stream.describe_frame', 'live.stream.summarize', 'live.viewers.reply', 'live.chat.insight', 'live.hero.slogans', 'live.stream.recap', 'network.site_copy']) {
'live.translate', 'live.stream.describe_frame', 'live.stream.summarize', 'live.viewers.reply', 'live.chat.insight', 'live.hero.slogans', 'live.stream.recap', 'network.site_copy']) {
assert.ok(keys.includes(k), `missing ${k}`);
assert.ok(keys.includes(k), `missing ${k}`);
}
}
- for (const k of ['default.chat', 'default.json', 'default.embedding', 'default.image', 'wiki.generate', 'blog.draft', 'news.summarize', 'reviews.summarize', 'deals.enrich', 'coupons.extract', 'trade.summarize', 'codes.generate_docs', 'games.generate_lore', 'tools.describe', 'moderation.classify', 'food.suggest', 'rent.screen', 'chat.member']) {
+ for (const k of ['default.chat', 'default.json', 'default.embedding', 'default.image', 'wiki.generate', 'blog.draft', 'news.summarize', 'reviews.summarize', 'deals.enrich', 'coupons.extract', 'trade.summarize', 'codes.generate_docs', 'games.generate_lore', 'tools.describe', 'moderation.classify', 'food.suggest', 'food.vision', 'rent.screen', 'chat.member']) {
assert.ok(await h.registry.getRoute(k), `missing historical route ${k}`);
assert.ok(await h.registry.getRoute(k), `missing historical route ${k}`);
}
}
});
});