openvibers/openvibe.ai Commits

AI: pics.describe_image — alt text, a caption and tags for one OpenVibe.Pics picture (#95)

OP OpenVibers committed 28 days ago · 4fa1df5a06 · 4 files, +27 −1

A vision workflow on the JSON pool (route pics.vision, an alias of default.json like food.vision): alt
(one sentence, 140 characters), caption, up to six tags. Describes only what is visible: never names or
guesses a real person, never reads out personal details shown in the picture. cache_mode none (a
person's picture is never shared between callers).

Changes

ddiff --git a/server/workflows/hooks.js b/server/workflows/hooks.js
iindex 7fe958a..48aa4c7 100644
---- a/server/workflows/hooks.js
++++ b/server/workflows/hooks.js
@@@ -268,6 +268,9 @@ const PREPARE = {
      sources(input) {
          return { vars: { ...input, sources_block: sourcesBlock(input.sources), source_count: (input.sources || []).length } };
      },
++    'pics.describe_image'(input) {
++        return { vars: { title: clean(input.title, 120) || '' } };
++    },
      'food.read_pantry_photo'(input) {
          const foods = (Array.isArray(input.foods) ? input.foods : []).slice(0, 80);
          return { vars: { kind: input.kind === 'receipt' ? 'grocery receipt' : 'cupboard, fridge or shelf', foods_block: foods.map((f, i) => `${i}: ${clean(f, 120)}`).join('\n') } };
ddiff --git a/server/workflows/products.js b/server/workflows/products.js
iindex e249253..7aa547e 100644
---- a/server/workflows/products.js
++++ b/server/workflows/products.js
@@@ -243,6 +243,26 @@ workflows.push({
      metadata: { seed: true },
  });
  
++// OpenVibe.Pics: alt text, a caption and tags for one picture (accessibility, embeds and finding your own pictures).
++templates.push({
++    key: 'pics.describe_image', name: 'Pics: describe a picture',
++    description: 'Alt text, a short caption and tags for one picture on OpenVibe.Pics.',
++    input_schema: { type: 'object' }, output_schema: { type: 'object' },
++    system_prompt: 'You describe one picture for OpenVibe.Pics so people who cannot see it know what it shows. alt: one plain sentence of at most 140 characters saying what the picture shows (never "image of" or "picture of"). caption: one or two sentences with the detail a sighted person would notice first. tags: up to six short lowercase words for what is in it. Describe only what is visible. Never name or guess who a real person is, and never read out personal details shown in it (names, addresses, number plates, phone numbers, faces of children described by age or appearance beyond "a child"). If it shows text, say what kind of text it is rather than quoting personal parts. Output only JSON matching the schema.',
++    user_prompt: '{{#title}}The uploader titled it: {{title}}\n{{/title}}Describe the picture.',
++    default_route: 'pics.vision', owner: 'media-hub', metadata: { json_name: 'pics_describe_image' },
++});
++workflows.push({
++    key: 'pics.describe_image', name: 'Describe a picture', namespace: 'pics',
++    input_schema: OBJ({ image: IMAGE, title: STR(120) }, ['image']),
++    output_schema: OBJ({ alt: STR(160), caption: STR(400), tags: ARR(STR(30), 6), gaps: GAPS }),
++    steps: [{ kind: 'llm', template: 'pics.describe_image', operation: 'generate', output: 'json', image: 'image', image_max_width: 1024, prepare: 'pics.describe_image', params: { max_tokens: 400, temperature: 0.2 } }],
++    default_route: 'pics.vision',
++    // A person's picture: never shared between callers (the stored run keeps only the image's hash).
++    cache_mode: 'none',
++    metadata: { seed: true },
++});
++
  // OpenVibe.Blog's satire desk ("Is This Real?"): invented on purpose, so it is not a sourced evidence package and has its
  // own rules. The page labels every post as satire written by AI; the rules keep the jokes away from real people.
  templates.push({
ddiff --git a/server/workflows/seed.js b/server/workflows/seed.js
iindex a7a8193..0cd4990 100644
---- a/server/workflows/seed.js
++++ b/server/workflows/seed.js
@@@ -124,6 +124,8 @@ async function seed({ registry, quotas, config, env = process.env, db }) {
      for (const key of HISTORICAL_ROUTES) await registry.seedVersioned('route', key, { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: null, response_format: 'json', timeout_ms: null, alias_of: 'default.json' });
      // Food's cupboard photos and receipts (food.read_pantry_photo): JSON from an image, on the JSON pool (its models read images).
      await registry.seedVersioned('route', 'food.vision', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 1200, response_format: 'json', timeout_ms: 60000, alias_of: 'default.json' });
++    // OpenVibe.Pics' alt text (pics.describe_image): JSON from an image, on the JSON pool like food.vision.
++    await registry.seedVersioned('route', 'pics.vision', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 400, response_format: 'json', timeout_ms: 60000, alias_of: 'default.json' });
      // Chat's AI members (server/workflows/chat.js): conversation text, so an alias of the chat pool.
      await registry.seedVersioned('route', 'chat.member', { primary: { provider: 'shared', model: null }, fallbacks: [], options: {}, max_output_tokens: 600, response_format: 'text', timeout_ms: null, alias_of: 'default.chat' });
  
ddiff --git a/test/workflows.test.js b/test/workflows.test.js
iindex 9bd0232..9d8f82e 100644
---- a/test/workflows.test.js
++++ b/test/workflows.test.js
@@@ -52,6 +52,7 @@ const EXAMPLES = {
      'food.suggest_recipes': { pantry: ['rice', 'onions', 'canned tomatoes'], people: 2, sources: [{ source_type: 'food.food', source_id: 'lentils', title: 'Lentils, dried', content: '{"id":"lentils","unit":"kg"}' }] },
      'chat.member.reply': { ai_name: 'Pantry', persona: 'A cheerful cook who keeps it cheap.', room: 'Kim and Lee\'s kitchen', memory: 'Lee is vegetarian.', context: 'Pantry: rice, onions, canned tomatoes, lentils.', context_source: 'OpenVibe.Food household', transcript: 'kim: home late tonight\nlee: same', asked_by: 'kim', message: '@Pantry what can we cook in 20 minutes?' },
      'food.read_pantry_photo': { image: { data_url: PNG }, kind: 'receipt', foods: ['Large Eggs (1 dozen)', 'Whole Milk (1 gallon)'] },
++    'pics.describe_image': { image: { data_url: PNG }, title: 'The coast' },
      'tools.describe': { tool: 'img.convert', sources: [{ source_type: 'tools.tool', title: 'img.convert descriptor', content: '{"id":"img.convert","inputs":["file","format"]}' }] },
  };
  for (const k of new Set(Object.values(KIND_TO_WORKFLOW).concat('live.complete'))) EXAMPLES[k] = PASS;
@@@ -96,7 +97,7 @@ t.test('boot seeds every required workflow key', async () => {
      h = await boot({ env: { OV_MEDIA_INTERNAL_URL: mediaBase, AI_IMAGE_MODEL: 'gpt-image-1' } });
      const keys = (await h.registry.listWorkflows()).map(w => w.key);
      for (const k of ['wiki.generate_space', 'wiki.generate_page', 'blog.draft_post', 'blog.write_digest', 'blog.write_satire', 'news.summarize_story', 'news.compare_perspectives', 'reviews.summarize_entity', 'deals.enrich_deal',
--        'coupons.extract_coupon', 'trade.summarize_market_context', 'codes.generate_docs', 'games.generate_lore', 'moderation.classify', 'tools.describe', 'food.suggest_recipes', 'food.read_pantry_photo', 'rent.screen_listing', 'chat.member.reply',
++        'coupons.extract_coupon', 'trade.summarize_market_context', 'codes.generate_docs', 'games.generate_lore', 'moderation.classify', 'tools.describe', 'food.suggest_recipes', 'food.read_pantry_photo', 'pics.describe_image', 'rent.screen_listing', 'chat.member.reply',
          'live.translate', 'live.stream.describe_frame', 'live.stream.summarize', 'live.viewers.reply', 'live.chat.insight', 'live.hero.slogans', 'live.stream.recap', 'network.site_copy']) {
          assert.ok(keys.includes(k), `missing ${k}`);
      }