openvibers/openvibe.live Commits

AI: analyse screenshot pastes (they were all silently skipped)

OP OpenVibers committed 19 Aug 2026 · 636dfb400f · 3 files, +25 −6

Every paste still missing AI was a screenshot — 33 of them — while text pastes
had been summarising fine since the queue was wired up.

_normImage handled data URLs, raw base64 and file paths. A screenshot paste's
screenshot_url is "/p/<slug>/screenshot": relative, and the file lives on Media's
disk rather than ours, so it fell through to the file-path branch where
readFileSync threw. The caller caught it and returned null, so the paste was
skipped with no error anywhere.

  - _normImage now fetches http(s) URLs and is async (_complete already was), so
    every image path benefits, not just pastes
  - media-client gains pasteScreenshotUrl() to build the absolute Media URL
  - the backfill resolves the screenshot from the slug rather than passing the
    relative URL through

Co-Authored-By: Claude Opus 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Xw6Sf5zzSALw8m93XNu5JE

Changes

ddiff --git a/server/ai/ai-analysis.js b/server/ai/ai-analysis.js
iindex b833a857..0e8b4f7b 100644
---- a/server/ai/ai-analysis.js
++++ b/server/ai/ai-analysis.js
@@@ -37,13 +37,29 @@ function estimateCost(inTok, outTok) {
      return (inTok / 1e6) * num('ai_input_cost_per_mtok', 3) + (outTok / 1e6) * num('ai_output_cost_per_mtok', 15);
  }
  
--// Normalize an image input (data URL, raw base64, or file path) → {base64, mediaType}.
--function _normImage(image) {
++// Normalize an image input (data URL, raw base64, http(s) URL, or file path)
++// → {base64, mediaType}.
++//
++// URL support matters: paste screenshots live in Media, not on this disk, and their
++// image is only reachable over HTTP. Without this branch a URL fell through to the
++// file-path case, readFileSync threw, and the caller silently got null — which is why
++// every screenshot paste went un-analysed while text pastes worked.
++async function _normImage(image) {
      if (!image) return null;
      if (typeof image === 'string' && image.startsWith('data:')) {
          const m = image.match(/^data:([^;]+);base64,(.*)$/);
          if (m) return { base64: m[2], mediaType: m[1] };
      }
++    if (typeof image === 'string' && /^https?:\/\//i.test(image)) {
++        try {
++            const res = await fetch(image, { signal: AbortSignal.timeout(20000) });
++            if (!res.ok) { console.warn(`[AI] image fetch ${res.status} for ${image}`); return null; }
++            const buf = Buffer.from(await res.arrayBuffer());
++            if (!buf.length) return null;
++            const ct = String(res.headers.get('content-type') || 'image/jpeg').split(';')[0].trim();
++            return { base64: buf.toString('base64'), mediaType: /^image\//.test(ct) ? ct : 'image/jpeg' };
++        } catch (e) { console.warn('[AI] image fetch failed:', e.message); return null; }
++    }
      if (typeof image === 'string' && /^[A-Za-z0-9+/=]+$/.test(image.slice(0, 40))) {
          return { base64: image, mediaType: 'image/jpeg' };
      }
@@@ -105,7 +121,7 @@ async function _openai(prompt, img, maxTokens, temperature) {
  /** Core call: dispatches by provider, records usage/cost. Returns text or null. */
  async function _complete({ prompt, image = null, maxTokens = 400, kind, temperature = null, ownerUserId = null, source = null }) {
      if (!isEnabled() || !withinBudget()) return null;
--    const img = image ? _normImage(image) : null;
++    const img = image ? await _normImage(image) : null;
      if (image && !img) return null;
      const provider = s('ai_provider') === 'openai' ? _openai : _anthropic;
      let r;
ddiff --git a/server/ai/backfill-job.js b/server/ai/backfill-job.js
iindex 19f6a1ac..320f1ee4 100644
---- a/server/ai/backfill-job.js
++++ b/server/ai/backfill-job.js
@@@ -84,8 +84,10 @@ async function tick() {
                      if (!p || !p.slug) continue;
                      let r = null;
                      if (p.type === 'screenshot') {
--                        // Vision needs the actual image; skip when there is no URL to read.
--                        const img = p.screenshot_url || null;
++                        // screenshot_url is RELATIVE ("/p/<slug>/screenshot") and the file
++                        // lives on Media's disk, not ours — so it has to be fetched over
++                        // HTTP from Media's public base, not opened as a path.
++                        const img = p.slug ? media.pasteScreenshotUrl(p.slug) : null;
                          if (img) r = await ai.analyzeImagePaste(img, p.title).catch(() => null);
                      } else {
                          r = await ai.analyzeTextPaste(p.content || '', p.title).catch(() => null);
ddiff --git a/server/media-client.js b/server/media-client.js
iindex c72e5a76..b4f570df 100644
---- a/server/media-client.js
++++ b/server/media-client.js
@@@ -267,6 +267,7 @@ function generateThumbnail(kind, id, opts = {}) {
  function vodPlaybackUrl(id) { return `${MEDIA_PUBLIC_URL}/v/${id}`; }
  function clipUrl(id) { return `${MEDIA_PUBLIC_URL}/c/${id}`; }
  function pasteUrl(slug) { return `${MEDIA_PUBLIC_URL}/p/${encodeURIComponent(slug)}`; }
++function pasteScreenshotUrl(slug) { return `${MEDIA_PUBLIC_URL}/p/${encodeURIComponent(slug)}/screenshot`; }
  function pasteRawUrl(slug) { return `${MEDIA_PUBLIC_URL}/p/${encodeURIComponent(slug)}/raw`; }
  function thumbUrl(id) { return `${MEDIA_PUBLIC_URL}/t/${id}`; }
  function fileUrl(key) { return `${MEDIA_PUBLIC_URL}/f/${key}`; }
@@@ -352,5 +353,5 @@ module.exports = {
      uploadFile, getFileMeta, deleteFile,
      uploadThumbnail, generateThumbnail,
      // URL builders
--    vodPlaybackUrl, clipUrl, pasteUrl, pasteRawUrl, thumbUrl, fileUrl, screenshotUrl, publicUrl,
++    vodPlaybackUrl, clipUrl, pasteUrl, pasteScreenshotUrl, pasteRawUrl, thumbUrl, fileUrl, screenshotUrl, publicUrl,
  };