openvibers/openvibe.ai Commits

AI: serve llms-full.txt with every public page and the API (SEO-33) (#193)

OP OpenVibers committed 15 days ago · cad6f73e99 · 4 files, +50 −4

Changes

ddiff --git a/deploy/nginx/ai.openvibe.services.conf b/deploy/nginx/ai.openvibe.services.conf
iindex 56cce27..24f0d4b 100644
---- a/deploy/nginx/ai.openvibe.services.conf
++++ b/deploy/nginx/ai.openvibe.services.conf
@@@ -10,7 +10,7 @@
  ## The Node service (port 4700, systemd unit openvibe-ai) serves:
  ##   /           the product home for a browser (server/home.js), the text API index for anything else
  ##   /stats      public price and latency per provider and model (server-rendered, no sign-in)
--##   /shared/*   the home's OpenVibe Frame files (content-addressed); /robots.txt, /sitemap.xml, /llms.txt
++##   /shared/*   the home's OpenVibe Frame files (content-addressed); /robots.txt, /sitemap.xml, /llms.txt, /llms-full.txt
  ##   /api/health, /api/ready
  ##   /api/v1/{chat,generate,summarize,classify,extract,embed} (POST) and /api/v1/runs/:id (GET)
  ##               developer apps (ai.app.run, app tokens from OpenVibe.Network): open here, rate-limited
@@@ -154,6 +154,7 @@ server {
      location = /stats { proxy_pass http://127.0.0.1:4700; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header CF-Connecting-IP $remote_addr; }
      location = /sitemap.xml { proxy_pass http://127.0.0.1:4700; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header CF-Connecting-IP $remote_addr; }
      location = /llms.txt { proxy_pass http://127.0.0.1:4700; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header CF-Connecting-IP $remote_addr; }
++    location = /llms-full.txt { proxy_pass http://127.0.0.1:4700; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header CF-Connecting-IP $remote_addr; }
      location = /release.json { proxy_pass http://127.0.0.1:4700; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header CF-Connecting-IP $remote_addr; }
      # Open tabs report how their update went (openvibe-shared release-watch → POST /release-metrics, counted in /metrics).
      # The catch-all below passes GET and HEAD only, so every visitor's report was refused with a 403 (2026-10-24).
ddiff --git a/server/app.js b/server/app.js
iindex b022d8e..974b7f9 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -163,7 +163,9 @@ function createApp({ config, db, registry, pool, quotas, cache, runs, auth, keys
              const from = new Date(clock.now() - STATS_DAYS * 86400000).toISOString().slice(0, 10);
              const stats = await db.prepare('SELECT MAX(day) AS lastmod FROM provider_stats_daily WHERE day >= ?').get(from);
              const statsLastmod = [stats && stats.lastmod ? new Date(stats.lastmod).toISOString() : null, releaseLastmod].filter(Boolean).sort().pop();
--            const entries = [['/', releaseLastmod], ['/stats', statsLastmod], ['/updates', releaseLastmod]];
++            // The paths come from home.PUBLIC_PAGES, the same catalogue /llms-full.txt describes.
++            const lastmods = { '/': releaseLastmod, '/stats': statsLastmod, '/updates': releaseLastmod };
++            const entries = home.PUBLIC_PAGES.map((p) => [p.path, lastmods[p.path]]);
              res.type('application/xml').send(`<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${entries.map(([p, lastmod]) => `  <url><loc>${config.baseUrl}${p}</loc>${lastmod ? `<lastmod>${lastmod.slice(0, 10)}</lastmod>` : ''}</url>`).join('\n')}\n</urlset>\n`);
          } catch (e) { next(e); }
      });
@@@ -171,6 +173,11 @@ function createApp({ config, db, registry, pool, quotas, cache, runs, auth, keys
          res.setHeader('Cache-Control', 'public, max-age=3600');
          res.type('text/plain').send(home.llmsTxt({ siteUrl: config.baseUrl }));
      });
++    // The full AI map: the same page catalogue as the sitemap plus the developer API, in plain text.
++    app.get('/llms-full.txt', (_req, res) => {
++        res.setHeader('Cache-Control', 'public, max-age=3600');
++        res.type('text/plain').send(home.llmsFullTxt({ siteUrl: config.baseUrl }));
++    });
  
      // GET /: the home page for a browser (server/home.js), the API index for curl and API clients (`*/*`, no
      // Accept, or any non-HTML Accept). Vary: Accept so a shared cache never serves one to the other; the home is
ddiff --git a/server/home.js b/server/home.js
iindex 0fc9035..6e70095 100644
---- a/server/home.js
++++ b/server/home.js
@@@ -265,6 +265,37 @@ ${sections({ siteUrl, stats })}
      });
  }
  
++/**
++ * The public page catalogue: /sitemap.xml lists these paths and /llms-full.txt describes them, so the
++ * two artifacts cannot drift apart. The API reference is a section body rather than a page: it is
++ * generated from the contracts at openvibe.services/docs/api, not served from this origin.
++ */
++const PUBLIC_PAGES = [
++    { path: '/', title: 'OpenVibe.AI home', text: 'One API for chat, generation, summaries, classification, extraction and embeddings across many models, OpenAI-compatible. Every run says which model answered, why it was chosen and what it cost; developer apps, tokens and the sandbox are explained here.' },
++    { path: '/stats', title: 'Prices and measured latency', text: 'Rate cards per provider and model and the measured p50 and p95 latency from the last seven days, with the number of calls each model answered. Nothing per caller.' },
++    { path: '/updates', title: 'What shipped on OpenVibe.AI', text: 'The shared changelog for OpenVibe.AI, newest first.' },
++];
++
++function llmsFullTxt({ siteUrl }) {
++    return seo.llmsFull({
++        site: SITE_NAME,
++        summary: SUMMARY,
++        base: siteUrl,
++        maxBytes: 256 * 1024,
++        sections: [
++            { title: 'Pages', pages: PUBLIC_PAGES.map((p) => ({ url: `${siteUrl}${p.path}`, title: p.title, text: p.text })) },
++            { title: 'Developer API', body: [
++                `OpenAPI reference (generated from the published contracts): ${API_DOCS_URL}`,
++                `Developer guide (tokens, limits, billing, sandbox): ${GUIDE}`,
++                `Create a project and an app: ${PROJECTS_URL}`,
++                `POST ${siteUrl}/api/v1/chat, /generate, /summarize, /classify, /extract, /embed`,
++                `POST ${siteUrl}/v1/chat/completions and GET ${siteUrl}/v1/models (OpenAI-compatible)`,
++                `GET ${siteUrl}/api/v1/runs/:id for your own runs`,
++            ].join('\n') },
++        ],
++    });
++}
++
  function llmsTxt({ siteUrl }) {
      return [
          `# ${SITE_NAME}`,
@@@ -298,4 +329,4 @@ function renderNotFound() {
          '<h1>Not found</h1>', `<p>Nothing is here. <a href="/">${SITE_NAME}</a> · <a href="/stats">Prices and latency</a> · <a href="${NETWORK_URL}">OpenVibe</a></p>`, '</body>', '</html>'].join('\n');
  }
  
--module.exports = { renderHome, renderUpdates, llmsTxt, HOME_CSP, SITE_NAME, renderNotFound, FAQ, DESCRIPTION };
++module.exports = { renderHome, renderUpdates, llmsTxt, llmsFullTxt, PUBLIC_PAGES, HOME_CSP, SITE_NAME, renderNotFound, FAQ, DESCRIPTION };
ddiff --git a/test/home.test.js b/test/home.test.js
iindex 45e5288..a030bf6 100644
---- a/test/home.test.js
++++ b/test/home.test.js
@@@ -139,13 +139,20 @@ t.test('robots.txt, sitemap.xml and llms.txt name the public origin', async () =
      assert.match(llms.headers.get('content-type'), /^text\/plain/);
      assert.ok(llms.text.startsWith('# OpenVibe.AI'));
      assert.ok(llms.text.includes(`${SITE}/stats`) && llms.text.includes(`POST ${SITE}/api/v1/chat`));
++    const full = await request(h.base, 'GET', '/llms-full.txt');
++    assert.strictEqual(full.status, 200);
++    assert.match(full.headers.get('content-type'), /^text\/plain/);
++    assert.ok(full.text.startsWith('# OpenVibe.AI'));
++    assert.ok(full.text.length >= 500, `real content, not a stub (${full.text.length} characters)`);
++    for (const loc of [...map.text.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1])) assert.ok(full.text.includes(loc), `llms-full.txt names ${loc}`);
++    assert.ok(full.text.includes('OpenAPI reference'), 'the generated API reference is named');
  });
  
  const active = (file) => fs.readFileSync(path.join(__dirname, '..', 'deploy', 'nginx', file), 'utf8').split('\n').filter((l) => !/^\s*#/.test(l)).join('\n');
  
  t.test('the public vhost proxies the home, its files and exactly the developer-app API routes', () => {
      const conf = active('ai.openvibe.services.conf');
--    for (const loc of ['location = / {', 'location = /stats', 'location = /updates', 'location = /sitemap.xml', 'location = /llms.txt', 'location /shared/']) assert.ok(conf.includes(loc), loc);
++    for (const loc of ['location = / {', 'location = /stats', 'location = /updates', 'location = /sitemap.xml', 'location = /llms.txt', 'location = /llms-full.txt', 'location /shared/']) assert.ok(conf.includes(loc), loc);
      const ops = conf.indexOf('location ~ ^/api/v1/(chat|generate|summarize|classify|extract|embed)$ {');
      const run = conf.indexOf('location ~ ^/api/v1/runs/[A-Za-z0-9_-]+$ {');
      const completions = conf.indexOf('location = /v1/chat/completions {');