openvibers/openvibe.search Commits

Crawl files through openvibe-shared/seo (plan T11): /llms.txt (what Search is, its public pages and machine-readable endpoints), /robots.txt from the shared builder with every existing rule kept, /sitemap.xml whose lastmods come from the newest public document and the deployed release (never the clock), and WebSite JSON-LD with a SearchAction on the home page. Public data only. nginx proxies the two new paths. test/discovery.test.js fetches each. npm test green (verified).

OP OpenVibers committed 29 Sept 2026 · fb2eed31b2 · 6 files, +184 −14





Changes
ddiff --git a/deploy/nginx/search.openvibe.network.conf b/deploy/nginx/search.openvibe.network.conf
iindex 265efa6..c20329b 100644
---- a/deploy/nginx/search.openvibe.network.conf
++++ b/deploy/nginx/search.openvibe.network.conf
@@@ -6,7 +6,8 @@
  ##
  ## The Node service (port 4710, systemd unit openvibe-search) serves:
  ##   /                                the search page (HTML for browsers, a text route index
--##                                    otherwise) and /robots.txt. Public, no-store, result pages noindex.
++##                                    otherwise) and the crawl files: /robots.txt, /llms.txt,
++##                                    /sitemap.xml. Public, no-store, result pages noindex.
  ##   /api/v1/search, /api/v1/suggest, GET /api/v1/documents/...   public query API (anonymous =
  ##                                    public documents only; the ov_token cookie or a Bearer adds
  ##                                    the caller's ACL matches). Responses are Cache-Control: no-store.
@@@ -79,6 +80,8 @@ server {
          proxy_pass http://127.0.0.1:4710;
      }
      location = /robots.txt { proxy_pass http://127.0.0.1:4710; }
++    location = /llms.txt { proxy_pass http://127.0.0.1:4710; }
++    location = /sitemap.xml { proxy_pass http://127.0.0.1:4710; }
      # The OpenVibe Frame's same-origin init script, and what shipped on this site (the shared log).
      location = /frame-init.js { limit_except GET { deny all; } proxy_pass http://127.0.0.1:4710; }
      # This site's own pinned copy of the Frame's browser files (openvibe-shared/serve).
ddiff --git a/server/app.js b/server/app.js
iindex 22a0b03..930aeb0 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -57,8 +57,9 @@ function createApp({ config, db, store, engine, auth, outbox, relay, purges, pur
      app.use(documentsRouter({ store, auth, db, relay, purges, limits }));
      app.use(queryRouter({ config, store, engine, auth, searcher, limits }));
      app.use(savedRouter({ config, saved, searcher, auth, limits }));
--    // GET / (HTML search page for browsers, the text route index otherwise) and /robots.txt.
--    app.use(pageRouter({ searcher, auth, baseUrl: config.baseUrl }));
++    // GET / (HTML search page for browsers, the text route index otherwise), /robots.txt, /llms.txt and
++    // /sitemap.xml (whose lastmods come from the index and the deployed release, never from the clock).
++    app.use(pageRouter({ searcher, auth, baseUrl: config.baseUrl, newestPublic: () => store.newestPublic(), releasedAt: release.full().released_at }));
  
      app.use((req, res) => http.sendProblem(res, 404, 'search.not_found', { detail: `no route ${req.method} ${req.path}`, ctx: req.ov }));
  
ddiff --git a/server/store.js b/server/store.js
iindex 29e4d39..c195c47 100644
---- a/server/store.js
++++ b/server/store.js
@@@ -38,6 +38,7 @@ function createStore({ db, engine, outbox, purges = null, now = () => Date.now()
              SUM(CASE WHEN exposure = 3 THEN 1 ELSE 0 END) AS public_listed,
              SUM(CASE WHEN exposure = 2 THEN 1 ELSE 0 END) AS public_unlisted,
              SUM(CASE WHEN exposure = 1 THEN 1 ELSE 0 END) AS restricted FROM documents`),
++        newestPublic: db.prepare(`SELECT MAX(NULLIF(sort_at, '')) AS lastmod FROM documents WHERE deleted = 0 AND exposure = 3`),
      };
  
      function rowValues(doc, hash, { via, eventId }) {
@@@ -197,7 +198,14 @@ function createStore({ db, engine, outbox, purges = null, now = () => Date.now()
          };
      }
  
--    return { apply, remove, get, byRid, ownerPage, counts, EXPOSURE_NAME };
++    /** The newest content time among the public, indexable documents, ISO 8601, or null on an empty
++     *  index. This is what the sitemap's front-page lastmod is: a real time from the data, not now(). */
++    async function newestPublic() {
++        const row = await st.newestPublic.get();
++        return row && row.lastmod ? String(row.lastmod) : null;
++    }
++
++    return { apply, remove, get, byRid, ownerPage, counts, newestPublic, EXPOSURE_NAME };
  }
  
  module.exports = { createStore, EXPOSURE_NAME };
ddiff --git a/server/web/page.js b/server/web/page.js
iindex 604ce21..392a829 100644
---- a/server/web/page.js
++++ b/server/web/page.js
@@@ -5,16 +5,22 @@
   * indexable documents only; an ov_token cookie for this host adds its person's ACL matches).
   *
   *   GET /             search form; ?q= results, ?owner= / ?type= filters, ?cursor= next page
-- *   GET /robots.txt   the front page may be crawled, result pages and the API not
++ *   GET /robots.txt   the front page and /updates may be crawled, result pages and the API not
++ *   GET /llms.txt     what this site is, its public pages and its public endpoints (openvibe-shared/seo)
++ *   GET /sitemap.xml  the public pages, with lastmod from the newest public document (never the clock)
   *
   * Every answer is no-store (a document that leaves the index leaves this page at once) and result
   * pages are noindex. Clients that do not ask for HTML get the plain-text route index at /.
   */
  const express = require('express');
  const ovServe = require('openvibe-shared/serve');
++const seo = require('openvibe-shared/seo');
  const { AuthError, ANONYMOUS } = require('../auth');
  const { QueryError, one } = require('../api/query');
  
++const SITE_NAME = 'OpenVibe.Search';
++const SITE_DESCRIPTION = "Search what the OpenVibe network's services have published.";
++
  // The OpenVibe Frame (navbar, footer, "shipped" views, themes) comes from openvibe.network; its init is
  // /frame-init.js (same origin, no inline script), reading the JSON config in #ov-frame-config.
  const NETWORK = 'https://openvibe.network';
@@@ -76,7 +82,67 @@ const TEXT_INDEX = [
      '',
  ].join('\n');
  
--function layout({ title, q, owner, type, body, noindex, canonical }) {
++/**
++ * The home page's JSON-LD: the WebSite (with the sitelinks SearchAction that is this site's whole
++ * purpose) and OpenVibe.Search as the web application it is. Nothing private: both nodes describe
++ * the public front page only.
++ */
++function homeJsonLd(origin) {
++    return [
++        seo.jsonLd.website({ name: SITE_NAME, url: `${origin}/`, description: SITE_DESCRIPTION, searchUrl: `${origin}/?q={q}` }),
++        seo.jsonLd.softwareApp({ name: SITE_NAME, url: `${origin}/`, description: SITE_DESCRIPTION, category: 'SearchApplication' }),
++    ];
++}
++
++/** robots.txt: the same Disallows this site already had, plus the sitemap and the AI/search bots by name. */
++function robotsTxt(origin) {
++    return [
++        '# openvibe.search automated-consumer policy: search engines and AI crawlers may read the front',
++        '# page and the update log. Result pages carry their own X-Robots-Tag: noindex, and the API, the',
++        '# Events webhook and owner writes are not for crawling.',
++        seo.robotsTxt({ sitemaps: [`${origin}/sitemap.xml`], allow: ['/$', '/updates'], disallow: ['/?', '/api/', '/internal/'] }),
++    ].join('\n');
++}
++
++/** /llms.txt: what this site is, its public pages and its public machine-readable endpoints. */
++function llmsTxt(origin) {
++    return seo.llmsTxt({
++        name: SITE_NAME,
++        summary: `The search service of the OpenVibe network: one permission-aware index over what the network's services have published. ${SITE_DESCRIPTION}`,
++        details: 'The index holds only what the services have sent so far (Alpha: it is young). Anonymous callers see public, published, indexable documents only; a browser with an ov_token cookie also gets that person\'s own restricted matches. Private, draft and deleted documents are never listed or returned, and a document that is removed or made private leaves the results at once. Result pages are noindex; this file lists pages and public endpoints only, never a person\'s saved searches or the owner API.',
++        sections: [
++            { title: 'Pages', links: [
++                { title: 'Search', url: `${origin}/`, note: 'the search page; ?q= and ?owner=/?type= search, results are noindex' },
++                { title: 'What shipped', url: `${origin}/updates`, note: "the site's update log" },
++            ] },
++            { title: 'Machine-readable', links: [
++                { title: 'Search API', url: `${origin}/api/v1/search?q=`, note: 'JSON results; anonymous = public documents only' },
++                { title: 'Suggest API', url: `${origin}/api/v1/suggest?q=`, note: 'title suggestions (JSON)' },
++                { title: 'Sitemap', url: `${origin}/sitemap.xml` },
++                { title: 'robots.txt', url: `${origin}/robots.txt` },
++                { title: 'Release', url: `${origin}/release.json`, note: 'the deployed release manifest (JSON)' },
++            ] },
++            { title: 'The network', links: [
++                { title: 'OpenVibe.Network', url: 'https://openvibe.network/', note: 'accounts and the network directory' },
++                { title: 'Source', url: 'https://github.com/OpenVibers/OpenVibe.Search' },
++            ] },
++        ],
++    });
++}
++
++/**
++ * /sitemap.xml over the public pages. lastmod comes from the data, never from the clock: the front
++ * page carries the newest content time among the public, indexable documents (so an empty index
++ * leaves it off), and /updates carries the deployed release's date.
++ */
++function sitemapXml(origin, { indexLastmod = null, releasedAt = null } = {}) {
++    const entries = [{ loc: `${origin}/`, ...(indexLastmod ? { lastmod: indexLastmod } : {}), changefreq: 'hourly', priority: 1.0 }];
++    if (releasedAt) entries.push({ loc: `${origin}/updates`, lastmod: releasedAt, changefreq: 'weekly', priority: 0.5 });
++    else entries.push({ loc: `${origin}/updates` });
++    return seo.sitemapXml(entries);
++}
++
++function layout({ title, q, owner, type, body, noindex, canonical, jsonLd }) {
      return `<!doctype html>
  <html lang="en">
  <head>
@@@ -85,7 +151,7 @@ function layout({ title, q, owner, type, body, noindex, canonical }) {
  <title>${esc(title)}</title>
  ${noindex ? '<meta name="robots" content="noindex, nofollow">\n' : ''}${canonical && !noindex ? `<link rel="canonical" href="${esc(canonical)}">\n` : ''}<meta name="description" content="Search what the OpenVibe network's services have published.">
  <meta name="color-scheme" content="light dark">
--${require('openvibe-shared/app-icon').headTags({ site: 'network', iconBase: `${NETWORK}/assets` }).split('\n').filter((l) => l.startsWith('<link') && !/rel="manifest"/.test(l)).join('\n')}
++${jsonLd ? jsonLd + '\n' : ''}${require('openvibe-shared/app-icon').headTags({ site: 'network', iconBase: `${NETWORK}/assets` }).split('\n').filter((l) => l.startsWith('<link') && !/rel="manifest"/.test(l)).join('\n')}
  <script src="${ovServe.url('theme-loader.js')}" defer></script>
  <script src="${ovServe.url('navbar.js')}" defer></script>
  <script src="${ovServe.url('footer.js')}" defer></script>
@@@ -153,8 +219,10 @@ ${r.snippet_html ? `<p class="snip">${r.snippet_html}</p>` : r.summary ? `<p cla
      }).join('\n')}</ol>`;
  }
  
--/** baseUrl (config.baseUrl) names the canonical URL of the pages that may be indexed: the front page and /updates. */
--function pageRouter({ searcher, auth, baseUrl = 'https://search.openvibe.network' }) {
++/** baseUrl (config.baseUrl) names the canonical URL of the pages that may be indexed: the front page and /updates.
++ *  newestPublic() reads the newest content time among the public indexable documents (the sitemap's real
++ *  lastmod); releasedAt is the deployed release's date (/updates' lastmod). */
++function pageRouter({ searcher, auth, baseUrl = 'https://search.openvibe.network', newestPublic = null, releasedAt = null }) {
      const origin = String(baseUrl).replace(/\/+$/, '');
      const router = express.Router();
  
@@@ -170,9 +238,23 @@ function pageRouter({ searcher, auth, baseUrl = 'https://search.openvibe.network
          res.type('html').send(layout({ title: 'What shipped on OpenVibe.Search', q: '', owner: '', type: '', body: frame.updatesBody({ service: 'search', siteName: 'OpenVibe.Search' }) + `<script src="${ovServe.url('shipped.js')}" defer></script>`, canonical: `${origin}/updates` }));
      });
  
++    // Crawl files (openvibe-shared/seo): the existing robots rules kept, plus /llms.txt and /sitemap.xml.
      router.get('/robots.txt', (_req, res) => {
--        res.type('text/plain').set('Cache-Control', 'public, max-age=3600')
--            .send('User-agent: *\nAllow: /$\nAllow: /updates\nDisallow: /?\nDisallow: /api/\nDisallow: /internal/\n');
++        res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(robotsTxt(origin));
++    });
++
++    router.get('/llms.txt', (_req, res) => {
++        res.type('text/plain').set('Cache-Control', 'public, max-age=3600').send(llmsTxt(origin));
++    });
++
++    router.get('/sitemap.xml', async (_req, res, next) => {
++        let indexLastmod = null;
++        try {
++            indexLastmod = newestPublic ? await newestPublic() : null;
++        } catch (err) {
++            return next(err); // a sitemap that cannot read the index is not a sitemap of this site
++        }
++        res.type('application/xml').set('Cache-Control', 'public, max-age=3600').send(sitemapXml(origin, { indexLastmod, releasedAt }));
      });
  
      router.get('/', async (req, res, next) => {
@@@ -220,10 +302,12 @@ function pageRouter({ searcher, auth, baseUrl = 'https://search.openvibe.network
                  body = `<p class="empty">${esc(err.message)}.</p>`;
              }
          }
--        res.status(status).type('html').send(layout({ title: q ? `${q} · OpenVibe.Search` : 'OpenVibe.Search', q, owner, type, body, noindex: searching, canonical: `${origin}/` }));
++        // JSON-LD only where the page may be indexed: the plain front page, never a result page.
++        const jsonLd = searching || status !== 200 ? null : homeJsonLd(origin).map(seo.jsonLdTag).join('\n');
++        res.status(status).type('html').send(layout({ title: q ? `${q} · OpenVibe.Search` : 'OpenVibe.Search', q, owner, type, body, noindex: searching, canonical: `${origin}/`, jsonLd }));
      });
  
      return router;
  }
  
--module.exports = { pageRouter, TEXT_INDEX };
++module.exports = { pageRouter, TEXT_INDEX, robotsTxt, llmsTxt, sitemapXml, homeJsonLd };
ddiff --git a/test/discovery.test.js b/test/discovery.test.js
nnew file mode 100644
iindex 0000000..d14f626
---- /dev/null
++++ b/test/discovery.test.js
@@@ -0,0 +1,69 @@
++'use strict';
++/**
++ * The crawl files at the origin (server/web/page.js + openvibe-shared/seo): /llms.txt, /robots.txt
++ * and /sitemap.xml are served from the booted app, cached like the site's other public files, list
++ * the public pages and the newest public document's real time (never the clock), keep the robots
++ * rules the site already had, and never name a private path or a private document.
++ */
++const assert = require('assert');
++const { ids } = require('openvibe-contracts');
++const { boot, request, doc, suite } = require('./helpers');
++
++const t = suite('discovery');
++let svc;
++const alice = ids.newId('user');
++const origin = () => svc.config.baseUrl;
++
++t('boot', async () => {
++    svc = await boot();
++    // The public document is the newest content the sitemap may use; the private one is NEWER and
++    // must be ignored by every file.
++    await svc.store.apply(doc({ id: 'crawl-pub', title: 'Published page', canonical_url: 'https://openvibe.wiki/p/pub', updated_at: '2026-09-10T08:30:00Z' }));
++    await svc.store.apply(doc({ id: 'crawl-prv', visibility: 'private', acl: { subjects: [alice] }, title: 'Private diary', canonical_url: 'https://openvibe.wiki/p/secret', updated_at: '2026-09-20T08:30:00Z' }));
++});
++
++t('GET /llms.txt: what the site is, its public pages and its public endpoints, cached', async () => {
++    const r = await request(svc.base, 'GET', '/llms.txt');
++    assert.strictEqual(r.status, 200);
++    assert.match(r.headers.get('content-type'), /text\/plain/);
++    assert.strictEqual(r.headers.get('cache-control'), 'public, max-age=3600');
++    assert.match(r.text, /^# OpenVibe\.Search/m);
++    assert.ok(r.text.includes(`- [Search](${origin()}/)`), 'the front page');
++    assert.ok(r.text.includes(`(${origin()}/updates)`), 'the update log');
++    assert.ok(r.text.includes(`(${origin()}/api/v1/search?q=)`), 'the public query API');
++    assert.ok(r.text.includes(`(${origin()}/sitemap.xml)`));
++    // Nothing private or per-user: no saved searches, no owner API, no Events webhook.
++    assert.ok(!/saved-searches|\/api\/v1\/owners|\/internal\/events/.test(r.text));
++});
++
++t('GET /robots.txt: every existing Disallow kept, plus the sitemap and the AI/search bots', async () => {
++    const r = await request(svc.base, 'GET', '/robots.txt');
++    assert.strictEqual(r.status, 200);
++    assert.match(r.headers.get('content-type'), /text\/plain/);
++    assert.match(r.headers.get('cache-control'), /max-age=3600/);
++    for (const d of ['/?', '/api/', '/internal/']) assert.ok(r.text.includes(`Disallow: ${d}`), `kept Disallow: ${d}`);
++    assert.ok(r.text.includes('Allow: /$') && r.text.includes('Allow: /updates'), 'the crawlable pages');
++    assert.ok(r.text.includes(`Sitemap: ${origin()}/sitemap.xml`));
++    for (const bot of ['GPTBot', 'ClaudeBot', 'Googlebot']) assert.ok(r.text.includes(`User-agent: ${bot}`), `names ${bot}`);
++});
++
++t('GET /sitemap.xml: the public pages, lastmod from the newest public document, no private path', async () => {
++    const r = await request(svc.base, 'GET', '/sitemap.xml');
++    assert.strictEqual(r.status, 200);
++    assert.match(r.headers.get('content-type'), /application\/xml/);
++    assert.match(r.headers.get('cache-control'), /max-age=3600/);
++    assert.ok(r.text.includes(`<loc>${origin()}/</loc>`), 'the front page');
++    assert.ok(r.text.includes(`<loc>${origin()}/updates</loc>`), 'the update log');
++    // The front page's lastmod is the newest PUBLIC document's time — not the clock, and not the
++    // newer private document (2026-09-20), which never appears anywhere in the file.
++    const front = /<url><loc>([^<]+)<\/loc><lastmod>([^<]+)<\/lastmod>/.exec(r.text);
++    assert.ok(front, 'the front page has a lastmod');
++    assert.strictEqual(front[1], `${origin()}/`);
++    assert.strictEqual(front[2], '2026-09-10');
++    assert.ok(!r.text.includes('2026-09-20'), 'the private document never moves the sitemap');
++    for (const p of ['/api/', '/internal/', '/?', 'openvibe.wiki']) assert.ok(!r.text.includes(p), `no ${p}`);
++});
++
++t('shutdown', async () => { await svc.stop(); });
++
++t.run();
ddiff --git a/test/page.test.js b/test/page.test.js
iindex d925932..6e97ee7 100644
---- a/test/page.test.js
++++ b/test/page.test.js
@@@ -44,11 +44,16 @@ t('the front page is a search form, indexable, no-store, with a strict CSP', asy
      assert.ok(!/script-src[^;]*unsafe-inline/.test(csp), 'no inline script allowed');
      const scripts = [...r.text.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)];
      for (const [, attrs, body] of scripts) {
--        if (/type="application\/json"/.test(attrs)) continue;
++        if (/type="application\/(json|ld\+json)"/.test(attrs)) continue;
          assert.strictEqual(body.trim(), '', 'no inline JavaScript');
          const src = (/src="([^"]+)"/.exec(attrs) || [])[1];
          assert.ok(src && (src.startsWith('/') || src.startsWith('https://openvibe.network/')), `script from this site or openvibe.network: ${src}`);
      }
++    // JSON-LD on the front page only: the WebSite (with its SearchAction) and the search application.
++    const ld = [...r.text.matchAll(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g)].map((m) => JSON.parse(m[1]));
++    assert.deepStrictEqual(ld.map((o) => o['@type']), ['WebSite', 'WebApplication']);
++    assert.strictEqual(ld[0].potentialAction.target.urlTemplate.endsWith('/?q={search_term_string}'), true);
++    assert.ok(ld[0].url.endsWith('/') && ld[1].url.endsWith('/'));
      assert.ok(r.text.includes('<div id="navbar-mount"></div>') && r.text.includes('id="ov-footer"'), 'the OpenVibe Frame');
      assert.ok(r.text.includes('data-ov-shipped="latest" data-service="search" href="/updates"'), 'what shipped');
  });