openvibers/openvibe.search Commits

nginx: route the agent manifest and the twins to Search (#128)

OP OpenVibers committed 15 days ago · 59a6b8a6c5 · 3 files, +37 −2

Follow-up to #127. The vhost proxies an explicit allowlist and answers everything else with a static
JSON 404, so the agent layer was live in the app but unreachable through nginx: /.well-known/agent.json
and /index.md were the vhost's 404. Two locations fix it (both GET-only, on the same upstream): the
manifest, and the twins of the two pages that have one — /index.md and /updates.md (a result page's
twin answers on / with ?format=markdown, which the existing `location = /` already proxies). Patched on
the host the same day; this is the durable copy.

/updates now names its twin in the Link header as its head tags already did, and the page router's agent
gate is narrowed to the manifest's own path: it sent every /.well-known/ file to the agent layer, which
would swallow a security.txt served by the app (as it did on Chat). discovery.test.js proves the
manifest, both twins, both link forms and a result twin.

Changes

ddiff --git a/deploy/nginx/search.openvibe.network.conf b/deploy/nginx/search.openvibe.network.conf
iindex 5e4409a..8804526 100644
---- a/deploy/nginx/search.openvibe.network.conf
++++ b/deploy/nginx/search.openvibe.network.conf
@@@ -91,6 +91,10 @@ server {
      location = /llms.txt { proxy_pass http://127.0.0.1:4710; }
      location = /llms-full.txt { proxy_pass http://127.0.0.1:4710; }
      location = /sitemap.xml { proxy_pass http://127.0.0.1:4710; }
++    # The agent layer (openvibe-shared/agent-layer): the manifest an agent reads first, and the Markdown twins of
++    # the front page and /updates (a result page's twin answers on / with ?format=markdown, already proxied).
++    location = /.well-known/agent.json { limit_except GET { deny all; } proxy_pass http://127.0.0.1:4710; }
++    location ~ "^/(index|updates)\.md$" { limit_except GET { deny all; } proxy_pass http://127.0.0.1:4710; }
      # The IndexNow key file (8-128 letters and digits); Node answers 404 unless it is INDEXNOW_KEY.
      location ~ "^/[A-Za-z0-9]{8,128}\.txt$" { limit_except GET { deny all; } proxy_pass http://127.0.0.1:4710; }
      # This site's own pinned copy of the Frame's browser files (openvibe-shared/serve).
ddiff --git a/server/web/page.js b/server/web/page.js
iindex c75b04b..29ec3de 100644
---- a/server/web/page.js
++++ b/server/web/page.js
@@@ -411,10 +411,11 @@ function pageRouter({ searcher, auth, baseUrl = 'https://search.openvibe.network
      const origin = String(baseUrl).replace(/\/+$/, '');
      const router = express.Router();
      // Markdown twin requests and the agent manifest go to openvibe-shared/agent-layer, mounted after this router:
--    // .md paths, ?format=markdown, an explicit text/markdown preference, and /.well-known/.
++    // .md paths, ?format=markdown, an explicit text/markdown preference, and the manifest's own path (any other
++    // /.well-known/ file stays this router's, exactly as security.txt does on Chat).
      router.use((req, res, next) => {
          const p = String(req.path || '');
--        if (p.endsWith('.md') || p.startsWith('/.well-known/') || (req.query && req.query.format === 'markdown')
++        if (p === '/.well-known/agent.json' || p.endsWith('.md') || (req.query && req.query.format === 'markdown')
              || req.accepts(['html', 'text/markdown']) === 'text/markdown') return next('router');
          return next();
      });
@@@ -424,6 +425,8 @@ function pageRouter({ searcher, auth, baseUrl = 'https://search.openvibe.network
          res.setHeader('Content-Security-Policy', CSP);
          res.setHeader('X-Frame-Options', 'DENY');
          res.setHeader('Cache-Control', cache.htmlHeaders({ maxAge: 60 }));
++        // This page has a Markdown twin (/updates.md), so its header names it, as the head tags do.
++        res.setHeader('Link', seo.linkHeaderTags(`${origin}/updates`, `${origin}/llms.txt`).header);
          res.type('html').send(layout({ title: 'What shipped on OpenVibe.Search', q: '', owner: '', type: '', ownHeading: true, body: frame.updatesBody({ service: 'search', siteName: 'OpenVibe.Search' }) + `<script src="${ovServe.url('shipped.js')}" defer></script>`, canonical: `${origin}/updates` }));
      });
  
ddiff --git a/test/discovery.test.js b/test/discovery.test.js
iindex 99dc020..e1c386f 100644
---- a/test/discovery.test.js
++++ b/test/discovery.test.js
@@@ -121,6 +121,34 @@ t('the route index is built from routes only: every entry gets a line of text, n
      assert.ok(!/saved-searches|\/api\/v1\/owners|\/internal\/events|\/api\/v1\/documents/.test(text));
  });
  
++t('the agent layer: manifest, twins, and the pages that name them', async () => {
++    const m = await request(svc.base, 'GET', '/.well-known/agent.json');
++    assert.strictEqual(m.status, 200);
++    assert.match(m.headers.get('content-type'), /application\/json/);
++    const d = JSON.parse(m.text);
++    assert.strictEqual(d.id, 'search');
++    assert.strictEqual(d.home, 'https://search.openvibe.network');
++    assert.strictEqual(d.api, 'https://search.openvibe.network/api/v1');
++    assert.deepStrictEqual(d.policy.auth, ['none', 'Network OIDC']);
++    // A twin of the front page and of the update log, from the pages' own render. The twin's canonical is the
++    // manifest's home (the site's public address), not the test server's own base URL.
++    for (const [p, canonical] of [['/index.md', 'https://search.openvibe.network/'], ['/updates.md', 'https://search.openvibe.network/updates']]) {
++        const tw = await request(svc.base, 'GET', p);
++        assert.strictEqual(tw.status, 200, `${p} answers`);
++        assert.match(tw.headers.get('content-type'), /text\/markdown/, `${p} is markdown`);
++        assert.ok(tw.text.includes(`canonical: "${canonical}"`), `${p} names its page`);
++    }
++    // The HTML page names its twin in the Link header and the head; a result page is noindex but still twinned.
++    for (const p of ['/', '/updates']) {
++        const page = await request(svc.base, 'GET', p, { headers: { accept: 'text/html' } });
++        assert.strictEqual(page.status, 200);
++        assert.match(String(page.headers.get('link')), /rel="alternate"; type="text\/markdown"/, `${p} names its twin`);
++        assert.match(page.text, /<link rel="alternate" type="text\/markdown"/, `${p} carries the head tag`);
++    }
++    const results = await request(svc.base, 'GET', '/?q=page&format=markdown');
++    assert.match(results.headers.get('content-type'), /text\/markdown/, 'a result twin answers for an agent');
++});
++
  t('shutdown', async () => { await svc.stop(); });
  
  t.run();