openvibers/openvibe.tools Commits

One canonical address per tool: /tool/:id 301s to the primary host (#235)

OP OpenVibers committed 6 days ago · 1a35e58468 · 6 files, +75 −13

Changes

ddiff --git a/README.md b/README.md
iindex cc2fcd1..76f8b4f 100644
---- a/README.md
++++ b/README.md
@@@ -100,7 +100,7 @@ Domains; the gateway merges those overrides every minute. New custom domain on t
  `sudo deploy/scripts/add-custom-domain.sh <domain>`.
  
  Public, cacheable outputs on the apex: `/` (every tool by family, planned tools as placeholders),
--`/<family>-tools`, `/tool/<id>`, `/all-tools`, `/search?q=`, `/sitemap.xml`, `/robots.txt`,
++`/<family>-tools`, `/all-tools`, `/search?q=`, `/sitemap.xml`, `/robots.txt` (and `/tool/<id>`, which 301s to the tool's primary host),
  `/llms.txt`, `/api/catalog.json`, `/terms`, `/privacy`, `/dmca`. All of it renders without JavaScript.
  
  Deploy with `deploy/scripts/deploy.sh`: it runs `ovhost deploy tools` (OpenVibe.Host, strategy
ddiff --git a/apps/gateway/server/pages/agent-pages.js b/apps/gateway/server/pages/agent-pages.js
iindex d7b8386..77c56cf 100644
---- a/apps/gateway/server/pages/agent-pages.js
++++ b/apps/gateway/server/pages/agent-pages.js
@@@ -1,6 +1,7 @@
  'use strict';
--// The Markdown twins of openvibe.tools' public pages (openvibe-shared/agent-layer): the directory, every family,
--// /all-tools and each tool's page. They are the same pageIndex/pageFamily/pageTool/pageAll renderers the site serves,
++// The Markdown twins of openvibe.tools' public pages (openvibe-shared/agent-layer): the directory, every family and
++// /all-tools. A tool's twin lives in site.js's /tool/:id.md with the tool's primary host as canonical (a /tool page
++// would be a second canonical). They are the same pageIndex/pageFamily/pageAll renderers the site serves,
  // so a twin cannot drift from its page; /developers too. The noindex search page and the API pages have no twin.
  const site = require('./site');
  const registry = require('../registry');
@@@ -15,10 +16,6 @@ function createAgentPages({ opens = null } = {}) {
          if (p === '/all-tools') return { html: site.pageAll() };
          // The API guide (11-03: llms.txt listed it as HTML; for an agent it is the page that matters most).
          if (p === '/developers') return { html: site.pageDevelopers() };
--        if (p.startsWith('/tool/')) {
--            const t = registry.get().tools.find((x) => x.id === p.slice('/tool/'.length));
--            return t ? { html: site.pageTool(t) } : null;
--        }
          const f = registry.get().families.find((x) => x.path === p);
          return f ? { html: site.pageFamily(f) } : null;
      }
ddiff --git a/apps/gateway/server/pages/site.js b/apps/gateway/server/pages/site.js
iindex 7aca7ed..d060386 100644
---- a/apps/gateway/server/pages/site.js
++++ b/apps/gateway/server/pages/site.js
@@@ -3,7 +3,7 @@
  // openvibe.tools, server-rendered from the registry.
  //   /                 index: search, families, popular tools
  //   /<family-path>    one family: every tool, intro, FAQ
--//   /tool/<id>        one tool: what it does, where it lives (primary host + mirrors), related
++//   /tool/<id>        301 to the tool's primary host (one canonical address per tool)
  //   /all-tools        A–Z
  //   /search?q=        results (works as a plain form; JS only makes it instant)
  //   /sitemap.xml /robots.txt /llms.txt /llms-full.txt /api/catalog.json
@@@ -552,7 +552,7 @@ function pageAll() {
      const { tools, families } = registry.get();
      const sorted = [...tools].sort((a, b) => a.name.localeCompare(b.name));
      const body = `<nav class="crumbs" aria-label="Breadcrumb"><a href="/">Tools</a> › All tools</nav><div class="page-h">${icon('tools', 56)}<h1>All ${tools.length} tools, A to Z</h1></div>
--${searchForm('')}<section><ul class="az">${sorted.map(t => `<li><a href="${esc(t.url)}">${esc(t.name)}</a> <small><a href="/tool/${esc(t.id)}" aria-label="About ${esc(t.name)}">about</a></small></li>`).join('')}</ul></section>`;
++${searchForm('')}<section><ul class="az">${sorted.map(t => `<li><a href="${esc(t.url)}">${esc(t.name)}</a> <small><a href="${esc(t.url)}" aria-label="About ${esc(t.name)}">about</a></small></li>`).join('')}</ul></section>`;
      return shell({ families, agent: true, body, head: head({ title: `All ${tools.length} Online Tools, A to Z`, description: 'The complete list of tools on OpenVibe.Tools: converters, downloaders, PDF and image tools, text utilities, developer tools and network diagnostics.', canonical: SITE + '/all-tools',
          jsonLd: [seo.jsonLd.breadcrumbs([{ name: 'Tools', url: SITE + '/' }, { name: 'All tools', url: SITE + '/all-tools' }])] }) });
  }
@@@ -651,7 +651,7 @@ function sitemapEntries() {
      return [{ loc: SITE + '/', changefreq: 'daily', priority: 1 }, { loc: SITE + '/all-tools', changefreq: 'weekly', priority: 0.8 }, { loc: SITE + '/developers', changefreq: 'weekly', priority: 0.7 }, { loc: SITE + '/updates', changefreq: 'daily', priority: 0.4 },
          ...families.filter(f => f.path).map(f => ({ loc: SITE + f.path, changefreq: 'weekly', priority: 0.9 })),
          ...families.filter(f => f.hosts).map(f => ({ loc: f.url, changefreq: 'weekly', priority: 0.9 })),
--        ...tools.filter(t => !t.external).flatMap(t => [{ loc: t.url, changefreq: 'weekly', priority: 0.8 }, { loc: `${SITE}/tool/${t.id}`, changefreq: 'monthly', priority: 0.5 }]),
++        ...tools.filter(t => !t.external).map(t => ({ loc: t.url, changefreq: 'weekly', priority: 0.8 })),
      ].map(u => Object.assign({ lastmod: today }, u));
  }
  
@@@ -702,6 +702,22 @@ function llmsFullBody() {
  
  function createSiteRouter({ opens = null } = {}) {
      const router = express.Router();
++    // One canonical address per tool: its primary host. The directory's /tool/<id> page is only a pointer, so it
++    // 301s there (query kept) instead of competing with the tool's own self-canonical page. Its Markdown twin keeps
++    // the same canonical, so HTML, twin, sitemap and llms.txt all name one URL.
++    router.get('/tool/:id.md', (req, res, next) => {
++        const t = registry.get().tools.find((x) => x.id === req.params.id);
++        if (!t) return next();
++        const { markdownTwin } = require('openvibe-shared/agent-layer');
++        const body = markdownTwin({ html: pageTool(t), canonical: t.url });
++        return res.set('Content-Type', 'text/markdown; charset=utf-8').set('Cache-Control', 'public, max-age=300').send(body);
++    });
++    router.get('/tool/:id', (req, res, next) => {
++        const t = registry.get().tools.find((x) => x.id === req.params.id);
++        if (!t) return next();
++        const q = req.originalUrl.includes('?') ? req.originalUrl.slice(req.originalUrl.indexOf('?')) : '';
++        return res.redirect(301, `${t.url}${q}`);
++    });
      // Markdown twin and agent manifest requests go to openvibe-shared/agent-layer, mounted after this router.
      router.use((req, res, next) => {
          const p = String(req.path || '');
@@@ -724,7 +740,6 @@ function createSiteRouter({ opens = null } = {}) {
      });
      router.get('/all-tools', (req, res) => send(req, res, '/all-tools', pageAll));
      router.get('/search', (req, res) => { res.set('Cache-Control', 'no-store'); res.type('html').send(pageSearch(String(req.query.q || '').slice(0, 80))); });
--    router.get('/tool/:id', (req, res, next) => { const t = registry.get().tools.find(x => x.id === req.params.id); return t ? send(req, res, '/tool/' + t.id, () => pageTool(t)) : next(); });
      router.get('/sitemap.xml', (req, res) => send(req, res, 'sitemap', () => seo.sitemapXml(sitemapEntries()), 'application/xml; charset=utf-8'));
      router.get('/robots.txt', (req, res) => send(req, res, 'robots', () => seo.robotsTxt({ sitemaps: [SITE + '/sitemap.xml'], disallow: ['/api/', '/auth/'], allow: ['/api/catalog.json'], allowAI: true }) + `# Agent manifest: ${SITE}/.well-known/agent.json\n`, 'text/plain; charset=utf-8'));
      router.get('/llms.txt', (req, res) => send(req, res, 'llms', llmsTxtBody, 'text/plain; charset=utf-8'));
ddiff --git a/apps/gateway/test/registry.test.js b/apps/gateway/test/registry.test.js
iindex 0abe1fb..7231f74 100644
---- a/apps/gateway/test/registry.test.js
++++ b/apps/gateway/test/registry.test.js
@@@ -27,6 +27,7 @@ assert.equal(r('x.com').kind, 'unknown', 'override for an unknown tool is ignore
  assert.equal(r('openvibe.tools').kind, 'apex', 'the apex cannot be claimed');
  assert.ok(site.sitemapEntries().some(u => u.loc === 'https://youtubedownloadonline.com/'), 'sitemap uses the canonical host');
  assert.ok(!site.sitemapEntries().some(u => u.loc === 'https://yt.openvibe.tools/'), 'short hosts stay out of the sitemap');
++assert.ok(!site.sitemapEntries().some(u => u.loc.includes('/tool/')), 'the /tool/<id> pointer is not a second sitemap page');
  registry.setOverrides([]);
  assert.equal(registry.catalog().tools.find(t => t.id === 'yt').hosts.canonical, 'youtube-downloader.openvibe.tools');
  
ddiff --git a/apps/gateway/test/tool-canonical.test.js b/apps/gateway/test/tool-canonical.test.js
nnew file mode 100644
iindex 0000000..2bebdac
---- /dev/null
++++ b/apps/gateway/test/tool-canonical.test.js
@@@ -0,0 +1,47 @@
++'use strict';
++// One canonical address per tool: the directory's /tool/<id> is a pointer to the tool's primary host. This starts
++// the real gateway (not a router in a test) so the redirect and the Markdown twin are the ones production serves.
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++const { startApp } = require('../../_shared/test/spawn');
++
++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-tool-canonical-'));
++
++(async () => {
++    const gateway = await startApp('gateway', {
++        DATABASE_URL: '', DATABASE_DIRECT_URL: '', DATA_DIR: tmp,
++        OV_NETWORK_URL: 'http://127.0.0.1:9', OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9',
++        OV_DOMAINS_URL: 'http://127.0.0.1:9/api/domains', OV_REGISTRY_URL: 'http://127.0.0.1:9/registry',
++        TOOLS_SATELLITE_PORTS: 'yt=9,text=9,img=9,audio=9,docs=9,maps=9,food=9',
++    });
++    try {
++        const canonical = 'https://youtube-downloader.openvibe.tools/';
++
++        const get = (p) => fetch(gateway.base + p, { redirect: 'manual' });
++        const html = await get('/tool/yt');
++        assert.strictEqual(html.status, 301, await html.text());
++        assert.strictEqual(html.headers.get('location'), canonical);
++
++        const query = await get('/tool/yt?ref=directory');
++        assert.deepStrictEqual([query.status, query.headers.get('location')], [301, `${canonical}?ref=directory`]);
++
++        assert.strictEqual((await get('/tool/not-a-tool')).status, 404, 'unknown ids stay 404');
++
++        const twin = await get('/tool/yt.md');
++        const twinText = await twin.text();
++        assert.strictEqual(twin.status, 200, twinText);
++        assert.match(twin.headers.get('content-type') || '', /^text\/markdown/);
++        assert.ok(twinText.includes(`canonical: ${JSON.stringify(canonical)}`), 'the twin names the primary host');
++
++        const map = await get('/sitemap.xml');
++        const mapText = await map.text();
++        assert.strictEqual(map.status, 200);
++        assert.ok(mapText.includes(`<loc>${canonical}</loc>`), 'the primary is listed');
++        assert.ok(!mapText.includes('openvibe.tools/tool/'), 'no /tool/ duplicate is listed');
++        console.log('tool canonical redirect + sitemap: all checks passed');
++    } finally {
++        await gateway.kill();
++    }
++})().catch((err) => { console.error(err); process.exit(1); });
ddiff --git a/apps/gateway/test/tool-opens.test.js b/apps/gateway/test/tool-opens.test.js
iindex f5ddb7b..76ca758 100644
---- a/apps/gateway/test/tool-opens.test.js
++++ b/apps/gateway/test/tool-opens.test.js
@@@ -110,12 +110,14 @@ function fakeDb({ failFirst = false } = {}) {
                  assert.ok(home.includes('5 people this week') && home.includes('1 person this week'), 'each with its count');
              } finally { srv2.close(); }
  
--            const page = await (await fetch(`${base}/tool/dns`)).text();
++            // /tool/<id> is a 301 to the tool's primary host; the about page itself is still the site's renderer
++            // (and its .md twin), so read that rather than follow the redirect off this host.
++            const page = site.pageTool(registry.get().tools.find((t) => t.id === 'dns'));
              assert.ok(page.includes('Use it from code') && page.includes('https://openvibe.tools/api/v1/tools/dns/run'), 'a tool page names its request');
              assert.ok(page.includes('/api/v1/tools/dns/schema'), 'and its input schema');
              const file = registry.get().tools.find((t) => t.id === 'mergepdf');
              if (file) {
--                const pdf = await (await fetch(`${base}/tool/mergepdf`)).text();
++                const pdf = site.pageTool(file);
                  assert.ok(pdf.includes('-F &#39;[email protected]&#39;') || pdf.includes("-F '[email protected]'"), 'a file tool shows a multipart upload');
              }
          } finally { srv.close(); }