openvibers/openvibe.network Commits

SEO-9: publish /sites, the hub of every open OpenVibe site (#409)

OP OpenVibers committed 15 days ago · 23538c05ea · 7 files, +170 −3

One server-rendered page listing every site the registry marks open, under the
home's own groups with each site's own tagline, anchored at its registry id
(/sites#live) with a jump list for phones. A CollectionPage with an ItemList of
the same sites, a self-canonical and index, follow.

It is now in the sitemap and llms-full.txt, and the home's sites section links
it with the real count. One source (server/frame/sites.js), so the hub cannot
disagree with the home, and a site that is not open never appears.

Changes

ddiff --git a/server/home/render.js b/server/home/render.js
iindex 97a03141..f1f8a214 100644
---- a/server/home/render.js
++++ b/server/home/render.js
@@@ -262,7 +262,8 @@ function body(catalog, act) {
          .filter((x) => x.sites.length)
          .map(({ g, sites: list, also }) => `<h3 class="home-group">${esc(g.name)}</h3>${grid(list.map((site) => ({ icon: `ov:${site.icon}`, title: siteName(site), text: site.tagline ? `${asSentence(site.tagline)} ${site.what}` : site.what, href: `https://${site.host}/` })))}${also.length ? `<p class="home-also">Also here: ${also.map((p) => `<a href="https://${esc(p.host)}/">${esc(p.host)}</a> (${esc(p.tagline.toLowerCase())})`).join(', ')}</p>` : ''}`).join('\n');
      const sites = `<section class="sc-sec" id="network" aria-labelledby="h-sites"><h2 id="h-sites">${esc(sitesHeading())}</h2><p class="sc-lede">Stream, chat, build, share, play, write and store. Everything is open to visitors; signing in once carries your name, theme and notifications to all of it.</p>
--${groups}</section>`;
++${groups}
++<p class="home-more"><a href="/sites">All ${OPEN.length + 1} sites, grouped by what you came to do →</a></p></section>`;
      const tools = inside(showcase.features({
          id: 'tools', title: catalog.tools.length ? `${catalog.tools.length} tools that just open` : 'Tools that just open',
          lede: 'No installs and no sign-up wall. Every tool has its own short address, so yt.openvibe.tools or dns.openvibe.tools takes you straight there.',
@@@ -354,12 +355,13 @@ const LLMS_PAGE_TEXT = {
      '/': () => ['One OpenVibe account signs you in on every site:', ...OPEN.map((site) => `- ${siteName(site)} (https://${site.host}/): ${site.what}`)].join('\n'),
      '/updates': () => "Every change deployed to the OpenVibe network, newest first: each site's commits as they ship, and the Patch notes posts that gather them.",
      '/stats': () => "Open stats: page views, people, streams, OpenCoins and how fast each service answers, for every OpenVibe site, public and reusable as JSON (https://openvibe.network/api/v1/stats) or CSV. Counts only, never about a person.",
++    '/sites': () => require('../sites/page').llmsText(),
      '/bot': () => 'OpenVibeBot, OpenVibe\'s crawler: its user agent, what it reads (sitemaps, feeds, schema.org JSON-LD), what it obeys (robots.txt for OpenVibeBot or OpenVibeSources, Crawl-delay, noindex, Retry-After), and how to block it, slow it or offer a feed instead. As JSON: https://openvibe.network/bot.json',
      '/terms': () => 'The rules for using OpenVibe.Network, in plain language.',
      '/privacy': () => 'What OpenVibe.Network collects, why, and the choices you have.',
      '/dmca': () => 'How to report copyright infringement on OpenVibe.Network, and how to respond to a report.',
  };
--const LLMS_PAGE_TITLE = { '/': 'OpenVibe.Network', '/updates': 'Updates', '/stats': 'Open stats', '/bot': 'OpenVibeBot', '/terms': 'Terms of Service', '/privacy': 'Privacy Policy', '/dmca': 'DMCA' };
++const LLMS_PAGE_TITLE = { '/': 'OpenVibe.Network', '/updates': 'Updates', '/stats': 'Open stats', '/sites': 'All the sites', '/bot': 'OpenVibeBot', '/terms': 'Terms of Service', '/privacy': 'Privacy Policy', '/dmca': 'DMCA' };
  const LLMS_FULL_MAX_BYTES = 512 * 1024;
  
  function llmsFullTxt({ baseUrl = 'https://openvibe.network' } = {}) {
ddiff --git a/server/index.js b/server/index.js
iindex 5aba1dac..cd61e441 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -781,6 +781,12 @@ app.get(['/', '/index.html'], (req, res) => {
      if (req.path === '/index.html') return redirectWithoutHtml(req, res, '/');
      return require('./home/render').sendHome(req, res, { releaseId: release.release });
  });
++// The hub of the whole network (SEO-9): every open site, from the same registry the home reads, so the two cannot
++// disagree. Built at boot from the site list, so a deploy is the only thing that changes it.
++app.get('/sites', (_req, res) => {
++    res.set('Content-Type', 'text/html; charset=utf-8').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 }));
++    res.send(require('./sites/page').render({ releaseId: release.release }));
++});
  app.get('/llms.txt', (_req, res) => res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(require('./home/render').llmsTxt()));
  app.get('/llms-full.txt', (_req, res) => res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(require('./home/render').llmsFullTxt({ baseUrl: config.baseUrl })));
  // The crawl files the front door owes crawlers, built from openvibe-shared/seo (plan T11 lane D), ahead
ddiff --git a/server/seo/routes.js b/server/seo/routes.js
iindex 80192d22..39db1840 100644
---- a/server/seo/routes.js
++++ b/server/seo/routes.js
@@@ -16,6 +16,7 @@ const PAGES = [
      { path: '/', changefreq: 'daily', priority: 1.0 },
      { path: '/updates', changefreq: 'daily', priority: 0.8 },
      { path: '/stats', changefreq: 'daily', priority: 0.7 },
++    { path: '/sites', changefreq: 'weekly', priority: 0.8 },
      { path: '/bot', changefreq: 'monthly', priority: 0.4 },
      { path: '/terms', changefreq: 'yearly', priority: 0.3 },
      { path: '/privacy', changefreq: 'yearly', priority: 0.3 },
ddiff --git a/server/sites/page.js b/server/sites/page.js
nnew file mode 100644
iindex 00000000..c69a02ec
---- /dev/null
++++ b/server/sites/page.js
@@@ -0,0 +1,111 @@
++'use strict';
++// /sites: the front door to the whole network, for people and for crawlers (SEO-9). Every open site is listed under
++// the same six neighbourhoods the home page uses (server/frame/sites.js GROUPS), with the site's own tagline, and
++// each site is anchored at its registry id (/sites#live) so it can be linked directly. One source: the registry the
++// home reads, so the hub cannot disagree with the home, and a site that is not open is never listed.
++const seo = require('openvibe-shared/seo');
++const icons = require('openvibe-shared/ov-icons');
++const { SITES, GROUPS, groupOf, PRODUCTS, fullName } = require('../frame/sites');
++
++const esc = seo.esc;
++const siteName = (site) => (site.name.includes('.') ? site.name : fullName(site));
++// The one-line purpose, in the home's own words: the tagline it renders on every card (the llms.txt and ItemList
++// descriptions use `what`, the longer text).
++const siteLine = (site) => site.tagline || site.what;
++const openSites = (list = SITES) => list.filter((site) => site.status === 'open');
++// The home counts the front door itself in "N sites, one account"; the hub lists it too (it is an open site, with the
++// account and themes behind every other one).
++const groupsOf = (sites) => GROUPS.map((group) => ({ group, sites: sites.filter((site) => groupOf(site.id) === group.id), also: PRODUCTS.filter((p) => groupOf(p.id) === group.id) })).filter((x) => x.sites.length);
++
++const card = (site) => `<a class="sc-card" id="${esc(site.id)}" href="https://${esc(site.host)}/"><span class="sc-ic sc-ic-ov"><span class="ov-icon sc-ovi" data-icon="${esc(site.icon)}" style="--ovi-size:40px" aria-hidden="true">${icons.svg(site.icon)}</span></span><span><b>${esc(siteName(site))}</b><small>${esc(siteLine(site))}</small></span></a>`;
++
++function render({ releaseId = '', sites = SITES } = {}) {
++    const open = openSites(sites);
++    const groups = groupsOf(open);
++    const count = open.length;
++    const title = `All ${count} OpenVibe sites`;
++    const desc = `Every open OpenVibe site in one place, grouped by what people come to do, each with a link to its own home. One free account signs you in on all of them.`;
++    const intro = 'Every open OpenVibe site, grouped by what people come to do; signing in once carries your name, theme and notifications to all of them.';
++    const headings = new Map(groups.map(({ group }) => [group.id, `h-${group.id}`]));
++    // The group anchors are prefixed: a group id ("games") can be a site id too, and one page cannot hold both.
++    const jump = `<nav class="sites-jump" aria-labelledby="h-jump"><h2 id="h-jump">The groups</h2><ol>${groups.map(({ group, sites: list }) => `<li><a href="#group-${esc(group.id)}">${esc(group.name)}<small>${list.length}</small></a></li>`).join('')}</ol></nav>`;
++    const body = groups.map(({ group, sites: list, also }) => `<section class="sc-sec" id="group-${esc(group.id)}" aria-labelledby="${esc(headings.get(group.id))}"><h2 id="${esc(headings.get(group.id))}">${esc(group.name)}</h2><div class="sc-grid">${list.map(card).join('')}</div>${also.length ? `<p class="sites-also">Also here: ${also.map((p) => `<a href="https://${esc(p.host)}/">${esc(p.host)}</a> (${esc(p.tagline.toLowerCase())})`).join(', ')}</p>` : ''}</section>`).join('\n');
++    const ld = {
++        '@context': 'https://schema.org', '@type': 'CollectionPage',
++        name: title, description: desc, url: 'https://openvibe.network/sites',
++        isPartOf: { '@id': 'https://openvibe.network/#site' },
++        publisher: { '@type': 'Organization', '@id': 'https://openvibe.network/#org', name: 'OpenVibe' },
++        mainEntity: seo.jsonLd.itemList(title, open.map((site) => ({ name: siteName(site), url: `https://${site.host}/`, description: siteLine(site) }))),
++    };
++    return `<!DOCTYPE html>
++<html lang="en">
++<head>
++<meta charset="utf-8">
++<meta name="viewport" content="width=device-width, initial-scale=1">
++<title>${esc(title)} · OpenVibe.Network</title>
++<meta name="description" content="${esc(desc)}">
++<meta name="robots" content="index, follow">
++<link rel="canonical" href="https://openvibe.network/sites">
++<meta property="og:title" content="${esc(title)}"><meta property="og:description" content="${esc(desc)}"><meta property="og:type" content="website"><meta property="og:url" content="https://openvibe.network/sites">
++${require('../boost').meta(releaseId)}
++${require('openvibe-shared/app-icon').headTags({ site: 'network', iconBase: '/assets' })}
++<meta name="color-scheme" content="dark light">
++<link rel="stylesheet" href="/shared/showcase.css">
++<script src="/shared/theme-loader.js" defer></script>
++<script type="application/ld+json">${JSON.stringify(ld).replace(/</g, '\\u003c')}</script>
++<style>
++*,*::before,*::after{box-sizing:border-box}
++:root{--bg-primary:#0a0f1c;--bg-secondary:#101828;--bg-card:#0f1729;--border:#1f2d47;--border-light:#2c3d5c;--accent:#3b82f6;--accent-light:#60a5fa;--text-primary:#e6edf7;--text-secondary:#96a7c2;--text-muted:#7386a3}
++html,body{margin:0;background:var(--bg-primary);color:var(--text-primary);font-family:'Inter',-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif}
++a{color:var(--accent)}
++${icons.CSS}
++${CSS}
++</style>
++</head>
++<body>
++<div id="navbar-mount"></div>
++${require('openvibe-shared/frame').noscriptNav({ name: 'OpenVibe.Network', links: [{ label: 'Open stats', href: '/stats' }, { label: 'OpenVibeBot', href: '/bot' }] })}
++<main class="sites" id="main">
++<header class="sites-head">
++<p class="sc-eyebrow">The OpenVibe network</p>
++<h1>${esc(title)}</h1>
++<p class="sc-hero-lede">${esc(intro)}</p>
++${jump}
++</header>
++${body}
++</main>
++${require('openvibe-shared/footer').ssr({ service: 'network', variant: 'compact' })}
++<script src="/shared/navbar.js" defer></script>
++${require('../boost').SCRIPT}
++<script>document.addEventListener('DOMContentLoaded',function(){try{if(window.OpenVibeNavbar)OpenVibeNavbar.init({service:'network',apiBase:location.origin});}catch(e){}});</script>
++</body>
++</html>`;
++}
++
++// The same list as text, for /llms-full.txt: every site with the one-line purpose the page shows.
++function llmsText() {
++    return ['Every open OpenVibe site, grouped by what people come to do:', ...GROUPS.map((group) => {
++        const list = openSites().filter((site) => groupOf(site.id) === group.id);
++        return list.length ? `\n${group.name}:\n${list.map((site) => `- ${siteName(site)} (https://${site.host}/): ${siteLine(site)}`).join('\n')}` : '';
++    }).filter(Boolean)].join('\n');
++}
++
++const CSS = `
++.sites{padding:0 0 64px}
++.sites-head{max-width:1080px;margin:0 auto;padding:24px 24px 0}
++.sites-head h1{margin:6px 0 10px;font-size:clamp(1.7rem,4vw,2.5rem);letter-spacing:-.03em;line-height:1.1}
++.sites-head .sc-hero-lede{margin:0;max-width:760px}
++.sites-jump{margin:26px 0 0;padding:16px 18px;border-radius:16px;border:1px solid var(--border);background:var(--bg-card)}
++.sites-jump h2{margin:0 0 10px;font-size:12px;font-weight:800;letter-spacing:.7px;text-transform:uppercase;color:var(--text-muted)}
++.sites-jump ol{list-style:none;margin:0;padding:0;display:flex;flex-wrap:wrap;gap:8px}
++.sites-jump a{display:inline-flex;align-items:center;gap:7px;padding:6px 12px;border-radius:999px;border:1px solid var(--border);font-size:13.5px;font-weight:600;color:inherit;text-decoration:none}
++.sites-jump a:hover,.sites-jump a:focus-visible{border-color:var(--accent);outline:0}
++.sites-jump small{color:var(--text-muted);font-weight:700;font-size:12px;font-variant-numeric:tabular-nums}
++.sites .sc-sec:first-of-type{margin-top:40px}
++.sc-card[id]{scroll-margin-top:84px}
++.sites-also{margin:12px 0 0;font-size:13.5px;color:var(--text-secondary)}
++.sites-also a{color:var(--accent-light);font-weight:600;text-decoration:none}
++.sites-also a:hover{text-decoration:underline}
++`;
++
++module.exports = { render, llmsText, siteLine, openSites, CSS };
ddiff --git a/test/home-front-door.test.js b/test/home-front-door.test.js
iindex 04cc3c4a..33b82125 100644
---- a/test/home-front-door.test.js
++++ b/test/home-front-door.test.js
@@@ -103,6 +103,8 @@ assert.ok(!/[.?!]\. /.test(html.replace(/<[^>]+>/g, ' ')), 'no doubled sentence
  assert.ok(!/\b1 tools\b/.test(html), 'one tool is "1 tool"');
  assert.ok(/<h1>Go live, chat, play<br>/.test(html) && /Stream from a browser tab, no OBS\./.test(html), 'the hero still leads with going live from a browser');
  for (const s of open) assert.ok(html.includes(`<a class="sc-card" href="https://${s.host}/">`), `the sites section links ${s.host}`);
++// One plain link to the hub of every site (SEO-9), in the sites section the home already has, with the real count.
++assert.ok(html.includes(`<p class="home-more"><a href="/sites">All ${open.length + 1} sites, grouped by what you came to do →</a></p>`), 'the home links /sites with the real count');
  for (const href of ['https://openvibe.tools/api/catalog.json', '/llms.txt', 'https://openvibe.tools/llms.txt', 'https://github.com/OpenVibers', 'https://openvibe.codes/', 'https://yt.openvibe.tools/', 'https://dns.openvibe.tools/']) assert.ok(html.includes(`href="${href}"`), `the page still links ${href}`);
  assert.strictEqual((html.match(/<section[\s>]/g) || []).length, (html.match(/<\/section>/g) || []).length, 'every section is closed once');
  
ddiff --git a/test/seo-routes.test.js b/test/seo-routes.test.js
iindex 665d4d27..ed460293 100644
---- a/test/seo-routes.test.js
++++ b/test/seo-routes.test.js
@@@ -80,6 +80,51 @@ const cache = require('openvibe-shared/cache-policy');
              assert.ok(/<meta name="robots" content="index,\s*follow">/.test(head), `${p} says index, follow instead of relying on the crawler's default`);
          }
  
++        // ── /sites: the hub of the whole network (SEO-9) ──
++        const sitesPage = require('../server/sites/page');
++        const { SITES } = require('../server/frame/sites');
++        const openSites = SITES.filter((site) => site.status === 'open');
++        const hub = await get('/sites');
++        assert.strictEqual(hub.status, 200, '/sites answers 200 (it was a 404 before SEO-9)');
++        assert.ok(hub.type.startsWith('text/html'), '/sites is HTML');
++        const hubHead = hub.body.slice(0, hub.body.indexOf('</head>'));
++        assert.strictEqual((hub.body.match(/<h1[ >]/g) || []).length, 1, '/sites has exactly one <h1>');
++        assert.ok(/<link rel="canonical" href="https:\/\/openvibe\.network\/sites">/.test(hubHead), '/sites is its own canonical');
++        assert.ok(/<meta name="robots" content="index,\s*follow">/.test(hubHead), '/sites says index, follow');
++        assert.ok(hubHead.includes(`<title>All ${openSites.length} OpenVibe sites · OpenVibe.Network</title>`), 'the title carries the real count');
++        assert.ok(/<meta name="description" content="[^"]{50,170}">/.test(hubHead), '/sites has a real description');
++        const hubLd = JSON.parse(hub.body.match(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/)[1]);
++        assert.strictEqual(hubLd['@type'], 'CollectionPage', 'the JSON-LD is a CollectionPage');
++        assert.strictEqual(hubLd.mainEntity['@type'], 'ItemList', 'with an ItemList');
++        assert.strictEqual(hubLd.mainEntity.numberOfItems, openSites.length, 'the ItemList holds every open site');
++        const ids = new Set(openSites.map((site) => `https://${site.host}/`));
++        for (const item of hubLd.mainEntity.itemListElement) {
++            assert.ok(ids.has(item.url), `${item.url} is an open site`);
++            assert.ok(item.name && item.description, `${item.url} carries its name and its one-line purpose`);
++        }
++        assert.deepStrictEqual(hubLd.mainEntity.itemListElement.map((i) => i.url).sort(), [...ids].sort(), 'every open site is in the ItemList, and nothing else');
++        // The home's ids: each site is anchored at the registry id the home groups it by, and links to its own home.
++        for (const site of openSites) {
++            assert.ok(hub.body.includes(`class="sc-card" id="${site.id}" href="https://${site.host}/"`), `/sites#${site.id} links to https://${site.host}/`);
++        }
++        // The groups are the home's six neighbourhoods, in the home's order, with a jump list to each.
++        const { GROUPS, groupOf } = require('../server/frame/sites');
++        const groupIds = GROUPS.map((g) => g.id).filter((id) => openSites.some((site) => groupOf(site.id) === id));
++        assert.deepStrictEqual([...hub.body.matchAll(/<section class="sc-sec" id="(group-[a-z]+)" aria-labelledby=/g)].map((m) => m[1]), groupIds.map((id) => `group-${id}`), 'the groups appear once each, in the home order');
++        for (const id of groupIds) assert.ok(hub.body.includes(`<a href="#group-${id}">`), `the jump list reaches #group-${id}`);
++        // One page, one id: the site anchors and the group anchors never collide (a group "games" is a site "games").
++        const pageIds = [...hub.body.matchAll(/\bid="([^"]+)"/g)].map((m) => m[1]);
++        assert.deepStrictEqual(pageIds.filter((id, i) => pageIds.indexOf(id) !== i), [], 'no id repeats on the page');
++        // Only `open`: an entry that is not open never reaches the page or its ItemList.
++        const one = openSites[0];
++        const closed = sitesPage.render({ sites: SITES.map((site) => (site.id === one.id ? { ...site, status: 'soon' } : site)) });
++        assert.ok(!closed.includes(`id="${one.id}"`), 'a site that is not open is not anchored on the page');
++        assert.strictEqual(JSON.parse(closed.match(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/)[1]).mainEntity.numberOfItems, openSites.length - 1, 'and not in the ItemList');
++        // Findable: the sitemap lists it, and llms-full.txt carries its list.
++        assert.ok(locs.includes('https://openvibe.network/sites'), 'the sitemap lists /sites');
++        assert.ok(fullBody.includes('URL: https://openvibe.network/sites'), 'llms-full.txt lists /sites');
++        assert.ok(fullBody.includes(`${one.name} (https://${one.host}/)`) || fullBody.includes(`OpenVibe.${one.name} (https://${one.host}/)`), 'llms-full.txt carries the site list under /sites');
++
          // ── IndexNow: no INDEXNOW_KEY → no key file ──
          const KEY = require('crypto').randomBytes(16).toString('hex');
          assert.strictEqual((await get(`/${KEY}.txt`)).status, 404, 'no key file is served without INDEXNOW_KEY');
ddiff --git a/test/unknown-paths.test.js b/test/unknown-paths.test.js
iindex 19a516c3..b7f336ae 100644
---- a/test/unknown-paths.test.js
++++ b/test/unknown-paths.test.js
@@@ -15,7 +15,7 @@ const { ACCOUNT_HUB_PATHS } = require('../server/not-found');
          // ── Real pages keep 200 ──
          const real = [
              '/', '/login', '/forgot-password', '/reset-password', '/admin', '/admin/settings', '/admin/analytics/overview',
--            '/verify-email', '/status', '/sso/fanout', '/terms', '/privacy', '/dmca', '/llms.txt',
++            '/verify-email', '/status', '/sso/fanout', '/terms', '/privacy', '/dmca', '/llms.txt', '/sites',
              '/robots.txt', '/sitemap.xml', '/manifest.webmanifest', '/assets/logo.svg', '/js/login.js', '/shared/navbar.js', '/shared/v1/navbar.js',
              '/css/login.css', '/css/my.css', '/css/admin.css',
              '/openvibe-sw.js', '/release.json', '/.well-known/openvibe', '/.well-known/web-identity', '/api/health', '/api/brand',