openvibers/openvibe.community Commits

SEO-29: noindex pastes and send X-Robots-Tag on noindex pages (#178)

OP OpenVibers committed 15 days ago · 9f29ec05ad · 9 files, +57 −23

Pastes are link-shared content: Google's 2026-10-26 UGC guidance says to
noindex pages reached by a link rather than found by search. Every paste page
is now noindex (public ones were index,follow), no /p/:slug row is in the
sitemap, and every noindex page sends the matching X-Robots-Tag header.
Comments, /pastes browse/search and the forum stay as they were: the first
two are already noindex or meant for search; the forum is public discussion.

Changes

ddiff --git a/server/app.js b/server/app.js
iindex 59d7f96..d834eed 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -36,7 +36,7 @@ const catalog = require('./pastes/catalog');
  const source = require('./pastes/source');
  const discovery = require('./discovery');
  const pages = require('./render/pages');
--const { assetVersion, SITE_NAME, DEFAULT_DESCRIPTION } = require('./render/layout');
++const { assetVersion, SITE_NAME, DEFAULT_DESCRIPTION, setRobotsHeader } = require('./render/layout');
  const { createAuthClient, createAuthRoutes } = require('./auth/routes');
  const { getDb } = require('./db');
  const { createNetworkIdentity } = require('./identity/network');
@@@ -310,7 +310,10 @@ async function createApp(opts = {}) {
      // Pages are for browsers: the viewer (subject, staff) is resolved here too.
      const withUser = viewers.middleware({ services: false });
      const ctxOf = (req) => ({ token: req.token, ip: req.ip, userAgent: req.get('user-agent') || '', viewer: req.viewer });
--    const html = (res, body, status = 200) => res.status(status).type('html').set('Cache-Control', PAGE_CACHE).send(body);
++    const html = (res, body, status = 200) => {
++        setRobotsHeader(res, body);
++        return res.status(status).type('html').set('Cache-Control', PAGE_CACHE).send(body);
++    };
      // A public page with a Markdown twin names it in the Link header and varies on Accept. Only the
      // pages the agent layer twins (the home, /pastes and a paste) call this.
      const twinLink = (res, pathname) => {
ddiff --git a/server/comments/routes.js b/server/comments/routes.js
iindex b83910c..1cf2b6e 100644
---- a/server/comments/routes.js
++++ b/server/comments/routes.js
@@@ -16,6 +16,7 @@ const pages = require('../render/pages');
  const { threadPage } = require('../render/comments');
  const { ApiError } = require('../http/v1');
  const cache = require('openvibe-shared/cache-policy');
++const { setRobotsHeader } = require('../render/layout');
  
  const ACCESS_ID_RE = /^cth_[A-Za-z0-9_-]{22}$/;
  
@@@ -23,7 +24,10 @@ function createCommentPages({ comments, viewers, config }) {
      const router = express.Router();
      const withViewer = viewers.middleware({ services: false });
      const form = express.urlencoded({ extended: false, limit: '64kb' });
--    const html = (res, body, status = 200) => res.status(status).type('html').set('Cache-Control', cache.htmlHeaders({ private: true })).send(body);
++    const html = (res, body, status = 200) => {
++        setRobotsHeader(res, body);
++        return res.status(status).type('html').set('Cache-Control', cache.htmlHeaders({ private: true })).send(body);
++    };
      const wrap = (fn) => (req, res, next) => Promise.resolve(fn(req, res, next)).catch(next);
  
      function failPage(req, res, err, next) {
ddiff --git a/server/discovery.js b/server/discovery.js
iindex 219c317..c1ec78b 100644
---- a/server/discovery.js
++++ b/server/discovery.js
@@@ -66,10 +66,8 @@ async function buildSitemapRows() {
      add('/', null, 'hourly', '1.0');
      add('/pastes', null, 'hourly', '0.8');
      add('/pastes?sort=views', null, 'daily', '0.5');
--    for (const p of await catalog.recent()) {
--        if (Number(p.is_nsfw) || Number(p.burn_after_read)) continue;
--        add(`/p/${p.slug}`, isoDate(p.updated_at || p.created_at), 'weekly', '0.6');
--    }
++    // No /p/:slug rows: a paste is link-shared content and is noindex (SEO-29), so it must not be
++    // in a sitemap. The /pastes browse and search pages above are the surfaces meant for search.
      if (_forum) {
          add('/s', null, 'hourly', '0.8');
          add('/pulse', null, 'hourly', '0.5');
ddiff --git a/server/forum/routes.js b/server/forum/routes.js
iindex b8f27b9..0404a25 100644
---- a/server/forum/routes.js
++++ b/server/forum/routes.js
@@@ -24,7 +24,7 @@ const multer = require('multer');
  const seo = require('openvibe-shared/seo');
  const cache = require('openvibe-shared/cache-policy');
  const discovery = require('../discovery');
--const { SITE_NAME } = require('../render/layout');
++const { SITE_NAME, setRobotsHeader } = require('../render/layout');
  const pages = require('../render/pages');
  const forumPages = require('../render/forum');
  const boardPages = require('../render/board');
@@@ -51,7 +51,10 @@ function createForumRoutes({ forum, viewers, config }) {
          return ids;
      }
      const worthUploading = (title, body) => (title == null || String(title).trim().length >= 3) && String(body || '').trim().length > 0;
--    const html = (res, body, status = 200) => res.status(status).type('html').set('Cache-Control', cache.htmlHeaders({ private: true })).send(body);
++    const html = (res, body, status = 200) => {
++        setRobotsHeader(res, body);
++        return res.status(status).type('html').set('Cache-Control', cache.htmlHeaders({ private: true })).send(body);
++    };
      const login = (res, next) => res.redirect(303, `/auth/login?next=${encodeURIComponent(next)}`);
      const wrap = (fn) => (req, res, next) => Promise.resolve(fn(req, res, next)).catch(next);
  
ddiff --git a/server/render/layout.js b/server/render/layout.js
iindex 7a3c800..2b50a8d 100644
---- a/server/render/layout.js
++++ b/server/render/layout.js
@@@ -166,4 +166,16 @@ document.addEventListener('DOMContentLoaded', function () {
      return o.ogImage ? html : html.replace('<meta name="twitter:card" content="summary_large_image">', '<meta name="twitter:card" content="summary">');
  }
  
--module.exports = { renderPage, asset, assetVersion, abs, setRelease, SITE_NAME, DEFAULT_DESCRIPTION, DEFAULT_OG_IMAGE };
++/**
++ * Copy a page's noindex rule into an X-Robots-Tag response header (Google's 2026-10-26 UGC
++ * guidance asks for both a robots meta tag and the header on link-shared content). Called before
++ * send, so clients that never parse the head see it too. The value mirrors the meta tag
++ * ('noindex', 'noindex,nofollow'), so both say the same thing.
++ */
++function setRobotsHeader(res, html) {
++    const tag = /<meta[^>]*\bname=["']robots["'][^>]*>/i.exec(String(html || ''));
++    const content = tag && /\bcontent=["']([^"']*)["']/i.exec(tag[0]);
++    if (content && /(?:^|[\s,;])noindex(?:[\s,;]|$)/i.test(content[1])) res.set('X-Robots-Tag', content[1].replace(/\s+/g, ''));
++}
++
++module.exports = { renderPage, asset, assetVersion, abs, setRelease, SITE_NAME, DEFAULT_DESCRIPTION, DEFAULT_OG_IMAGE, setRobotsHeader };
ddiff --git a/server/render/pages.js b/server/render/pages.js
iindex 5feee0c..ac8b66d 100644
---- a/server/render/pages.js
++++ b/server/render/pages.js
@@@ -364,7 +364,6 @@ function pastePage({ paste: p, related, user, comments = null, commentSort = 'ne
      const title = p.title || (isShot ? 'Screenshot' : 'Untitled paste');
      const description = pasteDescription(p);
      const shot = isShot ? mediaAbs(p.screenshot_url) || screenshotFallback(p.slug) : null;
--    const indexable = (p.visibility === 'public' || p.visibility == null) && !Number(p.is_nsfw) && !Number(p.burn_after_read);
      // Pastes name their owner by Network subject; the projection cache supplies the display fields.
      const isOwner = !!(user && (p.is_owner === true
          || (p.owner_subject && user.subject_id && p.owner_subject === user.subject_id)
@@@ -442,7 +441,11 @@ ${related && related.length ? `<section class="section" id="related">
          description,
          canonicalPath: `/p/${p.slug}`,
          agentLayer: true,   // the public page has a Markdown twin (server/agent-pages.js)
--        robots: indexable ? 'index,follow' : 'noindex,follow',
++        // A paste is link-shared content: reached by the link its author shares, not found by
++        // search. Google's 2026-10-26 UGC guidance says to noindex those by default; the browse
++        // and search pages that are meant to be found keep index. Private, unlisted, NSFW and
++        // burn-after-read pastes were already noindex and stay so.
++        robots: 'noindex',
          ogType: 'article',
          ogImage: shot || undefined,
          published: ld.isoDate(p.created_at) || undefined,
ddiff --git a/test/seo.test.js b/test/seo.test.js
iindex 469ccc8..828ec04 100644
---- a/test/seo.test.js
++++ b/test/seo.test.js
@@@ -84,20 +84,31 @@ const { boot, check, done } = require('./helpers/app');
          }
      });
  
--    await check('sitemap.xml lists home, /pastes and every recent public paste — never unlisted/private', async () => {
++    await check('sitemap.xml lists the pages meant for search and never a link-shared paste (SEO-29)', async () => {
          const r = await t.get('/sitemap.xml');
          assert.strictEqual(r.status, 200);
          assert.ok(r.headers.get('content-type').includes('xml'));
          assert.ok(r.text.startsWith('<?xml version="1.0" encoding="UTF-8"?>'));
          assert.ok(r.text.includes('<loc>https://openvibe.community/</loc>'));
          assert.ok(r.text.includes('<loc>https://openvibe.community/pastes</loc>'));
--        assert.ok(r.text.includes('<loc>https://openvibe.community/p/amber-fox-42</loc>'));
--        assert.ok(r.text.includes('<lastmod>2026-09-15</lastmod>'));
--        assert.ok(r.text.includes('/p/fair-moon-23'));
--        assert.ok(!r.text.includes('dark-owl-55') && !r.text.includes('grim-vale-61'));
++        for (const slug of ['amber-fox-42', 'fair-moon-23', 'dark-owl-55', 'grim-vale-61']) {
++            assert.ok(!r.text.includes(`/p/${slug}`), `${slug} must not be in a sitemap`);
++        }
          assert.strictEqual(r.headers.get('cache-control'), cache.htmlHeaders({ maxAge: 3600 }));
      });
  
++    await check('a noindex page says so in the meta tag and in X-Robots-Tag; an indexable page sends no such header', async () => {
++        for (const path of ['/p/amber-fox-42', '/p/fair-moon-23', '/c/cth_0000000000000000000000', '/nope-not-a-page']) {
++            const r = await t.get(path);
++            assert.ok(r.headers.get('x-robots-tag') && /(?:^|,)\s*noindex/.test(r.headers.get('x-robots-tag')),
++                `${path}: X-Robots-Tag (got ${r.headers.get('x-robots-tag')})`);
++        }
++        assert.strictEqual((await t.get('/p/amber-fox-42')).headers.get('x-robots-tag'), 'noindex');
++        for (const path of ['/', '/pastes', '/updates']) {
++            assert.strictEqual((await t.get(path)).headers.get('x-robots-tag'), null, path);
++        }
++    });
++
      await check('feed.xml is a valid-looking RSS of the latest pastes', async () => {
          const r = await t.get('/feed.xml');
          assert.strictEqual(r.status, 200);
ddiff --git a/test/ssr-community.test.js b/test/ssr-community.test.js
iindex 64c1604..878c1e7 100644
---- a/test/ssr-community.test.js
++++ b/test/ssr-community.test.js
@@@ -49,7 +49,7 @@ const { boot, check, done } = require('./helpers/app');
  
      await check('unlisted: noindex and escaped; private: 404 for visitors, the owner gets the page and delete', async () => {
          const u = await t.get(`/p/${unl}`);
--        has(u.text, '<meta name="robots" content="noindex,follow">');
++        has(u.text, '<meta name="robots" content="noindex">');
          has(u.text, '&lt;script&gt;alert(1)&lt;/script&gt;');
          assert.ok(!u.text.includes('<script>alert(1)</script>'));
          assert.strictEqual((await t.get(`/p/${priv}`)).status, 404);
@@@ -156,13 +156,12 @@ const { boot, check, done } = require('./helpers/app');
          assert.strictEqual(api.status, 429, 'the API shares the budget');
      });
  
--    await check('feed and sitemap list store pastes', async () => {
++    await check('the feed links a paste, but no paste is in the sitemap (they are link-shared, noindex)', async () => {
          const feed = await t.get('/feed.xml');
          has(feed.text, `/p/${js}`);
          assert.ok(!feed.text.includes(priv));
          const map = await t.get('/sitemap.xml');
--        has(map.text, `/p/${js}`);
--        assert.ok(!map.text.includes(unl) && !map.text.includes(priv));
++        assert.ok(!map.text.includes(`/p/${js}`) && !map.text.includes(`/p/${unl}`) && !map.text.includes(`/p/${priv}`));
      });
  
      await t.close();
ddiff --git a/test/ssr.test.js b/test/ssr.test.js
iindex 3776e51..2c2da17 100644
---- a/test/ssr.test.js
++++ b/test/ssr.test.js
@@@ -109,7 +109,7 @@ const { boot, check, done } = require('./helpers/app');
          assert.strictEqual(r.status, 200);
          has(r.text, '<title>Hello world in JavaScript · OpenVibe.Community</title>');
          has(r.text, '<link rel="canonical" href="https://openvibe.community/p/amber-fox-42">');
--        has(r.text, '<meta name="robots" content="index,follow">');
++        has(r.text, '<meta name="robots" content="noindex">');
          has(r.text, '<meta property="og:type" content="article">');
          has(r.text, 'article:published_time');
          const ld = r.text.match(/<script type="application\/ld\+json">(.*?)<\/script>/g).map((s) => JSON.parse(s.replace(/<script[^>]*>|<\/script>/g, '')));
@@@ -136,7 +136,8 @@ const { boot, check, done } = require('./helpers/app');
      await check('unlisted paste is noindex and its content is escaped', async () => {
          const r = await t.get('/p/dark-owl-55');
          assert.strictEqual(r.status, 200);
--        has(r.text, '<meta name="robots" content="noindex,follow">');
++        has(r.text, '<meta name="robots" content="noindex">');
++        assert.strictEqual(r.headers.get('x-robots-tag'), 'noindex');
          has(r.text, '&lt;script&gt;alert(1)&lt;/script&gt;');
          assert.ok(!r.text.includes('<script>alert(1)</script>'));
      });