Video: tell search engines about new recordings and clips (IndexNow) (#58)

ddiff --git a/.env.example b/.env.example
iindex 8d72212..9a0e086 100644
---- a/.env.example
++++ b/.env.example
@@@ -64,3 +64,5 @@ MEDIAHUB_VIDEO_URL=https://openvibe.video
  MEDIAHUB_LIVE_URL=https://openvibe.live
  MEDIAHUB_LIVE_PUBLIC_URL=https://openvibe.live
  MEDIAHUB_CHAT_URL=https://openvibe.live
++# IndexNow for openvibe.video (8–128 letters and digits): new watch pages are sent to search engines. Unset: off.
++#INDEXNOW_KEY=
ddiff --git a/README.md b/README.md
iindex 2dc99b9..2b9e4f1 100644
---- a/README.md
++++ b/README.md
@@@ -171,6 +171,7 @@ does not sign in on the other.
  | `/v/:id/captions.vtt` | anyone | the AI transcript as WebVTT captions |
  | `/v/:id/chapters.vtt` | anyone | the AI moments as WebVTT chapters |
  | `/sitemap.xml`, `/robots.txt`, `/llms.txt` | anyone | crawl artifacts for the video host |
++| `/<INDEXNOW_KEY>.txt` | anyone | IndexNow's key file, only with a key: every 10 minutes a sweep of Live's public feed sends new watch pages, their channel pages and the sitemap to search engines ([server/video/indexnow.js](server/video/indexnow.js)) |
  
  Configuration: `MEDIAHUB_VIDEO_URL` (default `https://openvibe.video`; its hostname is the video host),
  `MEDIAHUB_LIVE_URL` (default `https://openvibe.live`; production points it at the loopback address),
ddiff --git a/server/app.js b/server/app.js
iindex 87f53f7..1f4ab12 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -40,6 +40,7 @@ const { createService } = require('./drive/service');
  const { createUploads } = require('./drive/upload');
  const { createAccount, createSender } = require('./account');
  const { createPlaylists } = require('./video/playlists');
++const { createIndexNow } = require('openvibe-shared/indexnow');
  
  const PUBLIC_DIR = path.join(__dirname, '..', 'public');
  const VERSION = require('../package.json').version;
@@@ -65,6 +66,14 @@ async function createApp(opts = {}) {
      const account = createAccount({ db: s.db, config, media, log });
      const playlists = createPlaylists({ s, config, log });
      const ctx = { config, s, store, keys, sso, principal, media, identity, drive, uploads, account, playlists, fetchImpl, log };
++    // IndexNow for openvibe.video: off without a key; a malformed key warns and stays off rather than stopping the site.
++    ctx.indexnow = opts.indexnow || (() => {
++        const warn = (...a) => (log.warn || console.warn)(...a);
++        try { return createIndexNow({ host: config.video.url, key: config.video.indexnowKey, fetch: fetchImpl, log: warn }); } catch (err) {
++            warn(`[MediaHub] ${err.message} — IndexNow is off until INDEXNOW_KEY is fixed`);
++            return createIndexNow({ host: config.video.url, key: '', fetch: fetchImpl, log: warn });
++        }
++    })();
  
      const app = express();
      app.disable('x-powered-by');
@@@ -130,6 +139,8 @@ async function createApp(opts = {}) {
      // the drive. On the video host the drive's pages, API and legal pages never answer.
      const videoHost = (() => { try { return new URL(config.video.url).hostname; } catch { return null; } })();
      const videoRoutes = createVideoRoutes(ctx);
++    // GET /<key>.txt on the video host: IndexNow's key file, only when a key is configured.
++    if (ctx.indexnow.enabled) app.use((req, res, next) => (videoHost && req.hostname === videoHost ? ctx.indexnow.keyFile(req, res, next) : next()));
      app.use((req, res, next) => {
          if (videoHost && req.hostname === videoHost) return videoRoutes(req, res, next);
          return next();
ddiff --git a/server/config.js b/server/config.js
iindex 20d1439..769b4be 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -66,6 +66,9 @@ function load(env = process.env) {
              // Playlists (step C): the per-person caps the pages and the API enforce together.
              playlistsMax: size(env.MEDIAHUB_VIDEO_PLAYLISTS_MAX, 100),
              playlistItemsMax: size(env.MEDIAHUB_VIDEO_PLAYLIST_ITEMS_MAX, 500),
++            // IndexNow (server/video/indexnow.js): with a key, new watch pages are sent to search engines and the key
++            // is served at https://openvibe.video/<key>.txt. Unset: off.
++            indexnowKey: String(env.INDEXNOW_KEY || '').trim(),
          },
  
          // What one person may keep and do here. Media's own quotas are per tenant (this whole service is one
ddiff --git a/server/index.js b/server/index.js
iindex 42e34f8..139e789 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -31,7 +31,12 @@ async function start() {
      // Subscribe to the two ADR-033 topics at OpenVibe.Events (idempotent; off without MEDIAHUB_EVENTS_URL and
      // MEDIAHUB_EVENTS_SECRET). The consumer itself is mounted in server/app.js.
      const subscriptions = require('./events-consumer').startSubscriptions({ config, port: config.port, secret: config.events.secrets[0] || '' });
--    createLifecycle({ server, ctx, extra: [() => { if (subscriptions) subscriptions.stop(); }] });
++    // New watch pages on openvibe.video reach search engines (server/video/indexnow.js; off without INDEXNOW_KEY).
++    const { createVideoIndexNow } = require('./video/indexnow');
++    const { createLiveClient } = require('./video/live-client');
++    const videoIndexNow = createVideoIndexNow({ config, live: createLiveClient({ config, fetchImpl: ctx.fetchImpl, log: ctx.log }), indexnow: ctx.indexnow, log: ctx.log });
++    videoIndexNow.start();
++    createLifecycle({ server, ctx, extra: [() => videoIndexNow.stop(), () => { if (subscriptions) subscriptions.stop(); }] });
      return { server, ctx };
  }
  
ddiff --git a/server/video/indexnow.js b/server/video/indexnow.js
nnew file mode 100644
iindex 0000000..326b4cd
---- /dev/null
++++ b/server/video/indexnow.js
@@@ -0,0 +1,72 @@
++'use strict';
++/**
++ * openvibe.video tells search engines about new watch pages (IndexNow, openvibe-shared/indexnow).
++ *
++ * Live publishes the recordings and clips, and nothing tells this service when one becomes public, so a sweep reads
++ * Live's public feed (the newest recordings and clips, signed out) every few minutes. The first read only remembers
++ * what is already there; each later read pings the pages it has not seen before: the watch page, its channel page and
++ * the sitemap, in one batch. A restart remembers afresh, so a page published while the process was down reaches the
++ * engines through the sitemap instead of a ping. A sweep that cannot read both feeds pings nothing and remembers
++ * nothing, so a Live outage never primes the memory with half a feed.
++ *
++ * Off without a key (INDEXNOW_KEY): start() does nothing and sweep() reads nothing.
++ */
++const { USERNAME_RE } = require('./live-client');
++
++function createVideoIndexNow({ config, live, indexnow, log = console, intervalMs = 10 * 60_000, limit = 50, maxRemembered = 5000 }) {
++    const base = config.video.url.replace(/\/$/, '');
++    const seen = new Set();
++    let primed = false;
++    let running = false;
++    let timer = null;
++
++    async function sweep() {
++        if (!indexnow || !indexnow.enabled || running) return { pinged: 0 };
++        running = true;
++        try {
++            let pages;
++            try {
++                pages = await Promise.all(['vods', 'clips'].map((type) => live.feed({ type, limit })));
++            } catch (err) {
++                log.warn(`[Video IndexNow] the feed could not be read: ${(err && err.message) || err}`);
++                return { pinged: 0, skipped: 'feed' };
++            }
++            const urls = [];
++            let fresh = 0;
++            for (const it of pages.flatMap((p) => (p && Array.isArray(p.items) ? p.items : []))) {
++                const id = Number(it && it.id);
++                if (!Number.isSafeInteger(id) || id <= 0) continue;
++                const path = `${it.kind === 'clip' ? '/c/' : '/v/'}${id}`;
++                if (seen.has(path)) continue;
++                seen.add(path);
++                if (!primed) continue;
++                fresh++;
++                urls.push(`${base}${path}`);
++                const u = it.channel && it.channel.username;
++                if (u && USERNAME_RE.test(u)) urls.push(`${base}/@${u}`);
++            }
++            primed = true;
++            while (seen.size > maxRemembered) seen.delete(seen.values().next().value);
++            if (urls.length) indexnow.pingSoon([...urls, `${base}/sitemap.xml`]);
++            return { pinged: fresh };
++        } finally { running = false; }
++    }
++
++    function start() {
++        if (!indexnow || !indexnow.enabled || timer) return false;
++        const tick = () => { sweep().catch((err) => log.warn(`[Video IndexNow] sweep failed: ${(err && err.message) || err}`)); };
++        tick();
++        timer = setInterval(tick, intervalMs);
++        if (timer.unref) timer.unref();
++        return true;
++    }
++
++    function stop() {
++        if (timer) clearInterval(timer);
++        timer = null;
++    }
++
++    return { sweep, start, stop };
++}
++
++module.exports = { createVideoIndexNow };
ddiff --git a/test/video-indexnow.test.js b/test/video-indexnow.test.js
nnew file mode 100644
iindex 0000000..264e57d
---- /dev/null
++++ b/test/video-indexnow.test.js
@@@ -0,0 +1,117 @@
++'use strict';
++/**
++ * openvibe.video's IndexNow (server/video/indexnow.js): the first sweep only remembers Live's feed; a later sweep
++ * pings each new watch page with its channel page and the sitemap, once; a feed Live cannot serve pings and
++ * remembers nothing; without a key nothing is read. The key file answers on the video host only, and only with a key.
++ */
++const assert = require('assert');
++const http = require('http');
++const { boot, check, done } = require('./helpers/boot');
++const { createVideoIndexNow } = require('../server/video/indexnow');
++
++const config = { video: { url: 'https://openvibe.video' } };
++const item = (kind, id, username) => ({ kind, id, title: `${kind} ${id}`, channel: { username } });
++
++function fakeLive(feeds) {
++    const live = { calls: 0, down: false };
++    live.feed = async ({ type }) => {
++        live.calls++;
++        if (live.down) throw new Error('Live did not answer');
++        return { items: feeds[type].slice(), next: null };
++    };
++    return live;
++}
++function fakeIndexNow(enabled = true) {
++    const pinged = [];
++    return { enabled, pinged, pingSoon: (urls) => { pinged.push(...urls); return urls.length; } };
++}
++const quiet = { warn() {}, log() {} };
++
++function request(base, path, host) {
++    return new Promise((resolve, reject) => {
++        const u = new URL(base + path);
++        const req = http.request({ hostname: u.hostname, port: u.port, path: u.pathname, method: 'GET', headers: { Host: host } }, (res) => {
++            const chunks = [];
++            res.on('data', (c) => chunks.push(c));
++            res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, text: Buffer.concat(chunks).toString('utf8') }));
++        });
++        req.on('error', reject);
++        req.end();
++    });
++}
++
++(async () => {
++    await check('the first sweep remembers what is there; a later one pings each new page, its channel and the sitemap, once', async () => {
++        const feeds = { vods: [item('vod', 101, 'streamer')], clips: [item('clip', 201, 'clipper')] };
++        const live = fakeLive(feeds);
++        const indexnow = fakeIndexNow();
++        const v = createVideoIndexNow({ config, live, indexnow, log: quiet });
++        assert.deepStrictEqual(await v.sweep(), { pinged: 0 });
++        assert.deepStrictEqual(indexnow.pinged, [], 'what was already there is not announced');
++
++        feeds.vods.unshift(item('vod', 102, 'streamer'));
++        feeds.clips.unshift(item('clip', 202, 'bad name!'));
++        assert.deepStrictEqual(await v.sweep(), { pinged: 2 });
++        assert.deepStrictEqual(indexnow.pinged, [
++            'https://openvibe.video/v/102', 'https://openvibe.video/@streamer',
++            'https://openvibe.video/c/202',
++            'https://openvibe.video/sitemap.xml',
++        ], 'a channel name that is not a username is left out');
++
++        indexnow.pinged.length = 0;
++        assert.deepStrictEqual(await v.sweep(), { pinged: 0 });
++        assert.deepStrictEqual(indexnow.pinged, [], 'a page is announced once');
++    });
++
++    await check('a feed Live cannot serve pings nothing and primes nothing', async () => {
++        const feeds = { vods: [item('vod', 101, 'streamer')], clips: [] };
++        const live = fakeLive(feeds);
++        const indexnow = fakeIndexNow();
++        const v = createVideoIndexNow({ config, live, indexnow, log: quiet });
++        live.down = true;
++        assert.deepStrictEqual(await v.sweep(), { pinged: 0, skipped: 'feed' });
++        live.down = false;
++        assert.deepStrictEqual(await v.sweep(), { pinged: 0 }, 'the first good read only remembers');
++        assert.deepStrictEqual(indexnow.pinged, []);
++    });
++
++    await check('without a key nothing is read and nothing starts', async () => {
++        const live = fakeLive({ vods: [], clips: [] });
++        const v = createVideoIndexNow({ config, live, indexnow: fakeIndexNow(false), log: quiet });
++        assert.deepStrictEqual(await v.sweep(), { pinged: 0 });
++        assert.strictEqual(v.start(), false);
++        assert.strictEqual(live.calls, 0);
++    });
++
++    await check('the memory stays bounded', async () => {
++        const feeds = { vods: [], clips: [] };
++        for (let i = 1; i <= 30; i++) feeds.vods.push(item('vod', i, 'streamer'));
++        const indexnow = fakeIndexNow();
++        const v = createVideoIndexNow({ config, live: fakeLive(feeds), indexnow, log: quiet, maxRemembered: 10 });
++        await v.sweep();
++        feeds.vods = [item('vod', 31, 'streamer')];
++        assert.deepStrictEqual(await v.sweep(), { pinged: 1 });
++    });
++
++    // The key file, served by the app on the video host only (a computed stand-in key).
++    const key = ['video', 'index', 'now'].join('') + '2468';
++    const t = await boot({ env: { MEDIAHUB_VIDEO_URL: 'https://openvibe.video', INDEXNOW_KEY: key } });
++    try {
++        await check('with a key, /<key>.txt on openvibe.video answers the key as text/plain', async () => {
++            const r = await request(t.base, `/${key}.txt`, 'openvibe.video');
++            assert.strictEqual(r.status, 200);
++            assert.match(r.headers['content-type'], /^text\/plain/);
++            assert.strictEqual(r.text, key);
++            const d = await request(t.base, `/${key}.txt`, 'openvibe.download');
++            assert.notStrictEqual(d.text, key, 'the drive host does not serve the video host\'s key');
++        });
++    } finally { await t.close(); }
++    const off = await boot({ env: { MEDIAHUB_VIDEO_URL: 'https://openvibe.video' } });
++    try {
++        await check('without a key there is no key file', async () => {
++            const r = await request(off.base, `/${key}.txt`, 'openvibe.video');
++            assert.strictEqual(r.status, 404);
++        });
++    } finally { await off.close(); }
++    done();
++})().catch((e) => { console.error(e); process.exit(1); });