openvibers/openvibe.shared indexnow.js

indexnow.js

main 7.4 KB · 146 lines · javascript

Raw HistoryBlame Repository
1'use strict';
2/**
3 * openvibe-shared/indexnow — tell search engines a page changed, in one POST, per the IndexNow
4 * protocol. A site mounts it once at boot (serving its key file) and calls pingSoon() wherever it
5 * publishes; batching and debouncing are the module's job, because one publish touches a page, a
6 * sitemap and a feed and no engine wants three separate requests for that.
7 *
8 * const indexnow = require('openvibe-shared/indexnow').createIndexNow({ host: 'openvibe.wiki', key: process.env.INDEXNOW_KEY });
9 * app.get(`/${key}.txt`, indexnow.keyFile); // engines fetch the key from here
10 * indexnow.pingSoon(['/wiki/Mars', '/sitemap.xml']);
11 *
12 * The protocol: POST { host, key, keyLocation, urlList } to api.indexnow.org. Only https URLs that
13 * are on `host` belong in the batch (a foreign host is a different key's job), the list is deduped,
14 * and one batch may carry at most 10,000 URLs — more are dropped, because a request that large is
15 * refused anyway. Nothing here throws: a failed ping must never take a publish down with it, so the
16 * failure is logged once and { sent: 0, status } says what happened.
17 *
18 * A key is 8–128 hex or alphanumeric characters (what IndexNow's own tools generate); anything else
19 * is refused at createIndexNow() so a typo cannot quietly produce batches every engine rejects.
20 */
21const DEFAULT_ENDPOINT = 'https://api.indexnow.org/indexnow';
22const MAX_URL_LIST = 10000;
23// The protocol allows 10,000 URLs a request, but api.indexnow.org answered 403 to a 939-URL list from a host whose key
24// it accepted for one URL (2026-10-16); 100 at a time were all 200. A ping sends CHUNK at a time, CHUNK_GAP_MS apart.
25const CHUNK = 100;
26const CHUNK_GAP_MS = 1000;
27const BATCH_MS = 30000;
28const KEY_RE = /^[A-Za-z0-9]{8,128}$/;
29
30/** Accepts 'openvibe.wiki', 'https://openvibe.wiki/' or a full URL; yields the bare host the protocol wants. */
31function normaliseHost(host) {
32 const s = String(host || '').trim();
33 try { return new URL(s).host.toLowerCase(); } catch { /* not a URL: a bare host */ }
34 return s.replace(/^[a-z]+:\/\//i, '').replace(/\/.*$/, '').toLowerCase();
35}
36
37/**
38 * @param {object} o
39 * @param {string} o.host the site's host, e.g. 'openvibe.wiki' (a full origin also works)
40 * @param {string} o.key the IndexNow key; 8–128 hex/alnum. Unset disables pinging entirely.
41 * @param {string} [o.keyLocation] where the key file is served; defaults to `host` + '/' + key + '.txt'
42 * @param {string} [o.endpoint] the API endpoint (test and self-hosted servers override it)
43 * @param {Function} [o.fetch] the fetch to POST with; defaults to globalThis.fetch
44 * @param {Function} [o.log] where a failure is reported once; defaults to console.warn
45 * @returns {{keyFile: Function, ping: Function, pingSoon: Function, flush: Function, enabled: boolean}}
46 */
47function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, fetch: fetchImpl, log, chunkGapMs = CHUNK_GAP_MS } = {}) {
48 const theHost = normaliseHost(host);
49 const doFetch = fetchImpl || ((...args) => globalThis.fetch(...args));
50 const warn = typeof log === 'function' ? log : (...args) => console.warn('[indexnow]', ...args);
51 const keySet = key != null && String(key).trim() !== '';
52 if (keySet && !KEY_RE.test(String(key).trim())) throw new Error('openvibe-shared/indexnow: key must be 8–128 hex or alphanumeric characters');
53 const theKey = keySet ? String(key).trim() : null;
54 const location = keyLocation || (theHost && theKey ? `https://${theHost}/${theKey}.txt` : null);
55
56 let queued = new Set();
57 let timer = null;
58
59 /** Keep only absolute https URLs on this host, deduped, in insertion order. */
60 function clean(urls) {
61 const list = Array.isArray(urls) ? urls : urls == null ? [] : [urls];
62 const kept = [];
63 for (const raw of list) {
64 let u;
65 try { u = new URL(String(raw)); } catch { continue; }
66 if (u.protocol !== 'https:') continue;
67 if (normaliseHost(u.host) !== theHost) continue;
68 if (queued.has(u.toString()) || kept.includes(u.toString())) continue;
69 kept.push(u.toString());
70 }
71 return kept;
72 }
73
74 /**
75 * POST one batch now. Never throws.
76 * @returns {Promise<{sent: number, status: number|string, skipped?: string}>}
77 */
78 async function ping(urls) {
79 const all = clean(urls).slice(0, MAX_URL_LIST);
80 if (!theKey) return { sent: 0, status: 0, skipped: 'no key' };
81 if (!all.length) return { sent: 0, status: 0, skipped: 'nothing to send' };
82 if (all.length <= CHUNK) return await pingOne(all);
83 // Several requests: the URLs sent, and the first refusal's status (200 when every chunk was taken).
84 let sent = 0; let status = 200;
85 for (let i = 0; i < all.length; i += CHUNK) {
86 if (i) await new Promise((ok) => setTimeout(ok, chunkGapMs));
87 const r = await pingOne(all.slice(i, i + CHUNK));
88 sent += r.sent;
89 if (status === 200 && r.status !== 200 && r.status !== 202) status = r.status;
90 }
91 return { sent, status };
92 }
93
94 async function pingOne(urlList) {
95 const body = { host: theHost, key: theKey, keyLocation: location, urlList };
96 try {
97 const res = await doFetch(endpoint, {
98 method: 'POST',
99 headers: { 'Content-Type': 'application/json; charset=utf-8' },
100 body: JSON.stringify(body),
101 });
102 const status = res && typeof res.status === 'number' ? res.status : 0;
103 // 200 and 202 are both accepted by the protocol; anything else is the engine saying no.
104 if (status !== 200 && status !== 202) warn(`ping rejected with status ${status} for ${urlList.length} url(s)`);
105 return { sent: status === 200 || status === 202 ? urlList.length : 0, status };
106 } catch (err) {
107 warn(`ping failed for ${urlList.length} url(s): ${err && err.message ? err.message : err}`);
108 return { sent: 0, status: 0 };
109 }
110 }
111
112 /** Sends everything queued now and clears the queue (used by the debounce timer and by tests). */
113 async function flush() {
114 if (!timer) return { sent: 0, status: 0, skipped: 'empty' };
115 clearTimeout(timer);
116 timer = null;
117 const batch = [...queued];
118 queued = new Set();
119 return ping(batch);
120 }
121
122 /** Queue URLs and send one batch in 30s, coalescing everything published in the meantime. */
123 function pingSoon(urls) {
124 if (!theKey) return 0;
125 for (const u of clean(urls)) queued.add(u);
126 if (queued.size && !timer) {
127 timer = setTimeout(() => { void flush(); }, BATCH_MS); // ping never rejects
128 if (timer.unref) timer.unref();
129 }
130 return queued.size;
131 }
132
133 /** GET /<key>.txt — the key file the protocol requires engines to be able to fetch. */
134 function keyFile(req, res, next) {
135 if (typeof next === 'function' && (!req || !req.url || !new RegExp(`^/${theKey}\\.txt/?$`).test(String(req.url).split('?')[0]))) return next();
136 if (!theKey) { res.statusCode = 404; return res.end('no key'); }
137 res.setHeader('Content-Type', 'text/plain; charset=utf-8');
138 res.setHeader('Cache-Control', 'public, max-age=3600');
139 return res.end(theKey);
140 }
141
142 return { keyFile, ping, pingSoon, flush, enabled: Boolean(theKey) };
143}
144
145module.exports = { createIndexNow, KEY_RE, MAX_URL_LIST, CHUNK, BATCH_MS, DEFAULT_ENDPOINT };
146