openvibers/openvibe.shared Commits

Shared 4.2.2: IndexNow sends 100 URLs a request (api.indexnow.org refused a 939-URL list) (#83)

OP OpenVibers committed 28 days ago · ce3bd52388 · 5 files, +43 −7

Changes

ddiff --git a/CHANGELOG.md b/CHANGELOG.md
iindex 9d2d367..fcb0bd0 100644
---- a/CHANGELOG.md
++++ b/CHANGELOG.md
@@@ -4,6 +4,13 @@ All notable changes to `openvibe-shared`. Versions follow [semver](https://semve
breaking change to any exported module, browser global or served file name is a new major.
A release is the git tag `vX.Y.Z`; consumers pin the tag's tarball (see README).
++## 4.2.2 — 2026-10-16
++
++- **IndexNow sends 100 URLs a request.** `ping()` (and the debounced `pingSoon()` flush) splits a list into requests of
++ 100 URLs, a second apart, and returns the URLs sent and the first refusal's status. api.indexnow.org answered 403 to
++ a 939-URL list from a host whose key it accepted for one URL; 100 at a time were all 200. `createIndexNow` takes
++ `chunkGapMs` (tests pass 0); `CHUNK` is exported.
++
## 4.2.1 — 2026-10-15
- **A showcase table that scrolls is a focusable region.** `compare()` and `limits()` wrap their table in
ddiff --git a/indexnow.js b/indexnow.js
iindex 8194afa..b7df044 100644
---- a/indexnow.js
++++ b/indexnow.js
@@@ -20,6 +20,10 @@
*/
const DEFAULT_ENDPOINT = 'https://api.indexnow.org/indexnow';
const MAX_URL_LIST = 10000;
++// The protocol allows 10,000 URLs a request, but api.indexnow.org answered 403 to a 939-URL list from a host whose key
++// it accepted for one URL (2026-10-16); 100 at a time were all 200. A ping sends CHUNK at a time, CHUNK_GAP_MS apart.
++const CHUNK = 100;
++const CHUNK_GAP_MS = 1000;
const BATCH_MS = 30000;
const KEY_RE = /^[A-Za-z0-9]{8,128}$/;
@@@ -40,7 +44,7 @@ function normaliseHost(host) {
* @param {Function} [o.log] where a failure is reported once; defaults to console.warn
* @returns {{keyFile: Function, ping: Function, pingSoon: Function, flush: Function, enabled: boolean}}
*/
--function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, fetch: fetchImpl, log } = {}) {
++function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, fetch: fetchImpl, log, chunkGapMs = CHUNK_GAP_MS } = {}) {
const theHost = normaliseHost(host);
const doFetch = fetchImpl || ((...args) => globalThis.fetch(...args));
const warn = typeof log === 'function' ? log : (...args) => console.warn('[indexnow]', ...args);
@@@ -72,9 +76,22 @@ function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, f
* @returns {Promise<{sent: number, status: number|string, skipped?: string}>}
*/
async function ping(urls) {
-- const urlList = clean(urls).slice(0, MAX_URL_LIST);
++ const all = clean(urls).slice(0, MAX_URL_LIST);
if (!theKey) return { sent: 0, status: 0, skipped: 'no key' };
-- if (!urlList.length) return { sent: 0, status: 0, skipped: 'nothing to send' };
++ if (!all.length) return { sent: 0, status: 0, skipped: 'nothing to send' };
++ if (all.length <= CHUNK) return await pingOne(all);
++ // Several requests: the URLs sent, and the first refusal's status (200 when every chunk was taken).
++ let sent = 0; let status = 200;
++ for (let i = 0; i < all.length; i += CHUNK) {
++ if (i) await new Promise((ok) => setTimeout(ok, chunkGapMs));
++ const r = await pingOne(all.slice(i, i + CHUNK));
++ sent += r.sent;
++ if (status === 200 && r.status !== 200 && r.status !== 202) status = r.status;
++ }
++ return { sent, status };
++ }
++
++ async function pingOne(urlList) {
const body = { host: theHost, key: theKey, keyLocation: location, urlList };
try {
const res = await doFetch(endpoint, {
@@@ -125,4 +142,4 @@ function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, f
return { keyFile, ping, pingSoon, flush, enabled: Boolean(theKey) };
}
--module.exports = { createIndexNow, KEY_RE, MAX_URL_LIST, BATCH_MS, DEFAULT_ENDPOINT };
++module.exports = { createIndexNow, KEY_RE, MAX_URL_LIST, CHUNK, BATCH_MS, DEFAULT_ENDPOINT };
ddiff --git a/package-lock.json b/package-lock.json
iindex a98c4df..0c4eed5 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -1,12 +1,12 @@
{
"name": "openvibe-shared",
-- "version": "4.2.1",
++ "version": "4.2.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "openvibe-shared",
-- "version": "4.2.1",
++ "version": "4.2.2",
"license": "MIT",
"dependencies": {
"jsonwebtoken": "^9.0.0"
ddiff --git a/package.json b/package.json
iindex e9f7ae1..be4a651 100644
---- a/package.json
++++ b/package.json
@@@ -1,6 +1,6 @@
{
"name": "openvibe-shared",
-- "version": "4.2.1",
++ "version": "4.2.2",
"description": "Shared navbar, footer, themes, icons, legal pages, SEO, auth and analytics for every OpenVibe site",
"main": "index.js",
"bin": {
ddiff --git a/test/indexnow.test.js b/test/indexnow.test.js
iindex c32f240..168c069 100644
---- a/test/indexnow.test.js
++++ b/test/indexnow.test.js
@@@ -95,5 +95,17 @@ function recorder(status = 200) {
let fell = false;
inw.keyFile({ url: '/wiki/Mars' }, { setHeader() {}, end() {} }, () => { fell = true; });
assert.ok(fell, 'a different path is not the key file');
++
++ // A big list goes out 100 URLs a request (api.indexnow.org refused a 939-URL list), the results summed; the first
++ // refusal is the status.
++ const big = recorder(200);
++ const many = Array.from({ length: 250 }, (_, i) => `https://openvibe.wiki/p/${i}`);
++ const bigOut = await createIndexNow({ host: 'openvibe.wiki', key: 'a1b2c3d4', fetch: big.fetch, chunkGapMs: 0 }).ping(many);
++ assert.deepStrictEqual(big.calls.map((c) => c.body.urlList.length), [100, 100, 50]);
++ assert.deepStrictEqual(bigOut, { sent: 250, status: 200 });
++ let n = 0;
++ const mixed = { fetch: async () => ({ status: ++n === 2 ? 403 : 200 }) };
++ const mixedOut = await createIndexNow({ host: 'openvibe.wiki', key: 'a1b2c3d4', fetch: mixed.fetch, chunkGapMs: 0, log: () => {} }).ping(many);
++ assert.deepStrictEqual(mixedOut, { sent: 150, status: 403 });
console.log('indexnow: all checks passed');
})().catch((e) => { console.error(e); process.exit(1); });