openvibers/openvibe.shared Commits
Shared 4.2.2: IndexNow sends 100 URLs a request (api.indexnow.org refused a 939-URL list) (#83)
Changes
@@@ -4,6 +4,13 @@ All notable changes to `openvibe-shared`. Versions follow [semver](https://semvebreaking change to any exported module, browser global or served file name is a new major.A release is the git tag `vX.Y.Z`; consumers pin the tag's tarball (see README).++## 4.2.2 — 2026-10-16++++- **IndexNow sends 100 URLs a request.** `ping()` (and the debounced `pingSoon()` flush) splits a list into requests of++ 100 URLs, a second apart, and returns the URLs sent and the first refusal's status. api.indexnow.org answered 403 to++ a 939-URL list from a host whose key it accepted for one URL; 100 at a time were all 200. `createIndexNow` takes++ `chunkGapMs` (tests pass 0); `CHUNK` is exported.++## 4.2.1 — 2026-10-15- **A showcase table that scrolls is a focusable region.** `compare()` and `limits()` wrap their table in@@@ -20,6 +20,10 @@*/const DEFAULT_ENDPOINT = 'https://api.indexnow.org/indexnow';const MAX_URL_LIST = 10000;++// The protocol allows 10,000 URLs a request, but api.indexnow.org answered 403 to a 939-URL list from a host whose key++// it accepted for one URL (2026-10-16); 100 at a time were all 200. A ping sends CHUNK at a time, CHUNK_GAP_MS apart.++const CHUNK = 100;++const CHUNK_GAP_MS = 1000;const BATCH_MS = 30000;const KEY_RE = /^[A-Za-z0-9]{8,128}$/;@@@ -40,7 +44,7 @@ function normaliseHost(host) {* @param {Function} [o.log] where a failure is reported once; defaults to console.warn* @returns {{keyFile: Function, ping: Function, pingSoon: Function, flush: Function, enabled: boolean}}*/--function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, fetch: fetchImpl, log } = {}) {++function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, fetch: fetchImpl, log, chunkGapMs = CHUNK_GAP_MS } = {}) {const theHost = normaliseHost(host);const doFetch = fetchImpl || ((...args) => globalThis.fetch(...args));const warn = typeof log === 'function' ? log : (...args) => console.warn('[indexnow]', ...args);@@@ -72,9 +76,22 @@ function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, f* @returns {Promise<{sent: number, status: number|string, skipped?: string}>}*/async function ping(urls) {-- const urlList = clean(urls).slice(0, MAX_URL_LIST);++ const all = clean(urls).slice(0, MAX_URL_LIST);if (!theKey) return { sent: 0, status: 0, skipped: 'no key' };-- if (!urlList.length) return { sent: 0, status: 0, skipped: 'nothing to send' };++ if (!all.length) return { sent: 0, status: 0, skipped: 'nothing to send' };++ if (all.length <= CHUNK) return await pingOne(all);++ // Several requests: the URLs sent, and the first refusal's status (200 when every chunk was taken).++ let sent = 0; let status = 200;++ for (let i = 0; i < all.length; i += CHUNK) {++ if (i) await new Promise((ok) => setTimeout(ok, chunkGapMs));++ const r = await pingOne(all.slice(i, i + CHUNK));++ sent += r.sent;++ if (status === 200 && r.status !== 200 && r.status !== 202) status = r.status;++ }++ return { sent, status };++ }++++ async function pingOne(urlList) {const body = { host: theHost, key: theKey, keyLocation: location, urlList };try {const res = await doFetch(endpoint, {@@@ -125,4 +142,4 @@ function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, freturn { keyFile, ping, pingSoon, flush, enabled: Boolean(theKey) };}--module.exports = { createIndexNow, KEY_RE, MAX_URL_LIST, BATCH_MS, DEFAULT_ENDPOINT };++module.exports = { createIndexNow, KEY_RE, MAX_URL_LIST, CHUNK, BATCH_MS, DEFAULT_ENDPOINT };@@@ -1,12 +1,12 @@{"name": "openvibe-shared",-- "version": "4.2.1",++ "version": "4.2.2","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "openvibe-shared",-- "version": "4.2.1",++ "version": "4.2.2","license": "MIT","dependencies": {"jsonwebtoken": "^9.0.0"@@@ -1,6 +1,6 @@{"name": "openvibe-shared",-- "version": "4.2.1",++ "version": "4.2.2","description": "Shared navbar, footer, themes, icons, legal pages, SEO, auth and analytics for every OpenVibe site","main": "index.js","bin": {@@@ -95,5 +95,17 @@ function recorder(status = 200) {let fell = false;inw.keyFile({ url: '/wiki/Mars' }, { setHeader() {}, end() {} }, () => { fell = true; });assert.ok(fell, 'a different path is not the key file');++++ // A big list goes out 100 URLs a request (api.indexnow.org refused a 939-URL list), the results summed; the first++ // refusal is the status.++ const big = recorder(200);++ const many = Array.from({ length: 250 }, (_, i) => `https://openvibe.wiki/p/${i}`);++ const bigOut = await createIndexNow({ host: 'openvibe.wiki', key: 'a1b2c3d4', fetch: big.fetch, chunkGapMs: 0 }).ping(many);++ assert.deepStrictEqual(big.calls.map((c) => c.body.urlList.length), [100, 100, 50]);++ assert.deepStrictEqual(bigOut, { sent: 250, status: 200 });++ let n = 0;++ const mixed = { fetch: async () => ({ status: ++n === 2 ? 403 : 200 }) };++ const mixedOut = await createIndexNow({ host: 'openvibe.wiki', key: 'a1b2c3d4', fetch: mixed.fetch, chunkGapMs: 0, log: () => {} }).ping(many);++ assert.deepStrictEqual(mixedOut, { sent: 150, status: 403 });console.log('indexnow: all checks passed');})().catch((e) => { console.error(e); process.exit(1); });
@@ -4,6 +4,13 @@ All notable changes to `openvibe-shared`. Versions follow [semver](https://semve
@@ -4,6 +4,13 @@ All notable changes to `openvibe-shared`. Versions follow [semver](https://semve
breaking change to any exported module, browser global or served file name is a new major.
breaking change to any exported module, browser global or served file name is a new major.
A release is the git tag `vX.Y.Z`; consumers pin the tag's tarball (see README).
A release is the git tag `vX.Y.Z`; consumers pin the tag's tarball (see README).
+## 4.2.2 — 2026-10-16
+
+- **IndexNow sends 100 URLs a request.** `ping()` (and the debounced `pingSoon()` flush) splits a list into requests of
+ 100 URLs, a second apart, and returns the URLs sent and the first refusal's status. api.indexnow.org answered 403 to
+ a 939-URL list from a host whose key it accepted for one URL; 100 at a time were all 200. `createIndexNow` takes
+ `chunkGapMs` (tests pass 0); `CHUNK` is exported.
+
## 4.2.1 — 2026-10-15
## 4.2.1 — 2026-10-15
- **A showcase table that scrolls is a focusable region.** `compare()` and `limits()` wrap their table in
- **A showcase table that scrolls is a focusable region.** `compare()` and `limits()` wrap their table in
@@ -20,6 +20,10 @@
@@ -20,6 +20,10 @@
*/
*/
const DEFAULT_ENDPOINT = 'https://api.indexnow.org/indexnow';
const DEFAULT_ENDPOINT = 'https://api.indexnow.org/indexnow';
const MAX_URL_LIST = 10000;
const MAX_URL_LIST = 10000;
+// The protocol allows 10,000 URLs a request, but api.indexnow.org answered 403 to a 939-URL list from a host whose key
+// it accepted for one URL (2026-10-16); 100 at a time were all 200. A ping sends CHUNK at a time, CHUNK_GAP_MS apart.
+const CHUNK = 100;
+const CHUNK_GAP_MS = 1000;
const BATCH_MS = 30000;
const BATCH_MS = 30000;
const KEY_RE = /^[A-Za-z0-9]{8,128}$/;
const KEY_RE = /^[A-Za-z0-9]{8,128}$/;
@@ -40,7 +44,7 @@ function normaliseHost(host) {
@@ -40,7 +44,7 @@ function normaliseHost(host) {
* @param {Function} [o.log] where a failure is reported once; defaults to console.warn
* @param {Function} [o.log] where a failure is reported once; defaults to console.warn
* @returns {{keyFile: Function, ping: Function, pingSoon: Function, flush: Function, enabled: boolean}}
* @returns {{keyFile: Function, ping: Function, pingSoon: Function, flush: Function, enabled: boolean}}
*/
*/
-function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, fetch: fetchImpl, log } = {}) {
+function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, fetch: fetchImpl, log, chunkGapMs = CHUNK_GAP_MS } = {}) {
const theHost = normaliseHost(host);
const theHost = normaliseHost(host);
const doFetch = fetchImpl || ((...args) => globalThis.fetch(...args));
const doFetch = fetchImpl || ((...args) => globalThis.fetch(...args));
const warn = typeof log === 'function' ? log : (...args) => console.warn('[indexnow]', ...args);
const warn = typeof log === 'function' ? log : (...args) => console.warn('[indexnow]', ...args);
@@ -72,9 +76,22 @@ function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, f
@@ -72,9 +76,22 @@ function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, f
* @returns {Promise<{sent: number, status: number|string, skipped?: string}>}
* @returns {Promise<{sent: number, status: number|string, skipped?: string}>}
*/
*/
async function ping(urls) {
async function ping(urls) {
- const urlList = clean(urls).slice(0, MAX_URL_LIST);
+ const all = clean(urls).slice(0, MAX_URL_LIST);
if (!theKey) return { sent: 0, status: 0, skipped: 'no key' };
if (!theKey) return { sent: 0, status: 0, skipped: 'no key' };
- if (!urlList.length) return { sent: 0, status: 0, skipped: 'nothing to send' };
+ if (!all.length) return { sent: 0, status: 0, skipped: 'nothing to send' };
+ if (all.length <= CHUNK) return await pingOne(all);
+ // Several requests: the URLs sent, and the first refusal's status (200 when every chunk was taken).
+ let sent = 0; let status = 200;
+ for (let i = 0; i < all.length; i += CHUNK) {
+ if (i) await new Promise((ok) => setTimeout(ok, chunkGapMs));
+ const r = await pingOne(all.slice(i, i + CHUNK));
+ sent += r.sent;
+ if (status === 200 && r.status !== 200 && r.status !== 202) status = r.status;
+ }
+ return { sent, status };
+ }
+
+ async function pingOne(urlList) {
const body = { host: theHost, key: theKey, keyLocation: location, urlList };
const body = { host: theHost, key: theKey, keyLocation: location, urlList };
try {
try {
const res = await doFetch(endpoint, {
const res = await doFetch(endpoint, {
@@ -125,4 +142,4 @@ function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, f
@@ -125,4 +142,4 @@ function createIndexNow({ host, key, keyLocation, endpoint = DEFAULT_ENDPOINT, f
return { keyFile, ping, pingSoon, flush, enabled: Boolean(theKey) };
return { keyFile, ping, pingSoon, flush, enabled: Boolean(theKey) };
}
}
-module.exports = { createIndexNow, KEY_RE, MAX_URL_LIST, BATCH_MS, DEFAULT_ENDPOINT };
+module.exports = { createIndexNow, KEY_RE, MAX_URL_LIST, CHUNK, BATCH_MS, DEFAULT_ENDPOINT };
@@ -1,12 +1,12 @@
@@ -1,12 +1,12 @@
{
{
"name": "openvibe-shared",
"name": "openvibe-shared",
- "version": "4.2.1",
+ "version": "4.2.2",
"lockfileVersion": 3,
"lockfileVersion": 3,
"requires": true,
"requires": true,
"packages": {
"packages": {
"": {
"": {
"name": "openvibe-shared",
"name": "openvibe-shared",
- "version": "4.2.1",
+ "version": "4.2.2",
"license": "MIT",
"license": "MIT",
"dependencies": {
"dependencies": {
"jsonwebtoken": "^9.0.0"
"jsonwebtoken": "^9.0.0"
@@ -1,6 +1,6 @@
@@ -1,6 +1,6 @@
{
{
"name": "openvibe-shared",
"name": "openvibe-shared",
- "version": "4.2.1",
+ "version": "4.2.2",
"description": "Shared navbar, footer, themes, icons, legal pages, SEO, auth and analytics for every OpenVibe site",
"description": "Shared navbar, footer, themes, icons, legal pages, SEO, auth and analytics for every OpenVibe site",
"main": "index.js",
"main": "index.js",
"bin": {
"bin": {
@@ -95,5 +95,17 @@ function recorder(status = 200) {
@@ -95,5 +95,17 @@ function recorder(status = 200) {
let fell = false;
let fell = false;
inw.keyFile({ url: '/wiki/Mars' }, { setHeader() {}, end() {} }, () => { fell = true; });
inw.keyFile({ url: '/wiki/Mars' }, { setHeader() {}, end() {} }, () => { fell = true; });
assert.ok(fell, 'a different path is not the key file');
assert.ok(fell, 'a different path is not the key file');
+
+ // A big list goes out 100 URLs a request (api.indexnow.org refused a 939-URL list), the results summed; the first
+ // refusal is the status.
+ const big = recorder(200);
+ const many = Array.from({ length: 250 }, (_, i) => `https://openvibe.wiki/p/${i}`);
+ const bigOut = await createIndexNow({ host: 'openvibe.wiki', key: 'a1b2c3d4', fetch: big.fetch, chunkGapMs: 0 }).ping(many);
+ assert.deepStrictEqual(big.calls.map((c) => c.body.urlList.length), [100, 100, 50]);
+ assert.deepStrictEqual(bigOut, { sent: 250, status: 200 });
+ let n = 0;
+ const mixed = { fetch: async () => ({ status: ++n === 2 ? 403 : 200 }) };
+ const mixedOut = await createIndexNow({ host: 'openvibe.wiki', key: 'a1b2c3d4', fetch: mixed.fetch, chunkGapMs: 0, log: () => {} }).ping(many);
+ assert.deepStrictEqual(mixedOut, { sent: 150, status: 403 });
console.log('indexnow: all checks passed');
console.log('indexnow: all checks passed');
})().catch((e) => { console.error(e); process.exit(1); });
})().catch((e) => { console.error(e); process.exit(1); });