openvibers/openvibe.tips Commits
Add IndexNow key file and ping creator pages on change
Changes
@@@ -93,6 +93,12 @@ TIPS_EVENTS_SECRET=#TIPS_LIMITS_MINUTE=120#TIPS_LIMITS_HOUR=3000++# ── IndexNow ────────────────────────────────────────────────++# IndexNow (openvibe-shared/indexnow): 8–128 hex/alphanumeric characters. Set: the key file is served++# at /<key>.txt and a creator's public page appearing, changing or going away pings the engines.++# Unset: off, nothing is mounted and nothing is sent.++#INDEXNOW_KEY=++# ── Background jobs: effects worker, due transfers, overlay window, outbox relay ──#TIPS_JOBS=on#TIPS_JOBS_INTERVAL_MS=2000@@@ -22,6 +22,7 @@ const cookieParser = require('cookie-parser');const { http } = require('openvibe-contracts');const { isLoopbackDirect } = require('openvibe-shared/metrics');const cache = require('openvibe-shared/cache-policy');++const { createIndexNow } = require('openvibe-shared/indexnow');const { loadConfig } = require('./config');const { openDb } = require('./db');const { createKeyProvider, createUserAuth } = require('./network');@@@ -55,7 +56,11 @@ function createApp(opts = {}) {const billing = opts.billing || createBillingClient(config, { fetchImpl });const adapters = opts.adapters || createAdapters(config, { fetchImpl });const outbox = opts.outbox || createTipsOutbox({ db, config, fetchImpl: opts.eventsFetch, now, log });-- const domain = createDomain({ db, config, outbox, billing, adapters, valkey, now, log });++ // IndexNow (openvibe-shared/indexnow): created once at boot from INDEXNOW_KEY; unset → off (nothing++ // mounted, nothing sent). The key file is served at /<key>.txt and the profile change pings the++ // engines when a creator's public page appears, changes or goes away.++ const indexnow = opts.indexnow || createIndexNow({ host: config.baseUrl, key: config.indexnow.key, ...(fetchImpl ? { fetch: fetchImpl } : {}), log });++ const domain = createDomain({ db, config, outbox, billing, adapters, valkey, now, log, indexnow });const apiAuth = createApiAuth({ config, keys, userAuth });const release = require('openvibe-shared/release').createRelease({ service: 'tips', root: path.join(__dirname, '..') });require('./web/layout').setRelease(release.release);@@@ -112,6 +117,8 @@ function createApp(opts = {}) {// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).app.use('/shared', require('openvibe-shared/serve').handler());++ // GET /<key>.txt — the IndexNow key file (only when a key is configured; it serves itself).++ if (indexnow.enabled) app.use(indexnow.keyFile);app.use(express.static(PUBLIC_DIR, {index: false, redirect: false,setHeaders(res, filePath) {@@@ -141,7 +148,7 @@ function createApp(opts = {}) {return res.status(500).type('html').send(layout.page({ title: 'Error', robots: 'noindex', body: pages.errorPage({ status: 500, title: 'Something went wrong', message: 'This one is on us. Please try again.' }) }));});-- Object.assign(app.locals, { config, db, valkey, domain, keys, outbox, adapters, billing, consumer, metrics });++ Object.assign(app.locals, { config, db, valkey, domain, keys, outbox, adapters, billing, consumer, metrics, indexnow });return app;}@@@ -110,6 +110,11 @@ function loadConfig(env = process.env) {maxStreamsPerToken: int(env.TIPS_OVERLAY_MAX_STREAMS, 10),},++ // IndexNow (openvibe-shared/indexnow): when INDEXNOW_KEY is set the key file is served at++ // /<key>.txt and a creator's public page appearing, changing or going away pings the engines.++ // Unset: off — nothing is mounted and nothing is sent.++ indexnow: { key: env.INDEXNOW_KEY || '' },++limits: {maxBits: int(env.TIPS_MAX_BITS, 10_000_000), // Billing's maximummessageChars: int(env.TIPS_MESSAGE_CHARS, 300), // Live's donation message limit@@@ -36,8 +36,8 @@ const { createModeration } = require('./moderation');const MONEY = { isolation: 'serializable' };--function createDomain({ db, config, outbox, billing, adapters, valkey = null, now = () => Date.now(), log = console }) {-- const ctx = { db, config, now, log, outbox, billing, adapters, valkey, MONEY };++function createDomain({ db, config, outbox, billing, adapters, valkey = null, now = () => Date.now(), log = console, indexnow = null }) {++ const ctx = { db, config, now, log, outbox, billing, adapters, valkey, indexnow, MONEY };const runHook = (h) => { Promise.resolve().then(h).catch((e) => log.warn('[Tips] after-commit:', e.message)); };ctx.tx = async (fn, opts = {}) => {@@@ -97,8 +97,10 @@ function createProfiles(ctx) {}const handle = normalizeHandle(username) && normalizeHandle(username) !== existing.handle ? await freeHandle(db, subject, username) : existing.handle;if (handle !== existing.handle || name !== existing.display_name || (safeUrl(avatarUrl) || null) !== (existing.avatar_url || null)) {-- return parse(await db.one(sql`UPDATE creator_tip_profiles SET handle = ${handle}, display_name = ${name}, avatar_url = ${safeUrl(avatarUrl)}, updated_at = ${at}++ const next = parse(await db.one(sql`UPDATE creator_tip_profiles SET handle = ${handle}, display_name = ${name}, avatar_url = ${safeUrl(avatarUrl)}, updated_at = ${at}WHERE creator_subject = ${subject} RETURNING *`));++ pingPublic(existing, next);++ return next;}return existing;}@@@ -169,8 +171,24 @@ function createProfiles(ctx) {set.display_name = n.replace(/[<>]/g, '');}if (!Object.keys(set).length) return p;-- return parse(await t.one(sql`UPDATE creator_tip_profiles SET ${sql.set({ ...set, updated_at: iso(ctx.now()) })}, revision = revision + 1++ const next = parse(await t.one(sql`UPDATE creator_tip_profiles SET ${sql.set({ ...set, updated_at: iso(ctx.now()) })}, revision = revision + 1WHERE creator_subject = ${subject} RETURNING *`));++ pingPublic(p, next);++ return next;++ }++++ /**++ * IndexNow: a creator's public page appeared, changed or went away — ping its URL and the sitemap++ * that lists it. Queued here (the shared module batches and debounces) and only when a public page++ * was on one side of the change; pingSoon never throws and is a no-op without a key.++ */++ function pingPublic(before, after) {++ const inx = ctx.indexnow;++ if (!inx || !inx.enabled) return;++ const visible = (before && before.page_enabled) || (after && after.page_enabled);++ const handle = (after && after.handle) || (before && before.handle);++ if (!visible || !handle) return;++ inx.pingSoon([`${ctx.config.baseUrl}/${handle}`, `${ctx.config.baseUrl}/sitemap.xml`]);}/** Public shape; `owner` adds the settings only the creator sees. */nnew file mode 100644@@@ -0,0 +1,68 @@++'use strict';++/**++ * IndexNow (openvibe-shared/indexnow): INDEXNOW_KEY unset → the feature is off (no key route, nothing++ * sent). With a key, the key file answers at /<key>.txt as text/plain and a creator's public page++ * appearing or changing pings its URL and the sitemap. A switched-off page never pings, and the++ * crawler map (/llms.txt, /llms-full.txt) is untouched by the feature either way.++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/app');++++const KEY = 'k'.repeat(32);++++(async () => {++ const off = await boot();++ await check('without a key IndexNow is off: the key route 404s and the maps still answer', async () => {++ assert.strictEqual(off.app.locals.indexnow.enabled, false);++ const res = await off.call('GET', `/${KEY}.txt`);++ assert.strictEqual(res.status, 404, res.text);++ assert.strictEqual((await off.call('GET', '/llms.txt')).status, 200);++ assert.strictEqual((await off.call('GET', '/llms-full.txt')).status, 200);++ });++ await off.close();++++ const on = await boot({ env: { INDEXNOW_KEY: KEY } });++ await check('with a key the key file answers /<key>.txt as text/plain, and the maps still answer', async () => {++ assert.strictEqual(on.app.locals.indexnow.enabled, true);++ const res = await on.call('GET', `/${KEY}.txt`);++ assert.strictEqual(res.status, 200, res.text);++ assert.match(res.headers.get('content-type'), /text\/plain/);++ assert.strictEqual(res.text, KEY);++ assert.strictEqual((await on.call('GET', '/llms.txt')).status, 200);++ assert.strictEqual((await on.call('GET', '/llms-full.txt')).status, 200);++ });++ await on.close();++++ // A spy in place of the module's HTTP send: records every pingSoon batch.++ const pings = [];++ const spy = {++ enabled: true,++ keyFile: (_req, _res, next) => next(),++ pingSoon: (urls) => { const a = Array.isArray(urls) ? urls : [urls]; pings.push(...a); return a.length; },++ };++ const t = await boot({ appOpts: { indexnow: spy } });++ let alex;++++ await check('switching a creator page on pings its URL and the sitemap', async () => {++ alex = await t.creator('alex', { settings: { headline: 'Late-night synth streams' } });++ assert.ok(pings.includes('http://tips.test/alex'), JSON.stringify(pings));++ assert.ok(pings.includes('http://tips.test/sitemap.xml'), JSON.stringify(pings));++ });++++ await check('changing a public page pings again', async () => {++ pings.length = 0;++ const r = await t.call('PATCH', '/api/v1/profiles/me', { user: alex, body: { headline: 'Now with text-to-speech' } });++ assert.strictEqual(r.status, 200, r.text);++ assert.ok(pings.includes('http://tips.test/alex'), JSON.stringify(pings));++ assert.ok(pings.includes('http://tips.test/sitemap.xml'), JSON.stringify(pings));++ });++++ await check('a switched-off page never pings', async () => {++ pings.length = 0;++ await t.creator('bob', { settings: { page_enabled: false } });++ assert.ok(!pings.some((u) => u.includes('/bob')), JSON.stringify(pings));++ });++++ await t.close();++ done();++})().catch((err) => { console.error(err); process.exit(1); });
@@ -93,6 +93,12 @@ TIPS_EVENTS_SECRET=
@@ -93,6 +93,12 @@ TIPS_EVENTS_SECRET=
#TIPS_LIMITS_MINUTE=120
#TIPS_LIMITS_MINUTE=120
#TIPS_LIMITS_HOUR=3000
#TIPS_LIMITS_HOUR=3000
+# ── IndexNow ────────────────────────────────────────────────
+# IndexNow (openvibe-shared/indexnow): 8–128 hex/alphanumeric characters. Set: the key file is served
+# at /<key>.txt and a creator's public page appearing, changing or going away pings the engines.
+# Unset: off, nothing is mounted and nothing is sent.
+#INDEXNOW_KEY=
+
# ── Background jobs: effects worker, due transfers, overlay window, outbox relay ──
# ── Background jobs: effects worker, due transfers, overlay window, outbox relay ──
#TIPS_JOBS=on
#TIPS_JOBS=on
#TIPS_JOBS_INTERVAL_MS=2000
#TIPS_JOBS_INTERVAL_MS=2000
@@ -22,6 +22,7 @@ const cookieParser = require('cookie-parser');
@@ -22,6 +22,7 @@ const cookieParser = require('cookie-parser');
const { http } = require('openvibe-contracts');
const { http } = require('openvibe-contracts');
const { isLoopbackDirect } = require('openvibe-shared/metrics');
const { isLoopbackDirect } = require('openvibe-shared/metrics');
const cache = require('openvibe-shared/cache-policy');
const cache = require('openvibe-shared/cache-policy');
+const { createIndexNow } = require('openvibe-shared/indexnow');
const { loadConfig } = require('./config');
const { loadConfig } = require('./config');
const { openDb } = require('./db');
const { openDb } = require('./db');
const { createKeyProvider, createUserAuth } = require('./network');
const { createKeyProvider, createUserAuth } = require('./network');
@@ -55,7 +56,11 @@ function createApp(opts = {}) {
@@ -55,7 +56,11 @@ function createApp(opts = {}) {
const billing = opts.billing || createBillingClient(config, { fetchImpl });
const billing = opts.billing || createBillingClient(config, { fetchImpl });
const adapters = opts.adapters || createAdapters(config, { fetchImpl });
const adapters = opts.adapters || createAdapters(config, { fetchImpl });
const outbox = opts.outbox || createTipsOutbox({ db, config, fetchImpl: opts.eventsFetch, now, log });
const outbox = opts.outbox || createTipsOutbox({ db, config, fetchImpl: opts.eventsFetch, now, log });
- const domain = createDomain({ db, config, outbox, billing, adapters, valkey, now, log });
+ // IndexNow (openvibe-shared/indexnow): created once at boot from INDEXNOW_KEY; unset → off (nothing
+ // mounted, nothing sent). The key file is served at /<key>.txt and the profile change pings the
+ // engines when a creator's public page appears, changes or goes away.
+ const indexnow = opts.indexnow || createIndexNow({ host: config.baseUrl, key: config.indexnow.key, ...(fetchImpl ? { fetch: fetchImpl } : {}), log });
+ const domain = createDomain({ db, config, outbox, billing, adapters, valkey, now, log, indexnow });
const apiAuth = createApiAuth({ config, keys, userAuth });
const apiAuth = createApiAuth({ config, keys, userAuth });
const release = require('openvibe-shared/release').createRelease({ service: 'tips', root: path.join(__dirname, '..') });
const release = require('openvibe-shared/release').createRelease({ service: 'tips', root: path.join(__dirname, '..') });
require('./web/layout').setRelease(release.release);
require('./web/layout').setRelease(release.release);
@@ -112,6 +117,8 @@ function createApp(opts = {}) {
@@ -112,6 +117,8 @@ function createApp(opts = {}) {
// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).
// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).
app.use('/shared', require('openvibe-shared/serve').handler());
app.use('/shared', require('openvibe-shared/serve').handler());
+ // GET /<key>.txt — the IndexNow key file (only when a key is configured; it serves itself).
+ if (indexnow.enabled) app.use(indexnow.keyFile);
app.use(express.static(PUBLIC_DIR, {
app.use(express.static(PUBLIC_DIR, {
index: false, redirect: false,
index: false, redirect: false,
setHeaders(res, filePath) {
setHeaders(res, filePath) {
@@ -141,7 +148,7 @@ function createApp(opts = {}) {
@@ -141,7 +148,7 @@ function createApp(opts = {}) {
return res.status(500).type('html').send(layout.page({ title: 'Error', robots: 'noindex', body: pages.errorPage({ status: 500, title: 'Something went wrong', message: 'This one is on us. Please try again.' }) }));
return res.status(500).type('html').send(layout.page({ title: 'Error', robots: 'noindex', body: pages.errorPage({ status: 500, title: 'Something went wrong', message: 'This one is on us. Please try again.' }) }));
});
});
- Object.assign(app.locals, { config, db, valkey, domain, keys, outbox, adapters, billing, consumer, metrics });
+ Object.assign(app.locals, { config, db, valkey, domain, keys, outbox, adapters, billing, consumer, metrics, indexnow });
return app;
return app;
}
}
@@ -110,6 +110,11 @@ function loadConfig(env = process.env) {
@@ -110,6 +110,11 @@ function loadConfig(env = process.env) {
maxStreamsPerToken: int(env.TIPS_OVERLAY_MAX_STREAMS, 10),
maxStreamsPerToken: int(env.TIPS_OVERLAY_MAX_STREAMS, 10),
},
},
+ // IndexNow (openvibe-shared/indexnow): when INDEXNOW_KEY is set the key file is served at
+ // /<key>.txt and a creator's public page appearing, changing or going away pings the engines.
+ // Unset: off — nothing is mounted and nothing is sent.
+ indexnow: { key: env.INDEXNOW_KEY || '' },
+
limits: {
limits: {
maxBits: int(env.TIPS_MAX_BITS, 10_000_000), // Billing's maximum
maxBits: int(env.TIPS_MAX_BITS, 10_000_000), // Billing's maximum
messageChars: int(env.TIPS_MESSAGE_CHARS, 300), // Live's donation message limit
messageChars: int(env.TIPS_MESSAGE_CHARS, 300), // Live's donation message limit
@@ -36,8 +36,8 @@ const { createModeration } = require('./moderation');
@@ -36,8 +36,8 @@ const { createModeration } = require('./moderation');
const MONEY = { isolation: 'serializable' };
const MONEY = { isolation: 'serializable' };
-function createDomain({ db, config, outbox, billing, adapters, valkey = null, now = () => Date.now(), log = console }) {
- const ctx = { db, config, now, log, outbox, billing, adapters, valkey, MONEY };
+function createDomain({ db, config, outbox, billing, adapters, valkey = null, now = () => Date.now(), log = console, indexnow = null }) {
+ const ctx = { db, config, now, log, outbox, billing, adapters, valkey, indexnow, MONEY };
const runHook = (h) => { Promise.resolve().then(h).catch((e) => log.warn('[Tips] after-commit:', e.message)); };
const runHook = (h) => { Promise.resolve().then(h).catch((e) => log.warn('[Tips] after-commit:', e.message)); };
ctx.tx = async (fn, opts = {}) => {
ctx.tx = async (fn, opts = {}) => {
@@ -97,8 +97,10 @@ function createProfiles(ctx) {
@@ -97,8 +97,10 @@ function createProfiles(ctx) {
}
}
const handle = normalizeHandle(username) && normalizeHandle(username) !== existing.handle ? await freeHandle(db, subject, username) : existing.handle;
const handle = normalizeHandle(username) && normalizeHandle(username) !== existing.handle ? await freeHandle(db, subject, username) : existing.handle;
if (handle !== existing.handle || name !== existing.display_name || (safeUrl(avatarUrl) || null) !== (existing.avatar_url || null)) {
if (handle !== existing.handle || name !== existing.display_name || (safeUrl(avatarUrl) || null) !== (existing.avatar_url || null)) {
- return parse(await db.one(sql`UPDATE creator_tip_profiles SET handle = ${handle}, display_name = ${name}, avatar_url = ${safeUrl(avatarUrl)}, updated_at = ${at}
+ const next = parse(await db.one(sql`UPDATE creator_tip_profiles SET handle = ${handle}, display_name = ${name}, avatar_url = ${safeUrl(avatarUrl)}, updated_at = ${at}
WHERE creator_subject = ${subject} RETURNING *`));
WHERE creator_subject = ${subject} RETURNING *`));
+ pingPublic(existing, next);
+ return next;
}
}
return existing;
return existing;
}
}
@@ -169,8 +171,24 @@ function createProfiles(ctx) {
@@ -169,8 +171,24 @@ function createProfiles(ctx) {
set.display_name = n.replace(/[<>]/g, '');
set.display_name = n.replace(/[<>]/g, '');
}
}
if (!Object.keys(set).length) return p;
if (!Object.keys(set).length) return p;
- return parse(await t.one(sql`UPDATE creator_tip_profiles SET ${sql.set({ ...set, updated_at: iso(ctx.now()) })}, revision = revision + 1
+ const next = parse(await t.one(sql`UPDATE creator_tip_profiles SET ${sql.set({ ...set, updated_at: iso(ctx.now()) })}, revision = revision + 1
WHERE creator_subject = ${subject} RETURNING *`));
WHERE creator_subject = ${subject} RETURNING *`));
+ pingPublic(p, next);
+ return next;
+ }
+
+ /**
+ * IndexNow: a creator's public page appeared, changed or went away — ping its URL and the sitemap
+ * that lists it. Queued here (the shared module batches and debounces) and only when a public page
+ * was on one side of the change; pingSoon never throws and is a no-op without a key.
+ */
+ function pingPublic(before, after) {
+ const inx = ctx.indexnow;
+ if (!inx || !inx.enabled) return;
+ const visible = (before && before.page_enabled) || (after && after.page_enabled);
+ const handle = (after && after.handle) || (before && before.handle);
+ if (!visible || !handle) return;
+ inx.pingSoon([`${ctx.config.baseUrl}/${handle}`, `${ctx.config.baseUrl}/sitemap.xml`]);
}
}
/** Public shape; `owner` adds the settings only the creator sees. */
/** Public shape; `owner` adds the settings only the creator sees. */
new file mode 100644
new file mode 100644
@@ -0,0 +1,68 @@
@@ -0,0 +1,68 @@
+'use strict';
+/**
+ * IndexNow (openvibe-shared/indexnow): INDEXNOW_KEY unset → the feature is off (no key route, nothing
+ * sent). With a key, the key file answers at /<key>.txt as text/plain and a creator's public page
+ * appearing or changing pings its URL and the sitemap. A switched-off page never pings, and the
+ * crawler map (/llms.txt, /llms-full.txt) is untouched by the feature either way.
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/app');
+
+const KEY = 'k'.repeat(32);
+
+(async () => {
+ const off = await boot();
+ await check('without a key IndexNow is off: the key route 404s and the maps still answer', async () => {
+ assert.strictEqual(off.app.locals.indexnow.enabled, false);
+ const res = await off.call('GET', `/${KEY}.txt`);
+ assert.strictEqual(res.status, 404, res.text);
+ assert.strictEqual((await off.call('GET', '/llms.txt')).status, 200);
+ assert.strictEqual((await off.call('GET', '/llms-full.txt')).status, 200);
+ });
+ await off.close();
+
+ const on = await boot({ env: { INDEXNOW_KEY: KEY } });
+ await check('with a key the key file answers /<key>.txt as text/plain, and the maps still answer', async () => {
+ assert.strictEqual(on.app.locals.indexnow.enabled, true);
+ const res = await on.call('GET', `/${KEY}.txt`);
+ assert.strictEqual(res.status, 200, res.text);
+ assert.match(res.headers.get('content-type'), /text\/plain/);
+ assert.strictEqual(res.text, KEY);
+ assert.strictEqual((await on.call('GET', '/llms.txt')).status, 200);
+ assert.strictEqual((await on.call('GET', '/llms-full.txt')).status, 200);
+ });
+ await on.close();
+
+ // A spy in place of the module's HTTP send: records every pingSoon batch.
+ const pings = [];
+ const spy = {
+ enabled: true,
+ keyFile: (_req, _res, next) => next(),
+ pingSoon: (urls) => { const a = Array.isArray(urls) ? urls : [urls]; pings.push(...a); return a.length; },
+ };
+ const t = await boot({ appOpts: { indexnow: spy } });
+ let alex;
+
+ await check('switching a creator page on pings its URL and the sitemap', async () => {
+ alex = await t.creator('alex', { settings: { headline: 'Late-night synth streams' } });
+ assert.ok(pings.includes('http://tips.test/alex'), JSON.stringify(pings));
+ assert.ok(pings.includes('http://tips.test/sitemap.xml'), JSON.stringify(pings));
+ });
+
+ await check('changing a public page pings again', async () => {
+ pings.length = 0;
+ const r = await t.call('PATCH', '/api/v1/profiles/me', { user: alex, body: { headline: 'Now with text-to-speech' } });
+ assert.strictEqual(r.status, 200, r.text);
+ assert.ok(pings.includes('http://tips.test/alex'), JSON.stringify(pings));
+ assert.ok(pings.includes('http://tips.test/sitemap.xml'), JSON.stringify(pings));
+ });
+
+ await check('a switched-off page never pings', async () => {
+ pings.length = 0;
+ await t.creator('bob', { settings: { page_enabled: false } });
+ assert.ok(!pings.some((u) => u.includes('/bob')), JSON.stringify(pings));
+ });
+
+ await t.close();
+ done();
+})().catch((err) => { console.error(err); process.exit(1); });