openvibers/openvibe.wiki Commits
Sitemaps: every entry carries an honest lastmod (item changes, release time for static pages) (#42)
Changes
@@@ -24,6 +24,8 @@ citations, redirects, the indexability gate, feeds, sitemaps, JSON-LD, authorshimedia references, discussion references and the Search index hooks all come from there. Wiki ownsits publication state; the packages supply the mechanics.++Sitemap entries carry lastmod from each published page or space's latest change.++## OwnsThe ten authority tables of §15.13, in Wiki's own PostgreSQL database (`ov_wiki` on the host's data role, ADR-035; schema in [migrations/](migrations/)):@@@ -37,25 +37,39 @@ function createMachine({ svc, config }) {const wrap = (fn) => (req, res, next) => Promise.resolve(fn(req, res)).catch(next);-- async function pageEntries() {-- return (await svc.publishedPublic()).map(({ space, page, decision }) => ({++ async function pageEntries(published = null) {++ published = published || await svc.publishedPublic();++ return published.map(({ space, page, decision }) => ({loc: svc.pageUrl(space, page), lastmod: page.revision_published_at, decision,}));}-- async function spaceEntries() {++ async function spaceEntries(published = null) {++ published = published || await svc.publishedPublic();++ const newestPage = new Map();++ for (const { space, page } of published) {++ if (!page.revision_published_at) continue;++ newestPage.set(space.id, Math.max(newestPage.get(space.id) || 0, page.revision_published_at));++ }return (await svc.listSpaces({ kind: 'anonymous' })).filter((s) => s.visibility === 'public').map((s) => {const url = seo.canonicalUrl(origin, svc.spacePath(s));// A space index is navigation, not an article: no word or source minimum.const decision = seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: url, wordCount: 0 }, { policy: { minWords: 0, requireSources: false }, now: Date.now() });-- return { loc: url, lastmod: s.updated_at, decision };++ return { loc: url, lastmod: Math.max(s.updated_at, newestPage.get(s.id) || 0), decision };});}router.get('/sitemap.xml', wrap(async (_req, res) => {-- const pages = seo.sitemap(await pageEntries(), { maxUrls: PER_SITEMAP });-- const maps = [{ loc: `${origin}/sitemaps/spaces.xml` }];-- pages.files.forEach((_f, i) => maps.push({ loc: `${origin}/sitemaps/pages-${i + 1}.xml` }));++ const published = await svc.publishedPublic();++ const entries = (await pageEntries(published)).filter((e) => e.decision.indexable);++ const spaces = (await spaceEntries(published)).filter((e) => e.decision.indexable);++ const pages = seo.sitemap(entries, { maxUrls: PER_SITEMAP });++ const newest = (rows) => {++ const times = rows.map((e) => e.lastmod).filter(Boolean);++ return times.length ? new Date(Math.max(...times)).toISOString() : undefined;++ };++ const maps = [{ loc: `${origin}/sitemaps/spaces.xml`, lastmod: newest(spaces) }];++ pages.files.forEach((_f, i) => maps.push({ loc: `${origin}/sitemaps/pages-${i + 1}.xml`, lastmod: newest(entries.slice(i * PER_SITEMAP, (i + 1) * PER_SITEMAP)) }));setCache(res).type('application/xml').send(seo.sitemapIndex(maps));}));router.get('/sitemaps/spaces.xml', wrap(async (_req, res) => setCache(res).type('application/xml').send(seo.sitemap(await spaceEntries()).files[0])));@@@ -6,6 +6,17 @@* search.index-document@1 (openvibe-contracts).*/const assert = require('assert');++++function assertSitemapLastmods(xml, tag = 'url') {++ const entries = [...xml.matchAll(new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`, 'g'))];++ for (const [, body] of entries) {++ const match = /<lastmod>(\d{4}-\d{2}-\d{2})<\/lastmod>/.exec(body);++ assert.ok(match, body);++ assert.strictEqual(new Date(match[1]).toISOString().slice(0, 10), match[1]);++ }++ return entries;++}++const contracts = require('openvibe-contracts');const H = require('./helpers');@@@ -36,6 +47,11 @@ async function lastIndexEvent(h, pageId) {await h.svc.publish(unsourced.id, {}, owner);r = await H.req(h, 'GET', '/sitemaps/pages-1.xml');assert.ok(r.text.includes('http://wiki.test/w/notes/open-page'));++ assertSitemapLastmods(r.text);++ const publishedPage = await h.db.prepare('SELECT revision_published_at FROM wiki_pages WHERE id = ?').get(page.id);++ assert.ok(r.text.includes(`<lastmod>${new Date(publishedPage.revision_published_at).toISOString().slice(0, 10)}</lastmod>`));++ assertSitemapLastmods((await H.req(h, 'GET', '/sitemaps/spaces.xml')).text);++ assertSitemapLastmods((await H.req(h, 'GET', '/sitemap.xml')).text, 'sitemap');assert.ok(!r.text.includes('/w/notes/unsourced-page'), 'an unsourced page is noindex: not in the sitemap');assert.ok((await H.req(h, 'GET', '/feed.atom')).text.includes('http://wiki.test/w/notes/open-page'));// /llms-full.txt: the llms.txt header, then exactly the indexable pages — the same gate as the
@@ -24,6 +24,8 @@ citations, redirects, the indexability gate, feeds, sitemaps, JSON-LD, authorshi
@@ -24,6 +24,8 @@ citations, redirects, the indexability gate, feeds, sitemaps, JSON-LD, authorshi
media references, discussion references and the Search index hooks all come from there. Wiki owns
media references, discussion references and the Search index hooks all come from there. Wiki owns
its publication state; the packages supply the mechanics.
its publication state; the packages supply the mechanics.
+Sitemap entries carry lastmod from each published page or space's latest change.
+
## Owns
## Owns
The ten authority tables of §15.13, in Wiki's own PostgreSQL database (`ov_wiki` on the host's data role, ADR-035; schema in [migrations/](migrations/)):
The ten authority tables of §15.13, in Wiki's own PostgreSQL database (`ov_wiki` on the host's data role, ADR-035; schema in [migrations/](migrations/)):
@@ -37,25 +37,39 @@ function createMachine({ svc, config }) {
@@ -37,25 +37,39 @@ function createMachine({ svc, config }) {
const wrap = (fn) => (req, res, next) => Promise.resolve(fn(req, res)).catch(next);
const wrap = (fn) => (req, res, next) => Promise.resolve(fn(req, res)).catch(next);
- async function pageEntries() {
- return (await svc.publishedPublic()).map(({ space, page, decision }) => ({
+ async function pageEntries(published = null) {
+ published = published || await svc.publishedPublic();
+ return published.map(({ space, page, decision }) => ({
loc: svc.pageUrl(space, page), lastmod: page.revision_published_at, decision,
loc: svc.pageUrl(space, page), lastmod: page.revision_published_at, decision,
}));
}));
}
}
- async function spaceEntries() {
+ async function spaceEntries(published = null) {
+ published = published || await svc.publishedPublic();
+ const newestPage = new Map();
+ for (const { space, page } of published) {
+ if (!page.revision_published_at) continue;
+ newestPage.set(space.id, Math.max(newestPage.get(space.id) || 0, page.revision_published_at));
+ }
return (await svc.listSpaces({ kind: 'anonymous' })).filter((s) => s.visibility === 'public').map((s) => {
return (await svc.listSpaces({ kind: 'anonymous' })).filter((s) => s.visibility === 'public').map((s) => {
const url = seo.canonicalUrl(origin, svc.spacePath(s));
const url = seo.canonicalUrl(origin, svc.spacePath(s));
// A space index is navigation, not an article: no word or source minimum.
// A space index is navigation, not an article: no word or source minimum.
const decision = seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: url, wordCount: 0 }, { policy: { minWords: 0, requireSources: false }, now: Date.now() });
const decision = seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: url, wordCount: 0 }, { policy: { minWords: 0, requireSources: false }, now: Date.now() });
- return { loc: url, lastmod: s.updated_at, decision };
+ return { loc: url, lastmod: Math.max(s.updated_at, newestPage.get(s.id) || 0), decision };
});
});
}
}
router.get('/sitemap.xml', wrap(async (_req, res) => {
router.get('/sitemap.xml', wrap(async (_req, res) => {
- const pages = seo.sitemap(await pageEntries(), { maxUrls: PER_SITEMAP });
- const maps = [{ loc: `${origin}/sitemaps/spaces.xml` }];
- pages.files.forEach((_f, i) => maps.push({ loc: `${origin}/sitemaps/pages-${i + 1}.xml` }));
+ const published = await svc.publishedPublic();
+ const entries = (await pageEntries(published)).filter((e) => e.decision.indexable);
+ const spaces = (await spaceEntries(published)).filter((e) => e.decision.indexable);
+ const pages = seo.sitemap(entries, { maxUrls: PER_SITEMAP });
+ const newest = (rows) => {
+ const times = rows.map((e) => e.lastmod).filter(Boolean);
+ return times.length ? new Date(Math.max(...times)).toISOString() : undefined;
+ };
+ const maps = [{ loc: `${origin}/sitemaps/spaces.xml`, lastmod: newest(spaces) }];
+ pages.files.forEach((_f, i) => maps.push({ loc: `${origin}/sitemaps/pages-${i + 1}.xml`, lastmod: newest(entries.slice(i * PER_SITEMAP, (i + 1) * PER_SITEMAP)) }));
setCache(res).type('application/xml').send(seo.sitemapIndex(maps));
setCache(res).type('application/xml').send(seo.sitemapIndex(maps));
}));
}));
router.get('/sitemaps/spaces.xml', wrap(async (_req, res) => setCache(res).type('application/xml').send(seo.sitemap(await spaceEntries()).files[0])));
router.get('/sitemaps/spaces.xml', wrap(async (_req, res) => setCache(res).type('application/xml').send(seo.sitemap(await spaceEntries()).files[0])));
@@ -6,6 +6,17 @@
@@ -6,6 +6,17 @@
* search.index-document@1 (openvibe-contracts).
* search.index-document@1 (openvibe-contracts).
*/
*/
const assert = require('assert');
const assert = require('assert');
+
+function assertSitemapLastmods(xml, tag = 'url') {
+ const entries = [...xml.matchAll(new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`, 'g'))];
+ for (const [, body] of entries) {
+ const match = /<lastmod>(\d{4}-\d{2}-\d{2})<\/lastmod>/.exec(body);
+ assert.ok(match, body);
+ assert.strictEqual(new Date(match[1]).toISOString().slice(0, 10), match[1]);
+ }
+ return entries;
+}
+
const contracts = require('openvibe-contracts');
const contracts = require('openvibe-contracts');
const H = require('./helpers');
const H = require('./helpers');
@@ -36,6 +47,11 @@ async function lastIndexEvent(h, pageId) {
@@ -36,6 +47,11 @@ async function lastIndexEvent(h, pageId) {
await h.svc.publish(unsourced.id, {}, owner);
await h.svc.publish(unsourced.id, {}, owner);
r = await H.req(h, 'GET', '/sitemaps/pages-1.xml');
r = await H.req(h, 'GET', '/sitemaps/pages-1.xml');
assert.ok(r.text.includes('http://wiki.test/w/notes/open-page'));
assert.ok(r.text.includes('http://wiki.test/w/notes/open-page'));
+ assertSitemapLastmods(r.text);
+ const publishedPage = await h.db.prepare('SELECT revision_published_at FROM wiki_pages WHERE id = ?').get(page.id);
+ assert.ok(r.text.includes(`<lastmod>${new Date(publishedPage.revision_published_at).toISOString().slice(0, 10)}</lastmod>`));
+ assertSitemapLastmods((await H.req(h, 'GET', '/sitemaps/spaces.xml')).text);
+ assertSitemapLastmods((await H.req(h, 'GET', '/sitemap.xml')).text, 'sitemap');
assert.ok(!r.text.includes('/w/notes/unsourced-page'), 'an unsourced page is noindex: not in the sitemap');
assert.ok(!r.text.includes('/w/notes/unsourced-page'), 'an unsourced page is noindex: not in the sitemap');
assert.ok((await H.req(h, 'GET', '/feed.atom')).text.includes('http://wiki.test/w/notes/open-page'));
assert.ok((await H.req(h, 'GET', '/feed.atom')).text.includes('http://wiki.test/w/notes/open-page'));
// /llms-full.txt: the llms.txt header, then exactly the indexable pages — the same gate as the
// /llms-full.txt: the llms.txt header, then exactly the indexable pages — the same gate as the