openvibers/openvibe.reviews Commits
Reviews: llms-full.txt describes every public page (SEO-33) (#136)
Changes
@@@ -19,6 +19,28 @@ const { SITE_SUMMARY } = require('../render/layout');const PER_SITEMAP = 45000;const ALL_SUMMARIES = 50000;++/**++ * The public page catalogue: /sitemaps/pages.xml takes the entries marked for the sitemap and++ * /llms-full.txt describes every entry, so the two artifacts cannot drift apart. Dynamic pages++ * (entity pages and /k/:kind) are described by their route pattern; the entity pages themselves++ * are appended from the database below.++ */++const PUBLIC_PAGES = [++ { path: '/', sitemap: true, title: 'OpenVibe.Reviews home', text: 'Review signals from named sources resolved to entities, with provenance for every number; editor-reviewed summaries, members\' reviews, and every other review site in one search.' },++ { path: '/about', sitemap: true, title: 'How it works', text: 'Where the signals come from, how aggregates are computed, what a summary is and how corrections and editor review work.' },++ { path: '/updates', title: 'What shipped on OpenVibe.Reviews', text: 'The shared changelog for OpenVibe.Reviews, newest first.' },++ { path: '/everywhere', title: 'Every review site, one search', text: 'Search an employer, restaurant or other entity across the other review sites OpenVibe.Reviews knows about, without retyping the name.' },++ { path: '/review', title: 'Write a review', text: 'Name what you want to review and write about it on its page; members\' reviews are separate from source signals.' },++ { path: '/e/:slug', title: 'One entity page', text: 'Every signal for one entity (an employer, restaurant, game, product or place), its provenance, member reviews, editor-reviewed summaries and where else to read about it; add .json for the same data as JSON.' },++ { path: '/k/:kind', title: 'Entities of one kind', text: 'Every entity of one kind, the most reviewed by members first.' },++];++++/** The legal pages every OpenVibe site serves, named from the shared kit rather than retyped here. */++function legalPages() {++ const labels = { '/terms': 'Terms of use', '/privacy': 'Privacy policy', '/dmca': 'DMCA and takedown policy' };++ return require('openvibe-shared/legal').PATHS.map((path) => ({ path, title: labels[path] || path.slice(1), text: `The ${(labels[path] || path.slice(1)).toLowerCase()} for OpenVibe.Reviews, served by the site itself.` }));++}++function createMachine({ svc, config, releaseLastmod }) {const router = express.Router();const origin = config.baseUrl;@@@ -52,7 +74,7 @@ function createMachine({ svc, config, releaseLastmod }) {const updated = (await entityEntries()).filter((e) => e.decision.indexable && e.lastmod).map((e) => e.lastmod);const homeLastmod = updated.length ? new Date(Math.max(...updated)).toISOString() : releaseLastmod;const policy = { minWords: 0, requireSources: false };-- const entries = ['/', '/about'].map((p) => ({ loc: seo.canonicalUrl(origin, p), lastmod: p === '/' ? homeLastmod : releaseLastmod, decision: seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: seo.canonicalUrl(origin, p), wordCount: 0 }, { policy, now: Date.now() }) }));++ const entries = PUBLIC_PAGES.filter((p) => p.sitemap).map((p) => ({ loc: seo.canonicalUrl(origin, p.path), lastmod: p.path === '/' ? homeLastmod : releaseLastmod, decision: seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: seo.canonicalUrl(origin, p.path), wordCount: 0 }, { policy, now: Date.now() }) }));setCache(res).type('application/xml').send(seo.sitemap(entries).files[0]);});router.get('/sitemaps/entities-:n.xml', async (req, res) => {@@@ -107,17 +129,18 @@ function createMachine({ svc, config, releaseLastmod }) {router.get('/llms-full.txt', async (_req, res) => {// The /llms.txt header plus a section of only indexable reviews — the same gate the sitemap// applies — each as its short summary. Review text is never stored, so no body can appear.-- const pages = (await svc.recentSummaries(ALL_SUMMARIES)).filter(({ decision }) => decision.indexable).map(({ entity, rev }) => ({++ const reviews = (await svc.recentSummaries(ALL_SUMMARIES)).filter(({ decision }) => decision.indexable).map(({ entity, rev }) => ({title: entity.name,url: svc.entityUrl(entity),text: [rev.content || '', ...(rev.fields.pros || []).map((p) => p.text), ...(rev.fields.cons || []).map((p) => p.text)].join('\n').slice(0, 1000),}));++ const pages = [...PUBLIC_PAGES, ...legalPages()].map((p) => ({ url: `${origin}${p.path}`, title: p.title, text: p.text }));setCache(res, 3600).type('text/plain').send(sharedSeo.llmsFull({site: 'OpenVibe.Reviews',summary: SITE_SUMMARY,base: origin,maxBytes: 512 * 1024,-- sections: [{ title: 'Reviews', pages }],++ sections: [{ title: 'Pages', pages }, { title: 'Reviews', pages: reviews }],}));});@@@ -164,6 +164,10 @@ t('/llms-full.txt carries only indexable reviews and the home page an AI summaryassert.match(full.headers.get('content-type'), /^text\/plain/);assert.strictEqual(full.headers.get('cache-control'), 'public, max-age=3600, stale-while-revalidate=3600');assert.ok(full.text.includes('OpenVibe.Reviews'), 'the llms.txt header');++ assert.ok(full.text.length >= 500, `real content, not a stub (${full.text.length} characters)`);++ for (const p of ['/', '/about', '/updates', '/everywhere', '/review', '/e/:slug', '/k/:kind', '/terms', '/privacy', '/dmca']) {++ assert.ok(full.text.includes(`http://reviews.test${p}`), `llms-full.txt names the public route ${p}`);++ }assert.ok(full.text.includes(`http://reviews.test/e/${e.slug}`), 'an indexable review is listed');assert.match(full.text, /One recent review recommends it/, 'its short summary is the text');assert.ok(!full.text.includes(`/e/${thin.slug}`), 'a thin, non-indexable review is never listed');
@@ -19,6 +19,28 @@ const { SITE_SUMMARY } = require('../render/layout');
@@ -19,6 +19,28 @@ const { SITE_SUMMARY } = require('../render/layout');
const PER_SITEMAP = 45000;
const PER_SITEMAP = 45000;
const ALL_SUMMARIES = 50000;
const ALL_SUMMARIES = 50000;
+/**
+ * The public page catalogue: /sitemaps/pages.xml takes the entries marked for the sitemap and
+ * /llms-full.txt describes every entry, so the two artifacts cannot drift apart. Dynamic pages
+ * (entity pages and /k/:kind) are described by their route pattern; the entity pages themselves
+ * are appended from the database below.
+ */
+const PUBLIC_PAGES = [
+ { path: '/', sitemap: true, title: 'OpenVibe.Reviews home', text: 'Review signals from named sources resolved to entities, with provenance for every number; editor-reviewed summaries, members\' reviews, and every other review site in one search.' },
+ { path: '/about', sitemap: true, title: 'How it works', text: 'Where the signals come from, how aggregates are computed, what a summary is and how corrections and editor review work.' },
+ { path: '/updates', title: 'What shipped on OpenVibe.Reviews', text: 'The shared changelog for OpenVibe.Reviews, newest first.' },
+ { path: '/everywhere', title: 'Every review site, one search', text: 'Search an employer, restaurant or other entity across the other review sites OpenVibe.Reviews knows about, without retyping the name.' },
+ { path: '/review', title: 'Write a review', text: 'Name what you want to review and write about it on its page; members\' reviews are separate from source signals.' },
+ { path: '/e/:slug', title: 'One entity page', text: 'Every signal for one entity (an employer, restaurant, game, product or place), its provenance, member reviews, editor-reviewed summaries and where else to read about it; add .json for the same data as JSON.' },
+ { path: '/k/:kind', title: 'Entities of one kind', text: 'Every entity of one kind, the most reviewed by members first.' },
+];
+
+/** The legal pages every OpenVibe site serves, named from the shared kit rather than retyped here. */
+function legalPages() {
+ const labels = { '/terms': 'Terms of use', '/privacy': 'Privacy policy', '/dmca': 'DMCA and takedown policy' };
+ return require('openvibe-shared/legal').PATHS.map((path) => ({ path, title: labels[path] || path.slice(1), text: `The ${(labels[path] || path.slice(1)).toLowerCase()} for OpenVibe.Reviews, served by the site itself.` }));
+}
+
function createMachine({ svc, config, releaseLastmod }) {
function createMachine({ svc, config, releaseLastmod }) {
const router = express.Router();
const router = express.Router();
const origin = config.baseUrl;
const origin = config.baseUrl;
@@ -52,7 +74,7 @@ function createMachine({ svc, config, releaseLastmod }) {
@@ -52,7 +74,7 @@ function createMachine({ svc, config, releaseLastmod }) {
const updated = (await entityEntries()).filter((e) => e.decision.indexable && e.lastmod).map((e) => e.lastmod);
const updated = (await entityEntries()).filter((e) => e.decision.indexable && e.lastmod).map((e) => e.lastmod);
const homeLastmod = updated.length ? new Date(Math.max(...updated)).toISOString() : releaseLastmod;
const homeLastmod = updated.length ? new Date(Math.max(...updated)).toISOString() : releaseLastmod;
const policy = { minWords: 0, requireSources: false };
const policy = { minWords: 0, requireSources: false };
- const entries = ['/', '/about'].map((p) => ({ loc: seo.canonicalUrl(origin, p), lastmod: p === '/' ? homeLastmod : releaseLastmod, decision: seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: seo.canonicalUrl(origin, p), wordCount: 0 }, { policy, now: Date.now() }) }));
+ const entries = PUBLIC_PAGES.filter((p) => p.sitemap).map((p) => ({ loc: seo.canonicalUrl(origin, p.path), lastmod: p.path === '/' ? homeLastmod : releaseLastmod, decision: seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: seo.canonicalUrl(origin, p.path), wordCount: 0 }, { policy, now: Date.now() }) }));
setCache(res).type('application/xml').send(seo.sitemap(entries).files[0]);
setCache(res).type('application/xml').send(seo.sitemap(entries).files[0]);
});
});
router.get('/sitemaps/entities-:n.xml', async (req, res) => {
router.get('/sitemaps/entities-:n.xml', async (req, res) => {
@@ -107,17 +129,18 @@ function createMachine({ svc, config, releaseLastmod }) {
@@ -107,17 +129,18 @@ function createMachine({ svc, config, releaseLastmod }) {
router.get('/llms-full.txt', async (_req, res) => {
router.get('/llms-full.txt', async (_req, res) => {
// The /llms.txt header plus a section of only indexable reviews — the same gate the sitemap
// The /llms.txt header plus a section of only indexable reviews — the same gate the sitemap
// applies — each as its short summary. Review text is never stored, so no body can appear.
// applies — each as its short summary. Review text is never stored, so no body can appear.
- const pages = (await svc.recentSummaries(ALL_SUMMARIES)).filter(({ decision }) => decision.indexable).map(({ entity, rev }) => ({
+ const reviews = (await svc.recentSummaries(ALL_SUMMARIES)).filter(({ decision }) => decision.indexable).map(({ entity, rev }) => ({
title: entity.name,
title: entity.name,
url: svc.entityUrl(entity),
url: svc.entityUrl(entity),
text: [rev.content || '', ...(rev.fields.pros || []).map((p) => p.text), ...(rev.fields.cons || []).map((p) => p.text)].join('\n').slice(0, 1000),
text: [rev.content || '', ...(rev.fields.pros || []).map((p) => p.text), ...(rev.fields.cons || []).map((p) => p.text)].join('\n').slice(0, 1000),
}));
}));
+ const pages = [...PUBLIC_PAGES, ...legalPages()].map((p) => ({ url: `${origin}${p.path}`, title: p.title, text: p.text }));
setCache(res, 3600).type('text/plain').send(sharedSeo.llmsFull({
setCache(res, 3600).type('text/plain').send(sharedSeo.llmsFull({
site: 'OpenVibe.Reviews',
site: 'OpenVibe.Reviews',
summary: SITE_SUMMARY,
summary: SITE_SUMMARY,
base: origin,
base: origin,
maxBytes: 512 * 1024,
maxBytes: 512 * 1024,
- sections: [{ title: 'Reviews', pages }],
+ sections: [{ title: 'Pages', pages }, { title: 'Reviews', pages: reviews }],
}));
}));
});
});
@@ -164,6 +164,10 @@ t('/llms-full.txt carries only indexable reviews and the home page an AI summary
@@ -164,6 +164,10 @@ t('/llms-full.txt carries only indexable reviews and the home page an AI summary
assert.match(full.headers.get('content-type'), /^text\/plain/);
assert.match(full.headers.get('content-type'), /^text\/plain/);
assert.strictEqual(full.headers.get('cache-control'), 'public, max-age=3600, stale-while-revalidate=3600');
assert.strictEqual(full.headers.get('cache-control'), 'public, max-age=3600, stale-while-revalidate=3600');
assert.ok(full.text.includes('OpenVibe.Reviews'), 'the llms.txt header');
assert.ok(full.text.includes('OpenVibe.Reviews'), 'the llms.txt header');
+ assert.ok(full.text.length >= 500, `real content, not a stub (${full.text.length} characters)`);
+ for (const p of ['/', '/about', '/updates', '/everywhere', '/review', '/e/:slug', '/k/:kind', '/terms', '/privacy', '/dmca']) {
+ assert.ok(full.text.includes(`http://reviews.test${p}`), `llms-full.txt names the public route ${p}`);
+ }
assert.ok(full.text.includes(`http://reviews.test/e/${e.slug}`), 'an indexable review is listed');
assert.ok(full.text.includes(`http://reviews.test/e/${e.slug}`), 'an indexable review is listed');
assert.match(full.text, /One recent review recommends it/, 'its short summary is the text');
assert.match(full.text, /One recent review recommends it/, 'its short summary is the text');
assert.ok(!full.text.includes(`/e/${thin.slug}`), 'a thin, non-indexable review is never listed');
assert.ok(!full.text.includes(`/e/${thin.slug}`), 'a thin, non-indexable review is never listed');