openvibers/openvibe.news Commits
Add /llms-full.txt for indexable stories and adopt publishing v1.3.0
Changes
@@@ -4,3 +4,5 @@- Every page is rendered through `openvibe-publishing/layout` (v1.2.0, on `openvibe-shared/shell` v2.6.0): the head, the Frame, the noscript navigation, the footer and its init come from the shared document; robots and the canonical still come from the indexability gate's decision. The shell adds `web-runtime.js`, so the home page's JS budget is raised to 5 files, 245 KB, 59 KB brotli.- The `openvibe-publishing` v1.2.0 lock entry carries its sha512 integrity again (the bump had dropped it), and a test fails if any codeload pin loses its hash.++- `openvibe-publishing` v1.3.0 (from v1.2.0): `layout.renderDocument` now forwards `summary`, `facts`, `updated` and `url` to the shell's AI-summary block, so the home page carries the same one-line summary as `<meta name="ai-summary">` and a WebPage JSON-LD twin.++- `GET /llms-full.txt` sits beside `/llms.txt`: the same header plus one Stories section of only the indexable stories, each with its editor's opening summary as text (never a source's full article text), capped at 512 KiB.@@@ -9,7 +9,7 @@"domain": "openvibe.news","contracts": "openvibe-contracts v0.79.0","packages": [-- "openvibe-publishing v1.2.0",++ "openvibe-publishing v1.3.0","openvibe-contracts v0.79.0","openvibe-shared v2.6.0","openvibe-sdk v0.25.0"@@@ -16,7 +16,7 @@"helmet": "^8.1.0","iovalkey": "^0.4.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",-- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.2.0",++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.3.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.26.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.6.0","pg": "^8.23.0"@@@ -1163,9 +1163,9 @@}},"node_modules/openvibe-publishing": {-- "version": "1.2.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.2.0",-- "integrity": "sha512-n/20yxXo5jRs0wi7KsR+R9jDkGsoBak3gqElHV6XyU+i1t3XCzaoz+sDb10G2cbc4uEaNB/F09VtQKrQzLp6YQ==",++ "version": "1.3.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.3.0",++ "integrity": "sha512-dIXLK5JpFqUEztywlWgSupUa9NEkovVczR2qeIvpst5evOyKZlwGzQktArvWUQn1ly6CoLL5mn/FnKiqoPvZFA==","license": "MIT","engines": {"node": ">=20"@@@ -30,7 +30,7 @@"helmet": "^8.1.0","iovalkey": "^0.4.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",-- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.2.0",++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.3.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.26.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.6.0","pg": "^8.23.0"@@@ -5,6 +5,8 @@** GET /robots.txt sitemap location + explicit automated-consumer policy* GET /llms.txt orientation for language models++ * GET /llms-full.txt the same orientation plus one section per indexable story, each with++ * the editor's opening summary (never the source's full article text)* GET /sitemap.xml sitemap index over the two sections below* GET /sitemaps/stories.xml published, INDEXABLE stories only (the gate decides: retracted,* unsupported, AI-unreviewed and noindex stories never appear);@@@ -18,6 +20,9 @@ const seo = require('openvibe-publishing/seo');const sharedSeo = require('openvibe-shared/seo');const cache = require('openvibe-shared/cache-policy');++// The one-line site summary /llms.txt, /llms-full.txt and the home page all carry.++const SITE_SUMMARY = 'A source-backed publication: stories written by OpenVibe.News editors from source items collected by OpenVibe.Sources. Every paragraph cites the sources it rests on.';++function createDiscoveryRoutes({ config, store, publication, topics }) {const router = express.Router();const { db } = store;@@@ -49,7 +54,7 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {router.get('/llms.txt', (_req, res) => {res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(sharedSeo.llmsTxt({name: 'OpenVibe.News',-- summary: 'A source-backed publication: stories written by OpenVibe.News editors from source items collected by OpenVibe.Sources. Every paragraph cites the sources it rests on.',++ summary: SITE_SUMMARY,details: 'OpenVibe.News is not an automatic headline generator. Each story page lists its sources (headline, outlet, author and date as the source published them, with links), a timeline, editor-assigned perspective groupings and its full correction history. Full article text from sources is never republished; at most a short summary where the source’s terms allow it. AI-assisted drafts are never published or indexed before a person reviews them. Retracted stories stay at their URL with a retraction notice and are noindex. Append .json to any story URL for the same content as data, including every claim’s source numbers and the Sources item ids.',sections: [{ title: 'Start here', links: [@@@ -66,6 +71,26 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {}));});++ router.get('/llms-full.txt', async (_req, res) => {++ // The gate decides: only indexable stories, and only their editor's opening summary — the++ // source's full article text is never stored, so it can never appear here.++ const entries = (await storyEntries()).filter((e) => e.decision.indexable);++ res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(sharedSeo.llmsFull({++ site: 'OpenVibe.News',++ summary: SITE_SUMMARY,++ base: config.baseUrl,++ maxBytes: 512 * 1024,++ sections: [{++ title: 'Stories',++ pages: entries.map((e) => ({++ title: e.rev.fields.headline,++ url: publication.storyUrl(e.story),++ text: e.rev.fields.paragraphs && e.rev.fields.paragraphs[0] ? e.rev.fields.paragraphs[0].text : '',++ })),++ }],++ }));++ });++router.get('/sitemap.xml', async (_req, res) => {const entries = (await storyEntries()).filter((e) => e.decision.indexable);const newest = entries.length ? entries.map((e) => e.rev.createdAt).sort().pop() : null;@@@ -100,4 +125,4 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {return router;}--module.exports = { createDiscoveryRoutes };++module.exports = { createDiscoveryRoutes, SITE_SUMMARY };@@@ -28,6 +28,7 @@ const pages = require('../render/pages');const access = require('../domain/access');const { csrfToken, checkCsrf } = require('../auth/forms');const cache = require('openvibe-shared/cache-policy');++const { SITE_SUMMARY } = require('./discovery');const PER_PAGE = 20;@@@ -80,6 +81,7 @@ function createPublicRoutes(ctx) {send(req, res, 200, {title: topic ? topic.name : null,description: topic ? (topic.description || `Stories on ${topic.name}.`) : 'Source-backed stories: every paragraph cites the reports it rests on.',++ summary: topic ? undefined : SITE_SUMMARY,decision: pageDecision(canonical, { empty: total === 0, query: ['page'] }),canonical, feeds,prev: pager.prev ? pager.prev.href : null, next: pager.next ? pager.next.href : null,@@@ -41,7 +41,8 @@ function setRelease(id) { if (id) RELEASE = String(id); }/*** o: title, description, decision (required), canonical, type ('website'|'article'), image,* jsonLd [], feeds [{ type, href, title }], body (HTML), viewer, config, csrf,-- * published, modified, author, prev, next, bodyClass++ * published, modified, author, prev, next, bodyClass,++ * summary, facts, updated, url (the shell's ai-summary inputs; openvibe-publishing v1.3.0)*/function renderPage(o) {if (!o.decision) throw new TypeError('renderPage needs the gate decision');@@@ -74,6 +75,12 @@ function renderPage(o) {description: o.description || 'Source-backed news: every claim cites the reports it rests on.',canonical: o.canonical,decision: o.decision,++ // The shell builds the AI summary (seo.pageSummary): summary puts <meta name="ai-summary">++ // and a WebPage JSON-LD tag in <head>; facts/updated/url feed it. Nothing is invented.++ summary: o.summary,++ facts: o.facts,++ updated: o.updated,++ url: o.url,type: o.type || 'website',image: o.image,author: o.author,@@@ -107,6 +107,20 @@ async function storyOn(t, headline, items) {assert.match(await sitemap(), new RegExp(`<loc>https://openvibe.news/stories/${story.slug}</loc>`));assert.strictEqual((await indexEvent(story)).payload.indexability.decision, 'index');});++++ await check('/llms-full.txt lists only indexable stories, as the editor summary, within maxBytes', async () => {++ const hidden = await storyOn(t, 'Noindex story kept out of the full file', [feed1]);++ const listed = await storyOn(t, 'Indexable story carried in the full file', [solo, wire]);++ const r = await t.get('/llms-full.txt');++ assert.strictEqual(r.status, 200);++ assert.match(r.headers.get('content-type'), /^text\/plain/);++ assert.ok(r.text.includes('Indexable story carried in the full file'), 'the indexable headline is a page title');++ assert.ok(r.text.includes(`/stories/${listed.slug}`), 'the indexable story URL is listed');++ assert.ok(!r.text.includes('Noindex story kept out of the full file'), 'a noindex headline never appears');++ assert.ok(!r.text.includes(`/stories/${hidden.slug}`), 'a noindex story URL never appears');++ assert.ok(r.text.includes('Officials confirmed on Tuesday'), 'the editor summary text is the page text');++ assert.ok(Buffer.byteLength(r.text, 'utf8') <= 512 * 1024, 'within maxBytes');++ });await t.close();const one = await boot({ env: { NEWS_MIN_INDEPENDENT_SOURCES: '1' } });@@@ -86,7 +86,9 @@ const count = (html, re) => (html.match(re) || []).length;const head = home.slice(0, home.indexOf('</head>'));assert.strictEqual(count(head, /<title>/g), 1);assert.ok(head.includes('<link rel="canonical" href="https://openvibe.news/">'));-- assert.strictEqual(count(head, /<script type="application\/ld\+json">/g), 1, 'one JSON-LD script on the home page');++ assert.strictEqual(count(head, /<script type="application\/ld\+json">/g), 2, 'the home WebSite JSON-LD and the AI-summary WebPage JSON-LD');++ assert.ok(head.includes('"@type":"WebSite"'), 'the site JSON-LD');++ assert.ok(head.includes('"@type":"WebPage"'), 'the AI summary WebPage JSON-LD');assert.ok(head.includes('<link rel="alternate" type="application/atom+xml" href="/atom.xml" title="Atom">'));assert.ok(head.includes('<link rel="alternate" type="application/feed+json" href="/feed.json" title="JSON Feed">'));assert.ok(home.includes('Recently shipped on OpenVibe.News'), 'the shipped line stays on the home page');@@@ -74,7 +74,7 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.reawait check('no public surface, API response, event or Search document carries licensed text beyond the allowance', async () => {const paths = ['/', '/topics', '/topics/space', `/stories/${story.slug}`, `/stories/${story.slug}.json`, `/stories/${story.slug}?group=outlet`, `/stories/${story.slug}?group=perspective`,'/feed.xml', '/atom.xml', '/feed.json', '/topics/space/feed.xml', '/topics/space/atom.xml', '/topics/space/feed.json',-- '/sitemap.xml', '/sitemaps/stories.xml', '/sitemaps/topics.xml', '/robots.txt', '/llms.txt', '/api/v1/stories', `/api/v1/stories/${story.id}`, '/api/v1/topics'];++ '/sitemap.xml', '/sitemaps/stories.xml', '/sitemaps/topics.xml', '/robots.txt', '/llms.txt', '/llms-full.txt', '/api/v1/stories', `/api/v1/stories/${story.id}`, '/api/v1/topics'];let seen = '';for (const p of paths) {const res = await t.get(p);@@@ -39,6 +39,13 @@ const { boot, check, done } = require('./helpers/boot');assert.match(llms, /^# OpenVibe\.News/);assert.match(llms, /not an automatic headline generator/);assert.match(llms, /\.json/);++ const full = await t.get('/llms-full.txt');++ assert.strictEqual(full.status, 200);++ assert.match(full.headers.get('content-type'), /^text\/plain/);++ assert.strictEqual(full.headers.get('cache-control'), 'public, max-age=3600, stale-while-revalidate=3600');++ assert.match(full.text, /^# OpenVibe\.News/);++ assert.match(full.text, /source-backed publication/);++ assert.ok(Buffer.byteLength(full.text, 'utf8') <= 512 * 1024, 'no stories: within maxBytes');const idx = (await t.get('/sitemap.xml')).text;assert.match(idx, /<sitemapindex/);assert.match(idx, /sitemaps\/stories\.xml/);@@@ -47,6 +54,8 @@ const { boot, check, done } = require('./helpers/boot');assert.doesNotMatch(empty, /<url>/, 'no stories, no URLs');const home = await t.get('/');assert.match(home.text, /<meta name="robots" content="noindex, follow">/, 'an empty front page is not offered for indexing');++ assert.match(home.text, /<meta name="ai-summary" content="[^"]+">/, 'the home carries the site AI summary');++ assert.match(home.text, /"@type":"WebPage"/, 'and its WebPage JSON-LD twin');});await check('legal pages, 404s and API problems', async () => {
@@ -4,3 +4,5 @@
@@ -4,3 +4,5 @@
- Every page is rendered through `openvibe-publishing/layout` (v1.2.0, on `openvibe-shared/shell` v2.6.0): the head, the Frame, the noscript navigation, the footer and its init come from the shared document; robots and the canonical still come from the indexability gate's decision. The shell adds `web-runtime.js`, so the home page's JS budget is raised to 5 files, 245 KB, 59 KB brotli.
- Every page is rendered through `openvibe-publishing/layout` (v1.2.0, on `openvibe-shared/shell` v2.6.0): the head, the Frame, the noscript navigation, the footer and its init come from the shared document; robots and the canonical still come from the indexability gate's decision. The shell adds `web-runtime.js`, so the home page's JS budget is raised to 5 files, 245 KB, 59 KB brotli.
- The `openvibe-publishing` v1.2.0 lock entry carries its sha512 integrity again (the bump had dropped it), and a test fails if any codeload pin loses its hash.
- The `openvibe-publishing` v1.2.0 lock entry carries its sha512 integrity again (the bump had dropped it), and a test fails if any codeload pin loses its hash.
+- `openvibe-publishing` v1.3.0 (from v1.2.0): `layout.renderDocument` now forwards `summary`, `facts`, `updated` and `url` to the shell's AI-summary block, so the home page carries the same one-line summary as `<meta name="ai-summary">` and a WebPage JSON-LD twin.
+- `GET /llms-full.txt` sits beside `/llms.txt`: the same header plus one Stories section of only the indexable stories, each with its editor's opening summary as text (never a source's full article text), capped at 512 KiB.
@@ -9,7 +9,7 @@
@@ -9,7 +9,7 @@
"domain": "openvibe.news",
"domain": "openvibe.news",
"contracts": "openvibe-contracts v0.79.0",
"contracts": "openvibe-contracts v0.79.0",
"packages": [
"packages": [
- "openvibe-publishing v1.2.0",
+ "openvibe-publishing v1.3.0",
"openvibe-contracts v0.79.0",
"openvibe-contracts v0.79.0",
"openvibe-shared v2.6.0",
"openvibe-shared v2.6.0",
"openvibe-sdk v0.25.0"
"openvibe-sdk v0.25.0"
@@ -16,7 +16,7 @@
@@ -16,7 +16,7 @@
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",
- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.2.0",
+ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.3.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.26.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.26.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.6.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.6.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -1163,9 +1163,9 @@
@@ -1163,9 +1163,9 @@
}
}
},
},
"node_modules/openvibe-publishing": {
"node_modules/openvibe-publishing": {
- "version": "1.2.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.2.0",
- "integrity": "sha512-n/20yxXo5jRs0wi7KsR+R9jDkGsoBak3gqElHV6XyU+i1t3XCzaoz+sDb10G2cbc4uEaNB/F09VtQKrQzLp6YQ==",
+ "version": "1.3.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.3.0",
+ "integrity": "sha512-dIXLK5JpFqUEztywlWgSupUa9NEkovVczR2qeIvpst5evOyKZlwGzQktArvWUQn1ly6CoLL5mn/FnKiqoPvZFA==",
"license": "MIT",
"license": "MIT",
"engines": {
"engines": {
"node": ">=20"
"node": ">=20"
@@ -30,7 +30,7 @@
@@ -30,7 +30,7 @@
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",
- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.2.0",
+ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.3.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.26.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.26.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.6.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.6.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -5,6 +5,8 @@
@@ -5,6 +5,8 @@
*
*
* GET /robots.txt sitemap location + explicit automated-consumer policy
* GET /robots.txt sitemap location + explicit automated-consumer policy
* GET /llms.txt orientation for language models
* GET /llms.txt orientation for language models
+ * GET /llms-full.txt the same orientation plus one section per indexable story, each with
+ * the editor's opening summary (never the source's full article text)
* GET /sitemap.xml sitemap index over the two sections below
* GET /sitemap.xml sitemap index over the two sections below
* GET /sitemaps/stories.xml published, INDEXABLE stories only (the gate decides: retracted,
* GET /sitemaps/stories.xml published, INDEXABLE stories only (the gate decides: retracted,
* unsupported, AI-unreviewed and noindex stories never appear);
* unsupported, AI-unreviewed and noindex stories never appear);
@@ -18,6 +20,9 @@ const seo = require('openvibe-publishing/seo');
@@ -18,6 +20,9 @@ const seo = require('openvibe-publishing/seo');
const sharedSeo = require('openvibe-shared/seo');
const sharedSeo = require('openvibe-shared/seo');
const cache = require('openvibe-shared/cache-policy');
const cache = require('openvibe-shared/cache-policy');
+// The one-line site summary /llms.txt, /llms-full.txt and the home page all carry.
+const SITE_SUMMARY = 'A source-backed publication: stories written by OpenVibe.News editors from source items collected by OpenVibe.Sources. Every paragraph cites the sources it rests on.';
+
function createDiscoveryRoutes({ config, store, publication, topics }) {
function createDiscoveryRoutes({ config, store, publication, topics }) {
const router = express.Router();
const router = express.Router();
const { db } = store;
const { db } = store;
@@ -49,7 +54,7 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
@@ -49,7 +54,7 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
router.get('/llms.txt', (_req, res) => {
router.get('/llms.txt', (_req, res) => {
res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(sharedSeo.llmsTxt({
res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(sharedSeo.llmsTxt({
name: 'OpenVibe.News',
name: 'OpenVibe.News',
- summary: 'A source-backed publication: stories written by OpenVibe.News editors from source items collected by OpenVibe.Sources. Every paragraph cites the sources it rests on.',
+ summary: SITE_SUMMARY,
details: 'OpenVibe.News is not an automatic headline generator. Each story page lists its sources (headline, outlet, author and date as the source published them, with links), a timeline, editor-assigned perspective groupings and its full correction history. Full article text from sources is never republished; at most a short summary where the source’s terms allow it. AI-assisted drafts are never published or indexed before a person reviews them. Retracted stories stay at their URL with a retraction notice and are noindex. Append .json to any story URL for the same content as data, including every claim’s source numbers and the Sources item ids.',
details: 'OpenVibe.News is not an automatic headline generator. Each story page lists its sources (headline, outlet, author and date as the source published them, with links), a timeline, editor-assigned perspective groupings and its full correction history. Full article text from sources is never republished; at most a short summary where the source’s terms allow it. AI-assisted drafts are never published or indexed before a person reviews them. Retracted stories stay at their URL with a retraction notice and are noindex. Append .json to any story URL for the same content as data, including every claim’s source numbers and the Sources item ids.',
sections: [
sections: [
{ title: 'Start here', links: [
{ title: 'Start here', links: [
@@ -66,6 +71,26 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
@@ -66,6 +71,26 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
}));
}));
});
});
+ router.get('/llms-full.txt', async (_req, res) => {
+ // The gate decides: only indexable stories, and only their editor's opening summary — the
+ // source's full article text is never stored, so it can never appear here.
+ const entries = (await storyEntries()).filter((e) => e.decision.indexable);
+ res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(sharedSeo.llmsFull({
+ site: 'OpenVibe.News',
+ summary: SITE_SUMMARY,
+ base: config.baseUrl,
+ maxBytes: 512 * 1024,
+ sections: [{
+ title: 'Stories',
+ pages: entries.map((e) => ({
+ title: e.rev.fields.headline,
+ url: publication.storyUrl(e.story),
+ text: e.rev.fields.paragraphs && e.rev.fields.paragraphs[0] ? e.rev.fields.paragraphs[0].text : '',
+ })),
+ }],
+ }));
+ });
+
router.get('/sitemap.xml', async (_req, res) => {
router.get('/sitemap.xml', async (_req, res) => {
const entries = (await storyEntries()).filter((e) => e.decision.indexable);
const entries = (await storyEntries()).filter((e) => e.decision.indexable);
const newest = entries.length ? entries.map((e) => e.rev.createdAt).sort().pop() : null;
const newest = entries.length ? entries.map((e) => e.rev.createdAt).sort().pop() : null;
@@ -100,4 +125,4 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
@@ -100,4 +125,4 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
return router;
return router;
}
}
-module.exports = { createDiscoveryRoutes };
+module.exports = { createDiscoveryRoutes, SITE_SUMMARY };
@@ -28,6 +28,7 @@ const pages = require('../render/pages');
@@ -28,6 +28,7 @@ const pages = require('../render/pages');
const access = require('../domain/access');
const access = require('../domain/access');
const { csrfToken, checkCsrf } = require('../auth/forms');
const { csrfToken, checkCsrf } = require('../auth/forms');
const cache = require('openvibe-shared/cache-policy');
const cache = require('openvibe-shared/cache-policy');
+const { SITE_SUMMARY } = require('./discovery');
const PER_PAGE = 20;
const PER_PAGE = 20;
@@ -80,6 +81,7 @@ function createPublicRoutes(ctx) {
@@ -80,6 +81,7 @@ function createPublicRoutes(ctx) {
send(req, res, 200, {
send(req, res, 200, {
title: topic ? topic.name : null,
title: topic ? topic.name : null,
description: topic ? (topic.description || `Stories on ${topic.name}.`) : 'Source-backed stories: every paragraph cites the reports it rests on.',
description: topic ? (topic.description || `Stories on ${topic.name}.`) : 'Source-backed stories: every paragraph cites the reports it rests on.',
+ summary: topic ? undefined : SITE_SUMMARY,
decision: pageDecision(canonical, { empty: total === 0, query: ['page'] }),
decision: pageDecision(canonical, { empty: total === 0, query: ['page'] }),
canonical, feeds,
canonical, feeds,
prev: pager.prev ? pager.prev.href : null, next: pager.next ? pager.next.href : null,
prev: pager.prev ? pager.prev.href : null, next: pager.next ? pager.next.href : null,
@@ -41,7 +41,8 @@ function setRelease(id) { if (id) RELEASE = String(id); }
@@ -41,7 +41,8 @@ function setRelease(id) { if (id) RELEASE = String(id); }
/**
/**
* o: title, description, decision (required), canonical, type ('website'|'article'), image,
* o: title, description, decision (required), canonical, type ('website'|'article'), image,
* jsonLd [], feeds [{ type, href, title }], body (HTML), viewer, config, csrf,
* jsonLd [], feeds [{ type, href, title }], body (HTML), viewer, config, csrf,
- * published, modified, author, prev, next, bodyClass
+ * published, modified, author, prev, next, bodyClass,
+ * summary, facts, updated, url (the shell's ai-summary inputs; openvibe-publishing v1.3.0)
*/
*/
function renderPage(o) {
function renderPage(o) {
if (!o.decision) throw new TypeError('renderPage needs the gate decision');
if (!o.decision) throw new TypeError('renderPage needs the gate decision');
@@ -74,6 +75,12 @@ function renderPage(o) {
@@ -74,6 +75,12 @@ function renderPage(o) {
description: o.description || 'Source-backed news: every claim cites the reports it rests on.',
description: o.description || 'Source-backed news: every claim cites the reports it rests on.',
canonical: o.canonical,
canonical: o.canonical,
decision: o.decision,
decision: o.decision,
+ // The shell builds the AI summary (seo.pageSummary): summary puts <meta name="ai-summary">
+ // and a WebPage JSON-LD tag in <head>; facts/updated/url feed it. Nothing is invented.
+ summary: o.summary,
+ facts: o.facts,
+ updated: o.updated,
+ url: o.url,
type: o.type || 'website',
type: o.type || 'website',
image: o.image,
image: o.image,
author: o.author,
author: o.author,
@@ -107,6 +107,20 @@ async function storyOn(t, headline, items) {
@@ -107,6 +107,20 @@ async function storyOn(t, headline, items) {
assert.match(await sitemap(), new RegExp(`<loc>https://openvibe.news/stories/${story.slug}</loc>`));
assert.match(await sitemap(), new RegExp(`<loc>https://openvibe.news/stories/${story.slug}</loc>`));
assert.strictEqual((await indexEvent(story)).payload.indexability.decision, 'index');
assert.strictEqual((await indexEvent(story)).payload.indexability.decision, 'index');
});
});
+
+ await check('/llms-full.txt lists only indexable stories, as the editor summary, within maxBytes', async () => {
+ const hidden = await storyOn(t, 'Noindex story kept out of the full file', [feed1]);
+ const listed = await storyOn(t, 'Indexable story carried in the full file', [solo, wire]);
+ const r = await t.get('/llms-full.txt');
+ assert.strictEqual(r.status, 200);
+ assert.match(r.headers.get('content-type'), /^text\/plain/);
+ assert.ok(r.text.includes('Indexable story carried in the full file'), 'the indexable headline is a page title');
+ assert.ok(r.text.includes(`/stories/${listed.slug}`), 'the indexable story URL is listed');
+ assert.ok(!r.text.includes('Noindex story kept out of the full file'), 'a noindex headline never appears');
+ assert.ok(!r.text.includes(`/stories/${hidden.slug}`), 'a noindex story URL never appears');
+ assert.ok(r.text.includes('Officials confirmed on Tuesday'), 'the editor summary text is the page text');
+ assert.ok(Buffer.byteLength(r.text, 'utf8') <= 512 * 1024, 'within maxBytes');
+ });
await t.close();
await t.close();
const one = await boot({ env: { NEWS_MIN_INDEPENDENT_SOURCES: '1' } });
const one = await boot({ env: { NEWS_MIN_INDEPENDENT_SOURCES: '1' } });
@@ -86,7 +86,9 @@ const count = (html, re) => (html.match(re) || []).length;
@@ -86,7 +86,9 @@ const count = (html, re) => (html.match(re) || []).length;
const head = home.slice(0, home.indexOf('</head>'));
const head = home.slice(0, home.indexOf('</head>'));
assert.strictEqual(count(head, /<title>/g), 1);
assert.strictEqual(count(head, /<title>/g), 1);
assert.ok(head.includes('<link rel="canonical" href="https://openvibe.news/">'));
assert.ok(head.includes('<link rel="canonical" href="https://openvibe.news/">'));
- assert.strictEqual(count(head, /<script type="application\/ld\+json">/g), 1, 'one JSON-LD script on the home page');
+ assert.strictEqual(count(head, /<script type="application\/ld\+json">/g), 2, 'the home WebSite JSON-LD and the AI-summary WebPage JSON-LD');
+ assert.ok(head.includes('"@type":"WebSite"'), 'the site JSON-LD');
+ assert.ok(head.includes('"@type":"WebPage"'), 'the AI summary WebPage JSON-LD');
assert.ok(head.includes('<link rel="alternate" type="application/atom+xml" href="/atom.xml" title="Atom">'));
assert.ok(head.includes('<link rel="alternate" type="application/atom+xml" href="/atom.xml" title="Atom">'));
assert.ok(head.includes('<link rel="alternate" type="application/feed+json" href="/feed.json" title="JSON Feed">'));
assert.ok(head.includes('<link rel="alternate" type="application/feed+json" href="/feed.json" title="JSON Feed">'));
assert.ok(home.includes('Recently shipped on OpenVibe.News'), 'the shipped line stays on the home page');
assert.ok(home.includes('Recently shipped on OpenVibe.News'), 'the shipped line stays on the home page');
@@ -74,7 +74,7 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.re
@@ -74,7 +74,7 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.re
await check('no public surface, API response, event or Search document carries licensed text beyond the allowance', async () => {
await check('no public surface, API response, event or Search document carries licensed text beyond the allowance', async () => {
const paths = ['/', '/topics', '/topics/space', `/stories/${story.slug}`, `/stories/${story.slug}.json`, `/stories/${story.slug}?group=outlet`, `/stories/${story.slug}?group=perspective`,
const paths = ['/', '/topics', '/topics/space', `/stories/${story.slug}`, `/stories/${story.slug}.json`, `/stories/${story.slug}?group=outlet`, `/stories/${story.slug}?group=perspective`,
'/feed.xml', '/atom.xml', '/feed.json', '/topics/space/feed.xml', '/topics/space/atom.xml', '/topics/space/feed.json',
'/feed.xml', '/atom.xml', '/feed.json', '/topics/space/feed.xml', '/topics/space/atom.xml', '/topics/space/feed.json',
- '/sitemap.xml', '/sitemaps/stories.xml', '/sitemaps/topics.xml', '/robots.txt', '/llms.txt', '/api/v1/stories', `/api/v1/stories/${story.id}`, '/api/v1/topics'];
+ '/sitemap.xml', '/sitemaps/stories.xml', '/sitemaps/topics.xml', '/robots.txt', '/llms.txt', '/llms-full.txt', '/api/v1/stories', `/api/v1/stories/${story.id}`, '/api/v1/topics'];
let seen = '';
let seen = '';
for (const p of paths) {
for (const p of paths) {
const res = await t.get(p);
const res = await t.get(p);
@@ -39,6 +39,13 @@ const { boot, check, done } = require('./helpers/boot');
@@ -39,6 +39,13 @@ const { boot, check, done } = require('./helpers/boot');
assert.match(llms, /^# OpenVibe\.News/);
assert.match(llms, /^# OpenVibe\.News/);
assert.match(llms, /not an automatic headline generator/);
assert.match(llms, /not an automatic headline generator/);
assert.match(llms, /\.json/);
assert.match(llms, /\.json/);
+ const full = await t.get('/llms-full.txt');
+ assert.strictEqual(full.status, 200);
+ assert.match(full.headers.get('content-type'), /^text\/plain/);
+ assert.strictEqual(full.headers.get('cache-control'), 'public, max-age=3600, stale-while-revalidate=3600');
+ assert.match(full.text, /^# OpenVibe\.News/);
+ assert.match(full.text, /source-backed publication/);
+ assert.ok(Buffer.byteLength(full.text, 'utf8') <= 512 * 1024, 'no stories: within maxBytes');
const idx = (await t.get('/sitemap.xml')).text;
const idx = (await t.get('/sitemap.xml')).text;
assert.match(idx, /<sitemapindex/);
assert.match(idx, /<sitemapindex/);
assert.match(idx, /sitemaps\/stories\.xml/);
assert.match(idx, /sitemaps\/stories\.xml/);
@@ -47,6 +54,8 @@ const { boot, check, done } = require('./helpers/boot');
@@ -47,6 +54,8 @@ const { boot, check, done } = require('./helpers/boot');
assert.doesNotMatch(empty, /<url>/, 'no stories, no URLs');
assert.doesNotMatch(empty, /<url>/, 'no stories, no URLs');
const home = await t.get('/');
const home = await t.get('/');
assert.match(home.text, /<meta name="robots" content="noindex, follow">/, 'an empty front page is not offered for indexing');
assert.match(home.text, /<meta name="robots" content="noindex, follow">/, 'an empty front page is not offered for indexing');
+ assert.match(home.text, /<meta name="ai-summary" content="[^"]+">/, 'the home carries the site AI summary');
+ assert.match(home.text, /"@type":"WebPage"/, 'and its WebPage JSON-LD twin');
});
});
await check('legal pages, 404s and API problems', async () => {
await check('legal pages, 404s and API problems', async () => {