openvibers/openvibe.trade Commits

Trade: /sources leaves the sitemap, llms-full names every public page (SEO-18, SEO-33) (#134)

OP OpenVibers committed 15 days ago · 62cb270ab9 · 2 files, +36 −7

Changes

ddiff --git a/server/http/discovery.js b/server/http/discovery.js
iindex c153c45..ccf6ba7 100644
---- a/server/http/discovery.js
++++ b/server/http/discovery.js
@@@ -8,7 +8,7 @@
* GET /llms-full.txt the same map, with a short summary of every indexable
* instrument (never the source's full text)
* GET /sitemap.xml sitemap index
-- * GET /sitemaps/pages.xml the front page, /instruments, /filings and /sources
++ * GET /sitemaps/pages.xml the front page, /instruments and /filings (not /sources: it is noindex)
* GET /sitemaps/instruments.xml INDEXABLE instrument pages only (the gate decides:
* person-reviewed context, no stale price); lastmod = the
* latest instrument, observation or document change
@@@ -25,6 +25,26 @@ const cache = require('openvibe-shared/cache-policy');
const { define } = require('./routes');
const { DISCLAIMER, SITE_NAME, SITE_SUMMARY } = require('../render/layout');
++/**
++ * The public page catalogue: /sitemaps/pages.xml lists the entries marked for the sitemap and
++ * /llms-full.txt describes every entry, so the two artifacts cannot drift apart. Entity pages are
++ * appended from the database below. /sources is public but noindex — it is freshness telemetry, not
++ * a landing page — so it is described for AI readers and deliberately left out of the sitemap.
++ */
++const PUBLIC_PAGES = [
++ { path: '/', sitemap: true, title: 'OpenVibe.Trade home', text: 'AI briefs written only from SEC numbers and filings, euro reference rates, new annual and quarterly reports, and the newest filings by form. Information only; there is no trading here.' },
++ { path: '/instruments', sitemap: true, title: 'Companies and currencies', text: 'Every instrument OpenVibe.Trade covers, the newest filing or figure first; each page states its observations, their source and when they were retrieved.' },
++ { path: '/filings', sitemap: true, title: 'Filings', text: 'The newest SEC filings across instruments; /filings?form=10-K filters to one form type.' },
++ { path: '/sources', sitemap: false, title: 'Source freshness', text: 'The freshness of every data source, its last successful sync and any error. Public but noindex: a status page, not a landing page.' },
++ { path: '/i/:symbol', title: 'One instrument page', text: 'One company or currency: observations with observed_at, retrieved_at and their source, the reviewed context, filings and source freshness. Append .json for the same data as JSON; /i/<SYMBOL>/documents.xml, .atom and .json are its document feeds.' },
++];
++
++/** The legal pages every OpenVibe site serves, named from the shared kit rather than retyped here. */
++function legalPages() {
++ const labels = { '/terms': 'Terms of use', '/privacy': 'Privacy policy', '/dmca': 'DMCA and takedown policy' };
++ return require('openvibe-shared/legal').PATHS.map((path) => ({ path, title: labels[path] || path.slice(1), text: `The ${(labels[path] || path.slice(1)).toLowerCase()} for OpenVibe.Trade, served by the site itself.` }));
++}
++
function createDiscovery(ctx) {
const { config, store, instruments, documents, indexing, context, urls, common } = ctx;
const router = express.Router();
@@@ -96,13 +116,14 @@ function createDiscovery(ctx) {
define(router, 'get', '/llms-full.txt', 'llmsFullTxt', async (_req, res) => {
const active = await instruments.indexCandidates();
const decisions = await Promise.all(active.map(async (i) => await indexing.decide(i)));
-- const pages = await Promise.all(active.filter((_i, n) => decisions[n].indexable).map(async (i) => {
++ const instrumentsPages = await Promise.all(active.filter((_i, n) => decisions[n].indexable).map(async (i) => {
const rev = await context.published(i);
return { title: `${i.symbol} — ${i.name}`, url: urls.instrument(i), text: rev ? String(rev.text || '').replace(/\s+/g, ' ').trim().slice(0, 300) : '' };
}));
++ const pages = [...PUBLIC_PAGES, ...legalPages()].map((p) => ({ url: abs(p.path), title: p.title, text: p.text }));
res.type('text/plain').set('Cache-Control', cache.htmlHeaders({ maxAge: 3600 })).send(sharedSeo.llmsFull({
site: SITE_NAME, summary: SITE_SUMMARY, base: config.baseUrl, maxBytes: 512 * 1024,
-- sections: [{ title: 'Instruments', pages }],
++ sections: [{ title: 'Pages', pages }, { title: 'Instruments', pages: instrumentsPages }],
}));
});
@@@ -128,13 +149,13 @@ function createDiscovery(ctx) {
]));
});
-- // The pages a person browses: the front page, every company and currency, the filings, source freshness. Their
-- // lastmod is the newest filing's (they list it); /sources has none of its own.
++ // The pages a person browses: the front page, every company and currency, the filings. Their lastmod is the
++ // newest filing's (they list it). /sources is public but noindex, so it never enters the sitemap.
define(router, 'get', '/sitemaps/pages.xml', 'pagesSitemap', async (_req, res) => {
const newest = (await documents.recent(1))[0];
const lastmod = newest ? Math.max(newest.first_seen_at, newest.published_at || 0) : null;
const decide = (path) => seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: abs(path), wordCount: 0 }, { policy: { minWords: 0 } });
-- xml(res, seo.sitemap(['/', '/instruments', '/filings', '/sources'].map((path) => ({ loc: abs(path), lastmod: path === '/sources' ? null : lastmod, decision: decide(path) }))).files[0]);
++ xml(res, seo.sitemap(PUBLIC_PAGES.filter((p) => p.sitemap).map((p) => ({ loc: abs(p.path), lastmod, decision: decide(p.path) }))).files[0]);
});
define(router, 'get', '/sitemaps/instruments.xml', 'instrumentSitemap', async (_req, res) => {
ddiff --git a/test/pages.test.js b/test/pages.test.js
iindex 8fcf13e..4142325 100644
---- a/test/pages.test.js
++++ b/test/pages.test.js
@@@ -127,6 +127,10 @@ const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');
assert.ok(/text\/plain/.test(before.headers.get('content-type')));
assert.ok(before.text.startsWith('# OpenVibe.Trade'));
assert.ok(before.text.includes('not investment advice'));
++ assert.ok(before.text.length >= 500, `real content, not a stub (${before.text.length} characters)`);
++ for (const p of ['/', '/instruments', '/filings', '/sources', '/i/:symbol', '/terms', '/privacy', '/dmca']) {
++ assert.ok(before.text.includes(`https://openvibe.trade${p}`), `llms-full.txt names the public route ${p}`);
++ }
assert.ok(!before.text.includes('https://openvibe.trade/i/ACME'), 'nothing indexable is listed yet');
assert.ok(Buffer.byteLength(before.text) <= 512 * 1024, 'llms-full.txt stays within maxBytes');
@@@ -183,7 +187,11 @@ const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');
assert.ok(tenK.text.includes('Annual reports (10-K)') && /name="robots" content="noindex/.test(tenK.text), 'one form: noindex, canonical /filings');
assert.ok((await t.get('/sitemap.xml')).text.includes('https://openvibe.trade/sitemaps/pages.xml'));
const pages = (await t.get('/sitemaps/pages.xml')).text;
-- for (const p of ['/', '/instruments', '/filings', '/sources']) assert.ok(pages.includes(`<loc>https://openvibe.trade${p}</loc>`), p);
++ for (const p of ['/', '/instruments', '/filings']) assert.ok(pages.includes(`<loc>https://openvibe.trade${p}</loc>`), p);
++ assert.ok(!pages.includes('<loc>https://openvibe.trade/sources</loc>'), 'SEO-18: /sources is not in the sitemap');
++ const sources = await t.get('/sources');
++ assert.strictEqual(sources.status, 200);
++ assert.match(sources.headers.get('x-robots-tag') || '', /noindex/, 'SEO-18: the page stays noindex');
});
await check('the front page\'s euro rates: each currency\'s newest rate, its date and source, and the change from the same source', async () => {