openvibers/openvibe.trade Commits
Sitemaps: every entry carries an honest lastmod (item changes, release time for static pages) (#46)
Changes
@@@ -138,13 +138,15 @@ deployment has no instruments (editors add them), no documents until the source| `/api/health`, `/api/ready`, `/release.json`, `POST /release-metrics`, `/metrics` | health, readiness, release, open tabs' update reports (counted in `/metrics`), metrics (loopback only) || `POST /internal/events` | signed OpenVibe.Events deliveries (`sources.*`, and the two ADR-033 account events), host-local |++Sitemap entries carry lastmod from each instrument's latest recorded change.++### Discoverability (roadmap §32)- Instrument pages go through the `openvibe-publishing/seo` gate. Financial information is a**sensitive** category: a page is `noindex` (`unreviewed_sensitive`) until a person-reviewedcontext is published, `noindex` when its newest monetary observation is older than`TRADE_PRICE_MAX_AGE_SEC` (`stale_price`), and `thin` below `TRADE_GATE_MIN_WORDS` of context.--- The sitemap lists indexable instrument pages only, `lastmod` = the context's real publication time.++- The sitemap lists indexable instrument pages only, `lastmod` = the instrument's latest recorded change.- JSON-LD: `WebPage` (+ `Corporation` with `tickerSymbol` for equities) and breadcrumbs, from realfields only. No price, offer, rating or invented date.- Every instrument page has a JSON twin with the same data (`/i/:symbol.json`).@@@ -10,7 +10,7 @@* GET /sitemap.xml sitemap index* GET /sitemaps/instruments.xml INDEXABLE instrument pages only (the gate decides:* person-reviewed context, no stale price); lastmod = the-- * context's real publication time++ * latest instrument, observation or document change* GET /feed.xml | /atom.xml | /feed.json recent public documents across instruments* GET /i/:symbol/documents.xml|.atom|.json one instrument's documents*@@@ -103,18 +103,29 @@ function createDiscovery(ctx) {}));});++ async function pageChanges() {++ const rows = await store.db.prepare(`SELECT instrument_id, MAX(changed_at) AS changed_at FROM (++ SELECT instrument_id, recorded_at AS changed_at FROM trade_market_observations++ UNION ALL SELECT instrument_id, updated_at AS changed_at FROM trade_source_documents++ ) changes GROUP BY instrument_id`).all();++ return new Map(rows.map((r) => [r.instrument_id, r.changed_at]));++ }++ const pageLastmod = (instrument, changes) => Math.max(instrument.updated_at || 0, instrument.context_published_at || 0, changes.get(instrument.id) || 0);++define(router, 'get', '/sitemap.xml', 'sitemapIndex', async (_req, res) => {const active = await instruments.active();const decisions = await Promise.all(active.map(async (i) => await indexing.decide(i)));-- const times = active.filter((i, n) => decisions[n].indexable && i.context_published_at).map((i) => i.context_published_at);++ const changes = await pageChanges();++ const times = active.filter((i, n) => decisions[n].indexable).map((i) => pageLastmod(i, changes)).filter(Boolean);const newest = times.length ? new Date(Math.max(...times)).toISOString() : null;xml(res, seo.sitemapIndex([{ loc: abs('/sitemaps/instruments.xml'), ...(newest ? { lastmod: newest } : {}) }]));});define(router, 'get', '/sitemaps/instruments.xml', 'instrumentSitemap', async (_req, res) => {++ const changes = await pageChanges();const entries = (await Promise.all((await instruments.active()).map(async (i) => ({loc: urls.instrument(i),-- ...(i.context_published_at ? { lastmod: new Date(i.context_published_at).toISOString() } : {}),++ ...(pageLastmod(i, changes) ? { lastmod: new Date(pageLastmod(i, changes)).toISOString() } : {}),decision: await indexing.decide(i),}))));xml(res, seo.sitemap(entries).files[0]);@@@ -5,6 +5,17 @@* endpoints answer; the historical marketplace branch is recorded as an open question, not built.*/const assert = require('assert');++++function assertSitemapLastmods(xml, tag = 'url') {++ const entries = [...xml.matchAll(new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`, 'g'))];++ for (const [, body] of entries) {++ const match = /<lastmod>(\d{4}-\d{2}-\d{2})<\/lastmod>/.exec(body);++ assert.ok(match, body);++ assert.strictEqual(new Date(match[1]).toISOString().slice(0, 10), match[1]);++ }++ return entries;++}++const fs = require('fs');const path = require('path');const { boot, check, done } = require('./helpers/boot');@@@ -72,6 +83,7 @@ const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');assert.ok(/automated-consumer policy/.test(robots));assert.ok((await t.get('/sitemap.xml')).text.includes('https://openvibe.trade/sitemaps/instruments.xml'));assert.ok(!(await t.get('/sitemaps/instruments.xml')).text.includes('<loc>'), 'no indexable instrument yet');++ assertSitemapLastmods((await t.get('/sitemaps/instruments.xml')).text);assert.ok((await t.get('/llms.txt')).text.startsWith('# OpenVibe.Trade'));});@@@ -122,6 +134,14 @@ const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');await t.instrument({ symbol: 'OTHR', name: 'Other Co' });const pub = await t.get('/editor/i/ACME/context', { as: t.editor, form: { _csrf: t.csrf(t.editor), body: `Acme's price is stated by the source. ${WORDS(70)}`, cite: `observation:${observation.id}`, publish: '1' } });assert.ok(/n=context_published/.test(pub.headers.get('location')), 'the context published');++ const map = (await t.get('/sitemaps/instruments.xml')).text;++ assert.strictEqual(assertSitemapLastmods(map).length, 1);++ const instrument = await t.ctx.store.db.prepare("SELECT id, updated_at, context_published_at FROM trade_instruments WHERE symbol = 'ACME'").get();++ const observationTime = await t.ctx.store.db.prepare('SELECT MAX(recorded_at) AS at FROM trade_market_observations WHERE instrument_id = ?').get(instrument.id);++ const documentTime = await t.ctx.store.db.prepare('SELECT MAX(updated_at) AS at FROM trade_source_documents WHERE instrument_id = ?').get(instrument.id);++ const expected = new Date(Math.max(instrument.updated_at, instrument.context_published_at || 0, observationTime.at || 0, documentTime.at || 0)).toISOString().slice(0, 10);++ assert.ok(map.includes(`<loc>https://openvibe.trade/i/ACME</loc><lastmod>${expected}</lastmod>`));++ assertSitemapLastmods((await t.get('/sitemap.xml')).text, 'sitemap');const after = await t.get('/llms-full.txt');assert.ok(after.text.includes('https://openvibe.trade/i/ACME'));assert.ok(after.text.includes('### ACME — Acme Corp'));
@@ -138,13 +138,15 @@ deployment has no instruments (editors add them), no documents until the source
@@ -138,13 +138,15 @@ deployment has no instruments (editors add them), no documents until the source
| `/api/health`, `/api/ready`, `/release.json`, `POST /release-metrics`, `/metrics` | health, readiness, release, open tabs' update reports (counted in `/metrics`), metrics (loopback only) |
| `/api/health`, `/api/ready`, `/release.json`, `POST /release-metrics`, `/metrics` | health, readiness, release, open tabs' update reports (counted in `/metrics`), metrics (loopback only) |
| `POST /internal/events` | signed OpenVibe.Events deliveries (`sources.*`, and the two ADR-033 account events), host-local |
| `POST /internal/events` | signed OpenVibe.Events deliveries (`sources.*`, and the two ADR-033 account events), host-local |
+Sitemap entries carry lastmod from each instrument's latest recorded change.
+
### Discoverability (roadmap §32)
### Discoverability (roadmap §32)
- Instrument pages go through the `openvibe-publishing/seo` gate. Financial information is a
- Instrument pages go through the `openvibe-publishing/seo` gate. Financial information is a
**sensitive** category: a page is `noindex` (`unreviewed_sensitive`) until a person-reviewed
**sensitive** category: a page is `noindex` (`unreviewed_sensitive`) until a person-reviewed
context is published, `noindex` when its newest monetary observation is older than
context is published, `noindex` when its newest monetary observation is older than
`TRADE_PRICE_MAX_AGE_SEC` (`stale_price`), and `thin` below `TRADE_GATE_MIN_WORDS` of context.
`TRADE_PRICE_MAX_AGE_SEC` (`stale_price`), and `thin` below `TRADE_GATE_MIN_WORDS` of context.
-- The sitemap lists indexable instrument pages only, `lastmod` = the context's real publication time.
+- The sitemap lists indexable instrument pages only, `lastmod` = the instrument's latest recorded change.
- JSON-LD: `WebPage` (+ `Corporation` with `tickerSymbol` for equities) and breadcrumbs, from real
- JSON-LD: `WebPage` (+ `Corporation` with `tickerSymbol` for equities) and breadcrumbs, from real
fields only. No price, offer, rating or invented date.
fields only. No price, offer, rating or invented date.
- Every instrument page has a JSON twin with the same data (`/i/:symbol.json`).
- Every instrument page has a JSON twin with the same data (`/i/:symbol.json`).
@@ -10,7 +10,7 @@
@@ -10,7 +10,7 @@
* GET /sitemap.xml sitemap index
* GET /sitemap.xml sitemap index
* GET /sitemaps/instruments.xml INDEXABLE instrument pages only (the gate decides:
* GET /sitemaps/instruments.xml INDEXABLE instrument pages only (the gate decides:
* person-reviewed context, no stale price); lastmod = the
* person-reviewed context, no stale price); lastmod = the
- * context's real publication time
+ * latest instrument, observation or document change
* GET /feed.xml | /atom.xml | /feed.json recent public documents across instruments
* GET /feed.xml | /atom.xml | /feed.json recent public documents across instruments
* GET /i/:symbol/documents.xml|.atom|.json one instrument's documents
* GET /i/:symbol/documents.xml|.atom|.json one instrument's documents
*
*
@@ -103,18 +103,29 @@ function createDiscovery(ctx) {
@@ -103,18 +103,29 @@ function createDiscovery(ctx) {
}));
}));
});
});
+ async function pageChanges() {
+ const rows = await store.db.prepare(`SELECT instrument_id, MAX(changed_at) AS changed_at FROM (
+ SELECT instrument_id, recorded_at AS changed_at FROM trade_market_observations
+ UNION ALL SELECT instrument_id, updated_at AS changed_at FROM trade_source_documents
+ ) changes GROUP BY instrument_id`).all();
+ return new Map(rows.map((r) => [r.instrument_id, r.changed_at]));
+ }
+ const pageLastmod = (instrument, changes) => Math.max(instrument.updated_at || 0, instrument.context_published_at || 0, changes.get(instrument.id) || 0);
+
define(router, 'get', '/sitemap.xml', 'sitemapIndex', async (_req, res) => {
define(router, 'get', '/sitemap.xml', 'sitemapIndex', async (_req, res) => {
const active = await instruments.active();
const active = await instruments.active();
const decisions = await Promise.all(active.map(async (i) => await indexing.decide(i)));
const decisions = await Promise.all(active.map(async (i) => await indexing.decide(i)));
- const times = active.filter((i, n) => decisions[n].indexable && i.context_published_at).map((i) => i.context_published_at);
+ const changes = await pageChanges();
+ const times = active.filter((i, n) => decisions[n].indexable).map((i) => pageLastmod(i, changes)).filter(Boolean);
const newest = times.length ? new Date(Math.max(...times)).toISOString() : null;
const newest = times.length ? new Date(Math.max(...times)).toISOString() : null;
xml(res, seo.sitemapIndex([{ loc: abs('/sitemaps/instruments.xml'), ...(newest ? { lastmod: newest } : {}) }]));
xml(res, seo.sitemapIndex([{ loc: abs('/sitemaps/instruments.xml'), ...(newest ? { lastmod: newest } : {}) }]));
});
});
define(router, 'get', '/sitemaps/instruments.xml', 'instrumentSitemap', async (_req, res) => {
define(router, 'get', '/sitemaps/instruments.xml', 'instrumentSitemap', async (_req, res) => {
+ const changes = await pageChanges();
const entries = (await Promise.all((await instruments.active()).map(async (i) => ({
const entries = (await Promise.all((await instruments.active()).map(async (i) => ({
loc: urls.instrument(i),
loc: urls.instrument(i),
- ...(i.context_published_at ? { lastmod: new Date(i.context_published_at).toISOString() } : {}),
+ ...(pageLastmod(i, changes) ? { lastmod: new Date(pageLastmod(i, changes)).toISOString() } : {}),
decision: await indexing.decide(i),
decision: await indexing.decide(i),
}))));
}))));
xml(res, seo.sitemap(entries).files[0]);
xml(res, seo.sitemap(entries).files[0]);
@@ -5,6 +5,17 @@
@@ -5,6 +5,17 @@
* endpoints answer; the historical marketplace branch is recorded as an open question, not built.
* endpoints answer; the historical marketplace branch is recorded as an open question, not built.
*/
*/
const assert = require('assert');
const assert = require('assert');
+
+function assertSitemapLastmods(xml, tag = 'url') {
+ const entries = [...xml.matchAll(new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`, 'g'))];
+ for (const [, body] of entries) {
+ const match = /<lastmod>(\d{4}-\d{2}-\d{2})<\/lastmod>/.exec(body);
+ assert.ok(match, body);
+ assert.strictEqual(new Date(match[1]).toISOString().slice(0, 10), match[1]);
+ }
+ return entries;
+}
+
const fs = require('fs');
const fs = require('fs');
const path = require('path');
const path = require('path');
const { boot, check, done } = require('./helpers/boot');
const { boot, check, done } = require('./helpers/boot');
@@ -72,6 +83,7 @@ const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');
@@ -72,6 +83,7 @@ const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');
assert.ok(/automated-consumer policy/.test(robots));
assert.ok(/automated-consumer policy/.test(robots));
assert.ok((await t.get('/sitemap.xml')).text.includes('https://openvibe.trade/sitemaps/instruments.xml'));
assert.ok((await t.get('/sitemap.xml')).text.includes('https://openvibe.trade/sitemaps/instruments.xml'));
assert.ok(!(await t.get('/sitemaps/instruments.xml')).text.includes('<loc>'), 'no indexable instrument yet');
assert.ok(!(await t.get('/sitemaps/instruments.xml')).text.includes('<loc>'), 'no indexable instrument yet');
+ assertSitemapLastmods((await t.get('/sitemaps/instruments.xml')).text);
assert.ok((await t.get('/llms.txt')).text.startsWith('# OpenVibe.Trade'));
assert.ok((await t.get('/llms.txt')).text.startsWith('# OpenVibe.Trade'));
});
});
@@ -122,6 +134,14 @@ const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');
@@ -122,6 +134,14 @@ const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');
await t.instrument({ symbol: 'OTHR', name: 'Other Co' });
await t.instrument({ symbol: 'OTHR', name: 'Other Co' });
const pub = await t.get('/editor/i/ACME/context', { as: t.editor, form: { _csrf: t.csrf(t.editor), body: `Acme's price is stated by the source. ${WORDS(70)}`, cite: `observation:${observation.id}`, publish: '1' } });
const pub = await t.get('/editor/i/ACME/context', { as: t.editor, form: { _csrf: t.csrf(t.editor), body: `Acme's price is stated by the source. ${WORDS(70)}`, cite: `observation:${observation.id}`, publish: '1' } });
assert.ok(/n=context_published/.test(pub.headers.get('location')), 'the context published');
assert.ok(/n=context_published/.test(pub.headers.get('location')), 'the context published');
+ const map = (await t.get('/sitemaps/instruments.xml')).text;
+ assert.strictEqual(assertSitemapLastmods(map).length, 1);
+ const instrument = await t.ctx.store.db.prepare("SELECT id, updated_at, context_published_at FROM trade_instruments WHERE symbol = 'ACME'").get();
+ const observationTime = await t.ctx.store.db.prepare('SELECT MAX(recorded_at) AS at FROM trade_market_observations WHERE instrument_id = ?').get(instrument.id);
+ const documentTime = await t.ctx.store.db.prepare('SELECT MAX(updated_at) AS at FROM trade_source_documents WHERE instrument_id = ?').get(instrument.id);
+ const expected = new Date(Math.max(instrument.updated_at, instrument.context_published_at || 0, observationTime.at || 0, documentTime.at || 0)).toISOString().slice(0, 10);
+ assert.ok(map.includes(`<loc>https://openvibe.trade/i/ACME</loc><lastmod>${expected}</lastmod>`));
+ assertSitemapLastmods((await t.get('/sitemap.xml')).text, 'sitemap');
const after = await t.get('/llms-full.txt');
const after = await t.get('/llms-full.txt');
assert.ok(after.text.includes('https://openvibe.trade/i/ACME'));
assert.ok(after.text.includes('https://openvibe.trade/i/ACME'));
assert.ok(after.text.includes('### ACME — Acme Corp'));
assert.ok(after.text.includes('### ACME — Acme Corp'));