openvibers/openvibe.news Commits

News topic pages carry the AI briefs and the wire on their topic (a deterministic source-and-words match); a quiet topic says so and shows the latest wire, never "no stories have been published yet" (#71)

OP OpenVibers committed 28 days ago · 919a9de7d9 · 5 files, +114 −3

Changes

ddiff --git a/server/domain/clusters.js b/server/domain/clusters.js
iindex 0aaca66..b685dd0 100644
---- a/server/domain/clusters.js
++++ b/server/domain/clusters.js
@@@ -26,6 +26,7 @@
const { ids } = require('openvibe-contracts');
const { normalize } = require('openvibe-publishing/ingest');
const { ApiError } = require('../http/errors');
++const { topicsFor } = require('./topic-match');
const newClusterId = (now) => `clu_${ids.ulid(now)}`;
const newAuditId = (now) => `cla_${ids.ulid(now)}`;
@@@ -263,6 +264,34 @@ function createClusters({ store, config, outbox }) {
return out;
},
++ /**
++ * The wire for one topic (server/domain/topic-match.js): open clusters with activity in the last `days`, newest
++ * first, that the topic's sources or words place in it; shaped like wire(). One query, then matched in memory.
++ */
++ async topicWire(slug, { limit = 20, days = 30 } = {}) {
++ const since = store.now() - days * 86400000;
++ const rows = await db.prepare(`SELECT c.id AS cid, c.label, c.window_end, i.id, i.headline, i.canonical_url, i.outlet, i.published_at, i.source_key
++ FROM news_story_clusters c JOIN news_source_items i ON i.cluster_id = c.id
++ WHERE c.status = 'open' AND c.window_end >= ? AND i.status = 'active' AND i.removed_at IS NULL
++ ORDER BY c.window_end DESC, c.id DESC`).all(since);
++ const byCluster = new Map();
++ for (const r of rows) {
++ if (!byCluster.has(r.cid)) byCluster.set(r.cid, { id: r.cid, label: r.label, updated: r.window_end, items: [], sources: new Set() });
++ const c = byCluster.get(r.cid);
++ c.sources.add(r.source_key);
++ if (r.canonical_url) c.items.push({ id: r.id, headline: r.headline, url: r.canonical_url, outlet: r.outlet, published_at: r.published_at });
++ }
++ const out = [];
++ for (const c of byCluster.values()) {
++ if (!c.items.length) continue;
++ if (!topicsFor({ label: c.label, headlines: c.items.map((i) => i.headline), sources: [...c.sources] }).includes(slug)) continue;
++ c.items.sort((a, b) => String(b.published_at || '').localeCompare(String(a.published_at || '')));
++ out.push({ id: c.id, label: c.label, updated: c.updated, items: c.items });
++ if (out.length >= limit) break;
++ }
++ return out;
++ },
++
/** Open clusters, newest activity first (the editor dashboard). */
async recent({ limit = 50 } = {}) {
return await db.prepare("SELECT * FROM news_story_clusters WHERE status = 'open' ORDER BY window_end DESC, id DESC LIMIT ?").all(limit);
ddiff --git a/server/domain/topic-match.js b/server/domain/topic-match.js
nnew file mode 100644
iindex 0000000..b3a2933
---- /dev/null
++++ b/server/domain/topic-match.js
@@@ -0,0 +1,47 @@
++'use strict';
++/**
++ * Which topics a wire cluster belongs to, so a topic page carries the AI briefs and reports on it (not only the
++ * stories editors filed under it). Deterministic and explainable, like clustering:
++ *
++ * a source can place everything it reports (NASA's releases are Space, the FDA's recalls are Health);
++ * otherwise a topic's words must appear: once in the cluster's label, or twice (two different words) across the
++ * label and its reports' headlines.
++ *
++ * topicsFor({ label, headlines, sources }) → [slug, …] (several topics can match; none is fine)
++ */
++const RULES = Object.freeze({
++ space: { sources: ['nasa-news-releases'], words: ['nasa', 'spacecraft', 'astronaut', 'astronauts', 'orbit', 'orbital', 'rocket', 'mars', 'moon', 'lunar', 'satellite', 'satellites', 'telescope', 'space station', 'asteroid', 'artemis', 'esa', 'spaceflight', 'launch site', 'uk space agency'] },
++ science: { words: ['research', 'researchers', 'scientists', 'scientific', 'science', 'study', 'discovery', 'physics', 'chemistry', 'biology', 'laboratory', 'genome', 'fossil', 'quantum', 'experiment'] },
++ health: { sources: ['fda-food-recalls'], words: ['health', 'hospital', 'hospitals', 'nhs', 'disease', 'vaccine', 'vaccines', 'virus', 'outbreak', 'cancer', 'medicine', 'medicines', 'medical', 'patients', 'mental health', 'food safety', 'allergen', 'salmonella', 'listeria', 'recall', 'pharmaceutical', 'care homes', 'gp'] },
++ climate: { words: ['climate', 'emissions', 'net zero', 'carbon', 'flood', 'flooding', 'floods', 'heatwave', 'drought', 'wildfire', 'wildfires', 'storm', 'environment', 'environmental', 'renewable', 'renewables', 'wind farm', 'solar', 'biodiversity', 'pollution', 'energy transition', 'green'] },
++ business: { words: ['economy', 'economic', 'business', 'businesses', 'trade', 'tariff', 'tariffs', 'market', 'markets', 'investment', 'investors', 'inflation', 'jobs', 'employment', 'industry', 'industrial', 'budget', 'tax', 'taxes', 'bank', 'banks', 'competition', 'merger', 'consumers', 'retail', 'exports', 'growth', 'small firms'] },
++ security: { words: ['cyber', 'cybersecurity', 'cyber security', 'hacker', 'hackers', 'hacked', 'ransomware', 'vulnerability', 'vulnerabilities', 'data breach', 'breach', 'malware', 'phishing', 'ncsc', 'cisa', 'enisa', 'fraud', 'scam', 'scams'] },
++ technology: { words: ['technology', 'tech', 'digital', 'artificial intelligence', 'ai', 'software', 'semiconductor', 'semiconductors', 'chips', 'broadband', '5g', 'computing', 'robotics', 'data centre', 'data centres', 'data center', 'innovation', 'startups'] },
++ 'internet-policy': { words: ['online safety', 'digital services act', 'digital markets act', 'data protection', 'gdpr', 'privacy', 'net neutrality', 'online platforms', 'ofcom', 'copyright', 'content moderation', 'encryption', 'disinformation', 'age verification'] },
++ 'open-source': { words: ['open source', 'open-source', 'linux', 'open standards', 'open data', 'github'] },
++ games: { words: ['video game', 'video games', 'videogame', 'gaming', 'esports', 'playstation', 'xbox', 'nintendo', 'steam'] },
++ streaming: { words: ['streaming', 'streamer', 'streamers', 'twitch', 'youtube', 'netflix', 'broadcaster', 'broadcasting', 'tv licence', 'live stream', 'livestream'] },
++ world: { sources: ['eu-press-corner'], words: ['international', 'foreign', 'ukraine', 'russia', 'china', 'united nations', 'nato', 'g7', 'g20', 'summit', 'ambassador', 'embassy', 'diplomatic', 'refugees', 'humanitarian', 'middle east', 'gaza', 'israel', 'africa', 'india', 'sanctions', 'ceasefire'] },
++});
++
++const escape = (w) => w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
++const MATCHERS = Object.fromEntries(Object.entries(RULES).map(([slug, r]) => [slug, {
++ sources: new Set(r.sources || []),
++ words: r.words.map((w) => ({ w, re: new RegExp(`(?:^|[^a-z0-9])${escape(w)}(?:[^a-z0-9]|$)`, 'i') })),
++}]));
++
++function topicsFor({ label = '', headlines = [], sources = [] } = {}) {
++ const lab = String(label || '');
++ const all = `${lab}\n${headlines.slice(0, 12).join('\n')}`;
++ const out = [];
++ for (const [slug, m] of Object.entries(MATCHERS)) {
++ if (sources.some((s) => m.sources.has(s))) { out.push(slug); continue; }
++ if (m.words.some(({ re }) => re.test(lab))) { out.push(slug); continue; }
++ let hits = 0;
++ for (const { re } of m.words) { if (re.test(all) && ++hits >= 2) break; }
++ if (hits >= 2) out.push(slug);
++ }
++ return out;
++}
++
++module.exports = { topicsFor, RULES };
ddiff --git a/server/http/public.js b/server/http/public.js
iindex ab81525..e0b8fc5 100644
---- a/server/http/public.js
++++ b/server/http/public.js
@@@ -91,6 +91,16 @@ function createPublicRoutes(ctx) {
});
}
++ // A topic page also carries the AI briefs and the wire on it (server/domain/topic-match.js); with nothing on it in
++ // the last 30 days, the latest from the wire overall, said as such.
++ async function topicExtras(topic) {
++ const wire = await clusters.topicWire(topic.slug, { limit: 20 });
++ const ids = new Set(wire.map((c) => c.id));
++ const topicBriefs = briefs && wire.length ? (await briefs.latest(200)).filter((b) => b.kind === 'summary' && !b.stub && b.paragraphs.length && ids.has(b.cluster_id)).slice(0, 6) : [];
++ const briefed = new Set(topicBriefs.map((b) => b.cluster_id));
++ return { topicBriefs, topicWire: wire.filter((c) => !briefed.has(c.id)).slice(0, 12), latestWire: wire.length ? [] : await clusters.wire({ limit: 6 }) };
++ }
++
async function listing(req, res, { topic = null }) {
const page = pageNumber(req);
const path = topic ? publication.topicPath(topic) : '/';
@@@ -115,7 +125,7 @@ function createPublicRoutes(ctx) {
: [{ '@context': 'https://schema.org', '@type': 'WebSite', name: 'OpenVibe.News', url: publication.abs('/') }],
// The front page opens with what OpenVibe.News is for (openvibe-shared/showcase), then the stories.
...(!topic && pager.page === 1 ? { styles: [showcase.STYLESHEET] } : {}),
-- body: topic ? pages.topicPage({ topic, items, pager, feeds, breadcrumbs: crumbs })
++ body: topic ? pages.topicPage({ topic, items, pager, feeds, breadcrumbs: crumbs, ...(pager.page === 1 ? await topicExtras(topic) : {}) })
: (pager.page === 1 ? newsShowcase() : '') + pages.home({ items, pager, topics: await topics.list(), feeds, showcase: pager.page === 1, wire: pages.wireSection(wire), briefs: pager.page === 1 && briefs ? (await briefs.latest(12)).filter((b) => b.kind === 'summary' && !b.stub && b.paragraphs.length).slice(0, 4) : [] }),
}, { cacheable: true });
}
ddiff --git a/server/render/pages.js b/server/render/pages.js
iindex 9dbf1b8..cb42ea5 100644
---- a/server/render/pages.js
++++ b/server/render/pages.js
@@@ -101,10 +101,17 @@ function topicsIndex({ topics }) {
<ul class="topic-list wide">${topics.map((t) => h`<li><a href="/topics/${t.slug}">${t.name}</a>${t.description ? h` <span class="meta">— ${t.description}</span>` : ''}</li>`)}</ul>`);
}
--function topicPage({ topic, items, pager, feeds, breadcrumbs }) {
++function topicPage({ topic, items, pager, feeds, breadcrumbs, topicBriefs = [], topicWire = [], latestWire = [] }) {
++ const first = !pager || pager.page === 1;
++ const quiet = first && !items.length && !topicBriefs.length && !topicWire.length;
return String(h`${raw(ssr.breadcrumbsHtml(breadcrumbs))}
<header class="site-header"><h1>${topic.name}</h1>${topic.description ? h`<p class="lede">${topic.description}</p>` : ''}${raw(feedLinksHtml(feeds))}</header>
--<section aria-label="Stories">${storyList(items, `No stories on ${topic.name} have been published yet.`)}</section>
++${items.length ? h`<section aria-label="Stories">${storyList(items, '')}</section>` : ''}
++${raw(briefsSection(topicBriefs))}
++${topicWire.length ? h`<section class="wire" aria-labelledby="topic-wire-h"><h2 id="topic-wire-h">From the wire on ${topic.name}</h2>
++<p class="meta">${WIRE_NOTE}</p><ol class="wire-list">${topicWire.map((c) => wireCluster(c, { brief: true }))}</ol></section>` : ''}
++${quiet ? h`<p class="meta">The sources OpenVibe.News follows have not reported on ${topic.name} in the last 30 days. The latest from the wire:</p>
++${latestWire.length ? h`<ol class="wire-list">${latestWire.map((c) => wireCluster(c, { brief: true }))}</ol>` : ''}<p><a href="/wire">The whole wire</a> · <a href="/topics">Every topic</a></p>` : ''}
${raw(ssr.paginationHtml(pager))}`);
}
ddiff --git a/test/wire.test.js b/test/wire.test.js
iindex 174b785..d1f4541 100644
---- a/test/wire.test.js
++++ b/test/wire.test.js
@@@ -44,6 +44,24 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
assert.ok(!r.text.includes(B));
assert.ok(r.text.includes(A));
});
++
++ await check('a topic page carries the wire on its topic; a quiet topic says so and shows the latest wire', async () => {
++ const { topicsFor } = require('../server/domain/topic-match');
++ assert.deepStrictEqual(topicsFor({ label: 'europa clipper · nasa', headlines: [] }), ['space']);
++ assert.deepStrictEqual(topicsFor({ label: 'recall', headlines: [], sources: ['fda-food-recalls'] }), ['health']);
++ assert.deepStrictEqual(topicsFor({ label: 'bakery · prize', headlines: ['Local bakery wins regional sourdough prize'] }), [], 'one stray word places nothing');
++ assert.deepStrictEqual(topicsFor({ label: 'minister', headlines: ['New tariffs on steel', 'Trade talks resume as inflation eases'] }), ['business'], 'two different words across the headlines');
++ const space = await t.get('/topics/space');
++ assert.strictEqual(space.status, 200);
++ assert.ok(space.text.includes('From the wire on Space'));
++ assert.ok(space.text.includes('Europa Clipper'));
++ assert.ok(!space.text.includes('sourdough'), 'only the topic\'s clusters');
++ assert.ok(!/have been published yet/.test(space.text));
++ const games = await t.get('/topics/games');
++ assert.strictEqual(games.status, 200);
++ assert.ok(games.text.includes('have not reported on Games in the last 30 days. The latest from the wire:'));
++ assert.ok(games.text.includes('Europa Clipper') && games.text.includes('href="/wire"'));
++ });
} finally {
await t.close();
}