openvibers/openvibe.news Commits
No AI summary until two independent outlets report a story; a brief says how many did (#167)
Changes
@@@ -15,3 +15,6 @@@keyframes live-pulse { 50% { opacity: .35; } }@media (prefers-reduced-motion: reduce) { .live-dot { animation: none; } }.wire-cluster.is-breaking { border-left: 3px solid #ef4444; padding-left: 10px; }++/* A story too few outlets covered: its reports, and why there is no summary (market outcome 4) */++.badge-reports { background: color-mix(in srgb, var(--text-primary, #e6edf7) 9%, transparent); color: var(--text-secondary, #96a7c2); }++.brief-held { margin-top: 10px; padding: 10px 14px; border-left: 3px solid var(--warning, #f59e0b); border-radius: 0 10px 10px 0; background: color-mix(in srgb, var(--warning, #f59e0b) 8%, transparent); color: var(--text-secondary, #96a7c2); }@@@ -104,7 +104,7 @@ async function createApp(opts = {}) {secureCookies: config.cookies.secure,});const viewers = createViewerResolver({ auth, config, people });-- const briefs = createBriefs({ store, ai: config.briefs.enabled ? ai : null, log });++ const briefs = createBriefs({ store, ai: config.briefs.enabled ? ai : null, log, minOutlets: config.indexing && config.indexing.minIndependentSources });const worker = createWorker({ config, ingest, outbox, briefs, log });const ctx = { config, store, accountData, accountSend, outbox, publication, clusters, sources, ai, community, discussion, ingest, stories, topics, people, reading, auth, viewers, worker, indexnow, briefs };@@@ -10,6 +10,10 @@* for every cluster with at least MIN_REPORTS reports) or explained (made when a reader asks)* perspectives news.compare_perspectives: how the outlets frame it (when a reader asks; two outlets or more)*++ * Below minOutlets independent outlets (the market research's lesson from Ground News: withhold the summary when too++ * few distinct outlets covered a story) no summary is written, made or shown: the page lists the reports and says why.++ * It is the same number that decides whether a story may be indexed (config.indexing.minIndependentSources).++ ** Only News' stored fields go to AI (headline, outlet, date, URL, and the short summary its terms allow). A paragraph the* model did not cite is dropped (clients/ai.js draftFromOutput) and listed as a gap; a run with no cited text makes no* brief. The stub provider's synthetic output is stored but never shown. A brief made from a cluster that has since@@@ -27,7 +31,8 @@ const SWEEP_WINDOW_MS = 72 * 3_600_000;const parse = (v, d) => { try { return typeof v === 'string' ? JSON.parse(v) : (v == null ? d : v); } catch { return d; } };--function createBriefs({ store, ai, log = console }) {++function createBriefs({ store, ai, log = console, minOutlets = 2 }) {++ const MIN_OUTLETS = Math.max(1, Number(minOutlets) || 2);const db = store.db;const q = {cluster: db.prepare("SELECT * FROM news_story_clusters WHERE id = ? AND status = 'open'"),@@@ -38,11 +43,14 @@ function createBriefs({ store, ai, log = console }) {ON CONFLICT (cluster_id, kind, tone) DO UPDATE SET headline = excluded.headline, paragraphs = excluded.paragraphs, source_ids = excluded.source_ids,gaps = excluded.gaps, run_id = excluded.run_id, model = excluded.model, stub = excluded.stub, item_count = excluded.item_count,cluster_updated_at = excluded.cluster_updated_at, created_at = excluded.created_at`),-- latest: db.prepare(`SELECT b.*, c.label, c.window_end FROM news_cluster_briefs b JOIN news_story_clusters c ON c.id = b.cluster_id-- WHERE b.kind = 'summary' AND b.tone = 'neutral' AND b.stub = false AND c.status = 'open' ORDER BY c.window_end DESC, b.created_at DESC LIMIT ?`),++ latest: db.prepare(`SELECT * FROM (SELECT b.*, c.label, c.window_end,++ (SELECT count(DISTINCT i.outlet) FROM news_source_items i WHERE i.cluster_id = c.id AND i.status = 'active' AND i.removed_at IS NULL) AS outlets++ FROM news_cluster_briefs b JOIN news_story_clusters c ON c.id = b.cluster_id++ WHERE b.kind = 'summary' AND b.tone = 'neutral' AND b.stub = false AND c.status = 'open') x++ WHERE x.outlets >= ? ORDER BY x.window_end DESC, x.created_at DESC LIMIT ?`),// Open clusters active in the window with at least MIN_REPORTS live reports, newest activity first.due: db.prepare(`SELECT c.id, c.updated_at, count(i.id) AS n FROM news_story_clusters c JOIN news_source_items i ON i.cluster_id = c.id AND i.status = 'active' AND i.removed_at IS NULL-- WHERE c.status = 'open' AND c.window_end >= ? GROUP BY c.id, c.updated_at, c.window_end HAVING count(i.id) >= ? ORDER BY c.window_end DESC LIMIT 200`),++ WHERE c.status = 'open' AND c.window_end >= ? GROUP BY c.id, c.updated_at, c.window_end HAVING count(i.id) >= ? AND count(DISTINCT i.outlet) >= ? ORDER BY c.window_end DESC LIMIT 200`),};const fresh = (row, cluster, n) => Boolean(row && Number(row.item_count) === n && Number(row.cluster_updated_at) === Number(cluster.updated_at));@@@ -53,7 +61,8 @@ function createBriefs({ store, ai, log = console }) {if (!cluster) return null;const reports = await q.reports.all(cluster.id);const row = await q.get.get(cluster.id, kind, tone);-- return { cluster, reports, brief: row ? shape(row) : null, fresh: fresh(row, cluster, reports.length) };++ const outlets = new Set(reports.map((r) => r.outlet)).size;++ return { cluster, reports, brief: row ? shape(row) : null, fresh: fresh(row, cluster, reports.length), outlets, enough: outlets >= MIN_OUTLETS, minOutlets: MIN_OUTLETS };}function shape(row) {@@@ -73,6 +82,7 @@ function createBriefs({ store, ai, log = console }) {const reports = await q.reports.all(cluster.id);if (!reports.length) throw Object.assign(new Error('This cluster has no live reports'), { code: 'brief.no_reports', status: 422 });const outlets = new Set(reports.map((r) => r.outlet));++ if (kind === 'summary' && outlets.size < MIN_OUTLETS) throw Object.assign(new Error(`Only ${outlets.size === 1 ? 'one outlet has' : `${outlets.size} outlets have`} reported this; a summary needs ${MIN_OUTLETS}`), { code: 'brief.too_few_outlets', status: 422 });if (kind === 'perspectives' && outlets.size < 2) throw Object.assign(new Error('Comparing how outlets frame it needs two outlets or more'), { code: 'brief.one_outlet', status: 422 });const workflow = kind === 'perspectives' ? 'news.compare_perspectives' : 'news.summarize_story';const result = await ai.run(workflow, {@@@ -92,7 +102,7 @@ function createBriefs({ store, ai, log = console }) {async function sweep({ limit = SWEEP_PER_TICK } = {}) {if (!ai || !ai.enabled) return 0;let made = 0;-- for (const c of await q.due.all(store.now() - SWEEP_WINDOW_MS, MIN_REPORTS)) {++ for (const c of await q.due.all(store.now() - SWEEP_WINDOW_MS, MIN_REPORTS, MIN_OUTLETS)) {if (made >= limit) break;const row = await q.get.get(c.id, 'summary', 'neutral');if (fresh(row, c, Number(c.n))) continue;@@@ -104,9 +114,9 @@ function createBriefs({ store, ai, log = console }) {return made;}-- const latest = async (limit = 12) => (await q.latest.all(limit)).map((r) => ({ ...shape(r), label: r.label, window_end: Number(r.window_end) }));++ const latest = async (limit = 12) => (await q.latest.all(MIN_OUTLETS, limit)).map((r) => ({ ...shape(r), label: r.label, window_end: Number(r.window_end), outlets: Number(r.outlets) }));-- return { TONES, TONE_TEXT, KINDS, MIN_REPORTS, get, make, sweep, latest };++ return { TONES, TONE_TEXT, KINDS, MIN_REPORTS, MIN_OUTLETS, get, make, sweep, latest };}module.exports = { createBriefs, TONES, TONE_TEXT, MIN_REPORTS };@@@ -185,8 +185,8 @@ function createPublicRoutes(ctx) {const persp = await briefs.get(id, { kind: 'perspectives', tone: 'neutral' });const notice = req.query.made === '1' ? 'Written just now.' : (req.query.failed ? String(req.query.failed).slice(0, 200) : null);send(req, res, 200, {-- title: `${got.brief && got.brief.headline ? got.brief.headline : got.cluster.label} (AI brief)`,-- description: 'An AI summary of the reports on this subject, every paragraph linked to its reports.',++ title: got.enough ? `${got.brief && got.brief.headline ? got.brief.headline : got.cluster.label} (AI brief)` : `${got.cluster.label} (reports)`,++ description: got.enough ? 'An AI summary of the reports on this subject, every paragraph linked to its reports.' : 'The reports on this subject so far. Too few outlets have covered it for an AI summary.',decision: noindex(`/briefs/${id}`), canonical: `${config.baseUrl}/briefs/${id}`, localStyles: ['css/briefs.css'],body: briefPages.briefPage({ ...got, tone, perspectives: persp ? persp.brief : null, canMake: canMake(), notice }),}, { cacheable: !notice });@@@ -32,15 +32,32 @@ function makeButton(clusterId, { kind, tone, label }) {return h`<form class="brief-make" method="post" action="/briefs/${clusterId}/make"><input type="hidden" name="kind" value="${kind}"><input type="hidden" name="tone" value="${tone}"><button type="submit">${label}</button></form>`;}++const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;++/** "3 reports by 2 outlets: Reuters, AP" — how many covered it, and who (market outcome 2). */++const coverage = (reports, outlets) => `${plural(reports.length, 'report')} by ${plural(outlets.length, 'outlet')}${outlets.length ? `: ${outlets.slice(0, 6).join(', ')}${outlets.length > 6 ? ` and ${outlets.length - 6} more` : ''}` : ''}`;++++/**++ * A story fewer outlets than the threshold have covered: the reports, and why there is no summary (market outcome 4).++ * One outlet's three reports are one outlet's account; a summary of it would read as more than it is.++ */++function reportsOnlyPage({ cluster, reports, outlets, minOutlets }) {++ return String(h`<header class="site-header"><p class="eyebrow"><span class="badge badge-reports">Reports</span> ${coverage(reports, outlets)}</p>++<h1>${cluster.label}</h1>++<p class="meta brief-held">No AI summary: ${outlets.length === 1 ? 'only one outlet has' : `only ${outlets.length} outlets have`} reported this, and we write one only once ${minOutlets} independent outlets have. Until then, here is what was reported, as each outlet put it.</p></header>++<section aria-labelledby="reports-h"><h2 id="reports-h">The reports</h2>${sourceTable(reports)}</section>++<p class="meta"><a href="/briefs">All AI briefs</a> · <a href="/live">Live</a> · <a href="/wire">The wire</a></p>`);++}++/** /briefs/:id */--function briefPage({ cluster, reports, brief, fresh, tone, perspectives, canMake, notice = null }) {++function briefPage({ cluster, reports, brief, fresh, tone, perspectives, canMake, notice = null, enough = true, minOutlets = 2 }) {const outlets = [...new Set(reports.map((r) => r.outlet))];++ if (!enough) return reportsOnlyPage({ cluster, reports, outlets, minOutlets });const shown = brief && !brief.stub ? brief : null;const tabs = h`<nav class="brief-tones" aria-label="How to tell it">${TONES.map(([t, label]) => (t === tone? h`<span class="brief-tone on" aria-current="page">${label}</span>`: h`<a class="brief-tone" href="/briefs/${cluster.id}${t === 'neutral' ? '' : `?tone=${t}`}">${label}</a>`))}</nav>`;const persp = perspectives && !perspectives.stub ? perspectives : null;-- return String(h`<header class="site-header"><p class="eyebrow"><span class="badge badge-ai">AI brief</span> from ${reports.length} ${reports.length === 1 ? 'report' : 'reports'}${outlets.length ? h` (${outlets.slice(0, 5).join(', ')}${outlets.length > 5 ? ` and ${outlets.length - 5} more` : ''})` : ''}</p>++ return String(h`<header class="site-header"><p class="eyebrow"><span class="badge badge-ai">AI brief</span> from ${coverage(reports, outlets)}</p><h1>${shown && shown.headline ? shown.headline : cluster.label}</h1><p class="meta">${AI_NOTE}</p></header>${notice ? h`<p class="notice">${notice}</p>` : ''}@@@ -61,8 +78,8 @@ ${shown && shown.gaps.length ? h`<details class="brief-gaps"><summary>What the Afunction briefsIndex({ briefs }) {return String(h`<header class="site-header"><h1>AI briefs</h1><p class="lede">What the reports on a subject say, summarized by OpenVibe.AI with every paragraph linked to its reports, and labelled as AI. Editors' stories are on <a href="/">the front page</a>.</p></header>${briefs.length ? h`<ol class="story-list">${briefs.map((b) => h`<li class="story-item"><h2 class="story-item-title"><a href="/briefs/${b.cluster_id}">${b.headline || b.label}</a> <span class="badge badge-ai">AI</span></h2>--<p class="meta">${time(b.window_end)} · from ${b.item_count} reports</p>${b.paragraphs[0] ? h`<p class="summary">${b.paragraphs[0].text.slice(0, 280)}${b.paragraphs[0].text.length > 280 ? '…' : ''}</p>` : ''}</li>`)}</ol>`-- : h`<p class="empty">No briefs yet. They are written as reports on a subject come in from two or more of the sources.</p>`}`);++<p class="meta">${time(b.window_end)} · from ${b.item_count} reports${b.outlets ? ` by ${b.outlets} outlets` : ''}</p>${b.paragraphs[0] ? h`<p class="summary">${b.paragraphs[0].text.slice(0, 280)}${b.paragraphs[0].text.length > 280 ? '…' : ''}</p>` : ''}</li>`)}</ol>`++ : h`<p class="empty">No briefs yet. One is written once two or more independent outlets have reported a subject.</p>`}`);}/** /live: the newest reports as they arrive, grouped, with what is breaking. */@@@ -44,7 +44,7 @@ function statsPanel(stats) {function briefCard(b, i) {const first = b.paragraphs[0] ? b.paragraphs[0].text : '';-- return h`<li class="nh-card" data-i="${i}"><p class="nh-kicker"><span class="badge badge-ai">AI brief</span> from ${plural(b.item_count, 'report')}</p>++ return h`<li class="nh-card" data-i="${i}"><p class="nh-kicker"><span class="badge badge-ai">AI brief</span> from ${plural(b.item_count, 'report')}${b.outlets ? ` by ${plural(b.outlets, 'outlet')}` : ''}</p><h3><a href="/briefs/${b.cluster_id}">${b.headline || b.label}</a></h3><p class="meta">${time(b.window_end)}</p>${first ? h`<p class="nh-sum">${clip(first, 260)}</p>` : ''}</li>`;}@@@ -5,7 +5,7 @@* more reports; every paragraph cites its reports and uncited claims are dropped; the page labels it as AI, is never* indexed, and shows the reports; a reader's POST asks for another tone or the outlets' comparison (same-origin only);* a stub provider's output is never shown; a changed cluster's brief is stale and made again; /live refreshes itself-- * and marks what is breaking.++ * and marks what is breaking. Below two independent outlets nothing is summarized: the page lists the reports and says why.*/const assert = require('assert');const { boot, check, done, launchReports } = require('./helpers/boot');@@@ -90,6 +90,25 @@ const SAME = { 'sec-fetch-site': 'same-origin' };const home = await t.get('/');assert.ok(home.text.includes('href="/live"') && home.text.includes('href="/briefs"'), 'in the navigation');});++ await check('one outlet is not enough: no summary is written, made or shown; the reports are, with why (market outcome 4)', async () => {++ // The same story, now reported by one outlet only.++ await t.db().prepare("UPDATE news_source_items SET outlet = 'GOV.UK' WHERE cluster_id = ?").run(clusterId);++ const got = await t.ctx.briefs.get(clusterId);++ assert.deepStrictEqual([got.outlets, got.enough, got.minOutlets], [1, false, 2]);++ const r = await t.get(`/briefs/${clusterId}`);++ assert.strictEqual(r.status, 200);++ assert.match(r.text, /<span class="badge badge-reports">Reports<\/span> \d+ reports by 1 outlet: GOV\.UK/);++ assert.match(r.text, /No AI summary: only one outlet has reported this, and we write one only once 2 independent outlets have\./);++ assert.ok(!r.text.includes('Europa Clipper is on its way') && !r.text.includes('Written by OpenVibe.AI'), 'the stored summary is not shown');++ assert.ok(r.text.includes('id="source-1"'), 'the reports are');++ assert.ok(!r.text.includes('Write it with AI'), 'and nobody is offered to make one');++ assert.ok(!(await t.get('/briefs')).text.includes(`/briefs/${clusterId}`), 'not listed among the briefs');++ const before = t.ai.requests.length;++ await assert.rejects(() => t.ctx.briefs.make(clusterId, { kind: 'summary', tone: 'brief' }), (e) => e.code === 'brief.too_few_outlets' && e.status === 422);++ await t.db().prepare('UPDATE news_story_clusters SET updated_at = updated_at + 1 WHERE id = ?').run(clusterId);++ assert.strictEqual(await t.ctx.briefs.sweep(), 0, 'the sweep does not ask AI about it');++ assert.strictEqual(t.ai.requests.length, before, 'AI was never asked');++ });} finally { await t.close(); }done();})().catch((e) => { console.error(e); process.exit(1); });
@@ -15,3 +15,6 @@
@@ -15,3 +15,6 @@
@keyframes live-pulse { 50% { opacity: .35; } }
@keyframes live-pulse { 50% { opacity: .35; } }
@media (prefers-reduced-motion: reduce) { .live-dot { animation: none; } }
@media (prefers-reduced-motion: reduce) { .live-dot { animation: none; } }
.wire-cluster.is-breaking { border-left: 3px solid #ef4444; padding-left: 10px; }
.wire-cluster.is-breaking { border-left: 3px solid #ef4444; padding-left: 10px; }
+/* A story too few outlets covered: its reports, and why there is no summary (market outcome 4) */
+.badge-reports { background: color-mix(in srgb, var(--text-primary, #e6edf7) 9%, transparent); color: var(--text-secondary, #96a7c2); }
+.brief-held { margin-top: 10px; padding: 10px 14px; border-left: 3px solid var(--warning, #f59e0b); border-radius: 0 10px 10px 0; background: color-mix(in srgb, var(--warning, #f59e0b) 8%, transparent); color: var(--text-secondary, #96a7c2); }
@@ -104,7 +104,7 @@ async function createApp(opts = {}) {
@@ -104,7 +104,7 @@ async function createApp(opts = {}) {
secureCookies: config.cookies.secure,
secureCookies: config.cookies.secure,
});
});
const viewers = createViewerResolver({ auth, config, people });
const viewers = createViewerResolver({ auth, config, people });
- const briefs = createBriefs({ store, ai: config.briefs.enabled ? ai : null, log });
+ const briefs = createBriefs({ store, ai: config.briefs.enabled ? ai : null, log, minOutlets: config.indexing && config.indexing.minIndependentSources });
const worker = createWorker({ config, ingest, outbox, briefs, log });
const worker = createWorker({ config, ingest, outbox, briefs, log });
const ctx = { config, store, accountData, accountSend, outbox, publication, clusters, sources, ai, community, discussion, ingest, stories, topics, people, reading, auth, viewers, worker, indexnow, briefs };
const ctx = { config, store, accountData, accountSend, outbox, publication, clusters, sources, ai, community, discussion, ingest, stories, topics, people, reading, auth, viewers, worker, indexnow, briefs };
@@ -10,6 +10,10 @@
@@ -10,6 +10,10 @@
* for every cluster with at least MIN_REPORTS reports) or explained (made when a reader asks)
* for every cluster with at least MIN_REPORTS reports) or explained (made when a reader asks)
* perspectives news.compare_perspectives: how the outlets frame it (when a reader asks; two outlets or more)
* perspectives news.compare_perspectives: how the outlets frame it (when a reader asks; two outlets or more)
*
*
+ * Below minOutlets independent outlets (the market research's lesson from Ground News: withhold the summary when too
+ * few distinct outlets covered a story) no summary is written, made or shown: the page lists the reports and says why.
+ * It is the same number that decides whether a story may be indexed (config.indexing.minIndependentSources).
+ *
* Only News' stored fields go to AI (headline, outlet, date, URL, and the short summary its terms allow). A paragraph the
* Only News' stored fields go to AI (headline, outlet, date, URL, and the short summary its terms allow). A paragraph the
* model did not cite is dropped (clients/ai.js draftFromOutput) and listed as a gap; a run with no cited text makes no
* model did not cite is dropped (clients/ai.js draftFromOutput) and listed as a gap; a run with no cited text makes no
* brief. The stub provider's synthetic output is stored but never shown. A brief made from a cluster that has since
* brief. The stub provider's synthetic output is stored but never shown. A brief made from a cluster that has since
@@ -27,7 +31,8 @@ const SWEEP_WINDOW_MS = 72 * 3_600_000;
@@ -27,7 +31,8 @@ const SWEEP_WINDOW_MS = 72 * 3_600_000;
const parse = (v, d) => { try { return typeof v === 'string' ? JSON.parse(v) : (v == null ? d : v); } catch { return d; } };
const parse = (v, d) => { try { return typeof v === 'string' ? JSON.parse(v) : (v == null ? d : v); } catch { return d; } };
-function createBriefs({ store, ai, log = console }) {
+function createBriefs({ store, ai, log = console, minOutlets = 2 }) {
+ const MIN_OUTLETS = Math.max(1, Number(minOutlets) || 2);
const db = store.db;
const db = store.db;
const q = {
const q = {
cluster: db.prepare("SELECT * FROM news_story_clusters WHERE id = ? AND status = 'open'"),
cluster: db.prepare("SELECT * FROM news_story_clusters WHERE id = ? AND status = 'open'"),
@@ -38,11 +43,14 @@ function createBriefs({ store, ai, log = console }) {
@@ -38,11 +43,14 @@ function createBriefs({ store, ai, log = console }) {
ON CONFLICT (cluster_id, kind, tone) DO UPDATE SET headline = excluded.headline, paragraphs = excluded.paragraphs, source_ids = excluded.source_ids,
ON CONFLICT (cluster_id, kind, tone) DO UPDATE SET headline = excluded.headline, paragraphs = excluded.paragraphs, source_ids = excluded.source_ids,
gaps = excluded.gaps, run_id = excluded.run_id, model = excluded.model, stub = excluded.stub, item_count = excluded.item_count,
gaps = excluded.gaps, run_id = excluded.run_id, model = excluded.model, stub = excluded.stub, item_count = excluded.item_count,
cluster_updated_at = excluded.cluster_updated_at, created_at = excluded.created_at`),
cluster_updated_at = excluded.cluster_updated_at, created_at = excluded.created_at`),
- latest: db.prepare(`SELECT b.*, c.label, c.window_end FROM news_cluster_briefs b JOIN news_story_clusters c ON c.id = b.cluster_id
- WHERE b.kind = 'summary' AND b.tone = 'neutral' AND b.stub = false AND c.status = 'open' ORDER BY c.window_end DESC, b.created_at DESC LIMIT ?`),
+ latest: db.prepare(`SELECT * FROM (SELECT b.*, c.label, c.window_end,
+ (SELECT count(DISTINCT i.outlet) FROM news_source_items i WHERE i.cluster_id = c.id AND i.status = 'active' AND i.removed_at IS NULL) AS outlets
+ FROM news_cluster_briefs b JOIN news_story_clusters c ON c.id = b.cluster_id
+ WHERE b.kind = 'summary' AND b.tone = 'neutral' AND b.stub = false AND c.status = 'open') x
+ WHERE x.outlets >= ? ORDER BY x.window_end DESC, x.created_at DESC LIMIT ?`),
// Open clusters active in the window with at least MIN_REPORTS live reports, newest activity first.
// Open clusters active in the window with at least MIN_REPORTS live reports, newest activity first.
due: db.prepare(`SELECT c.id, c.updated_at, count(i.id) AS n FROM news_story_clusters c JOIN news_source_items i ON i.cluster_id = c.id AND i.status = 'active' AND i.removed_at IS NULL
due: db.prepare(`SELECT c.id, c.updated_at, count(i.id) AS n FROM news_story_clusters c JOIN news_source_items i ON i.cluster_id = c.id AND i.status = 'active' AND i.removed_at IS NULL
- WHERE c.status = 'open' AND c.window_end >= ? GROUP BY c.id, c.updated_at, c.window_end HAVING count(i.id) >= ? ORDER BY c.window_end DESC LIMIT 200`),
+ WHERE c.status = 'open' AND c.window_end >= ? GROUP BY c.id, c.updated_at, c.window_end HAVING count(i.id) >= ? AND count(DISTINCT i.outlet) >= ? ORDER BY c.window_end DESC LIMIT 200`),
};
};
const fresh = (row, cluster, n) => Boolean(row && Number(row.item_count) === n && Number(row.cluster_updated_at) === Number(cluster.updated_at));
const fresh = (row, cluster, n) => Boolean(row && Number(row.item_count) === n && Number(row.cluster_updated_at) === Number(cluster.updated_at));
@@ -53,7 +61,8 @@ function createBriefs({ store, ai, log = console }) {
@@ -53,7 +61,8 @@ function createBriefs({ store, ai, log = console }) {
if (!cluster) return null;
if (!cluster) return null;
const reports = await q.reports.all(cluster.id);
const reports = await q.reports.all(cluster.id);
const row = await q.get.get(cluster.id, kind, tone);
const row = await q.get.get(cluster.id, kind, tone);
- return { cluster, reports, brief: row ? shape(row) : null, fresh: fresh(row, cluster, reports.length) };
+ const outlets = new Set(reports.map((r) => r.outlet)).size;
+ return { cluster, reports, brief: row ? shape(row) : null, fresh: fresh(row, cluster, reports.length), outlets, enough: outlets >= MIN_OUTLETS, minOutlets: MIN_OUTLETS };
}
}
function shape(row) {
function shape(row) {
@@ -73,6 +82,7 @@ function createBriefs({ store, ai, log = console }) {
@@ -73,6 +82,7 @@ function createBriefs({ store, ai, log = console }) {
const reports = await q.reports.all(cluster.id);
const reports = await q.reports.all(cluster.id);
if (!reports.length) throw Object.assign(new Error('This cluster has no live reports'), { code: 'brief.no_reports', status: 422 });
if (!reports.length) throw Object.assign(new Error('This cluster has no live reports'), { code: 'brief.no_reports', status: 422 });
const outlets = new Set(reports.map((r) => r.outlet));
const outlets = new Set(reports.map((r) => r.outlet));
+ if (kind === 'summary' && outlets.size < MIN_OUTLETS) throw Object.assign(new Error(`Only ${outlets.size === 1 ? 'one outlet has' : `${outlets.size} outlets have`} reported this; a summary needs ${MIN_OUTLETS}`), { code: 'brief.too_few_outlets', status: 422 });
if (kind === 'perspectives' && outlets.size < 2) throw Object.assign(new Error('Comparing how outlets frame it needs two outlets or more'), { code: 'brief.one_outlet', status: 422 });
if (kind === 'perspectives' && outlets.size < 2) throw Object.assign(new Error('Comparing how outlets frame it needs two outlets or more'), { code: 'brief.one_outlet', status: 422 });
const workflow = kind === 'perspectives' ? 'news.compare_perspectives' : 'news.summarize_story';
const workflow = kind === 'perspectives' ? 'news.compare_perspectives' : 'news.summarize_story';
const result = await ai.run(workflow, {
const result = await ai.run(workflow, {
@@ -92,7 +102,7 @@ function createBriefs({ store, ai, log = console }) {
@@ -92,7 +102,7 @@ function createBriefs({ store, ai, log = console }) {
async function sweep({ limit = SWEEP_PER_TICK } = {}) {
async function sweep({ limit = SWEEP_PER_TICK } = {}) {
if (!ai || !ai.enabled) return 0;
if (!ai || !ai.enabled) return 0;
let made = 0;
let made = 0;
- for (const c of await q.due.all(store.now() - SWEEP_WINDOW_MS, MIN_REPORTS)) {
+ for (const c of await q.due.all(store.now() - SWEEP_WINDOW_MS, MIN_REPORTS, MIN_OUTLETS)) {
if (made >= limit) break;
if (made >= limit) break;
const row = await q.get.get(c.id, 'summary', 'neutral');
const row = await q.get.get(c.id, 'summary', 'neutral');
if (fresh(row, c, Number(c.n))) continue;
if (fresh(row, c, Number(c.n))) continue;
@@ -104,9 +114,9 @@ function createBriefs({ store, ai, log = console }) {
@@ -104,9 +114,9 @@ function createBriefs({ store, ai, log = console }) {
return made;
return made;
}
}
- const latest = async (limit = 12) => (await q.latest.all(limit)).map((r) => ({ ...shape(r), label: r.label, window_end: Number(r.window_end) }));
+ const latest = async (limit = 12) => (await q.latest.all(MIN_OUTLETS, limit)).map((r) => ({ ...shape(r), label: r.label, window_end: Number(r.window_end), outlets: Number(r.outlets) }));
- return { TONES, TONE_TEXT, KINDS, MIN_REPORTS, get, make, sweep, latest };
+ return { TONES, TONE_TEXT, KINDS, MIN_REPORTS, MIN_OUTLETS, get, make, sweep, latest };
}
}
module.exports = { createBriefs, TONES, TONE_TEXT, MIN_REPORTS };
module.exports = { createBriefs, TONES, TONE_TEXT, MIN_REPORTS };
@@ -185,8 +185,8 @@ function createPublicRoutes(ctx) {
@@ -185,8 +185,8 @@ function createPublicRoutes(ctx) {
const persp = await briefs.get(id, { kind: 'perspectives', tone: 'neutral' });
const persp = await briefs.get(id, { kind: 'perspectives', tone: 'neutral' });
const notice = req.query.made === '1' ? 'Written just now.' : (req.query.failed ? String(req.query.failed).slice(0, 200) : null);
const notice = req.query.made === '1' ? 'Written just now.' : (req.query.failed ? String(req.query.failed).slice(0, 200) : null);
send(req, res, 200, {
send(req, res, 200, {
- title: `${got.brief && got.brief.headline ? got.brief.headline : got.cluster.label} (AI brief)`,
- description: 'An AI summary of the reports on this subject, every paragraph linked to its reports.',
+ title: got.enough ? `${got.brief && got.brief.headline ? got.brief.headline : got.cluster.label} (AI brief)` : `${got.cluster.label} (reports)`,
+ description: got.enough ? 'An AI summary of the reports on this subject, every paragraph linked to its reports.' : 'The reports on this subject so far. Too few outlets have covered it for an AI summary.',
decision: noindex(`/briefs/${id}`), canonical: `${config.baseUrl}/briefs/${id}`, localStyles: ['css/briefs.css'],
decision: noindex(`/briefs/${id}`), canonical: `${config.baseUrl}/briefs/${id}`, localStyles: ['css/briefs.css'],
body: briefPages.briefPage({ ...got, tone, perspectives: persp ? persp.brief : null, canMake: canMake(), notice }),
body: briefPages.briefPage({ ...got, tone, perspectives: persp ? persp.brief : null, canMake: canMake(), notice }),
}, { cacheable: !notice });
}, { cacheable: !notice });
@@ -32,15 +32,32 @@ function makeButton(clusterId, { kind, tone, label }) {
@@ -32,15 +32,32 @@ function makeButton(clusterId, { kind, tone, label }) {
return h`<form class="brief-make" method="post" action="/briefs/${clusterId}/make"><input type="hidden" name="kind" value="${kind}"><input type="hidden" name="tone" value="${tone}"><button type="submit">${label}</button></form>`;
return h`<form class="brief-make" method="post" action="/briefs/${clusterId}/make"><input type="hidden" name="kind" value="${kind}"><input type="hidden" name="tone" value="${tone}"><button type="submit">${label}</button></form>`;
}
}
+const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
+/** "3 reports by 2 outlets: Reuters, AP" — how many covered it, and who (market outcome 2). */
+const coverage = (reports, outlets) => `${plural(reports.length, 'report')} by ${plural(outlets.length, 'outlet')}${outlets.length ? `: ${outlets.slice(0, 6).join(', ')}${outlets.length > 6 ? ` and ${outlets.length - 6} more` : ''}` : ''}`;
+
+/**
+ * A story fewer outlets than the threshold have covered: the reports, and why there is no summary (market outcome 4).
+ * One outlet's three reports are one outlet's account; a summary of it would read as more than it is.
+ */
+function reportsOnlyPage({ cluster, reports, outlets, minOutlets }) {
+ return String(h`<header class="site-header"><p class="eyebrow"><span class="badge badge-reports">Reports</span> ${coverage(reports, outlets)}</p>
+<h1>${cluster.label}</h1>
+<p class="meta brief-held">No AI summary: ${outlets.length === 1 ? 'only one outlet has' : `only ${outlets.length} outlets have`} reported this, and we write one only once ${minOutlets} independent outlets have. Until then, here is what was reported, as each outlet put it.</p></header>
+<section aria-labelledby="reports-h"><h2 id="reports-h">The reports</h2>${sourceTable(reports)}</section>
+<p class="meta"><a href="/briefs">All AI briefs</a> · <a href="/live">Live</a> · <a href="/wire">The wire</a></p>`);
+}
+
/** /briefs/:id */
/** /briefs/:id */
-function briefPage({ cluster, reports, brief, fresh, tone, perspectives, canMake, notice = null }) {
+function briefPage({ cluster, reports, brief, fresh, tone, perspectives, canMake, notice = null, enough = true, minOutlets = 2 }) {
const outlets = [...new Set(reports.map((r) => r.outlet))];
const outlets = [...new Set(reports.map((r) => r.outlet))];
+ if (!enough) return reportsOnlyPage({ cluster, reports, outlets, minOutlets });
const shown = brief && !brief.stub ? brief : null;
const shown = brief && !brief.stub ? brief : null;
const tabs = h`<nav class="brief-tones" aria-label="How to tell it">${TONES.map(([t, label]) => (t === tone
const tabs = h`<nav class="brief-tones" aria-label="How to tell it">${TONES.map(([t, label]) => (t === tone
? h`<span class="brief-tone on" aria-current="page">${label}</span>`
? h`<span class="brief-tone on" aria-current="page">${label}</span>`
: h`<a class="brief-tone" href="/briefs/${cluster.id}${t === 'neutral' ? '' : `?tone=${t}`}">${label}</a>`))}</nav>`;
: h`<a class="brief-tone" href="/briefs/${cluster.id}${t === 'neutral' ? '' : `?tone=${t}`}">${label}</a>`))}</nav>`;
const persp = perspectives && !perspectives.stub ? perspectives : null;
const persp = perspectives && !perspectives.stub ? perspectives : null;
- return String(h`<header class="site-header"><p class="eyebrow"><span class="badge badge-ai">AI brief</span> from ${reports.length} ${reports.length === 1 ? 'report' : 'reports'}${outlets.length ? h` (${outlets.slice(0, 5).join(', ')}${outlets.length > 5 ? ` and ${outlets.length - 5} more` : ''})` : ''}</p>
+ return String(h`<header class="site-header"><p class="eyebrow"><span class="badge badge-ai">AI brief</span> from ${coverage(reports, outlets)}</p>
<h1>${shown && shown.headline ? shown.headline : cluster.label}</h1>
<h1>${shown && shown.headline ? shown.headline : cluster.label}</h1>
<p class="meta">${AI_NOTE}</p></header>
<p class="meta">${AI_NOTE}</p></header>
${notice ? h`<p class="notice">${notice}</p>` : ''}
${notice ? h`<p class="notice">${notice}</p>` : ''}
@@ -61,8 +78,8 @@ ${shown && shown.gaps.length ? h`<details class="brief-gaps"><summary>What the A
@@ -61,8 +78,8 @@ ${shown && shown.gaps.length ? h`<details class="brief-gaps"><summary>What the A
function briefsIndex({ briefs }) {
function briefsIndex({ briefs }) {
return String(h`<header class="site-header"><h1>AI briefs</h1><p class="lede">What the reports on a subject say, summarized by OpenVibe.AI with every paragraph linked to its reports, and labelled as AI. Editors' stories are on <a href="/">the front page</a>.</p></header>
return String(h`<header class="site-header"><h1>AI briefs</h1><p class="lede">What the reports on a subject say, summarized by OpenVibe.AI with every paragraph linked to its reports, and labelled as AI. Editors' stories are on <a href="/">the front page</a>.</p></header>
${briefs.length ? h`<ol class="story-list">${briefs.map((b) => h`<li class="story-item"><h2 class="story-item-title"><a href="/briefs/${b.cluster_id}">${b.headline || b.label}</a> <span class="badge badge-ai">AI</span></h2>
${briefs.length ? h`<ol class="story-list">${briefs.map((b) => h`<li class="story-item"><h2 class="story-item-title"><a href="/briefs/${b.cluster_id}">${b.headline || b.label}</a> <span class="badge badge-ai">AI</span></h2>
-<p class="meta">${time(b.window_end)} · from ${b.item_count} reports</p>${b.paragraphs[0] ? h`<p class="summary">${b.paragraphs[0].text.slice(0, 280)}${b.paragraphs[0].text.length > 280 ? '…' : ''}</p>` : ''}</li>`)}</ol>`
- : h`<p class="empty">No briefs yet. They are written as reports on a subject come in from two or more of the sources.</p>`}`);
+<p class="meta">${time(b.window_end)} · from ${b.item_count} reports${b.outlets ? ` by ${b.outlets} outlets` : ''}</p>${b.paragraphs[0] ? h`<p class="summary">${b.paragraphs[0].text.slice(0, 280)}${b.paragraphs[0].text.length > 280 ? '…' : ''}</p>` : ''}</li>`)}</ol>`
+ : h`<p class="empty">No briefs yet. One is written once two or more independent outlets have reported a subject.</p>`}`);
}
}
/** /live: the newest reports as they arrive, grouped, with what is breaking. */
/** /live: the newest reports as they arrive, grouped, with what is breaking. */
@@ -44,7 +44,7 @@ function statsPanel(stats) {
@@ -44,7 +44,7 @@ function statsPanel(stats) {
function briefCard(b, i) {
function briefCard(b, i) {
const first = b.paragraphs[0] ? b.paragraphs[0].text : '';
const first = b.paragraphs[0] ? b.paragraphs[0].text : '';
- return h`<li class="nh-card" data-i="${i}"><p class="nh-kicker"><span class="badge badge-ai">AI brief</span> from ${plural(b.item_count, 'report')}</p>
+ return h`<li class="nh-card" data-i="${i}"><p class="nh-kicker"><span class="badge badge-ai">AI brief</span> from ${plural(b.item_count, 'report')}${b.outlets ? ` by ${plural(b.outlets, 'outlet')}` : ''}</p>
<h3><a href="/briefs/${b.cluster_id}">${b.headline || b.label}</a></h3><p class="meta">${time(b.window_end)}</p>${first ? h`<p class="nh-sum">${clip(first, 260)}</p>` : ''}</li>`;
<h3><a href="/briefs/${b.cluster_id}">${b.headline || b.label}</a></h3><p class="meta">${time(b.window_end)}</p>${first ? h`<p class="nh-sum">${clip(first, 260)}</p>` : ''}</li>`;
}
}
@@ -5,7 +5,7 @@
@@ -5,7 +5,7 @@
* more reports; every paragraph cites its reports and uncited claims are dropped; the page labels it as AI, is never
* more reports; every paragraph cites its reports and uncited claims are dropped; the page labels it as AI, is never
* indexed, and shows the reports; a reader's POST asks for another tone or the outlets' comparison (same-origin only);
* indexed, and shows the reports; a reader's POST asks for another tone or the outlets' comparison (same-origin only);
* a stub provider's output is never shown; a changed cluster's brief is stale and made again; /live refreshes itself
* a stub provider's output is never shown; a changed cluster's brief is stale and made again; /live refreshes itself
- * and marks what is breaking.
+ * and marks what is breaking. Below two independent outlets nothing is summarized: the page lists the reports and says why.
*/
*/
const assert = require('assert');
const assert = require('assert');
const { boot, check, done, launchReports } = require('./helpers/boot');
const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -90,6 +90,25 @@ const SAME = { 'sec-fetch-site': 'same-origin' };
@@ -90,6 +90,25 @@ const SAME = { 'sec-fetch-site': 'same-origin' };
const home = await t.get('/');
const home = await t.get('/');
assert.ok(home.text.includes('href="/live"') && home.text.includes('href="/briefs"'), 'in the navigation');
assert.ok(home.text.includes('href="/live"') && home.text.includes('href="/briefs"'), 'in the navigation');
});
});
+ await check('one outlet is not enough: no summary is written, made or shown; the reports are, with why (market outcome 4)', async () => {
+ // The same story, now reported by one outlet only.
+ await t.db().prepare("UPDATE news_source_items SET outlet = 'GOV.UK' WHERE cluster_id = ?").run(clusterId);
+ const got = await t.ctx.briefs.get(clusterId);
+ assert.deepStrictEqual([got.outlets, got.enough, got.minOutlets], [1, false, 2]);
+ const r = await t.get(`/briefs/${clusterId}`);
+ assert.strictEqual(r.status, 200);
+ assert.match(r.text, /<span class="badge badge-reports">Reports<\/span> \d+ reports by 1 outlet: GOV\.UK/);
+ assert.match(r.text, /No AI summary: only one outlet has reported this, and we write one only once 2 independent outlets have\./);
+ assert.ok(!r.text.includes('Europa Clipper is on its way') && !r.text.includes('Written by OpenVibe.AI'), 'the stored summary is not shown');
+ assert.ok(r.text.includes('id="source-1"'), 'the reports are');
+ assert.ok(!r.text.includes('Write it with AI'), 'and nobody is offered to make one');
+ assert.ok(!(await t.get('/briefs')).text.includes(`/briefs/${clusterId}`), 'not listed among the briefs');
+ const before = t.ai.requests.length;
+ await assert.rejects(() => t.ctx.briefs.make(clusterId, { kind: 'summary', tone: 'brief' }), (e) => e.code === 'brief.too_few_outlets' && e.status === 422);
+ await t.db().prepare('UPDATE news_story_clusters SET updated_at = updated_at + 1 WHERE id = ?').run(clusterId);
+ assert.strictEqual(await t.ctx.briefs.sweep(), 0, 'the sweep does not ask AI about it');
+ assert.strictEqual(t.ai.requests.length, before, 'AI was never asked');
+ });
} finally { await t.close(); }
} finally { await t.close(); }
done();
done();
})().catch((e) => { console.error(e); process.exit(1); });
})().catch((e) => { console.error(e); process.exit(1); });