openvibers/openvibe.news Commits

Drop future-dated reports from wire, live and briefs input (#173)

OP OpenVibers committed 12 days ago · 73cfa7d3e2 · 2 files, +15 −6

* Drop future-dated reports from wire, live and briefs input

* Merge remote-tracking branch 'origin/main' into fix/data-sweep-2026-11-01

Changes

ddiff --git a/server/domain/briefs.js b/server/domain/briefs.js
iindex 9c17cce..6066edb 100644
---- a/server/domain/briefs.js
++++ b/server/domain/briefs.js
@@@ -36,7 +36,7 @@ function createBriefs({ store, ai, log = console, minOutlets = 2 }) {
const db = store.db;
const q = {
cluster: db.prepare("SELECT * FROM news_story_clusters WHERE id = ? AND status = 'open'"),
-- reports: db.prepare("SELECT * FROM news_source_items WHERE cluster_id = ? AND status = 'active' AND removed_at IS NULL ORDER BY published_at, first_seen_at, seq"),
++ reports: db.prepare("SELECT * FROM news_source_items WHERE cluster_id = ? AND status = 'active' AND removed_at IS NULL AND (published_at IS NULL OR published_at <= ?) ORDER BY published_at, first_seen_at, seq"),
get: db.prepare('SELECT * FROM news_cluster_briefs WHERE cluster_id = ? AND kind = ? AND tone = ?'),
put: db.prepare(`INSERT INTO news_cluster_briefs (cluster_id, kind, tone, headline, paragraphs, source_ids, gaps, run_id, model, stub, item_count, cluster_updated_at, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
@@@ -49,7 +49,7 @@ function createBriefs({ store, ai, log = console, minOutlets = 2 }) {
WHERE b.kind = 'summary' AND b.tone = 'neutral' AND b.stub = false AND c.status = 'open') x
WHERE x.outlets >= ? ORDER BY x.window_end DESC, x.created_at DESC LIMIT ?`),
// Open clusters active in the window with at least MIN_REPORTS live reports, newest activity first.
-- due: db.prepare(`SELECT c.id, c.updated_at, count(i.id) AS n FROM news_story_clusters c JOIN news_source_items i ON i.cluster_id = c.id AND i.status = 'active' AND i.removed_at IS NULL
++ due: db.prepare(`SELECT c.id, c.updated_at, count(i.id) AS n FROM news_story_clusters c JOIN news_source_items i ON i.cluster_id = c.id AND i.status = 'active' AND i.removed_at IS NULL AND (i.published_at IS NULL OR i.published_at <= ?)
WHERE c.status = 'open' AND c.window_end >= ? GROUP BY c.id, c.updated_at, c.window_end HAVING count(i.id) >= ? AND count(DISTINCT i.outlet) >= ? ORDER BY c.window_end DESC LIMIT 200`),
};
@@@ -59,7 +59,8 @@ function createBriefs({ store, ai, log = console, minOutlets = 2 }) {
async function get(clusterId, { kind = 'summary', tone = 'neutral' } = {}) {
const cluster = await q.cluster.get(clusterId);
if (!cluster) return null;
-- const reports = await q.reports.all(cluster.id);
++ const cutoff = new Date(store.now() + 24 * 3_600_000).toISOString();
++ const reports = await q.reports.all(cluster.id, cutoff);
const row = await q.get.get(cluster.id, kind, tone);
const outlets = new Set(reports.map((r) => r.outlet)).size;
return { cluster, reports, brief: row ? shape(row) : null, fresh: fresh(row, cluster, reports.length), outlets, enough: outlets >= MIN_OUTLETS, minOutlets: MIN_OUTLETS };
@@@ -79,7 +80,8 @@ function createBriefs({ store, ai, log = console, minOutlets = 2 }) {
if (!KINDS.includes(kind) || !TONES.includes(tone)) throw Object.assign(new Error('kind or tone not known'), { code: 'brief.invalid', status: 422 });
const cluster = await q.cluster.get(clusterId);
if (!cluster) throw Object.assign(new Error('No such cluster'), { code: 'cluster.not_found', status: 404 });
-- const reports = await q.reports.all(cluster.id);
++ const cutoff = new Date(store.now() + 24 * 3_600_000).toISOString();
++ const reports = await q.reports.all(cluster.id, cutoff);
if (!reports.length) throw Object.assign(new Error('This cluster has no live reports'), { code: 'brief.no_reports', status: 422 });
const outlets = new Set(reports.map((r) => r.outlet));
if (kind === 'summary' && outlets.size < MIN_OUTLETS) throw Object.assign(new Error(`Only ${outlets.size === 1 ? 'one outlet has' : `${outlets.size} outlets have`} reported this; a summary needs ${MIN_OUTLETS}`), { code: 'brief.too_few_outlets', status: 422 });
@@@ -102,7 +104,8 @@ function createBriefs({ store, ai, log = console, minOutlets = 2 }) {
async function sweep({ limit = SWEEP_PER_TICK } = {}) {
if (!ai || !ai.enabled) return 0;
let made = 0;
-- for (const c of await q.due.all(store.now() - SWEEP_WINDOW_MS, MIN_REPORTS, MIN_OUTLETS)) {
++ const cutoff = new Date(store.now() + 24 * 3_600_000).toISOString();
++ for (const c of await q.due.all(cutoff, store.now() - SWEEP_WINDOW_MS, MIN_REPORTS, MIN_OUTLETS)) {
if (made >= limit) break;
const row = await q.get.get(c.id, 'summary', 'neutral');
if (fresh(row, c, Number(c.n))) continue;
ddiff --git a/server/domain/clusters.js b/server/domain/clusters.js
iindex b685dd0..f733641 100644
---- a/server/domain/clusters.js
++++ b/server/domain/clusters.js
@@@ -254,7 +254,10 @@ function createClusters({ store, config, outbox }) {
async wire({ limit = 30 } = {}) {
const out = [];
for (const c of await api.recent({ limit: limit * 2 })) {
-- const items = (await q.members.all(c.id)).filter((i) => i.status === 'active' && !i.removed_at && i.canonical_url)
++ const skewMs = 24 * 3_600_000;
++ const cutoff = store.now() + skewMs;
++ const items = (await q.members.all(c.id)).filter((i) => i.status === 'active' && !i.removed_at && i.canonical_url)
++ .filter((i) => !i.published_at || Date.parse(i.published_at) <= cutoff)
.map((i) => ({ id: i.id, headline: i.headline, url: i.canonical_url, outlet: i.outlet, published_at: i.published_at }));
if (!items.length) continue;
items.sort((a, b) => String(b.published_at || '').localeCompare(String(a.published_at || '')));
@@@ -275,7 +278,10 @@ function createClusters({ store, config, outbox }) {
WHERE c.status = 'open' AND c.window_end >= ? AND i.status = 'active' AND i.removed_at IS NULL
ORDER BY c.window_end DESC, c.id DESC`).all(since);
const byCluster = new Map();
++ const skewMs = 24 * 3_600_000;
++ const cutoff = store.now() + skewMs;
for (const r of rows) {
++ if (r.published_at && Date.parse(r.published_at) > cutoff) continue;
if (!byCluster.has(r.cid)) byCluster.set(r.cid, { id: r.cid, label: r.label, updated: r.window_end, items: [], sources: new Set() });
const c = byCluster.get(r.cid);
c.sources.add(r.source_key);