openvibers/openvibe.news Commits
Attribute the government feeds to their agency (#151)
Changes
@@@ -31,6 +31,13 @@ const { ids } = require('openvibe-contracts');const { createChangeCursor, pullChanges, normalize } = require('openvibe-publishing/ingest');const NEAR_TITLE = { anyOutlet: 0.9, sameOutlet: 0.75, windowMs: 48 * 3600 * 1000 };++// The agency behind a government feed: the outlet a reader sees for an item of that source, even before the Sources++// registry has taught News the source's own name. An item of another source keeps the name it learned.++const AGENCIES = Object.freeze({++ 'sec-press': 'US Securities and Exchange Commission',++ 'doj-news': 'US Department of Justice',++ 'dol-news': 'US Department of Labor',++});// Only an http(s) URL is kept as an item's link: a publisher's javascript: or data: URL would become a link on a page.const httpUrl = (u) => { try { const x = new URL(String(u || '')); return x.protocol === 'https:' || x.protocol === 'http:' ? x.href : null; } catch { return null; } };const newItemId = (now) => `nsi_${ids.ulid(now)}`;@@@ -74,7 +81,7 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = nasync function outletFor(item) {const st = await q.status.get(item.source_key);-- return (st && st.name) || normalize.hostOf(item.canonical_url) || item.source_key;++ return AGENCIES[item.source_key] || (st && st.name) || normalize.hostOf(item.canonical_url) || item.source_key;}/** The fields News keeps from a Sources item. Nothing else of the item is read. */nnew file mode 100644@@@ -0,0 +1,56 @@++'use strict';++/**++ * The three US government feeds News reads through OpenVibe.Sources (sec-press, doj-news, dol-news): each item is++ * attributed to the agency it comes from, links back to the agency, and keeps the headline and a short excerpt only —++ * never the release's full text (a recorded fixture: no live call here).++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++++const BODY = 'SECRET-GOVERNMENT-FULL-TEXT-51a2';++++(async () => {++ const t = await boot();++ try {++ await check('an item of each government feed is attributed to its agency and links back to it', async () => {++ t.sources.addSource('sec-press', {});++ t.sources.addSource('doj-news', {});++ t.sources.addSource('dol-news', {});++ const sec = t.sources.addItem({++ source_key: 'sec-press', title: 'SEC charges two brokers over misleading investor statements',++ url: 'https://www.sec.gov/news/pressreleases/2026-88', summary: 'The Securities and Exchange Commission charged two brokers over misleading investor statements.',++ terms_note: 'Keep the headline and a short summary of each release, never the whole release, and link back to the release on sec.gov.',++ fields: { body: BODY }, published_at: '2026-10-30T12:00:00Z', authors: [],++ });++ const doj = t.sources.addItem({++ source_key: 'doj-news', title: 'Justice Department reaches settlement in healthcare fraud case',++ url: 'https://www.justice.gov/news/press-releases/2026/10/settlement', summary: 'The Department of Justice announced a settlement in a healthcare fraud case today.',++ terms_note: 'Keep the headline and a short summary of each item, never the whole text, and link back to the justice.gov page.',++ fields: { body: BODY }, published_at: '2026-10-30T13:00:00Z', authors: [],++ });++ const dol = t.sources.addItem({++ source_key: 'dol-news', title: 'Department of Labor publishes new rule on worker classification',++ url: 'https://www.dol.gov/newsroom/releases/osec/2026-31', summary: 'The Department of Labor published a rule on worker classification today.',++ terms_note: 'Keep the headline and a short summary of each release, never the whole text, and link back to dol.gov.',++ fields: { body: BODY }, published_at: '2026-10-30T14:00:00Z', authors: [],++ });++ await t.pull();++ const rows = await t.db().many('SELECT source_key, headline, outlet, canonical_url, summary FROM news_source_items WHERE source_key IN (\'sec-press\',\'doj-news\',\'dol-news\') ORDER BY source_key');++ assert.deepStrictEqual(rows.map((r) => [r.source_key, r.outlet, r.canonical_url]), [++ ['doj-news', 'US Department of Justice', 'https://www.justice.gov/news/press-releases/2026/10/settlement'],++ ['dol-news', 'US Department of Labor', 'https://www.dol.gov/newsroom/releases/osec/2026-31'],++ ['sec-press', 'US Securities and Exchange Commission', 'https://www.sec.gov/news/pressreleases/2026-88'],++ ]);++ for (const r of rows) {++ assert.ok(r.headline && r.headline.length > 10, 'the headline is kept');++ assert.ok(r.summary && r.summary.length > 10, 'a short excerpt is kept');++ assert.ok(!r.summary.includes(BODY), 'the full text is not kept, only the excerpt the notes allow');++ }++ const all = await t.db().many('SELECT * FROM news_source_items');++ assert.ok(!JSON.stringify(all).includes(BODY), 'the release text never reaches the database');++ });++ } finally {++ await t.close();++ }++ done();++})().catch((err) => { console.error(err); process.exit(1); });
@@ -31,6 +31,13 @@ const { ids } = require('openvibe-contracts');
@@ -31,6 +31,13 @@ const { ids } = require('openvibe-contracts');
const { createChangeCursor, pullChanges, normalize } = require('openvibe-publishing/ingest');
const { createChangeCursor, pullChanges, normalize } = require('openvibe-publishing/ingest');
const NEAR_TITLE = { anyOutlet: 0.9, sameOutlet: 0.75, windowMs: 48 * 3600 * 1000 };
const NEAR_TITLE = { anyOutlet: 0.9, sameOutlet: 0.75, windowMs: 48 * 3600 * 1000 };
+// The agency behind a government feed: the outlet a reader sees for an item of that source, even before the Sources
+// registry has taught News the source's own name. An item of another source keeps the name it learned.
+const AGENCIES = Object.freeze({
+ 'sec-press': 'US Securities and Exchange Commission',
+ 'doj-news': 'US Department of Justice',
+ 'dol-news': 'US Department of Labor',
+});
// Only an http(s) URL is kept as an item's link: a publisher's javascript: or data: URL would become a link on a page.
// Only an http(s) URL is kept as an item's link: a publisher's javascript: or data: URL would become a link on a page.
const httpUrl = (u) => { try { const x = new URL(String(u || '')); return x.protocol === 'https:' || x.protocol === 'http:' ? x.href : null; } catch { return null; } };
const httpUrl = (u) => { try { const x = new URL(String(u || '')); return x.protocol === 'https:' || x.protocol === 'http:' ? x.href : null; } catch { return null; } };
const newItemId = (now) => `nsi_${ids.ulid(now)}`;
const newItemId = (now) => `nsi_${ids.ulid(now)}`;
@@ -74,7 +81,7 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@@ -74,7 +81,7 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
async function outletFor(item) {
async function outletFor(item) {
const st = await q.status.get(item.source_key);
const st = await q.status.get(item.source_key);
- return (st && st.name) || normalize.hostOf(item.canonical_url) || item.source_key;
+ return AGENCIES[item.source_key] || (st && st.name) || normalize.hostOf(item.canonical_url) || item.source_key;
}
}
/** The fields News keeps from a Sources item. Nothing else of the item is read. */
/** The fields News keeps from a Sources item. Nothing else of the item is read. */
new file mode 100644
new file mode 100644
@@ -0,0 +1,56 @@
@@ -0,0 +1,56 @@
+'use strict';
+/**
+ * The three US government feeds News reads through OpenVibe.Sources (sec-press, doj-news, dol-news): each item is
+ * attributed to the agency it comes from, links back to the agency, and keeps the headline and a short excerpt only —
+ * never the release's full text (a recorded fixture: no live call here).
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+
+const BODY = 'SECRET-GOVERNMENT-FULL-TEXT-51a2';
+
+(async () => {
+ const t = await boot();
+ try {
+ await check('an item of each government feed is attributed to its agency and links back to it', async () => {
+ t.sources.addSource('sec-press', {});
+ t.sources.addSource('doj-news', {});
+ t.sources.addSource('dol-news', {});
+ const sec = t.sources.addItem({
+ source_key: 'sec-press', title: 'SEC charges two brokers over misleading investor statements',
+ url: 'https://www.sec.gov/news/pressreleases/2026-88', summary: 'The Securities and Exchange Commission charged two brokers over misleading investor statements.',
+ terms_note: 'Keep the headline and a short summary of each release, never the whole release, and link back to the release on sec.gov.',
+ fields: { body: BODY }, published_at: '2026-10-30T12:00:00Z', authors: [],
+ });
+ const doj = t.sources.addItem({
+ source_key: 'doj-news', title: 'Justice Department reaches settlement in healthcare fraud case',
+ url: 'https://www.justice.gov/news/press-releases/2026/10/settlement', summary: 'The Department of Justice announced a settlement in a healthcare fraud case today.',
+ terms_note: 'Keep the headline and a short summary of each item, never the whole text, and link back to the justice.gov page.',
+ fields: { body: BODY }, published_at: '2026-10-30T13:00:00Z', authors: [],
+ });
+ const dol = t.sources.addItem({
+ source_key: 'dol-news', title: 'Department of Labor publishes new rule on worker classification',
+ url: 'https://www.dol.gov/newsroom/releases/osec/2026-31', summary: 'The Department of Labor published a rule on worker classification today.',
+ terms_note: 'Keep the headline and a short summary of each release, never the whole text, and link back to dol.gov.',
+ fields: { body: BODY }, published_at: '2026-10-30T14:00:00Z', authors: [],
+ });
+ await t.pull();
+ const rows = await t.db().many('SELECT source_key, headline, outlet, canonical_url, summary FROM news_source_items WHERE source_key IN (\'sec-press\',\'doj-news\',\'dol-news\') ORDER BY source_key');
+ assert.deepStrictEqual(rows.map((r) => [r.source_key, r.outlet, r.canonical_url]), [
+ ['doj-news', 'US Department of Justice', 'https://www.justice.gov/news/press-releases/2026/10/settlement'],
+ ['dol-news', 'US Department of Labor', 'https://www.dol.gov/newsroom/releases/osec/2026-31'],
+ ['sec-press', 'US Securities and Exchange Commission', 'https://www.sec.gov/news/pressreleases/2026-88'],
+ ]);
+ for (const r of rows) {
+ assert.ok(r.headline && r.headline.length > 10, 'the headline is kept');
+ assert.ok(r.summary && r.summary.length > 10, 'a short excerpt is kept');
+ assert.ok(!r.summary.includes(BODY), 'the full text is not kept, only the excerpt the notes allow');
+ }
+ const all = await t.db().many('SELECT * FROM news_source_items');
+ assert.ok(!JSON.stringify(all).includes(BODY), 'the release text never reaches the database');
+ });
+ } finally {
+ await t.close();
+ }
+ done();
+})().catch((err) => { console.error(err); process.exit(1); });