openvibers/openvibe.sources Commits

Sources: five more sources seeded disabled (GOV.UK, EU press corner, NOAA, Frankfurter, ECB) with their terms and robots notes (#18)

OP OpenVibers committed 10 Oct 2026 · 9a74b68759 · 3 files, +157 −3

Five more sources in `seeds/sources.json`, from the terms research's "enable first" lists. Every one is created **disabled** with `review_required` and `noindex`, like the original six:

| key | category | endpoint (checked 2026-10-10 with Sources' contactable user agent) |
|---|---|---|
| `govuk-news` | news | `https://www.gov.uk/search/news-and-communications.atom`: 200 Atom, OGL v3 |
| `eu-press-corner` | news | `https://ec.europa.eu/commission/presscorner/api/rss?language=en`: 200 RSS, CC BY 4.0 |
| `noaa-news` | news | `https://www.noaa.gov/rss.xml`: 200 RSS. NOAA's robots.txt answers 403 to bots, so it stays disabled until that is resolved with NOAA (recorded in `robots_note`) |
| `frankfurter-eur-rates` | trade (financial) | `https://api.frankfurter.dev/v2/rates?base=EUR&quotes=…`: the array route (the api adapter needs an array; `/latest` returns an object), one item per currency |
| `ecb-euro-reference-rates` | trade (financial) | `https://www.ecb.europa.eu/rss/fxref-usd.html`: 200 RSS 1.0, crawl-delay 5 honoured by the 60 s minimum interval |

Each `terms_note` names the required credit line and licence link, stores titles, links and short summaries only, and excludes third-party-credited items. Wikinews was considered and left out: its feeds live under `/w/`, which its robots.txt disallows.

Tests: new `test/seeds.test.js` runs every entry through `validateSource` and `createRegistry`, checks it is disabled and that endpoints are https. `test/proposals.test.js` now asks that every category is covered rather than exactly one source per category. Both pass. The entries were drafted by a DeepSeek job and reviewed here.

Changes

ddiff --git a/seeds/sources.json b/seeds/sources.json
iindex 649890d..22ac8c8 100644
---- a/seeds/sources.json
++++ b/seeds/sources.json
@@@ -1,5 +1,5 @@
{
-- "_about": "One real source per category, all DISABLED. Robots notes were checked by hand on 2026-09-22; terms must be re-verified by a person before a source is enabled (PATCH enabled: true). Seeding never overwrites or enables an existing source.",
++ "_about": "One real source per category in the original set, all DISABLED; further news and trade sources were added later, also all DISABLED with review_required. Robots notes were checked by hand on the date each one records (2026-09-22 for the original six, 2026-10-10 for those added later); terms must be re-verified by a person before a source is enabled (PATCH enabled: true). Seeding never overwrites or enables an existing source.",
"sources": [
{
"key": "nasa-news-releases",
@@@ -108,6 +108,111 @@
"review_required": true,
"sensitivity": "financial",
"default_indexability": "noindex"
++ },
++ {
++ "key": "govuk-news",
++ "name": "GOV.UK news and announcements",
++ "type": "atom",
++ "category": "news",
++ "homepage_url": "https://www.gov.uk/",
++ "endpoints": ["https://www.gov.uk/search/news-and-communications.atom"],
++ "robots_note": "robots.txt checked 2026-10-10: User-agent * disallows /*/print$ and /search/all*; the .atom feed at /search/news-and-communications.atom matches neither, so it is allowed. No crawl-delay for *.",
++ "terms_note": "Open Government Licence v3.0 (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/): copy, publish, distribute and exploit commercially, with acknowledgement and, where possible, a link to the licence. Credit the provider's attribution statement where given, otherwise the fallback 'Contains public sector information licensed under the Open Government Licence v3.0.' No implied official status or endorsement. Store titles, links and short summaries only; never full text or images; drop items marked 'except where otherwise stated' (third-party rights, logos, personal data). Re-verify before enabling.",
++ "license_note": "Contains public sector information licensed under the Open Government Licence v3.0 (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/).",
++ "min_interval_ms": 60000,
++ "poll_interval_sec": 3600,
++ "max_items": 50,
++ "enabled": false,
++ "review_required": true,
++ "sensitivity": "none",
++ "default_indexability": "noindex"
++ },
++ {
++ "key": "eu-press-corner",
++ "name": "European Commission press corner",
++ "type": "rss",
++ "category": "news",
++ "homepage_url": "https://ec.europa.eu/commission/presscorner/",
++ "endpoints": ["https://ec.europa.eu/commission/presscorner/api/rss?language=en"],
++ "robots_note": "robots.txt checked 2026-10-10: User-agent * lists many disallowed paths but none for /commission/presscorner/ or its /api/; the feed path is allowed. No crawl-delay.",
++ "terms_note": "Content owned by the EU is licensed under CC BY 4.0: reuse allowed with appropriate credit and an indication of changes; per-item notices ('unless otherwise indicated') override. Credit 'European Commission' and link the licence (https://creativecommons.org/licenses/by/4.0/). EU logos and names are excluded from the reuse policy. Store titles, links and short summaries only; never full text or images; drop third-party-credited items. Re-verify before enabling.",
++ "license_note": "Creative Commons Attribution 4.0 International (CC BY 4.0): https://creativecommons.org/licenses/by/4.0/. EU logos excluded.",
++ "min_interval_ms": 60000,
++ "poll_interval_sec": 3600,
++ "max_items": 50,
++ "enabled": false,
++ "review_required": true,
++ "sensitivity": "none",
++ "default_indexability": "noindex"
++ },
++ {
++ "key": "noaa-news",
++ "name": "NOAA news",
++ "type": "rss",
++ "category": "news",
++ "homepage_url": "https://www.noaa.gov/",
++ "endpoints": ["https://www.noaa.gov/rss.xml"],
++ "robots_note": "robots.txt checked 2026-10-10: https://www.noaa.gov/robots.txt answers 403 to our identifying User-Agent (NOAA's CDN blocks bot UAs on HTML paths), so its rules could not be read; the FEED https://www.noaa.gov/rss.xml itself answers 200/application/rss+xml to our identifying UA. Resolve the robots/User-Agent 403 with NOAA before enabling.",
++ "terms_note": "Most NOAA information is in the US public domain and may be used with credit; third-party-credited items, images and video are excluded, and use must not imply US-Government endorsement or affiliation. Credit 'NOAA' (National Oceanic and Atmospheric Administration) and link back. Store titles, links and short summaries only; never full text or images. Re-verify before enabling.",
++ "license_note": "US federal government work (public domain) with per-item credits; credit NOAA.",
++ "min_interval_ms": 60000,
++ "poll_interval_sec": 3600,
++ "max_items": 50,
++ "enabled": false,
++ "review_required": true,
++ "sensitivity": "none",
++ "default_indexability": "noindex"
++ },
++ {
++ "key": "frankfurter-eur-rates",
++ "name": "Frankfurter euro reference rates",
++ "type": "api",
++ "category": "trade",
++ "homepage_url": "https://frankfurter.dev/",
++ "endpoints": [
++ {
++ "url": "https://api.frankfurter.dev/v2/rates?base=EUR&quotes=USD,GBP,JPY,CHF,CAD,AUD,NZD,SEK,NOK,DKK,CZK,PLN,HUF,RON,TRY,CNY,HKD,SGD,KRW,INR,BRL,MXN,ZAR,THB,IDR,MYR,PHP,ILS",
++ "format": "json",
++ "items_path": "",
++ "item_kind": "rate",
++ "fields": {
++ "identity": "quote"
++ },
++ "extra": {
++ "rate": "rate",
++ "base": "base",
++ "date": "date"
++ }
++ }
++ ],
++ "robots_note": "robots.txt checked 2026-10-10: https://frankfurter.dev/robots.txt and https://api.frankfurter.dev/robots.txt both say User-agent * Allow /; the /v2/rates path is allowed. No crawl-delay.",
++ "terms_note": "Frankfurter is MIT-licensed software, free for commercial use, with no API key and no quota (only anti-abuse rate limiting). The rates themselves fall under each provider's terms, so attribute the contributing provider (ECB, TCMB, ...) and link https://frankfurter.dev/ and the provider's terms; for ECB-sourced rates cite 'European Central Bank'. The legacy object endpoint https://api.frankfurter.app/latest cannot be mapped by the api adapter (it requires an array at items_path), so the array endpoint https://api.frankfurter.dev/v2/rates is used and each row (date, base, quote, rate) becomes one item. Rates are informational observations only, never advice (ADR-025). Re-verify before enabling.",
++ "license_note": "Frankfurter software: MIT. Underlying rates: per provider (ECB reference rates reusable with attribution); see https://frankfurter.dev/license/.",
++ "min_interval_ms": 60000,
++ "poll_interval_sec": 86400,
++ "max_items": 40,
++ "enabled": false,
++ "review_required": true,
++ "sensitivity": "financial",
++ "default_indexability": "noindex"
++ },
++ {
++ "key": "ecb-euro-reference-rates",
++ "name": "ECB euro reference rates: US dollar",
++ "type": "rss",
++ "category": "trade",
++ "homepage_url": "https://www.ecb.europa.eu/stats/eurofxref/",
++ "endpoints": ["https://www.ecb.europa.eu/rss/fxref-usd.html"],
++ "robots_note": "robots.txt checked 2026-10-10: User-agent * with Crawl-delay 5 (honoured; min_interval_ms is well above it) and disallows only *_content.<lang>.html and a few video/data paths; the /rss/fxref-usd.html feed path is not disallowed.",
++ "terms_note": "ECB statistics may be used freely and must be reproduced accurately with the ECB cited as the source; where incorporated into a sold product, buyers must be told the information is available free of charge from the ECB website. Cite 'European Central Bank' and link https://www.ecb.europa.eu/services/disclaimer/html/index.en.html; no framing or logos. Store published rates as retrieved observations only — informational, never advice (ADR-025). Re-verify before enabling.",
++ "license_note": "ECB copyright and disclaimer: free use with citation; see https://www.ecb.europa.eu/services/disclaimer/html/index.en.html.",
++ "min_interval_ms": 60000,
++ "poll_interval_sec": 86400,
++ "max_items": 40,
++ "enabled": false,
++ "review_required": true,
++ "sensitivity": "financial",
++ "default_indexability": "noindex"
}
]
}
ddiff --git a/test/proposals.test.js b/test/proposals.test.js
iindex cbf5a60..55e0d6a 100644
---- a/test/proposals.test.js
++++ b/test/proposals.test.js
@@@ -57,9 +57,10 @@ t('the service manifest proposal is a valid registry.service-manifest@1', () =>
assert.deepStrictEqual([...m.capabilities].sort(), Object.values(CAPS).sort());
});
--t('seeds: one real source per category, all disabled, all valid, notes recorded', async () => {
++t('seeds: every category represented, all disabled, all valid, notes recorded', async () => {
const seeds = require('../seeds/sources.json').sources;
-- assert.deepStrictEqual(seeds.map(s => s.category).sort(), [...CATEGORIES].sort());
++ const represented = new Set(seeds.map(s => s.category));
++ for (const c of CATEGORIES) assert.ok(represented.has(c), `seeds must cover category ${c}`);
for (const s of seeds) {
const rec = validateSource(s);
assert.strictEqual(rec.enabled, false, s.key);
ddiff --git a/test/seeds.test.js b/test/seeds.test.js
nnew file mode 100644
iindex 0000000..7f9b0ee
---- /dev/null
++++ b/test/seeds.test.js
@@@ -0,0 +1,48 @@
++'use strict';
++/**
++ * The seed file is honest. Every entry in seeds/sources.json passes the registry's own validation
++ * (validateSource, registered through the create path too), every entry is DISABLED — seeding is
++ * never permission to fetch — and every endpoint is https. Mirrors test/proposals.test.js, which
++ * checks the seeds from the contract side.
++ *
++ * node test/seeds.test.js
++ */
++const assert = require('assert');
++const { validateSource, createRegistry } = require('../server/registry');
++const { suite } = require('./helpers');
++
++const t = suite('seeds');
++const seeds = require('../seeds/sources.json').sources;
++
++t('seeds/sources.json is a non-empty list of uniquely keyed sources', () => {
++ assert.ok(Array.isArray(seeds) && seeds.length > 0, 'seeds must be a non-empty array');
++ const keys = seeds.map((s) => s.key);
++ assert.strictEqual(new Set(keys).size, keys.length, 'seed keys must be unique');
++});
++
++t('every seed validates, is disabled, and keys only https endpoints', () => {
++ for (const s of seeds) {
++ const rec = validateSource(s); // throws on any invalid entry
++ assert.strictEqual(rec.enabled, false, `${s.key} must be disabled`);
++ assert.strictEqual(rec.review_required, true, `${s.key} must require review`);
++ assert.strictEqual(rec.default_indexability, 'noindex', `${s.key} must default to noindex`);
++ for (const ep of rec.endpoints) assert.ok(ep.url.startsWith('https://'), `${s.key}: endpoint must be https: ${ep.url}`);
++ }
++});
++
++t('every seed creates through the registry and stays disabled', async () => {
++ const fresh = await require('./db').testDb();
++ try {
++ const registry = createRegistry({ db: fresh.db });
++ for (const s of seeds) {
++ const row = await registry.create(s, 'seed'); // the real create path, not just validation
++ assert.strictEqual(Boolean(row.enabled), false, `${s.key} must be created disabled`);
++ }
++ const enabled = await fresh.db.prepare('SELECT COUNT(*) AS n FROM sources WHERE enabled = 1').get();
++ assert.strictEqual(enabled.n, 0, 'seeding must never enable a source');
++ } finally {
++ await fresh.close();
++ }
++});
++
++t.run();