openvibers/openvibe.sources Commits
Sources: sec-company-tickers, SEC's ticker/CIK/exchange list as instrument items (#62)
Changes
@@@ -59,7 +59,7 @@ VALKEY_PREFIX=#SOURCES_ALLOWED_PORTS=80,443# Exact hosts/IPs allowed to resolve to private/loopback addresses. Leave EMPTY in production.#SOURCES_ALLOW_PRIVATE_HOSTS=--#SOURCES_MAX_ITEMS_PER_FETCH=500++#SOURCES_MAX_ITEMS_PER_FETCH=12000# Source credentials: a registry entry names the variable (auth.env), never the value. Names# must start with SOURCES_CRED_. Example (none of the seeds needs one):@@@ -234,6 +234,7 @@ person before enabling:| `lowendtalk-offers` (2026-10-14) | hosting | rss | LowEndTalk's Offers board feed: OpenVibe.Host's offers page (headline, poster, date, link, the stated price) || `lowendbox-posts` (2026-10-14) | hosting | rss | LowEndBox's feed: Host keeps only the offer posts || `sec-xbrl-filings` | trade | rss | SEC EDGAR XBRL filings RSS (`/cgi-bin` is robots-disallowed, so the getcurrent feed is not used) |++| `sec-company-tickers` | trade | api | SEC's ticker, CIK and exchange list (`company_tickers_exchange.json`), one request a day; each row is an `instrument` item OpenVibe.Trade turns into a company page. Needs the per-fetch cap (`SOURCES_MAX_ITEMS_PER_FETCH`, default 12000) |Candidates rejected on 2026-09-22: CheapShark's API (`/api/1.0/` is disallowed in itsrobots.txt) and EDGAR's `getcurrent` Atom feed (`/cgi-bin` disallowed). The seeded feeds were@@@ -13,9 +13,9 @@ const { load } = require('../server/config');const { openDb } = require('../server/db');const { createRegistry } = require('../server/registry');--async function seed(db, { now = () => Date.now(), by = 'seed' } = {}) {++async function seed(db, { now = () => Date.now(), by = 'seed', maxItemsCap } = {}) {const seeds = require(path.join(__dirname, '..', 'seeds', 'sources.json')).sources;-- const registry = createRegistry({ db, now });++ const registry = createRegistry({ db, now, ...(maxItemsCap ? { maxItemsCap } : {}) });const out = { created: [], kept: [] };for (const s of seeds) {if (await registry.get(s.key)) { out.kept.push(s.key); continue; }@@@ -27,8 +27,9 @@ async function seed(db, { now = () => Date.now(), by = 'seed' } = {}) {if (require.main === module) {(async () => {-- const db = await openDb(load()); // DATABASE_URL (PostgreSQL); without it, the development PGlite-- const r = await seed(db);++ const config = load();++ const db = await openDb(config); // DATABASE_URL (PostgreSQL); without it, the development PGlite++ const r = await seed(db, { maxItemsCap: config.items.maxPerFetch });console.log(`seeded ${r.created.length} source(s)${r.created.length ? `: ${r.created.join(', ')}` : ''}; kept ${r.kept.length} existing. All seeds are disabled.`);await db.close();})().catch((err) => { console.error(`[seed] ${err.message}`); process.exit(1); });@@@ -829,6 +829,43 @@"sensitivity": "financial","default_indexability": "noindex"},++ {++ "key": "sec-company-tickers",++ "name": "SEC EDGAR: company tickers and exchanges",++ "type": "api",++ "category": "trade",++ "homepage_url": "https://www.sec.gov/search-filings/edgar-search-assistance/accessing-edgar-data",++ "endpoints": [++ {++ "url": "https://www.sec.gov/files/company_tickers_exchange.json",++ "format": "json",++ "items_path": "data",++ "item_kind": "instrument",++ "max_bytes": 4194304,++ "fields": {++ "identity": "2",++ "title": "1"++ },++ "extra": {++ "ticker": "2",++ "name": "1",++ "cik": "0",++ "exchange": "3"++ }++ }++ ],++ "robots_note": "robots.txt checked 2026-10-15: https://www.sec.gov/robots.txt does not disallow /files/; this is SEC's own published mapping of tickers to CIKs, linked from 'Accessing EDGAR Data'.",++ "terms_note": "Public EDGAR data. SEC fair-access policy: a User-Agent that names the requester and a contact (ours does) and at most 10 requests per second; this is one request a day. Each row [cik, name, ticker, exchange] becomes one instrument item: the company's name, ticker, exchange and CIK as SEC lists them. Informational only: no advice, no execution (ADR-025). Needs SOURCES_MAX_ITEMS_PER_FETCH of at least 11000 (about 10,450 rows in October 2026).",++ "license_note": "US government work: public domain (17 U.S.C. 105). Cite 'U.S. Securities and Exchange Commission'.",++ "min_interval_ms": 60000,++ "poll_interval_sec": 86400,++ "stale_after_sec": 259200,++ "max_items": 11000,++ "enabled": false,++ "review_required": true,++ "sensitivity": "financial",++ "default_indexability": "noindex"++ },{"key": "govuk-news","name": "GOV.UK news and announcements",@@@ -82,7 +82,9 @@ function load(env = process.env) {allowPrivateHosts: list(env.SOURCES_ALLOW_PRIVATE_HOSTS, []),},items: {-- maxPerFetch: int(env.SOURCES_MAX_ITEMS_PER_FETCH, 500),++ // The ceiling on any one source's max_items. Each source still keeps its own (most are 200 or less); 12000++ // lets sec-company-tickers take SEC's whole ticker list (about 10,450 rows) in its one daily request.++ maxPerFetch: int(env.SOURCES_MAX_ITEMS_PER_FETCH, 12000),summaryMax: 1000,},secrets: env,@@@ -51,7 +51,7 @@ const bool = (v, d, what) => {};/** Validate a full source definition → normalized record (no runtime state). */--function validateSource(input, { maxItemsCap = 500 } = {}) {++function validateSource(input, { maxItemsCap = 12000 } = {}) {if (!input || typeof input !== 'object' || Array.isArray(input)) throw new RegistryError('body must be a source object');const known = new Set(['key', 'name', 'type', 'category', 'homepage_url', 'endpoints', 'auth', 'robots_note', 'terms_note', 'license_note','min_interval_ms', 'poll_interval_sec', 'stale_after_sec', 'max_items', 'enabled', 'review_required', 'sensitivity',@@@ -142,7 +142,7 @@ function validateSource(input, { maxItemsCap = 500 } = {}) {return record;}--function createRegistry({ db, now = () => Date.now(), maxItemsCap = 500 }) {++function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {const st = {get: db.prepare('SELECT * FROM sources WHERE key = ?'),all: db.prepare('SELECT * FROM sources ORDER BY key'),nnew file mode 100644@@@ -0,0 +1,50 @@++'use strict';++/**++ * sec-company-tickers (seeds/sources.json): SEC's company_tickers_exchange.json is { fields, data: [[cik, name, ticker,++ * exchange], …] }, so the seed's endpoint reads rows by array index. Each row becomes one `instrument` item keyed by++ * its ticker, with the CIK, name and exchange as SEC lists them (a missing exchange is null, never a guess); the whole++ * list (about 10,450 rows) fits under the default per-fetch cap.++ */++const assert = require('assert');++const api = require('../server/adapters/api');++const { validateSource } = require('../server/registry');++const { load } = require('../server/config');++const { suite } = require('./helpers');++++const t = suite('sec-company-tickers');++const seed = require('../seeds/sources.json').sources.find((s) => s.key === 'sec-company-tickers');++++const BODY = JSON.stringify({++ fields: ['cik', 'name', 'ticker', 'exchange'],++ data: [++ [1045810, 'NVIDIA CORP', 'NVDA', 'Nasdaq'],++ [1652044, 'Alphabet Inc.', 'GOOGL', 'Nasdaq'],++ [1652044, 'Alphabet Inc.', 'GOOG', 'Nasdaq'],++ [1067983, 'BERKSHIRE HATHAWAY INC', 'BRK-B', 'NYSE'],++ [1999999, 'Shell Co', 'SHLL', null],++ ],++});++++t('the seed is a disabled, daily api source of instrument items that validates', () => {++ assert.ok(seed, 'sec-company-tickers is seeded');++ const rec = validateSource(seed);++ assert.strictEqual(rec.enabled, false);++ assert.strictEqual(rec.category, 'trade');++ assert.strictEqual(rec.poll_interval_sec, 86400);++ assert.ok(rec.max_items >= 10450 && rec.max_items <= load({}).items.maxPerFetch, 'the whole list fits under the default cap');++});++++t('each row is one instrument item keyed by ticker, with the CIK, name and exchange', () => {++ const r = api.parse(BODY, { url: seed.endpoints[0].url, endpoint: seed.endpoints[0], maxItems: seed.max_items });++ assert.strictEqual(r.items.length, 5);++ const [nvda, googl, goog, brk, shell] = r.items;++ assert.strictEqual(nvda.identity, 'NVDA');++ assert.strictEqual(nvda.kind, 'instrument');++ assert.strictEqual(nvda.title, 'NVIDIA CORP');++ assert.deepStrictEqual(nvda.fields, { ticker: 'NVDA', name: 'NVIDIA CORP', cik: 1045810, exchange: 'Nasdaq' });++ assert.strictEqual(googl.fields.cik, goog.fields.cik, 'two share classes, one company');++ assert.strictEqual(brk.identity, 'BRK-B');++ assert.strictEqual(shell.fields.exchange, null, 'a missing exchange stays null');++});++++t.run();
@@ -59,7 +59,7 @@ VALKEY_PREFIX=
@@ -59,7 +59,7 @@ VALKEY_PREFIX=
#SOURCES_ALLOWED_PORTS=80,443
#SOURCES_ALLOWED_PORTS=80,443
# Exact hosts/IPs allowed to resolve to private/loopback addresses. Leave EMPTY in production.
# Exact hosts/IPs allowed to resolve to private/loopback addresses. Leave EMPTY in production.
#SOURCES_ALLOW_PRIVATE_HOSTS=
#SOURCES_ALLOW_PRIVATE_HOSTS=
-#SOURCES_MAX_ITEMS_PER_FETCH=500
+#SOURCES_MAX_ITEMS_PER_FETCH=12000
# Source credentials: a registry entry names the variable (auth.env), never the value. Names
# Source credentials: a registry entry names the variable (auth.env), never the value. Names
# must start with SOURCES_CRED_. Example (none of the seeds needs one):
# must start with SOURCES_CRED_. Example (none of the seeds needs one):
@@ -234,6 +234,7 @@ person before enabling:
@@ -234,6 +234,7 @@ person before enabling:
| `lowendtalk-offers` (2026-10-14) | hosting | rss | LowEndTalk's Offers board feed: OpenVibe.Host's offers page (headline, poster, date, link, the stated price) |
| `lowendtalk-offers` (2026-10-14) | hosting | rss | LowEndTalk's Offers board feed: OpenVibe.Host's offers page (headline, poster, date, link, the stated price) |
| `lowendbox-posts` (2026-10-14) | hosting | rss | LowEndBox's feed: Host keeps only the offer posts |
| `lowendbox-posts` (2026-10-14) | hosting | rss | LowEndBox's feed: Host keeps only the offer posts |
| `sec-xbrl-filings` | trade | rss | SEC EDGAR XBRL filings RSS (`/cgi-bin` is robots-disallowed, so the getcurrent feed is not used) |
| `sec-xbrl-filings` | trade | rss | SEC EDGAR XBRL filings RSS (`/cgi-bin` is robots-disallowed, so the getcurrent feed is not used) |
+| `sec-company-tickers` | trade | api | SEC's ticker, CIK and exchange list (`company_tickers_exchange.json`), one request a day; each row is an `instrument` item OpenVibe.Trade turns into a company page. Needs the per-fetch cap (`SOURCES_MAX_ITEMS_PER_FETCH`, default 12000) |
Candidates rejected on 2026-09-22: CheapShark's API (`/api/1.0/` is disallowed in its
Candidates rejected on 2026-09-22: CheapShark's API (`/api/1.0/` is disallowed in its
robots.txt) and EDGAR's `getcurrent` Atom feed (`/cgi-bin` disallowed). The seeded feeds were
robots.txt) and EDGAR's `getcurrent` Atom feed (`/cgi-bin` disallowed). The seeded feeds were
@@ -13,9 +13,9 @@ const { load } = require('../server/config');
@@ -13,9 +13,9 @@ const { load } = require('../server/config');
const { openDb } = require('../server/db');
const { openDb } = require('../server/db');
const { createRegistry } = require('../server/registry');
const { createRegistry } = require('../server/registry');
-async function seed(db, { now = () => Date.now(), by = 'seed' } = {}) {
+async function seed(db, { now = () => Date.now(), by = 'seed', maxItemsCap } = {}) {
const seeds = require(path.join(__dirname, '..', 'seeds', 'sources.json')).sources;
const seeds = require(path.join(__dirname, '..', 'seeds', 'sources.json')).sources;
- const registry = createRegistry({ db, now });
+ const registry = createRegistry({ db, now, ...(maxItemsCap ? { maxItemsCap } : {}) });
const out = { created: [], kept: [] };
const out = { created: [], kept: [] };
for (const s of seeds) {
for (const s of seeds) {
if (await registry.get(s.key)) { out.kept.push(s.key); continue; }
if (await registry.get(s.key)) { out.kept.push(s.key); continue; }
@@ -27,8 +27,9 @@ async function seed(db, { now = () => Date.now(), by = 'seed' } = {}) {
@@ -27,8 +27,9 @@ async function seed(db, { now = () => Date.now(), by = 'seed' } = {}) {
if (require.main === module) {
if (require.main === module) {
(async () => {
(async () => {
- const db = await openDb(load()); // DATABASE_URL (PostgreSQL); without it, the development PGlite
- const r = await seed(db);
+ const config = load();
+ const db = await openDb(config); // DATABASE_URL (PostgreSQL); without it, the development PGlite
+ const r = await seed(db, { maxItemsCap: config.items.maxPerFetch });
console.log(`seeded ${r.created.length} source(s)${r.created.length ? `: ${r.created.join(', ')}` : ''}; kept ${r.kept.length} existing. All seeds are disabled.`);
console.log(`seeded ${r.created.length} source(s)${r.created.length ? `: ${r.created.join(', ')}` : ''}; kept ${r.kept.length} existing. All seeds are disabled.`);
await db.close();
await db.close();
})().catch((err) => { console.error(`[seed] ${err.message}`); process.exit(1); });
})().catch((err) => { console.error(`[seed] ${err.message}`); process.exit(1); });
@@ -829,6 +829,43 @@
@@ -829,6 +829,43 @@
"sensitivity": "financial",
"sensitivity": "financial",
"default_indexability": "noindex"
"default_indexability": "noindex"
},
},
+ {
+ "key": "sec-company-tickers",
+ "name": "SEC EDGAR: company tickers and exchanges",
+ "type": "api",
+ "category": "trade",
+ "homepage_url": "https://www.sec.gov/search-filings/edgar-search-assistance/accessing-edgar-data",
+ "endpoints": [
+ {
+ "url": "https://www.sec.gov/files/company_tickers_exchange.json",
+ "format": "json",
+ "items_path": "data",
+ "item_kind": "instrument",
+ "max_bytes": 4194304,
+ "fields": {
+ "identity": "2",
+ "title": "1"
+ },
+ "extra": {
+ "ticker": "2",
+ "name": "1",
+ "cik": "0",
+ "exchange": "3"
+ }
+ }
+ ],
+ "robots_note": "robots.txt checked 2026-10-15: https://www.sec.gov/robots.txt does not disallow /files/; this is SEC's own published mapping of tickers to CIKs, linked from 'Accessing EDGAR Data'.",
+ "terms_note": "Public EDGAR data. SEC fair-access policy: a User-Agent that names the requester and a contact (ours does) and at most 10 requests per second; this is one request a day. Each row [cik, name, ticker, exchange] becomes one instrument item: the company's name, ticker, exchange and CIK as SEC lists them. Informational only: no advice, no execution (ADR-025). Needs SOURCES_MAX_ITEMS_PER_FETCH of at least 11000 (about 10,450 rows in October 2026).",
+ "license_note": "US government work: public domain (17 U.S.C. 105). Cite 'U.S. Securities and Exchange Commission'.",
+ "min_interval_ms": 60000,
+ "poll_interval_sec": 86400,
+ "stale_after_sec": 259200,
+ "max_items": 11000,
+ "enabled": false,
+ "review_required": true,
+ "sensitivity": "financial",
+ "default_indexability": "noindex"
+ },
{
{
"key": "govuk-news",
"key": "govuk-news",
"name": "GOV.UK news and announcements",
"name": "GOV.UK news and announcements",
@@ -82,7 +82,9 @@ function load(env = process.env) {
@@ -82,7 +82,9 @@ function load(env = process.env) {
allowPrivateHosts: list(env.SOURCES_ALLOW_PRIVATE_HOSTS, []),
allowPrivateHosts: list(env.SOURCES_ALLOW_PRIVATE_HOSTS, []),
},
},
items: {
items: {
- maxPerFetch: int(env.SOURCES_MAX_ITEMS_PER_FETCH, 500),
+ // The ceiling on any one source's max_items. Each source still keeps its own (most are 200 or less); 12000
+ // lets sec-company-tickers take SEC's whole ticker list (about 10,450 rows) in its one daily request.
+ maxPerFetch: int(env.SOURCES_MAX_ITEMS_PER_FETCH, 12000),
summaryMax: 1000,
summaryMax: 1000,
},
},
secrets: env,
secrets: env,
@@ -51,7 +51,7 @@ const bool = (v, d, what) => {
@@ -51,7 +51,7 @@ const bool = (v, d, what) => {
};
};
/** Validate a full source definition → normalized record (no runtime state). */
/** Validate a full source definition → normalized record (no runtime state). */
-function validateSource(input, { maxItemsCap = 500 } = {}) {
+function validateSource(input, { maxItemsCap = 12000 } = {}) {
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new RegistryError('body must be a source object');
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new RegistryError('body must be a source object');
const known = new Set(['key', 'name', 'type', 'category', 'homepage_url', 'endpoints', 'auth', 'robots_note', 'terms_note', 'license_note',
const known = new Set(['key', 'name', 'type', 'category', 'homepage_url', 'endpoints', 'auth', 'robots_note', 'terms_note', 'license_note',
'min_interval_ms', 'poll_interval_sec', 'stale_after_sec', 'max_items', 'enabled', 'review_required', 'sensitivity',
'min_interval_ms', 'poll_interval_sec', 'stale_after_sec', 'max_items', 'enabled', 'review_required', 'sensitivity',
@@ -142,7 +142,7 @@ function validateSource(input, { maxItemsCap = 500 } = {}) {
@@ -142,7 +142,7 @@ function validateSource(input, { maxItemsCap = 500 } = {}) {
return record;
return record;
}
}
-function createRegistry({ db, now = () => Date.now(), maxItemsCap = 500 }) {
+function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {
const st = {
const st = {
get: db.prepare('SELECT * FROM sources WHERE key = ?'),
get: db.prepare('SELECT * FROM sources WHERE key = ?'),
all: db.prepare('SELECT * FROM sources ORDER BY key'),
all: db.prepare('SELECT * FROM sources ORDER BY key'),
new file mode 100644
new file mode 100644
@@ -0,0 +1,50 @@
@@ -0,0 +1,50 @@
+'use strict';
+/**
+ * sec-company-tickers (seeds/sources.json): SEC's company_tickers_exchange.json is { fields, data: [[cik, name, ticker,
+ * exchange], …] }, so the seed's endpoint reads rows by array index. Each row becomes one `instrument` item keyed by
+ * its ticker, with the CIK, name and exchange as SEC lists them (a missing exchange is null, never a guess); the whole
+ * list (about 10,450 rows) fits under the default per-fetch cap.
+ */
+const assert = require('assert');
+const api = require('../server/adapters/api');
+const { validateSource } = require('../server/registry');
+const { load } = require('../server/config');
+const { suite } = require('./helpers');
+
+const t = suite('sec-company-tickers');
+const seed = require('../seeds/sources.json').sources.find((s) => s.key === 'sec-company-tickers');
+
+const BODY = JSON.stringify({
+ fields: ['cik', 'name', 'ticker', 'exchange'],
+ data: [
+ [1045810, 'NVIDIA CORP', 'NVDA', 'Nasdaq'],
+ [1652044, 'Alphabet Inc.', 'GOOGL', 'Nasdaq'],
+ [1652044, 'Alphabet Inc.', 'GOOG', 'Nasdaq'],
+ [1067983, 'BERKSHIRE HATHAWAY INC', 'BRK-B', 'NYSE'],
+ [1999999, 'Shell Co', 'SHLL', null],
+ ],
+});
+
+t('the seed is a disabled, daily api source of instrument items that validates', () => {
+ assert.ok(seed, 'sec-company-tickers is seeded');
+ const rec = validateSource(seed);
+ assert.strictEqual(rec.enabled, false);
+ assert.strictEqual(rec.category, 'trade');
+ assert.strictEqual(rec.poll_interval_sec, 86400);
+ assert.ok(rec.max_items >= 10450 && rec.max_items <= load({}).items.maxPerFetch, 'the whole list fits under the default cap');
+});
+
+t('each row is one instrument item keyed by ticker, with the CIK, name and exchange', () => {
+ const r = api.parse(BODY, { url: seed.endpoints[0].url, endpoint: seed.endpoints[0], maxItems: seed.max_items });
+ assert.strictEqual(r.items.length, 5);
+ const [nvda, googl, goog, brk, shell] = r.items;
+ assert.strictEqual(nvda.identity, 'NVDA');
+ assert.strictEqual(nvda.kind, 'instrument');
+ assert.strictEqual(nvda.title, 'NVIDIA CORP');
+ assert.deepStrictEqual(nvda.fields, { ticker: 'NVDA', name: 'NVIDIA CORP', cik: 1045810, exchange: 'Nasdaq' });
+ assert.strictEqual(googl.fields.cik, goog.fields.cik, 'two share classes, one company');
+ assert.strictEqual(brk.identity, 'BRK-B');
+ assert.strictEqual(shell.fields.exchange, null, 'a missing exchange stays null');
+});
+
+t.run();