openvibers/openvibe.sources Commits

Sources: sec-company-tickers, SEC's ticker/CIK/exchange list as instrument items (#62)

OP OpenVibers committed 29 days ago · 00e875b4b3 · 7 files, +99 −8

Changes

ddiff --git a/.env.example b/.env.example
iindex 4da3a05..2320471 100644
---- a/.env.example
++++ b/.env.example
@@@ -59,7 +59,7 @@ VALKEY_PREFIX=
#SOURCES_ALLOWED_PORTS=80,443
# Exact hosts/IPs allowed to resolve to private/loopback addresses. Leave EMPTY in production.
#SOURCES_ALLOW_PRIVATE_HOSTS=
--#SOURCES_MAX_ITEMS_PER_FETCH=500
++#SOURCES_MAX_ITEMS_PER_FETCH=12000
# Source credentials: a registry entry names the variable (auth.env), never the value. Names
# must start with SOURCES_CRED_. Example (none of the seeds needs one):
ddiff --git a/README.md b/README.md
iindex 008d123..f773094 100644
---- a/README.md
++++ b/README.md
@@@ -234,6 +234,7 @@ person before enabling:
| `lowendtalk-offers` (2026-10-14) | hosting | rss | LowEndTalk's Offers board feed: OpenVibe.Host's offers page (headline, poster, date, link, the stated price) |
| `lowendbox-posts` (2026-10-14) | hosting | rss | LowEndBox's feed: Host keeps only the offer posts |
| `sec-xbrl-filings` | trade | rss | SEC EDGAR XBRL filings RSS (`/cgi-bin` is robots-disallowed, so the getcurrent feed is not used) |
++| `sec-company-tickers` | trade | api | SEC's ticker, CIK and exchange list (`company_tickers_exchange.json`), one request a day; each row is an `instrument` item OpenVibe.Trade turns into a company page. Needs the per-fetch cap (`SOURCES_MAX_ITEMS_PER_FETCH`, default 12000) |
Candidates rejected on 2026-09-22: CheapShark's API (`/api/1.0/` is disallowed in its
robots.txt) and EDGAR's `getcurrent` Atom feed (`/cgi-bin` disallowed). The seeded feeds were
ddiff --git a/scripts/seed.js b/scripts/seed.js
iindex ba42436..339929b 100644
---- a/scripts/seed.js
++++ b/scripts/seed.js
@@@ -13,9 +13,9 @@ const { load } = require('../server/config');
const { openDb } = require('../server/db');
const { createRegistry } = require('../server/registry');
--async function seed(db, { now = () => Date.now(), by = 'seed' } = {}) {
++async function seed(db, { now = () => Date.now(), by = 'seed', maxItemsCap } = {}) {
const seeds = require(path.join(__dirname, '..', 'seeds', 'sources.json')).sources;
-- const registry = createRegistry({ db, now });
++ const registry = createRegistry({ db, now, ...(maxItemsCap ? { maxItemsCap } : {}) });
const out = { created: [], kept: [] };
for (const s of seeds) {
if (await registry.get(s.key)) { out.kept.push(s.key); continue; }
@@@ -27,8 +27,9 @@ async function seed(db, { now = () => Date.now(), by = 'seed' } = {}) {
if (require.main === module) {
(async () => {
-- const db = await openDb(load()); // DATABASE_URL (PostgreSQL); without it, the development PGlite
-- const r = await seed(db);
++ const config = load();
++ const db = await openDb(config); // DATABASE_URL (PostgreSQL); without it, the development PGlite
++ const r = await seed(db, { maxItemsCap: config.items.maxPerFetch });
console.log(`seeded ${r.created.length} source(s)${r.created.length ? `: ${r.created.join(', ')}` : ''}; kept ${r.kept.length} existing. All seeds are disabled.`);
await db.close();
})().catch((err) => { console.error(`[seed] ${err.message}`); process.exit(1); });
ddiff --git a/seeds/sources.json b/seeds/sources.json
iindex 1d45c2d..2fca52d 100644
---- a/seeds/sources.json
++++ b/seeds/sources.json
@@@ -829,6 +829,43 @@
"sensitivity": "financial",
"default_indexability": "noindex"
},
++ {
++ "key": "sec-company-tickers",
++ "name": "SEC EDGAR: company tickers and exchanges",
++ "type": "api",
++ "category": "trade",
++ "homepage_url": "https://www.sec.gov/search-filings/edgar-search-assistance/accessing-edgar-data",
++ "endpoints": [
++ {
++ "url": "https://www.sec.gov/files/company_tickers_exchange.json",
++ "format": "json",
++ "items_path": "data",
++ "item_kind": "instrument",
++ "max_bytes": 4194304,
++ "fields": {
++ "identity": "2",
++ "title": "1"
++ },
++ "extra": {
++ "ticker": "2",
++ "name": "1",
++ "cik": "0",
++ "exchange": "3"
++ }
++ }
++ ],
++ "robots_note": "robots.txt checked 2026-10-15: https://www.sec.gov/robots.txt does not disallow /files/; this is SEC's own published mapping of tickers to CIKs, linked from 'Accessing EDGAR Data'.",
++ "terms_note": "Public EDGAR data. SEC fair-access policy: a User-Agent that names the requester and a contact (ours does) and at most 10 requests per second; this is one request a day. Each row [cik, name, ticker, exchange] becomes one instrument item: the company's name, ticker, exchange and CIK as SEC lists them. Informational only: no advice, no execution (ADR-025). Needs SOURCES_MAX_ITEMS_PER_FETCH of at least 11000 (about 10,450 rows in October 2026).",
++ "license_note": "US government work: public domain (17 U.S.C. 105). Cite 'U.S. Securities and Exchange Commission'.",
++ "min_interval_ms": 60000,
++ "poll_interval_sec": 86400,
++ "stale_after_sec": 259200,
++ "max_items": 11000,
++ "enabled": false,
++ "review_required": true,
++ "sensitivity": "financial",
++ "default_indexability": "noindex"
++ },
{
"key": "govuk-news",
"name": "GOV.UK news and announcements",
ddiff --git a/server/config.js b/server/config.js
iindex 72e49e0..7834546 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -82,7 +82,9 @@ function load(env = process.env) {
allowPrivateHosts: list(env.SOURCES_ALLOW_PRIVATE_HOSTS, []),
},
items: {
-- maxPerFetch: int(env.SOURCES_MAX_ITEMS_PER_FETCH, 500),
++ // The ceiling on any one source's max_items. Each source still keeps its own (most are 200 or less); 12000
++ // lets sec-company-tickers take SEC's whole ticker list (about 10,450 rows) in its one daily request.
++ maxPerFetch: int(env.SOURCES_MAX_ITEMS_PER_FETCH, 12000),
summaryMax: 1000,
},
secrets: env,
ddiff --git a/server/registry.js b/server/registry.js
iindex 8574aae..912618c 100644
---- a/server/registry.js
++++ b/server/registry.js
@@@ -51,7 +51,7 @@ const bool = (v, d, what) => {
};
/** Validate a full source definition → normalized record (no runtime state). */
--function validateSource(input, { maxItemsCap = 500 } = {}) {
++function validateSource(input, { maxItemsCap = 12000 } = {}) {
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new RegistryError('body must be a source object');
const known = new Set(['key', 'name', 'type', 'category', 'homepage_url', 'endpoints', 'auth', 'robots_note', 'terms_note', 'license_note',
'min_interval_ms', 'poll_interval_sec', 'stale_after_sec', 'max_items', 'enabled', 'review_required', 'sensitivity',
@@@ -142,7 +142,7 @@ function validateSource(input, { maxItemsCap = 500 } = {}) {
return record;
}
--function createRegistry({ db, now = () => Date.now(), maxItemsCap = 500 }) {
++function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {
const st = {
get: db.prepare('SELECT * FROM sources WHERE key = ?'),
all: db.prepare('SELECT * FROM sources ORDER BY key'),
ddiff --git a/test/sec-company-tickers.test.js b/test/sec-company-tickers.test.js
nnew file mode 100644
iindex 0000000..46c1fae
---- /dev/null
++++ b/test/sec-company-tickers.test.js
@@@ -0,0 +1,50 @@
++'use strict';
++/**
++ * sec-company-tickers (seeds/sources.json): SEC's company_tickers_exchange.json is { fields, data: [[cik, name, ticker,
++ * exchange], …] }, so the seed's endpoint reads rows by array index. Each row becomes one `instrument` item keyed by
++ * its ticker, with the CIK, name and exchange as SEC lists them (a missing exchange is null, never a guess); the whole
++ * list (about 10,450 rows) fits under the default per-fetch cap.
++ */
++const assert = require('assert');
++const api = require('../server/adapters/api');
++const { validateSource } = require('../server/registry');
++const { load } = require('../server/config');
++const { suite } = require('./helpers');
++
++const t = suite('sec-company-tickers');
++const seed = require('../seeds/sources.json').sources.find((s) => s.key === 'sec-company-tickers');
++
++const BODY = JSON.stringify({
++ fields: ['cik', 'name', 'ticker', 'exchange'],
++ data: [
++ [1045810, 'NVIDIA CORP', 'NVDA', 'Nasdaq'],
++ [1652044, 'Alphabet Inc.', 'GOOGL', 'Nasdaq'],
++ [1652044, 'Alphabet Inc.', 'GOOG', 'Nasdaq'],
++ [1067983, 'BERKSHIRE HATHAWAY INC', 'BRK-B', 'NYSE'],
++ [1999999, 'Shell Co', 'SHLL', null],
++ ],
++});
++
++t('the seed is a disabled, daily api source of instrument items that validates', () => {
++ assert.ok(seed, 'sec-company-tickers is seeded');
++ const rec = validateSource(seed);
++ assert.strictEqual(rec.enabled, false);
++ assert.strictEqual(rec.category, 'trade');
++ assert.strictEqual(rec.poll_interval_sec, 86400);
++ assert.ok(rec.max_items >= 10450 && rec.max_items <= load({}).items.maxPerFetch, 'the whole list fits under the default cap');
++});
++
++t('each row is one instrument item keyed by ticker, with the CIK, name and exchange', () => {
++ const r = api.parse(BODY, { url: seed.endpoints[0].url, endpoint: seed.endpoints[0], maxItems: seed.max_items });
++ assert.strictEqual(r.items.length, 5);
++ const [nvda, googl, goog, brk, shell] = r.items;
++ assert.strictEqual(nvda.identity, 'NVDA');
++ assert.strictEqual(nvda.kind, 'instrument');
++ assert.strictEqual(nvda.title, 'NVIDIA CORP');
++ assert.deepStrictEqual(nvda.fields, { ticker: 'NVDA', name: 'NVIDIA CORP', cik: 1045810, exchange: 'Nasdaq' });
++ assert.strictEqual(googl.fields.cik, goog.fields.cik, 'two share classes, one company');
++ assert.strictEqual(brk.identity, 'BRK-B');
++ assert.strictEqual(shell.fields.exchange, null, 'a missing exchange stays null');
++});
++
++t.run();