openvibers/openvibe.trade Commits

Trade: companies from SEC's ticker list (sec-company-tickers), so filings land (#68)

OP OpenVibers committed 29 days ago · 17bf11f84a · 9 files, +188 −15

Changes

ddiff --git a/README.md b/README.md
iindex d61a083..89971eb 100644
---- a/README.md
++++ b/README.md
@@@ -33,7 +33,7 @@ The nine charter tables (roadmap §15.13) live in Trade's own PostgreSQL databas
| Charter table | What it is |
|---|---|
--| `trade_instruments` | Symbol (canonical, upper case), name, kind, exchange, SEC CIK, trading currency, status, and which context revision is published. |
++| `trade_instruments` | Symbol (canonical, upper case), name, kind, exchange, SEC CIK, trading currency, status, and which context revision is published. Companies come from SEC's own ticker list (Sources `sec-company-tickers`, created by `svc:trade`: one per CIK, other share classes as ticker aliases); currencies from the rates; anything else from an editor. |
| `trade_instrument_aliases` | Tickers, CIKs and names with their normalised resolution key. A ticker or CIK names exactly one instrument (unique index); a name may be shared (resolution is then ambiguous). |
| `trade_watchlists` | Private, one owner (`usr_…`). |
| `trade_watchlist_items` | Instruments on a watchlist, with an optional note. |
ddiff --git a/server/domain/indexing.js b/server/domain/indexing.js
iindex 5fa4c41..fc2834f 100644
---- a/server/domain/indexing.js
++++ b/server/domain/indexing.js
@@@ -101,7 +101,9 @@ function createIndexing({ store, config, ctx, indexnow }) {
OWNER,
decide,
refresh,
-- refreshAll: async () => await store.tx(async () => (await Promise.all((await ctx.instruments.active()).map(async (i) => await refresh(i)))).filter(Boolean)),
++ // Every minute (the freshness tick): only the instruments that can have a Search document (instruments.js
++ // indexCandidates), not the whole SEC list.
++ refreshAll: async () => await store.tx(async () => (await Promise.all((await ctx.instruments.indexCandidates()).map(async (i) => await refresh(i)))).filter(Boolean)),
};
}
ddiff --git a/server/domain/ingest.js b/server/domain/ingest.js
iindex 6dfe7ae..2949483 100644
---- a/server/domain/ingest.js
++++ b/server/domain/ingest.js
@@@ -14,11 +14,16 @@
* removed item → the document is hidden (the row stays)
* observation item → an observation (domain/observations.js; idempotent per item revision)
* filing / keyed item → a document (domain/documents.js; new → document alerts)
++ * instrument item → a listed company (registerInstrument below)
* no instrument matches → skipped (counted by reason; nothing is created for it)
*
-- * The one instrument the sync ever creates is a currency a rate states (frankfurter/ECB name the
-- * quote, e.g. USD): an ISO code needs no editorial naming, and a rate with no instrument to hang on
-- * would otherwise never appear. Every other item still requires an editor-created instrument.
++ * The sync creates two kinds of instrument, both named by their source rather than by an editor:
++ * - a currency a rate states (frankfurter/ECB name the quote, e.g. USD), named by its ISO 4217 code;
++ * - a listed company from SEC's own ticker list (sec-company-tickers): one instrument per CIK, its
++ * first ticker the symbol and any other share class (GOOG beside GOOGL) a ticker alias, so the
++ * SEC filings feed (which names the CIK) finds it. SEC's name and exchange are kept current on
++ * the instruments the sync created; an editor's instrument is never renamed, only given its CIK
++ * when it had none.
*
* A failed call is recorded in trade_sync_state (last_error) and changes nothing else; freshness
* then decays on Trade's own clock, so pages show the data as stale instead of pretending.
@@@ -94,11 +99,48 @@ function createIngest({ store, config, ctx, sources, log = console }) {
}
}
++ const SYNC = 'svc:trade';
++
++ /** A listed company from SEC's ticker list → its instrument (created, given a ticker, or kept current). */
++ async function registerInstrument(m) {
++ const want = m.instrument;
++ const taken = (err) => { if (err && err.status === 409) return 'skipped_alias_taken'; throw err; };
++ const byCik = await ctx.instruments.resolve(want.cik, { kind: 'cik' });
++ if (byCik.status === 'resolved') {
++ const have = byCik.instrument;
++ const tickers = (await ctx.instruments.aliases(have)).filter((a) => a.kind === 'ticker').map((a) => a.normalized);
++ let out = 'instruments_unchanged';
++ if (have.symbol !== want.symbol && !tickers.includes(want.symbol)) {
++ try { await ctx.instruments.addAlias(have, 'ticker', want.symbol, SYNC); out = 'instruments_ticker_added'; } catch (err) { return taken(err); }
++ }
++ if (have.created_by === SYNC && (have.name !== want.name || (have.exchange || null) !== want.exchange) && have.symbol === want.symbol) {
++ await ctx.instruments.update(have, { name: want.name, exchange: want.exchange });
++ out = 'instruments_updated';
++ }
++ return out;
++ }
++ const bySymbol = await ctx.instruments.resolve(want.symbol, { kind: 'ticker' });
++ if (bySymbol.status === 'resolved') {
++ const have = bySymbol.instrument;
++ // An editor's equity without a CIK is this company: give it the CIK so its filings arrive. Anything else
++ // holding the ticker (a currency code, another company) keeps it.
++ if (have.kind === 'equity' && !have.cik) {
++ try { await ctx.instruments.update(have, { cik: want.cik }); return 'instruments_updated'; } catch (err) { return taken(err); }
++ }
++ return 'skipped_symbol_taken';
++ }
++ try {
++ await ctx.instruments.create(want, SYNC);
++ return 'instruments_created';
++ } catch (err) { return taken(err); }
++ }
++
/** Apply one item. → a counter name. Inside the page transaction. */
async function applyItem(item) {
if (item.removed) return await ctx.documents.remove(item.id, { at: parseTime(item.removed.at), reason: item.removed.reason }) ? 'documents_removed' : 'removed_unknown';
const m = mapItem(item);
if (m.type === 'skip') return `skipped_${m.reason}`;
++ if (m.type === 'instrument') return await registerInstrument(m);
const instrument = await instrumentFor(m);
if (!instrument) return 'skipped_no_matching_instrument';
if (instrument.status !== 'active') return 'skipped_archived_instrument';
ddiff --git a/server/domain/instruments.js b/server/domain/instruments.js
iindex acd449c..37d26b7 100644
---- a/server/domain/instruments.js
++++ b/server/domain/instruments.js
@@@ -65,8 +65,21 @@ function createInstruments({ store }) {
insertAlias: db.prepare(`INSERT INTO trade_instrument_aliases (instrument_id, kind, value, normalized, added_by, created_at)
VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING`),
count: db.prepare("SELECT COUNT(*) AS n FROM trade_instruments WHERE status = 'active'"),
-- page: db.prepare("SELECT * FROM trade_instruments WHERE status = 'active' ORDER BY symbol LIMIT ? OFFSET ?"),
++ // The home's list: the instruments with the newest document or observation first, then the rest by symbol (about
++ // 10,000 companies from SEC's list, most of them quiet on any given day).
++ page: db.prepare(`SELECT i.* FROM trade_instruments i
++ LEFT JOIN (SELECT instrument_id, MAX(t) AS t FROM (
++ SELECT instrument_id, retrieved_at AS t FROM trade_source_documents WHERE removed_at IS NULL
++ UNION ALL SELECT instrument_id, observed_at AS t FROM trade_market_observations) x GROUP BY instrument_id) a ON a.instrument_id = i.id
++ WHERE i.status = 'active' ORDER BY a.t DESC NULLS LAST, i.symbol LIMIT ? OFFSET ?`),
active: db.prepare("SELECT * FROM trade_instruments WHERE status = 'active' ORDER BY symbol"),
++ // Only these can have a Search document or change one: a published context (the gate needs it) or a document
++ // Search already holds (it may need a tombstone). Every other instrument is noindex and stays out of Search.
++ indexCandidates: db.prepare(`SELECT i.* FROM trade_instruments i WHERE i.status = 'active' AND (i.context_published_revision IS NOT NULL
++ OR EXISTS (SELECT 1 FROM trade_index_revisions r WHERE r.owner = 'trade' AND r.type = 'instrument' AND r.id = i.id)) ORDER BY i.symbol`),
++ withPending: db.prepare(`SELECT i.* FROM trade_instruments i WHERE i.status = 'active' AND EXISTS (SELECT 1 FROM trade_context_revisions r
++ WHERE r.entity_id = i.id AND r.number > COALESCE(i.context_published_revision, 0)) ORDER BY i.symbol`),
++ editorAdded: db.prepare("SELECT * FROM trade_instruments WHERE status = 'active' AND created_by LIKE 'usr\\_%' ORDER BY symbol"),
};
function fields(input, { partial = false } = {}) {
@@@ -114,6 +127,9 @@ function createInstruments({ store }) {
bySymbol: async (symbol) => { const s = normSymbol(symbol); return s ? await q.bySymbol.get(s) || null : null; },
aliases: async (instrument) => await q.aliasesOf.all(instrument.id),
active: async () => await q.active.all(),
++ indexCandidates: async () => await q.indexCandidates.all(),
++ withPending: async () => await q.withPending.all(),
++ editorAdded: async () => await q.editorAdded.all(),
async page({ limit = 50, offset = 0 } = {}) {
return { total: (await q.count.get()).n, instruments: await q.page.all(limit, offset) };
},
ddiff --git a/server/domain/mapping.js b/server/domain/mapping.js
iindex 4e0245a..4a318db 100644
---- a/server/domain/mapping.js
++++ b/server/domain/mapping.js
@@@ -21,6 +21,9 @@
* else fields.form_type; the filer is the item title.
* Any other item with fields.cik or fields.symbol/ticker is a document too
* (kind 'record' for record items, else 'article').
++ * instrument a listed company from SEC's own ticker list (sec-company-tickers, kind 'instrument'): fields
++ * { ticker, name, cik, exchange } exactly as SEC lists them. All three of ticker, name and CIK
++ * are required; the exchange may be null.
* skip everything else, with a reason (counted in the sync state, never guessed at).
*/
const { parseTime, parseDecimal, iso } = require('./helpers');
@@@ -91,7 +94,7 @@ function statedObservation(item, f) {
return null;
}
--/** → { type: 'observation', keys, observation } | { type: 'document', keys, document } | { type: 'skip', reason } */
++/** → { type: 'observation' | 'document' | 'instrument', keys, observation | document | instrument } | { type: 'skip', reason } */
function mapItem(item) {
if (!item || typeof item !== 'object' || !item.id) return { type: 'skip', reason: 'malformed_item' };
const f = item.fields && typeof item.fields === 'object' ? item.fields : {};
@@@ -103,6 +106,18 @@ function mapItem(item) {
const retrievedAt = parseTime(prov.retrieved_at);
if (retrievedAt == null) return { type: 'skip', reason: 'no_retrieval_time' };
++ if (item.kind === 'instrument') {
++ const symbol = tickerish(str1(f.ticker) || str1(item.identity));
++ const cik = f.cik != null && /^\d{1,10}$/.test(String(f.cik).trim()) ? String(f.cik).trim().padStart(10, '0') : null;
++ const name = str1(f.name) || str1(item.title);
++ if (!symbol || !cik || !name) return { type: 'skip', reason: 'incomplete_instrument' };
++ return {
++ type: 'instrument', keys: { cik, symbol },
++ // SEC lists companies on US exchanges (and OTC): they trade in US dollars.
++ instrument: { symbol, name: name.slice(0, 200), cik, exchange: str1(f.exchange) ? str1(f.exchange).slice(0, 40) : null, kind: 'equity', currency: 'USD' },
++ };
++ }
++
const obs = statedObservation(item, f);
if (obs) {
const symbol = obs.symbol || keys.symbol;
ddiff --git a/server/http/discovery.js b/server/http/discovery.js
iindex 937652d..89aed98 100644
---- a/server/http/discovery.js
++++ b/server/http/discovery.js
@@@ -91,7 +91,7 @@ function createDiscovery(ctx) {
// page — the same gate the sitemap uses. The summary is the reviewed context, clipped; the
// source's own full text is never reproduced (llms.txt states the rule).
define(router, 'get', '/llms-full.txt', 'llmsFullTxt', async (_req, res) => {
-- const active = await instruments.active();
++ const active = await instruments.indexCandidates();
const decisions = await Promise.all(active.map(async (i) => await indexing.decide(i)));
const pages = await Promise.all(active.filter((_i, n) => decisions[n].indexable).map(async (i) => {
const rev = await context.published(i);
@@@ -113,7 +113,7 @@ function createDiscovery(ctx) {
const pageLastmod = (instrument, changes) => Math.max(instrument.updated_at || 0, instrument.context_published_at || 0, changes.get(instrument.id) || 0);
define(router, 'get', '/sitemap.xml', 'sitemapIndex', async (_req, res) => {
-- const active = await instruments.active();
++ const active = await instruments.indexCandidates();
const decisions = await Promise.all(active.map(async (i) => await indexing.decide(i)));
const changes = await pageChanges();
const times = active.filter((i, n) => decisions[n].indexable).map((i) => pageLastmod(i, changes)).filter(Boolean);
@@@ -123,7 +123,8 @@ function createDiscovery(ctx) {
define(router, 'get', '/sitemaps/instruments.xml', 'instrumentSitemap', async (_req, res) => {
const changes = await pageChanges();
-- const entries = (await Promise.all((await instruments.active()).map(async (i) => ({
++ // Without a published context an instrument page is noindex (indexing.js), so only the candidates are weighed.
++ const entries = (await Promise.all((await instruments.indexCandidates()).map(async (i) => ({
loc: urls.instrument(i),
...(pageLastmod(i, changes) ? { lastmod: new Date(pageLastmod(i, changes)).toISOString() } : {}),
decision: await indexing.decide(i),
ddiff --git a/server/http/editor.js b/server/http/editor.js
iindex 2154ffe..a7716b5 100644
---- a/server/http/editor.js
++++ b/server/http/editor.js
@@@ -50,10 +50,10 @@ function createEditor(ctx) {
}
define(router, 'get', '/editor', 'showEditor', requireEditor, async (req, res) => {
-- const all = await instruments.active();
const pending = [];
-- for (const i of all) for (const p of await context.pending(i)) pending.push({ symbol: i.symbol, revision: p.revision, label: p.needs_review ? 'AI draft, needs review' : 'not published' });
-- common.page(req, res, { title: 'Editor', decision, personal: true, body: views.editorHome({ instruments: all, pending, csrf: common.csrf(req), notice: common.noticeOf(req), kinds: instruments.KINDS }) });
++ for (const i of await instruments.withPending()) for (const p of await context.pending(i)) pending.push({ symbol: i.symbol, revision: p.revision, label: p.needs_review ? 'AI draft, needs review' : 'not published' });
++ const added = await instruments.editorAdded();
++ common.page(req, res, { title: 'Editor', decision, personal: true, body: views.editorHome({ instruments: added, pending, csrf: common.csrf(req), notice: common.noticeOf(req), kinds: instruments.KINDS }) });
});
define(router, 'post', '/editor/instruments', 'createInstrumentForm', ...form, B('trade.instrument.manage'), async (req, res) => {
ddiff --git a/server/render/views.js b/server/render/views.js
iindex 07c064c..fda8d1a 100644
---- a/server/render/views.js
++++ b/server/render/views.js
@@@ -295,8 +295,9 @@ function editorHome({ instruments, pending, csrf, notice: n, kinds }) {
<button type="submit">Add instrument</button></form>
<h2>Context waiting for review</h2>
${pending.length ? html`<ul>${pending.map((p) => html`<li><a href="/editor/i/${encodeURIComponent(p.symbol)}">${p.symbol}</a>: revision ${p.revision} (${p.label})</li>`)}</ul>` : html`<p class="empty">Nothing is waiting.</p>`}
--<h2>Instruments</h2>
--<ul>${instruments.map((i) => html`<li><a href="/editor/i/${encodeURIComponent(i.symbol)}">${i.symbol}</a> — ${i.name}</li>`)}</ul>`;
++<h2>Instruments editors added</h2>
++${instruments.length ? html`<ul>${instruments.map((i) => html`<li><a href="/editor/i/${encodeURIComponent(i.symbol)}">${i.symbol}</a> — ${i.name}</li>`)}</ul>` : html`<p class="empty">None yet.</p>`}
++<p class="muted">Every other instrument (the companies on SEC's list, the currencies the rates name) opens at <code>/editor/i/SYMBOL</code>.</p>`;
}
async function editorInstrument({ instrument, aliases, revisions, documents, observations, csrf, notice: n, kinds, head }) {
ddiff --git a/test/sec-companies.test.js b/test/sec-companies.test.js
nnew file mode 100644
iindex 0000000..fc7098c
---- /dev/null
++++ b/test/sec-companies.test.js
@@@ -0,0 +1,96 @@
++'use strict';
++/**
++ * SEC's ticker list (Sources: sec-company-tickers, kind 'instrument', fields { ticker, name, cik, exchange }) becomes
++ * Trade's companies, so the SEC filings feed has something to land on:
++ * - one instrument per CIK, its first ticker the symbol, another share class a ticker alias;
++ * - SEC's later name or exchange updates an instrument the sync created, never an editor's;
++ * - an editor's equity with no CIK gets SEC's CIK; a currency code holding the ticker keeps it;
++ * - a filing for the company then becomes its document; a replay changes nothing;
++ * - the freshness tick, the sitemap and the editor's list skip the thousands of companies with no context.
++ */
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++
++(async () => {
++ const t = await boot();
++ const ret = t.iso(t.T0 - 5 * 60e3);
++ const db = t.ctx.store.db;
++ t.sources.setSource('sec-company-tickers', { name: 'SEC EDGAR: company tickers and exchanges', lastSuccessAt: ret, staleAfterSec: 259200 });
++ t.sources.setSource('sec-xbrl-filings', { name: 'SEC EDGAR: latest XBRL filings', lastSuccessAt: ret, staleAfterSec: 86400 });
++ const company = (ticker, name, cik, exchange) => t.sources.putItem({
++ source_key: 'sec-company-tickers', kind: 'instrument', identity: ticker, canonical_url: null, title: name, summary: null,
++ published_at: null, retrieved_at: ret, fields: { ticker, name, cik, exchange },
++ });
++ const editorMsft = await t.instrument({ symbol: 'MSFT', name: 'Microsoft (by an editor)', kind: 'equity' });
++
++ company('GOOGL', 'Alphabet Inc.', 1652044, 'Nasdaq');
++ company('GOOG', 'Alphabet Inc.', 1652044, 'Nasdaq');
++ company('BRK-B', 'BERKSHIRE HATHAWAY INC', 1067983, 'NYSE');
++ company('MSFT', 'MICROSOFT CORP', 789019, 'Nasdaq');
++ company('SHLL', 'Shell Co', 1999999, null);
++ company('USD', 'Some Fund Trust', 1888888, 'NYSE'); // the ticker is a currency code Trade already tracks
++ company('NOPE', 'No Cik Corp', null, 'OTC');
++ // A currency instrument named by a rate, as the ECB/Frankfurter sync creates it.
++ await t.ctx.instruments.create({ symbol: 'USD', name: 'US Dollar', kind: 'currency' }, null);
++
++ await check('a run registers one instrument per company, share classes as ticker aliases', async () => {
++ const r = await t.ctx.sync.run();
++ assert.strictEqual(r.ok, true, JSON.stringify(r));
++ assert.strictEqual(r.counts.instruments_created, 3, JSON.stringify(r.counts)); // GOOGL, BRK-B, SHLL
++ assert.strictEqual(r.counts.instruments_ticker_added, 1, 'GOOG on GOOGL');
++ assert.strictEqual(r.counts.instruments_updated, 1, 'the editor\'s MSFT gets its CIK');
++ assert.strictEqual(r.counts.skipped_symbol_taken, 1, 'USD stays the currency');
++ assert.strictEqual(r.counts.skipped_incomplete_instrument, 1, 'no CIK, no instrument');
++ const googl = await t.ctx.instruments.bySymbol('GOOGL');
++ assert.deepStrictEqual({ name: googl.name, kind: googl.kind, exchange: googl.exchange, cik: googl.cik, currency: googl.currency, by: googl.created_by },
++ { name: 'Alphabet Inc.', kind: 'equity', exchange: 'Nasdaq', cik: '0001652044', currency: 'USD', by: 'svc:trade' });
++ const goog = await t.ctx.instruments.resolve('GOOG');
++ assert.strictEqual(goog.status, 'resolved');
++ assert.strictEqual(goog.instrument.id, googl.id, 'GOOG resolves to Alphabet');
++ assert.strictEqual((await t.ctx.instruments.bySymbol('SHLL')).exchange, null);
++ const msft = await t.ctx.instruments.get(editorMsft.id);
++ assert.strictEqual(msft.cik, '0000789019');
++ assert.strictEqual(msft.name, 'Microsoft (by an editor)', 'an editor\'s name is never replaced');
++ assert.strictEqual((await t.ctx.instruments.bySymbol('USD')).kind, 'currency');
++ });
++
++ await check('SEC renaming a company updates the instrument the sync created; a replay changes nothing', async () => {
++ company('BRK-B', 'Berkshire Hathaway Inc.', 1067983, 'NYSE');
++ const r = await t.ctx.sync.run();
++ assert.strictEqual(r.counts.instruments_updated, 1, JSON.stringify(r.counts));
++ assert.strictEqual((await t.ctx.instruments.bySymbol('BRK-B')).name, 'Berkshire Hathaway Inc.');
++ const n = (await db.prepare('SELECT COUNT(*) AS n FROM trade_instruments').get()).n;
++ await db.prepare("UPDATE trade_ingest_cursor SET cursor = 0 WHERE name = 'sources.trade'").run();
++ const again = await t.ctx.sync.run();
++ assert.strictEqual(again.counts.instruments_created, undefined, JSON.stringify(again.counts));
++ assert.strictEqual((await db.prepare('SELECT COUNT(*) AS n FROM trade_instruments').get()).n, n);
++ });
++
++ await check('a filing for a registered company becomes its document', async () => {
++ t.sources.putItem({
++ source_key: 'sec-xbrl-filings', kind: 'article',
++ canonical_url: 'https://www.sec.gov/Archives/edgar/data/1652044/000165204426000090/0001652044-26-000090-index.htm',
++ identity: 'https://www.sec.gov/Archives/edgar/data/1652044/000165204426000090/0001652044-26-000090-xbrl.zip',
++ title: 'Alphabet Inc. (0001652044) (Filer)', summary: '10-Q', published_at: '2026-10-14T20:05:00.000Z', retrieved_at: ret,
++ fields: { categories: [], enclosure: null, comments_url: null },
++ });
++ const r = await t.ctx.sync.run();
++ assert.strictEqual(r.counts.documents_created, 1, JSON.stringify(r.counts));
++ const googl = await t.ctx.instruments.bySymbol('GOOGL');
++ const doc = await db.prepare('SELECT * FROM trade_source_documents WHERE instrument_id = ?').get(googl.id);
++ assert.strictEqual(doc.form_type, '10-Q');
++ const home = await t.get('/');
++ assert.ok(home.text.indexOf('GOOGL') < home.text.indexOf('SHLL'), 'the company with a new filing is listed first');
++ });
++
++ await check('the freshness tick, the sitemap and the editor list leave out companies with no context', async () => {
++ assert.deepStrictEqual(await t.ctx.instruments.indexCandidates(), [], 'nothing has a published context');
++ const sm = await t.get('/sitemaps/instruments.xml');
++ assert.strictEqual(sm.status, 200);
++ assert.ok(!sm.text.includes('/i/GOOGL'), 'a company with no context is not in the sitemap');
++ assert.deepStrictEqual((await t.ctx.instruments.editorAdded()).map((i) => i.symbol), ['MSFT']);
++ });
++
++ await t.close();
++ done();
++})();