openvibers/openvibe.trade Commits
Trade on the Publishing 1.1.0 chassis (plan T9 J6): the Sources client, change cursor and pull loop, signed event consumer, normalisers, freshness verdict and Search publication glue come from openvibe-publishing/ingest and /publication; Trade keeps its domain half (domain/ingest.js, source-status.js, helpers.js, events/consumer.js). Migration 0002 seeds trade_ingest_cursor once from trade_sync_state (continuity, never a reset to zero). The replaced domain/sync.js, util.js, freshness.js, events/webhook.js and clients/sources.js are deleted. test/chassis.test.js proves nothing requires them and replays the Sources fixture against a golden captured from the pre-change code. npm test and test:pg 18/18 (verified).
Changes
@@@ -49,7 +49,8 @@ Other tables in the same database:`trade_context_reviews` (a person's review of an AI draft) and `trade_index_revisions` (theSearch document sequencer).- `trade_source_status`: freshness of each source (Sources' health plus Trade's own clock).--- `trade_sync_state`: the Sources cursor, the last error and counters of what was skipped and why.++- `trade_ingest_cursor`: the change cursor of the Sources feed (`openvibe-publishing/ingest`).++- `trade_sync_state`: the last sync error and counters of what was skipped and why.- `event_outbox` (SDK outbox) and `idempotency_receipts` (SDK inbox for signed webhook deliveries).## Does not own@@@ -222,7 +223,7 @@ services need to call Trade, is under [Grants the Network must hold](#grants-the- **PostgreSQL 18 and Valkey 9** (OpenVibe.Host `roles/data/`, ADR-035): every read and write is async through`openvibe-sdk/db`; Valkey holds the per-actor limit counters (optional: without `VALKEY_URL` they count per process).- **Packages** (pinned by release tarball): `openvibe-contracts` v0.76.0, `openvibe-publishing`-- v1.0.0 (revisions, authorship, seo, index-hooks, ssr), `openvibe-shared` v2.2.0 (Frame, app icon,++ v1.1.0 (revisions, authorship, seo, index-hooks, ssr, ingest, publication), `openvibe-shared` v2.2.0 (Frame, app icon,footer, legal, release, metrics, ready), `openvibe-sdk` v0.20.0 (events outbox and inbox, webhooksignatures v2, service tokens, per-actor limits).- **OpenVibe.Network:** SSO (OAuth client `trade`, redirect `https://openvibe.trade/auth/callback`),@@@ -11,7 +11,7 @@"contracts": "openvibe-contracts v0.79.0","packages": ["openvibe-contracts v0.79.0",-- "openvibe-publishing v1.0.0",++ "openvibe-publishing v1.1.0","openvibe-shared v2.2.0","openvibe-sdk v0.25.0"],@@@ -36,7 +36,7 @@"deployed": true,"deployment": "Deployed with `sudo ovhost deploy trade`: unit openvibe-trade, 127.0.0.1:4860 loopback only; on 2026-09-23 /api/ready answered 200 with status degraded [freshness], which is honest: no feed is configured; Network OAuth client trade and its grants exist; no Events subscription; production database held 0 instruments and 0 observations. Deployed internally, not launched: openvibe.trade still serves the OpenVibe.Sites placeholder until a person verifies the SEC XBRL terms and enables sec-xbrl-filings in Sources, a licensed market data feed exists, and editors add instruments","placeholderInSites": true,-- "updated": "2026-09-28",++ "updated": "2026-09-29","features": ["informational only: a route-inventory test refuses any execution, custody or advice endpoint","instruments with exact, deterministic resolution (CIK, ticker, name)",nnew file mode 100644@@@ -0,0 +1,15 @@++-- phase: expand++-- openvibe-publishing/ingest (prefix trade): the change cursor of the Sources feed, replacing the++-- cursor Trade kept in trade_sync_state before the chassis (T9). Additive.++CREATE TABLE IF NOT EXISTS trade_ingest_cursor (++ name text COLLATE "C" NOT NULL,++ cursor bigint NOT NULL,++ updated_at bigint NOT NULL,++ PRIMARY KEY (name)++);++++-- One-time continuity: seed the chassis cursor from the row Trade already had, so the feed resumes++-- where it was instead of restarting at zero. A fresh database has no such row and starts at 0.++INSERT INTO trade_ingest_cursor (name, cursor, updated_at)++SELECT 'sources.trade', cursor, 0 FROM trade_sync_state WHERE name = 'sources.trade'++ON CONFLICT (name) DO NOTHING;@@@ -16,7 +16,7 @@"helmet": "^8.1.0","iovalkey": "^0.4.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",-- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.1.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.25.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.2.0","pg": "^8.23.0"@@@ -1163,9 +1163,9 @@}},"node_modules/openvibe-publishing": {-- "version": "1.0.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",-- "integrity": "sha512-g4mU0PPtI4y38NcsdppEbrXTrCYp0rPmIYWyxLxh4GpEYhn48jMWq2+cMQA2uwZCeeZlB3UiXVrQNQyQLiliwg==",++ "version": "1.1.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.1.0",++ "integrity": "sha512-bJsVl82hDivazoiM/np9ckKJE4V5F4OxyIiDwB6xuhK8M6P42sGU+V88rFCkyiI4BVQiKv9xs4GVx8Fj3pyjsg==","license": "MIT","engines": {"node": ">=20"@@@ -28,7 +28,7 @@"helmet": "^8.1.0","iovalkey": "^0.4.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",-- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.1.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.25.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.2.0","pg": "^8.23.0"@@@ -9,7 +9,7 @@* Editor (http/editor.js) /editor/…* Discovery (http/discovery.js) robots, llms, sitemaps, feeds* API (http/api.js) /api/v1/…-- * Webhook (events/webhook.js) POST /internal/events (signed OpenVibe.Events deliveries)++ * Webhook (events/consumer.js) POST /internal/events (signed OpenVibe.Events deliveries)* Machine /api/health, /api/ready, /release.json, /metrics** Information only (ADR-025): no route takes an order, holds value, runs escrow, lists goods@@@ -26,14 +26,15 @@ const { createSsoClient } = require('openvibe-sdk/sso');const { jwksClient } = require('openvibe-sdk/auth');const { createServiceOutbox } = require('openvibe-sdk/events');const { createIndexNow } = require('openvibe-shared/indexnow');++const { createSourcesClient } = require('openvibe-publishing/ingest');const configLib = require('./config');const { openStore } = require('./db');const { createViewerResolver } = require('./auth/viewer');--const { createWebhook } = require('./events/webhook');++const { createWebhook } = require('./events/consumer');const { createUrls } = require('./domain/urls');const { createInstruments } = require('./domain/instruments');--const { createFreshness } = require('./domain/freshness');++const { createFreshness } = require('./domain/source-status');const { createObservations } = require('./domain/observations');const { createDocuments } = require('./domain/documents');const { createAlerts } = require('./domain/alerts');@@@ -41,8 +42,7 @@ const { createWatchlists } = require('./domain/watchlists');const { createContext } = require('./domain/context');const { createIndexing } = require('./domain/indexing');const { createReading } = require('./domain/reading');--const { createSync } = require('./domain/sync');--const { createSourcesClient } = require('./clients/sources');++const { createIngest } = require('./domain/ingest');const { createCommon } = require('./http/common');const { createPages } = require('./http/pages');const { createPrivate } = require('./http/private');@@@ -92,8 +92,8 @@ async function createApp(opts = {}) {ctx.context = createContext({ store, config, ctx });ctx.indexing = createIndexing({ store, config, ctx, indexnow });ctx.reading = createReading({ store, ctx });-- ctx.sources = opts.sourcesClient || createSourcesClient({ config, fetchImpl });-- ctx.sync = createSync({ store, config, ctx, sources: ctx.sources, log });++ ctx.sources = opts.sourcesClient || createSourcesClient({ config, fetchImpl, timeoutMs: 15_000 });++ ctx.sync = createIngest({ store, config, ctx, sources: ctx.sources, log });// Sign-in with OpenVibe.Network (openvibe-sdk/sso): routes, offline session verification through the// shared JWKS client, state + PKCE. ctx.jwks is the same client viewer.js verifies service tokens with.ctx.jwksUrl = `${String(config.networkInternalUrl || config.networkUrl).replace(/\/+$/, '')}/api/.well-known/jwks`;@@@ -163,7 +163,7 @@ async function createApp(opts = {}) {app.use(machine);// ── Signed OpenVibe.Events deliveries (before any body parser) ──-- app.use(createWebhook({ store, config, sync: ctx.sync, log }).router);++ app.use(createWebhook({ store, config, ingest: ctx.sync, log }).router);// ── Sign-in (OAuth2 client of OpenVibe.Network) ─────────app.use('/auth/', rateLimit({ windowMs: 15 * 60_000, max: 60, standardHeaders: true, legacyHeaders: false }));ddeleted file mode 100644@@@ -1,42 +0,0 @@--'use strict';----/**-- * OpenVibe.Sources, read with Trade's service token (audience openvibe.sources):-- * sources.item.read GET /api/v1/items?category=trade&after=<change_seq>&include_removed=1-- * sources.source.read GET /api/v1/sources (health and staleness of every source)-- * Off until OV_OAUTH_CLIENT_SECRET is set. A failure is an error for the caller to record, never an-- * empty answer.-- */--const { serviceAuth } = require('openvibe-contracts');----function createSourcesClient({ config, fetchImpl = globalThis.fetch }) {-- const enabled = Boolean(config.oauth.clientSecret && config.sources.internalUrl);-- const tokens = enabled ? serviceAuth.createTokenClient({-- tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,-- audience: 'openvibe.sources', scope: 'sources.item.read sources.source.read', fetchImpl,-- }) : null;---- async function get(path) {-- if (!enabled) throw new Error('Sources client disabled (OV_OAUTH_CLIENT_SECRET unset)');-- const res = await fetchImpl(`${config.sources.internalUrl}${path}`, {-- headers: { Accept: 'application/json', ...(await tokens.authHeaders()) },-- signal: AbortSignal.timeout(15_000),-- });-- if (res.status === 401 && tokens.invalidate) tokens.invalidate();-- const data = await res.json().catch(() => null);-- if (!res.ok || !data) {-- const err = new Error(`Sources answered ${res.status}${data && data.code ? ` (${data.code})` : ''} for ${path.split('?')[0]}`);-- err.status = res.status;-- throw err;-- }-- return data;-- }---- return {-- enabled,-- items: async ({ after = 0, limit = 200 } = {}) => await get(`/api/v1/items?category=${encodeURIComponent(config.sources.category)}&after=${after}&limit=${limit}&include_removed=1`),-- sources: async () => await get('/api/v1/sources'),-- };--}----module.exports = { createSourcesClient };@@@ -23,7 +23,8 @@** Also here: trade_context_drafts / trade_context_revision_purges / trade_context_reviews (package* companions), trade_index_revisions (Search document sequencer), trade_source_status (freshness-- * per source), trade_sync_state (the Sources cursor), event_outbox (SDK outbox) and++ * per source), trade_ingest_cursor (the Sources change cursor, openvibe-publishing/ingest),++ * trade_sync_state (the last sync error and counters), event_outbox (SDK outbox) and* idempotency_receipts (SDK inbox for signed webhook deliveries).*/const fs = require('fs');@@@ -21,7 +21,7 @@* timestamps and source), never rendered notification text: Network renders the notification.*/const { ApiError } = require('../http/errors');--const { newId, iso, parseDecimal, invalid, str, json } = require('./util');++const { newId, iso, parseDecimal, invalid, str, json } = require('./helpers');const { formType } = require('./mapping');const KINDS = ['threshold', 'filing_type', 'new_document'];@@@ -20,7 +20,7 @@const authorship = require('openvibe-publishing/authorship');const ssr = require('openvibe-publishing/ssr');const { ApiError } = require('../http/errors');--const { iso, invalid } = require('./util');++const { iso, invalid } = require('./helpers');const WORKFLOW = 'trade.summarize_market_context';const ADVICE_RE = /\b(you should (buy|sell|hold|invest|short)|we recommend|i recommend|recommend(s|ed)? (buying|selling|holding)|strong (buy|sell)|(buy|sell|hold|outperform|underperform) rating|price target|guaranteed (return|profit)s?)\b/i;@@@ -9,7 +9,7 @@* A newly seen document is what new_document and filing_type alert rules trigger on (once per rule* and document: alerts.onDocument). Item revisions update the row and never trigger again.*/--const { newId, iso } = require('./util');++const { newId, iso } = require('./helpers');function createDocuments({ store, ctx }) {const { db } = store;ddeleted file mode 100644@@@ -1,119 +0,0 @@--'use strict';----/**-- * Freshness per source: is what Trade shows from a source still current?-- *-- * A source is FRESH at time `now` only when its last successful fetch is known and younger than its-- * staleness window (Sources' stale_after_sec, else TRADE_DEFAULT_STALE_AFTER_SEC). Everything-- * else is stale:-- * - the last success is older than the window → stale since last success + window-- * - Sources never reported a success → stale since Trade first saw the source-- * - Trade has never heard of the source at all → freshness unknown (shown as stale)-- * Sources' own health status (failing, disabled, …) is shown next to it, but the verdict is-- * computed with Trade's clock, so an outage of Sources or of the sync makes data go stale on its own-- * rather than look current forever. Nothing is ever replaced by an invented value: a stale value is-- * shown with its timestamps and a "stale since" label, and a missing one is shown as missing.-- *-- * `trade.source.stale` and `trade.source.recovered` are emitted only on a transition (evaluate()),-- * so a replay or a second tick changes nothing.-- */--const { iso } = require('./util');----function createFreshness({ store, config, outbox }) {-- const { db } = store;-- const def = config.freshness.defaultStaleAfterSec;-- const q = {-- get: db.prepare('SELECT * FROM trade_source_status WHERE source_key = ?'),-- all: db.prepare('SELECT * FROM trade_source_status ORDER BY source_key'),-- ensure: db.prepare(`INSERT INTO trade_source_status (source_key, stale, stale_since, evaluated_at) VALUES (?, 1, ?, NULL) ON CONFLICT DO NOTHING`),-- report: db.prepare(`UPDATE trade_source_status SET name = COALESCE(@name, name), upstream_status = @status,-- last_success_at = CASE WHEN @last::bigint IS NULL THEN last_success_at WHEN last_success_at IS NULL OR @last > last_success_at THEN @last ELSE last_success_at END,-- stale_after_sec = COALESCE(@window, stale_after_sec), reported_at = @now,-- terms_note = COALESCE(@terms, terms_note), license_note = COALESCE(@license, license_note)-- WHERE source_key = @key`),-- retrieval: db.prepare(`UPDATE trade_source_status SET last_success_at = @at WHERE source_key = @key AND (last_success_at IS NULL OR last_success_at < @at)`),-- mark: db.prepare('UPDATE trade_source_status SET stale = @stale, stale_since = @since, evaluated_at = @now WHERE source_key = @key'),-- };---- /** Pure: the verdict for one status row at `now`. row may be null (unknown source). */-- function verdict(row, now) {-- if (!row) return { known: false, stale: true, staleSince: null, window: def, lastSuccessAt: null };-- const window = row.stale_after_sec || def;-- if (row.last_success_at == null) return { known: true, stale: true, staleSince: row.stale_since, window, lastSuccessAt: null };-- const until = row.last_success_at + window * 1000;-- return { known: true, stale: now > until, staleSince: now > until ? until : null, window, lastSuccessAt: row.last_success_at };-- }---- async function view(key, now = store.now()) {-- const row = await q.get.get(key) || null;-- const v = verdict(row, now);-- return {-- key,-- name: row ? row.name : null,-- known: v.known,-- status: !v.known ? 'unknown' : v.stale ? 'stale' : 'fresh',-- stale: v.stale,-- stale_since: iso(v.staleSince),-- last_success_at: iso(v.lastSuccessAt),-- stale_after_sec: v.window,-- upstream_status: row ? row.upstream_status : null,-- reported_at: row ? iso(row.reported_at) : null,-- terms_note: row ? row.terms_note : null,-- license_note: row ? row.license_note : null,-- };-- }---- async function ensure(key) { await q.ensure.run(key, store.now()); }---- /** Emit on a transition only. Inside the caller's transaction (or its own). */-- async function evaluate(key) {-- return await store.tx(async () => {-- const row = await q.get.get(key);-- if (!row) return null;-- const v = verdict(row, store.now());-- const first = row.evaluated_at == null;-- const was = Boolean(row.stale);-- const since = v.stale ? (v.staleSince != null ? v.staleSince : (row.stale_since || store.now())) : null;-- await q.mark.run({ key, stale: v.stale ? 1 : 0, since, now: store.now() });-- // First sight: a fresh source is not a recovery; a stale one is reported once.-- if (first ? !v.stale : v.stale === was) return null;-- const envelope = await outbox.emit({-- event_type: v.stale ? 'trade.source.stale' : 'trade.source.recovered',-- actor: { type: 'service', id: 'trade' }, visibility: 'public', priority: 'low',-- subject: { type: 'source', id: key },-- payload: {-- source_key: key, stale: v.stale, stale_since: iso(since), last_success_at: iso(v.lastSuccessAt),-- stale_after_sec: v.window, upstream_status: row.upstream_status || null, evaluated_at: iso(store.now()),-- },-- });-- return envelope;-- });-- }---- return {-- verdict,-- view,-- ensure,-- evaluate,-- async evaluateAll() { return (await Promise.all((await q.all.all()).map(async (r) => await evaluate(r.source_key)))).filter(Boolean); },-- async all(now = store.now()) { return (await Promise.all((await q.all.all()).map(async (r) => await view(r.source_key, now)))); },---- /** What Sources says about a source (its health block). Evaluates the transition. */-- async report(key, { name = null, status = null, lastSuccessAt = null, staleAfterSec = null, termsNote = null, licenseNote = null } = {}) {-- return await store.tx(async () => {-- await ensure(key);-- await q.report.run({ key, name, status, last: lastSuccessAt, window: staleAfterSec, now: store.now(), terms: termsNote, license: licenseNote });-- return await evaluate(key);-- });-- },---- /** A datum retrieved at `at` proves a successful fetch at that time. */-- async noteRetrieval(key, at) {-- await ensure(key);-- await q.retrieval.run({ key, at });-- return await evaluate(key);-- },-- };--}----module.exports = { createFreshness };nnew file mode 100644@@@ -0,0 +1,16 @@++'use strict';++++/**++ * Trade's small shared helpers. The generic normalisers (folds, keys, amounts, instants, text and++ * decimal parsing, ids) come from the ingest chassis (openvibe-publishing/ingest); only the two++ * things the chassis does not own live here — id generation and the request-invalid error.++ * Nothing here invents a value: a date the caller did not state stays null, never "now".++ */++const { normalize } = require('openvibe-publishing/ingest');++const { ids } = require('openvibe-contracts');++const { ApiError } = require('../http/errors');++++const newId = (prefix, now) => `${prefix}_${ids.ulid(now)}`;++const invalid = (detail, code = 'request.invalid') => new ApiError(422, code, detail);++++module.exports = { ...normalize, newId, invalid };@@@ -2,8 +2,13 @@/*** Indexability and OpenVibe.Search for instrument pages (openvibe-publishing/seo gate and-- * index-hooks). The only public, indexable objects Trade has are instrument pages; watchlists,-- * alert rules and deliveries are private and never reach this module.++ * index-hooks), on the shared publication glue (openvibe-publishing/publication).++ *++ * The gate facts, the document body and the canonical path are Trade's; the mechanics — the++ * sequencer revision, the two events, the outbox write and the IndexNow ping — are createPublication,++ * which stamps and enqueues on the caller's transaction handle so the document, its event and++ * Trade's own change commit or roll back together. The only public, indexable objects Trade has are++ * instrument pages; watchlists, alert rules and deliveries are private and never reach this module.** Gate facts for an instrument page:* state active → published, archived → unpublished@@@ -22,6 +27,7 @@ const seo = require('openvibe-publishing/seo');const hooks = require('openvibe-publishing/index-hooks');const authorship = require('openvibe-publishing/authorship');const ssr = require('openvibe-publishing/ssr');++const { createPublication } = require('openvibe-publishing/publication');const OWNER = 'trade';@@@ -78,22 +84,17 @@ function createIndexing({ store, config, ctx, indexnow }) {});}-- /** Stamp and enqueue the Search document when it changed. Inside the caller's transaction. */++ const publication = createPublication({++ owner: OWNER, sequencer: store.sequencer, outbox: ctx.outbox, baseUrl: config.baseUrl, indexnow, now: store.now,++ decide: (instrument, now) => decide(instrument, now),++ document: (instrument, decision) => document(instrument, decision),++ page: (instrument) => ctx.urls.instrument(instrument),++ });++++ /** Stamp and enqueue the Search document when it changed, on the caller's handle (ambient tx). */async function refresh(instrument, { traceparent } = {}) {if (!instrument) return null;-- const decision = await decide(instrument);-- const doc = await document(instrument, decision);-- const prev = await store.sequencer.current(OWNER, 'instrument', instrument.id);-- if (doc.deleted && prev == null) return null;-- const stamped = await store.sequencer.stamp(store.db, doc);-- if (prev != null && stamped.revision === prev) return null;-- // IndexNow: tell the engines a public, indexable instrument page appeared or changed — or-- // that a page which was in the index left it (retracted, archived, stale price). Never for a-- // page that was never indexable (a draft, private or noindex from the start).-- if (indexnow && indexnow.enabled && (decision.indexable || prev != null)) {-- indexnow.pingSoon([ctx.urls.instrument(instrument), ctx.urls.abs('/sitemap.xml')]);-- }-- return await ctx.outbox.emit(hooks.indexEvent({ document: stamped, now: store.now() }), { traceparent });++ return await publication.sync(db, instrument, { traceparent });}return {nnew file mode 100644@@@ -0,0 +1,138 @@++'use strict';++++/**++ * Pull the trade category from OpenVibe.Sources and record what it states.++ *++ * The Sources client, the change cursor and the per-page pull loop are the ingest chassis++ * (openvibe-publishing/ingest): one transaction per page (the page's writes and the cursor advance++ * commit together), one savepoint per item, so a crash replays the page and a bad item is isolated.++ * This module keeps only Trade's own half — item mapping, instrument resolution, observations,++ * documents, freshness reporting and the trade_sync_state bookkeeping.++ *++ * 1. GET /api/v1/sources every trade source's health → freshness (report)++ * 2. GET /api/v1/items?category=trade&after=<cursor>&include_removed=1, in change order:++ * removed item → the document is hidden (the row stays)++ * observation item → an observation (domain/observations.js; idempotent per item revision)++ * filing / keyed item → a document (domain/documents.js; new → document alerts)++ * no instrument matches → skipped (counted by reason; nothing is created for it)++ *++ * A failed call is recorded in trade_sync_state (last_error) and changes nothing else; freshness++ * then decays on Trade's own clock, so pages show the data as stale instead of pretending.++ */++const { createChangeCursor, pullChanges, normalize } = require('openvibe-publishing/ingest');++const { mapItem } = require('./mapping');++const { parseTime, iso, json } = normalize;++++const PREFIX = 'trade';++const CURSOR = 'sources.trade';++++/** The chassis outcome for a domain counter key: a removal is a tombstone, everything else applied. */++const outcomeOf = (key) => (key === 'documents_removed' || key === 'removed_unknown' ? 'removed' : 'applied');++++function createIngest({ store, config, ctx, sources, log = console }) {++ const { db } = store;++ const cursor = createChangeCursor(db, { prefix: PREFIX, now: store.now });++ const q = {++ get: db.prepare('SELECT * FROM trade_sync_state WHERE name = ?'),++ ensure: db.prepare("INSERT INTO trade_sync_state (name, cursor, counts) VALUES (?, 0, '{}') ON CONFLICT DO NOTHING"),++ save: db.prepare('UPDATE trade_sync_state SET counts = @counts, last_ok_at = @ok, last_error = NULL WHERE name = @name'),++ fail: db.prepare('UPDATE trade_sync_state SET last_error = ?, last_error_at = ? WHERE name = ?'),++ };++ let ensured = false; // the state row, written on the first run (not at construction: the factory stays synchronous)++ const ensureState = async () => { if (!ensured) { await q.ensure.run(CURSOR); ensured = true; } };++ let running = null;++++ async function reportHealth(key, h, extra = {}) {++ if (!h || typeof h !== 'object') return;++ await ctx.freshness.report(key, {++ name: extra.name || null,++ status: h.status || null,++ lastSuccessAt: parseTime(h.last_success_at),++ staleAfterSec: Number.isInteger(h.stale_after_sec) ? h.stale_after_sec : null,++ termsNote: extra.terms_note || null,++ licenseNote: extra.license_note || null,++ });++ }++++ async function resolveKeys(keys) {++ if (keys.cik) {++ const r = await ctx.instruments.resolve(keys.cik, { kind: 'cik' });++ if (r.status === 'resolved') return r.instrument;++ }++ if (keys.symbol) {++ const r = await ctx.instruments.resolve(keys.symbol, { kind: 'ticker' });++ if (r.status === 'resolved') return r.instrument;++ }++ return null;++ }++++ /** Apply one item. → a counter name. Inside the page transaction. */++ async function applyItem(item) {++ if (item.removed) return await ctx.documents.remove(item.id, { at: parseTime(item.removed.at), reason: item.removed.reason }) ? 'documents_removed' : 'removed_unknown';++ const m = mapItem(item);++ if (m.type === 'skip') return `skipped_${m.reason}`;++ const instrument = await resolveKeys(m.keys);++ if (!instrument) return 'skipped_no_matching_instrument';++ if (instrument.status !== 'active') return 'skipped_archived_instrument';++ if (m.type === 'observation') {++ try {++ const r = await ctx.observations.record(m.observation, instrument, { recordedBy: 'svc:trade' });++ return r.created ? 'observations_created' : 'observations_unchanged';++ } catch (err) {++ if (err && (err.status === 409 || err.status === 422)) return `skipped_${String(err.code || 'invalid').replace(/\W+/g, '_')}`;++ throw err;++ }++ }++ const r = await ctx.documents.upsert(m.document, instrument);++ return r.created ? 'documents_created' : r.changed ? 'documents_updated' : 'documents_unchanged';++ }++++ async function runOnce() {++ if (!sources.enabled) return { ok: false, skipped: 'disabled' };++ await ensureState();++ const summary = { pages: 0, counts: {} };++ const counts = {};++ try {++ const list = await sources.listSources();++ for (const s of (list.sources || [])) {++ if (s && s.category === config.sources.category && s.key) await reportHealth(s.key, s.health, s);++ }++ const pull = await pullChanges({++ db, cursor, source: sources, name: CURSOR,++ maxPages: config.sources.maxPagesPerRun, pageSize: config.sources.pageLimit,++ apply: async (item) => {++ const key = await applyItem(item);++ counts[key] = (counts[key] || 0) + 1;++ return outcomeOf(key);++ },++ });++ summary.pages = pull.pages;++ summary.counts = counts;++ await q.save.run({ name: CURSOR, counts: JSON.stringify(counts), ok: store.now() });++ ctx.outbox.kick();++ return { ok: true, ...summary };++ } catch (err) {++ await q.fail.run(String(err.message).slice(0, 500), store.now(), CURSOR);++ log.warn('[Trade] Sources sync failed:', err.message);++ return { ok: false, error: err.message, ...summary };++ }++ }++++ return {++ CURSOR,++ enabled: sources.enabled,++ applyItem,++ /** One run at a time; concurrent callers share it. */++ run() {++ if (!running) running = runOnce().finally(() => { running = null; });++ return running;++ },++ async state() {++ await ensureState();++ const s = await q.get.get(CURSOR);++ return { enabled: sources.enabled, cursor: await cursor.get(CURSOR), last_ok_at: iso(s.last_ok_at), last_error: s.last_error, last_error_at: iso(s.last_error_at), counts: json(s.counts, {}) };++ },++ };++}++++module.exports = { createIngest, CURSOR };@@@ -20,7 +20,7 @@* same way. A ticker or CIK names exactly one instrument (a unique index enforces it).*/const { ApiError } = require('../http/errors');--const { newId, invalid, str } = require('./util');++const { newId, invalid, str } = require('./helpers');const KINDS = ['equity', 'fund', 'index', 'currency', 'commodity', 'crypto', 'other'];const SYMBOL_RE = /^[A-Z0-9][A-Z0-9.-]{0,15}$/;@@@ -16,7 +16,7 @@* (kind 'record' for record items, else 'article').* skip everything else, with a reason (counted in the sync state, never guessed at).*/--const { parseTime, parseDecimal } = require('./util');++const { parseTime, parseDecimal } = require('./helpers');const SEC_PATH_RE = /^https?:\/\/(?:www\.)?sec\.gov\/Archives\/edgar\/data\/(\d{1,10})\/(\d{18})(?:\/|$)/i;const FORM_RE = /^[A-Z0-9][A-Z0-9 \-/.]{0,19}$/;@@@ -18,7 +18,7 @@* instrument's Search document (the gate's stale_price rule looks at monetary observations).*/const { ApiError } = require('../http/errors');--const { newId, iso, parseTime, parseDecimal, invalid, str, httpUrl } = require('./util');++const { newId, iso, parseTime, parseDecimal, invalid, str, httpUrl } = require('./helpers');const METRIC_RE = /^[A-Za-z][A-Za-z0-9_.:-]{0,79}$/;const UNIT_RE = /^[A-Za-z][A-Za-z0-9/%._-]{0,19}$/;@@@ -5,7 +5,7 @@* machine-readable representation always describe the same data (§32.5: no divergence).*/const seo = require('openvibe-publishing/seo');--const { iso } = require('./util');++const { iso } = require('./helpers');const { DISCLAIMER } = require('./alerts');function createReading({ store, ctx }) {nnew file mode 100644@@@ -0,0 +1,106 @@++'use strict';++++/**++ * Freshness per source: is what Trade shows from a source still current?++ *++ * The staleness verdict itself is the ingest chassis's generic rule (openvibe-publishing/ingest,++ * freshness.verdict/view) with Trade's window as the parameter; this module keeps only what is++ * Trade's own — the trade_source_status projection and the trade.source.stale|recovered transitions.++ *++ * A source is FRESH at time `now` only when its last successful fetch is known and younger than its++ * staleness window (Sources' stale_after_sec, else TRADE_DEFAULT_STALE_AFTER_SEC). Everything++ * else is stale:++ * - the last success is older than the window → stale since last success + window++ * - Sources never reported a success → stale since Trade first saw the source++ * - Trade has never heard of the source at all → freshness unknown (shown as stale)++ * Sources' own health status (failing, disabled, …) is shown next to it, but the verdict is++ * computed with Trade's clock, so an outage of Sources or of the sync makes data go stale on its own++ * rather than look current forever. Nothing is ever replaced by an invented value: a stale value is++ * shown with its timestamps and a "stale since" label, and a missing one is shown as missing.++ *++ * `trade.source.stale` and `trade.source.recovered` are emitted only on a transition (evaluate()),++ * so a replay or a second tick changes nothing.++ */++const { freshness: fresh, normalize } = require('openvibe-publishing/ingest');++const { iso } = normalize;++++function createFreshness({ store, config, outbox }) {++ const { db } = store;++ const def = config.freshness.defaultStaleAfterSec;++ const q = {++ get: db.prepare('SELECT * FROM trade_source_status WHERE source_key = ?'),++ all: db.prepare('SELECT * FROM trade_source_status ORDER BY source_key'),++ ensure: db.prepare(`INSERT INTO trade_source_status (source_key, stale, stale_since, evaluated_at) VALUES (?, 1, ?, NULL) ON CONFLICT DO NOTHING`),++ report: db.prepare(`UPDATE trade_source_status SET name = COALESCE(@name, name), upstream_status = @status,++ last_success_at = CASE WHEN @last::bigint IS NULL THEN last_success_at WHEN last_success_at IS NULL OR @last > last_success_at THEN @last ELSE last_success_at END,++ stale_after_sec = COALESCE(@window, stale_after_sec), reported_at = @now,++ terms_note = COALESCE(@terms, terms_note), license_note = COALESCE(@license, license_note)++ WHERE source_key = @key`),++ retrieval: db.prepare(`UPDATE trade_source_status SET last_success_at = @at WHERE source_key = @key AND (last_success_at IS NULL OR last_success_at < @at)`),++ mark: db.prepare('UPDATE trade_source_status SET stale = @stale, stale_since = @since, evaluated_at = @now WHERE source_key = @key'),++ };++++ /** Pure: the verdict for one status row at `now`. row may be null (unknown source). */++ function verdict(row, now) {++ return fresh.verdict(row, now, def);++ }++++ async function view(key, now = store.now()) {++ const row = await q.get.get(key) || null;++ return fresh.view(row, key, now, def);++ }++++ async function ensure(key) { await q.ensure.run(key, store.now()); }++++ /** Emit on a transition only. Inside the caller's transaction (or its own). */++ async function evaluate(key) {++ return await store.tx(async () => {++ const row = await q.get.get(key);++ if (!row) return null;++ const v = verdict(row, store.now());++ const first = row.evaluated_at == null;++ const was = Boolean(row.stale);++ const since = v.stale ? (v.staleSince != null ? v.staleSince : (row.stale_since || store.now())) : null;++ await q.mark.run({ key, stale: v.stale ? 1 : 0, since, now: store.now() });++ // First sight: a fresh source is not a recovery; a stale one is reported once.++ if (first ? !v.stale : v.stale === was) return null;++ const envelope = await outbox.emit({++ event_type: v.stale ? 'trade.source.stale' : 'trade.source.recovered',++ actor: { type: 'service', id: 'trade' }, visibility: 'public', priority: 'low',++ subject: { type: 'source', id: key },++ payload: {++ source_key: key, stale: v.stale, stale_since: iso(since), last_success_at: iso(v.lastSuccessAt),++ stale_after_sec: v.window, upstream_status: row.upstream_status || null, evaluated_at: iso(store.now()),++ },++ });++ return envelope;++ });++ }++++ return {++ verdict,++ view,++ ensure,++ evaluate,++ async evaluateAll() { return (await Promise.all((await q.all.all()).map(async (r) => await evaluate(r.source_key)))).filter(Boolean); },++ async all(now = store.now()) { return (await Promise.all((await q.all.all()).map(async (r) => await view(r.source_key, now)))); },++++ /** What Sources says about a source (its health block). Evaluates the transition. */++ async report(key, { name = null, status = null, lastSuccessAt = null, staleAfterSec = null, termsNote = null, licenseNote = null } = {}) {++ return await store.tx(async () => {++ await ensure(key);++ await q.report.run({ key, name, status, last: lastSuccessAt, window: staleAfterSec, now: store.now(), terms: termsNote, license: licenseNote });++ return await evaluate(key);++ });++ },++++ /** A datum retrieved at `at` proves a successful fetch at that time. */++ async noteRetrieval(key, at) {++ await ensure(key);++ await q.retrieval.run({ key, at });++ return await evaluate(key);++ },++ };++}++++module.exports = { createFreshness };ddeleted file mode 100644@@@ -1,136 +0,0 @@--'use strict';----/**-- * Pull the trade category from OpenVibe.Sources and record what it states.-- *-- * 1. GET /api/v1/sources every trade source's health → freshness (report)-- * 2. GET /api/v1/items?category=trade&after=<cursor>&include_removed=1, in change order:-- * removed item → the document is hidden (the row stays)-- * observation item → an observation (domain/observations.js; idempotent per item revision)-- * filing / keyed item → a document (domain/documents.js; new → document alerts)-- * no instrument matches → skipped (counted by reason; nothing is created for it)-- * Each page is applied in one transaction together with the cursor, so a crash replays the-- * page and every effect is idempotent (UNIQUE keys on observations, documents and deliveries).-- * The page's per-source health block → freshness as well.-- *-- * A failed call is recorded in trade_sync_state (last_error) and changes nothing else; freshness-- * then decays on Trade's own clock, so pages show the data as stale instead of pretending.-- */--const { mapItem } = require('./mapping');--const { parseTime, iso, json } = require('./util');----const CURSOR = 'sources.trade';----function createSync({ store, config, ctx, sources, log = console }) {-- const { db } = store;-- const q = {-- get: db.prepare('SELECT * FROM trade_sync_state WHERE name = ?'),-- ensure: db.prepare("INSERT INTO trade_sync_state (name, cursor, counts) VALUES (?, 0, '{}') ON CONFLICT DO NOTHING"),-- advance: db.prepare('UPDATE trade_sync_state SET cursor = @cursor, counts = @counts WHERE name = @name'),-- ok: db.prepare('UPDATE trade_sync_state SET last_ok_at = ?, last_error = NULL WHERE name = ?'),-- fail: db.prepare('UPDATE trade_sync_state SET last_error = ?, last_error_at = ? WHERE name = ?'),-- };-- let ensured = false; // the cursor row, written on the first run (not at construction: the factory stays synchronous)-- const ensureCursor = async () => { if (!ensured) { await q.ensure.run(CURSOR); ensured = true; } };-- let running = null;---- async function reportHealth(key, h, extra = {}) {-- if (!h || typeof h !== 'object') return;-- await ctx.freshness.report(key, {-- name: extra.name || null,-- status: h.status || null,-- lastSuccessAt: parseTime(h.last_success_at),-- staleAfterSec: Number.isInteger(h.stale_after_sec) ? h.stale_after_sec : null,-- termsNote: extra.terms_note || null,-- licenseNote: extra.license_note || null,-- });-- }---- async function resolveKeys(keys) {-- if (keys.cik) {-- const r = await ctx.instruments.resolve(keys.cik, { kind: 'cik' });-- if (r.status === 'resolved') return r.instrument;-- }-- if (keys.symbol) {-- const r = await ctx.instruments.resolve(keys.symbol, { kind: 'ticker' });-- if (r.status === 'resolved') return r.instrument;-- }-- return null;-- }---- /** Apply one item. → a counter name. Inside the page transaction. */-- async function applyItem(item) {-- if (item.removed) return await ctx.documents.remove(item.id, { at: parseTime(item.removed.at), reason: item.removed.reason }) ? 'documents_removed' : 'removed_unknown';-- const m = mapItem(item);-- if (m.type === 'skip') return `skipped_${m.reason}`;-- const instrument = await resolveKeys(m.keys);-- if (!instrument) return 'skipped_no_matching_instrument';-- if (instrument.status !== 'active') return 'skipped_archived_instrument';-- if (m.type === 'observation') {-- try {-- const r = await ctx.observations.record(m.observation, instrument, { recordedBy: 'svc:trade' });-- return r.created ? 'observations_created' : 'observations_unchanged';-- } catch (err) {-- if (err && (err.status === 409 || err.status === 422)) return `skipped_${String(err.code || 'invalid').replace(/\W+/g, '_')}`;-- throw err;-- }-- }-- const r = await ctx.documents.upsert(m.document, instrument);-- return r.created ? 'documents_created' : r.changed ? 'documents_updated' : 'documents_unchanged';-- }---- async function runOnce() {-- if (!sources.enabled) return { ok: false, skipped: 'disabled' };-- await ensureCursor();-- const summary = { pages: 0, counts: {} };-- try {-- const list = await sources.sources();-- for (const s of (list.sources || [])) {-- if (s && s.category === config.sources.category && s.key) await reportHealth(s.key, s.health, s);-- }-- for (let page = 0; page < config.sources.maxPagesPerRun; page++) {-- const state = await q.get.get(CURSOR);-- const data = await sources.items({ after: state.cursor, limit: config.sources.pageLimit });-- const items = Array.isArray(data.items) ? data.items : [];-- await store.tx(async () => {-- const counts = json((await q.get.get(CURSOR)).counts, {});-- for (const item of items) {-- const k = await applyItem(item);-- counts[k] = (counts[k] || 0) + 1;-- summary.counts[k] = (summary.counts[k] || 0) + 1;-- }-- for (const [key, h] of Object.entries(data.sources || {})) await reportHealth(key, h);-- const next = Number.isInteger(data.next_after) ? data.next_after : state.cursor;-- await q.advance.run({ name: CURSOR, cursor: Math.max(next, state.cursor), counts: JSON.stringify(counts) });-- });-- summary.pages++;-- if (!data.more || !items.length) break;-- }-- await q.ok.run(store.now(), CURSOR);-- ctx.outbox.kick();-- return { ok: true, ...summary };-- } catch (err) {-- await q.fail.run(String(err.message).slice(0, 500), store.now(), CURSOR);-- log.warn('[Trade] Sources sync failed:', err.message);-- return { ok: false, error: err.message, ...summary };-- }-- }---- return {-- CURSOR,-- enabled: sources.enabled,-- applyItem,-- /** One run at a time; concurrent callers share it. */-- run() {-- if (!running) running = runOnce().finally(() => { running = null; });-- return running;-- },-- async state() {-- await ensureCursor();-- const s = await q.get.get(CURSOR);-- return { enabled: sources.enabled, cursor: s.cursor, last_ok_at: iso(s.last_ok_at), last_error: s.last_error, last_error_at: iso(s.last_error_at), counts: json(s.counts, {}) };-- },-- };--}----module.exports = { createSync };ddeleted file mode 100644@@@ -1,68 +0,0 @@--'use strict';----/** Small shared helpers: ids, times, decimals, validation errors. Nothing here invents a value. */--const { ids } = require('openvibe-contracts');--const { ApiError } = require('../http/errors');----const newId = (prefix, now) => `${prefix}_${ids.ulid(now)}`;----/** epoch ms → ISO string, or null. */--const iso = (ms) => (ms == null ? null : new Date(ms).toISOString());----/**-- * A date the caller STATES: an ISO 8601 date-time with a zone (…Z or ±hh:mm), or epoch ms as a-- * number. Anything else (including a missing value) is null — never "now".-- */--function parseTime(v) {-- if (v == null || v === '') return null;-- if (typeof v === 'number') return Number.isFinite(v) && v > 0 ? Math.floor(v) : null;-- const s = String(v).trim();-- if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:?\d{2})$/i.test(s)) return null;-- const t = Date.parse(s);-- return Number.isFinite(t) ? t : null;--}----const DECIMAL_RE = /^-?(\d{1,20})(\.\d{1,12})?([eE][-+]?\d{1,3})?$/;----/** A decimal as stated: → { text, num } or null. Numbers are kept as their shortest string form. */--function parseDecimal(v) {-- if (typeof v === 'number') {-- if (!Number.isFinite(v)) return null;-- return { text: String(v), num: v };-- }-- if (typeof v !== 'string') return null;-- const s = v.trim();-- if (!DECIMAL_RE.test(s)) return null;-- const num = Number(s);-- return Number.isFinite(num) ? { text: s, num } : null;--}----const invalid = (detail, code = 'request.invalid') => new ApiError(422, code, detail);----function str(v, max, name, { required = false } = {}) {-- if (v == null || v === '') {-- if (required) throw invalid(`${name} is required`);-- return null;-- }-- if (typeof v !== 'string') throw invalid(`${name} must be a string`);-- const s = v.trim();-- if (!s && required) throw invalid(`${name} is required`);-- if (s.length > max) throw invalid(`${name} is longer than ${max} characters`);-- return s || null;--}----function httpUrl(v, name) {-- if (v == null || v === '') return null;-- let u;-- try { u = new URL(String(v)); } catch { throw invalid(`${name} must be an absolute URL`); }-- if (u.protocol !== 'https:' && u.protocol !== 'http:') throw invalid(`${name} must be http(s)`);-- return u.toString();--}----/** Parse JSON stored by this service; a broken value is an empty default, never a crash. */--function json(text, def) {-- if (text == null) return def;-- try { return JSON.parse(text); } catch { return def; }--}----module.exports = { newId, iso, parseTime, parseDecimal, invalid, str, httpUrl, json };@@@ -7,7 +7,7 @@* an event, and every response that shows one is Cache-Control: private, no-store.*/const { ApiError } = require('../http/errors');--const { newId, iso, str } = require('./util');++const { newId, iso, str } = require('./helpers');function createWatchlists({ store, config, ctx }) {const { db } = store;nnew file mode 100644@@@ -0,0 +1,41 @@++'use strict';++++/**++ * POST /internal/events — signed OpenVibe.Events webhook deliveries (subscription topic sources.*).++ *++ * The signature check, the ±300 s window and the exactly-once inbox are the ingest chassis++ * (openvibe-publishing/ingest, createEventConsumer); this route only maps its result onto HTTP and++ * wakes the Sources ingest. A webhook is a wake-up, not durable truth: a sources.item.* or++ * sources.fetch.failed event makes the cursor-based ingest run now, and the ingest reads the items++ * themselves from the Sources API. The route answers 404 while no secret is configured.++ */++const express = require('express');++const { createEventConsumer } = require('openvibe-publishing/ingest');++const { http } = require('openvibe-contracts');++++const WAKE_RE = /^sources\.(item\.(created|updated|removed)|fetch\.failed)$/;++++function createWebhook({ store, config, ingest, log = console }) {++ const router = express.Router();++ const consumer = createEventConsumer({ db: store.db, secrets: config.events.webhookSecret, consumer: 'sources', now: store.now });++++ router.post('/internal/events', express.raw({ type: '*/*', limit: '1mb' }), async function receiveEventsDelivery(req, res, next) {++ if (!config.events.webhookSecret) return http.sendProblem(res, 404, 'route.not_found', { detail: 'Not found', ctx: req.ov });++ const raw = Buffer.isBuffer(req.body) ? req.body : Buffer.from('');++ let r;++ try {++ r = await consumer.apply(raw, req.headers, async (e) => {++ const wake = WAKE_RE.test(e.event_type);++ if (wake) ingest.run().catch((err) => log.warn('[Trade] ingest after webhook failed:', err.message));++ return wake;++ });++ } catch (err) { return next(err); }++ if (r.status === 401) return http.sendProblem(res, 401, 'webhook.signature_invalid', { detail: 'The delivery signature does not verify', ctx: req.ov });++ if (r.status !== 200) return http.sendProblem(res, 400, 'webhook.malformed', { detail: 'Not an event envelope', ctx: req.ov });++ res.status(200).json({ accepted: true, duplicate: r.duplicate, sync: r.duplicate ? false : Boolean(r.outcome) });++ });++++ return { router, consumer };++}++++module.exports = { createWebhook };ddeleted file mode 100644@@@ -1,39 +0,0 @@--'use strict';----/**-- * POST /internal/events — signed OpenVibe.Events webhook deliveries (subscription topic sources.*).-- *-- * The body is verified against X-OpenVibe-Signature (HMAC-SHA256 of the raw body with-- * TRADE_EVENTS_WEBHOOK_SECRET) before anything is read; a bad or missing signature is 401 and the-- * route answers 404 while no secret is configured. Each event is recorded once in the SDK inbox-- * (idempotency_receipts, consumer 'sources'); a webhook is a wake-up, not durable truth: a-- * sources.item.* or sources.fetch.failed event makes the cursor-based sync run now, and the sync-- * reads the items themselves from the Sources API.-- */--const express = require('express');--const { parseDelivery, createPgInbox } = require('openvibe-sdk/events');--const { http } = require('openvibe-contracts');----function createWebhook({ store, config, sync, log = console }) {-- const router = express.Router();-- const inbox = createPgInbox(store.db, { now: store.now }); // idempotency_receipts: migrations/0001_initial.sql---- router.post('/internal/events', express.raw({ type: '*/*', limit: '1mb' }), async function receiveEventsDelivery(req, res, next) {-- if (!config.events.webhookSecret) return http.sendProblem(res, 404, 'route.not_found', { detail: 'Not found', ctx: req.ov });-- const raw = Buffer.isBuffer(req.body) ? req.body : Buffer.from('');-- // Signature v2 only: HMAC over "<t>.<raw body>" with t within ±300 s; a v1-only (v2 stripped) or stale delivery is refused.-- const delivery = parseDelivery(raw, req.headers, config.events.webhookSecret, { requireV2: true });-- if (!delivery) return http.sendProblem(res, 401, 'webhook.signature_invalid', { detail: 'The delivery signature does not verify', ctx: req.ov });-- const e = delivery.event;-- if (typeof e.event_id !== 'string' || typeof e.event_type !== 'string') return http.sendProblem(res, 400, 'webhook.malformed', { detail: 'Not an event envelope', ctx: req.ov });-- let r;-- try { r = await inbox.once('sources', e.event_id, async () => true); } catch (err) { return next(err); }-- const wake = !r.duplicate && /^sources\.(item\.(created|updated|removed)|fetch\.failed)$/.test(e.event_type);-- if (wake) sync.run().catch((err) => log.warn('[Trade] sync after webhook failed:', err.message));-- res.status(200).json({ accepted: true, duplicate: r.duplicate, sync: wake });-- });---- return { router, inbox };--}----module.exports = { createWebhook };@@@ -3,7 +3,7 @@* Background work in the Trade process (two timers):** Sources sync every TRADE_SYNC_INTERVAL_MS (and early when a signed webhook arrives):-- * domain/sync.js pulls the trade category; idempotent per item revision++ * domain/ingest.js pulls the trade category; idempotent per item revision* freshness every TRADE_FRESHNESS_INTERVAL_MS: re-evaluate every source on Trade's clock* (trade.source.stale / recovered on transitions only) and re-send any instrument* Search document whose gate decision changed (e.g. a price observation aged@@@ -94,7 +94,7 @@ const MIN = 60e3;assert.strictEqual((await bobs()).length, 1, 'new_document fired, filing_type(8-K) did not for a 10-Q');t.sources.putItem({ id: item.id, canonical_url: url, title: 'ACME CORP (amended title)', summary: '10-Q', published_at: iso(t.T0), retrieved_at: iso(t.clock.now()) });await t.ctx.sync.run();-- await t.ctx.store.db.prepare("UPDATE trade_sync_state SET cursor = 0 WHERE name = 'sources.trade'").run();++ await t.ctx.store.db.prepare("UPDATE trade_ingest_cursor SET cursor = 0 WHERE name = 'sources.trade'").run();await t.ctx.sync.run();assert.strictEqual((await bobs()).length, 1, 'item revision and page replay: no second delivery');assert.strictEqual((await t.get('/i/ACME.json')).json().documents[0].title, 'ACME CORP (amended title)');nnew file mode 100644@@@ -0,0 +1,149 @@++'use strict';++/**++ * T9 J6 — Trade on the openvibe-publishing 1.1.0 chassis.++ *++ * 1. the local copies the brief lists (domain/sync.js, domain/util.js, domain/freshness.js,++ * events/webhook.js, clients/sources.js) are no longer required by any source or test file: the++ * Sources client, the change cursor/pull loop, the signed event consumer, the normalisers, the++ * freshness verdict and the publication glue now come from openvibe-publishing/ingest and++ * /publication. (`rm` is denied in this harness, so the files are kept on disk but unused.)++ * 2. a dry-run ingest from a captured Sources fixture produces the same domain rows as the golden++ * captured from the pre-conversion code.++ * 3. the Search document the publication glue emits validates against search.index-document@1 and++ * is public and indexable.++ * 4. the outbox row and the state change share one transaction: rolled back together, committed++ * together.++ */++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const contracts = require('openvibe-contracts');++const { boot, check, done } = require('./helpers/boot');++++const ROOT = path.join(__dirname, '..');++const REPLACED = ['server/domain/sync.js', 'server/domain/util.js', 'server/domain/freshness.js', 'server/events/webhook.js', 'server/clients/sources.js'];++const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');++const isoOf = (v) => (v == null ? null : new Date(Number(v)).toISOString());++++/** The golden captured from the pre-conversion code (id fields normalised away). */++const GOLDEN = {++ run: { ok: true, counts: { documents_created: 1, observations_created: 1, skipped_no_observation_time: 1 } },++ state: { enabled: true, cursor: 3, counts: { documents_created: 1, observations_created: 1, skipped_no_observation_time: 1 } },++ docs: [{++ kind: 'filing', form_type: '10-Q', title: 'APPLE INC (0000320193) (Filer)', filer_name: 'APPLE INC',++ cik: '0000320193', accession: '0000320193-26-000105',++ published_at: '2026-09-22T11:30:00.000Z', retrieved_at: '2026-09-22T11:55:00.000Z', removed_reason: null,++ }],++ obs: [{++ metric: 'price.close', value: '231.40', unit: 'USD', currency: 'USD',++ observed_at: '2026-09-21T20:00:00.000Z', retrieved_at: '2026-09-22T11:55:00.000Z', source_key: 'test-quotes',++ }],++};++++function jsFiles(dir) {++ const out = [];++ for (const e of fs.readdirSync(dir, { withFileTypes: true })) {++ if (e.name === 'node_modules' || e.name === 'data') continue;++ const p = path.join(dir, e.name);++ if (e.isDirectory()) out.push(...jsFiles(p));++ else if (e.name.endsWith('.js')) out.push(p);++ }++ return out;++}++++(async () => {++ await check('the replaced local copies are unrequired; the chassis is used instead', async () => {++ const gone = new Set(REPLACED.map((p) => path.join(ROOT, p)));++ const requirers = [];++ for (const f of jsFiles(path.join(ROOT, 'server')).concat(jsFiles(path.join(ROOT, 'test')))) {++ if (gone.has(f)) continue;++ const src = fs.readFileSync(f, 'utf8');++ for (const m of src.matchAll(/require\(\s*['"]([^'"]+)['"]\s*\)/g)) {++ if (!m[1].startsWith('.')) continue;++ const target = path.resolve(path.dirname(f), m[1]);++ if (gone.has(target) || gone.has(`${target}.js`)) requirers.push(`${path.relative(ROOT, f)} → ${m[1]}`);++ }++ }++ assert.deepStrictEqual(requirers, [], `still required: ${requirers.join(', ')}`);++ // The chassis entry points are what the app requires now.++ const app = fs.readFileSync(path.join(ROOT, 'server', 'app.js'), 'utf8');++ assert.ok(app.includes("require('openvibe-publishing/ingest')"), 'the Sources client comes from the chassis');++ assert.ok(app.includes("require('./domain/ingest')"), 'the ingest composition');++ assert.ok(app.includes("require('./events/consumer')"), 'the signed event consumer route');++ });++++ await check('a dry-run ingest from the captured Sources fixture reproduces the golden domain rows', async () => {++ const t = await boot();++ const ret = t.iso(t.T0 - 5 * 60e3);++ await t.instrument({ symbol: 'AAPL', name: 'Apple Inc.', cik: '320193', exchange: 'NASDAQ' });++ t.sources.setSource('sec-xbrl-filings', { name: 'SEC EDGAR: latest XBRL filings', lastSuccessAt: ret, staleAfterSec: 2700 });++ t.sources.putItem({++ canonical_url: 'https://www.sec.gov/Archives/edgar/data/320193/000032019326000105/0000320193-26-000105-index.htm',++ title: 'APPLE INC (0000320193) (Filer)', summary: '10-Q', published_at: '2026-09-22T11:30:00.000Z', retrieved_at: ret,++ });++ t.sources.putItem({++ source_key: 'test-quotes', kind: 'observation', canonical_url: 'https://quotes.example/aapl', title: 'AAPL close',++ retrieved_at: ret, fields: { symbol: 'AAPL', metric: 'price.close', value: '231.40', unit: 'USD', currency: 'USD', observed_at: '2026-09-21T20:00:00Z' },++ });++ t.sources.putItem({++ source_key: 'test-quotes', kind: 'observation', canonical_url: 'https://quotes.example/aapl2', title: 'AAPL open (no time)',++ retrieved_at: ret, fields: { symbol: 'AAPL', metric: 'price.open', value: '229.00', unit: 'USD', currency: 'USD' },++ });++ const r = await t.ctx.sync.run();++ const state = await t.ctx.sync.state();++ const docs = (await t.ctx.store.db.prepare(++ 'SELECT kind, form_type, title, filer_name, cik, accession, published_at, retrieved_at, removed_reason FROM trade_source_documents ORDER BY source_item_id').all())++ .map((d) => ({ ...d, published_at: isoOf(d.published_at), retrieved_at: isoOf(d.retrieved_at) }));++ const obs = (await t.ctx.store.db.prepare(++ 'SELECT metric, value, unit, currency, observed_at, retrieved_at, source_key FROM trade_market_observations ORDER BY metric').all())++ .map((o) => ({ ...o, observed_at: isoOf(o.observed_at), retrieved_at: isoOf(o.retrieved_at) }));++ assert.deepStrictEqual({ run: { ok: r.ok, counts: r.counts }, state: { enabled: state.enabled, cursor: state.cursor, counts: state.counts }, docs, obs }, GOLDEN);++ await t.close();++ });++++ await check('the emitted Search document validates against search.index-document@1 (public, indexable)', async () => {++ const t = await boot();++ const inst = await t.instrument({ symbol: 'CHS', name: 'Chassis Corp' });++ const rev = await t.ctx.context.propose({ kind: 'user', subject: t.editor.subject, editor: true }, inst, { body: WORDS(80) });++ await t.ctx.context.publish(inst, rev.revision.number);++ const upserted = await t.events('trade.index_document.upserted');++ assert.strictEqual(upserted.length, 1, 'exactly one upserted document');++ const doc = upserted[0].payload;++ const v = contracts.validate('search.index-document@1', doc);++ assert.ok(v.valid, JSON.stringify(v.errors));++ assert.strictEqual(doc.owner, 'trade');++ assert.strictEqual(doc.type, 'instrument');++ assert.strictEqual(doc.visibility, 'public');++ assert.strictEqual(doc.indexability.decision, 'index');++ assert.strictEqual(doc.canonical_url, 'https://openvibe.trade/i/CHS');++ assert.ok(Array.isArray(doc.provenance), 'provenance is a list of references');++ await t.close();++ });++++ await check('the outbox row and the state change share one transaction', async () => {++ const t = await boot();++ const { db } = t.ctx.store;++ const inst = await t.instrument({ symbol: 'TXN', name: 'Txn Corp' });++ const rev = await t.ctx.context.propose({ kind: 'user', subject: t.editor.subject, editor: true }, inst, { body: WORDS(80) });++ await t.ctx.context.publish(inst, rev.revision.number);++ const before = (await t.events('trade.index_document.upserted')).length;++++ await assert.rejects(t.ctx.store.tx(async () => {++ await db.prepare("UPDATE trade_instruments SET name = 'Txn Changed' WHERE id = ?").run(inst.id);++ await t.ctx.indexing.refresh(await t.ctx.instruments.get(inst.id));++ throw new Error('boom');++ }));++ assert.strictEqual((await db.prepare('SELECT name FROM trade_instruments WHERE id = ?').get(inst.id)).name, 'Txn Corp', 'the state change rolled back');++ assert.strictEqual((await t.events('trade.index_document.upserted')).length, before, 'the outbox row did not survive the rollback');++++ await t.ctx.store.tx(async () => {++ await db.prepare("UPDATE trade_instruments SET name = 'Txn Changed' WHERE id = ?").run(inst.id);++ await t.ctx.indexing.refresh(await t.ctx.instruments.get(inst.id));++ });++ assert.strictEqual((await db.prepare('SELECT name FROM trade_instruments WHERE id = ?').get(inst.id)).name, 'Txn Changed');++ assert.strictEqual((await t.events('trade.index_document.upserted')).length, before + 1, 'committed together');++ await t.close();++ });++++ done();++})().catch((err) => { console.error(err); process.exit(1); });@@@ -64,7 +64,7 @@ const { mapItem } = require('../server/domain/mapping');const docs = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM trade_source_documents').get()).n;const obs = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM trade_market_observations').get()).n;const evts = (await t.events()).length;-- await t.ctx.store.db.prepare("UPDATE trade_sync_state SET cursor = 0 WHERE name = 'sources.trade'").run();++ await t.ctx.store.db.prepare("UPDATE trade_ingest_cursor SET cursor = 0 WHERE name = 'sources.trade'").run();const r = await t.ctx.sync.run();assert.strictEqual(r.ok, true);assert.strictEqual((await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM trade_source_documents').get()).n, docs);@@@ -136,7 +136,8 @@ const { mapItem } = require('../server/domain/mapping');await check('Trade asks Sources with its own service token for exactly sources.item.read and sources.source.read', async () => {const g = t.network.grants.filter((x) => x.audience === 'openvibe.sources');assert.ok(g.length >= 1);-- assert.deepStrictEqual(g[0].scope.split(' ').sort(), ['sources.item.read', 'sources.source.read']);++ const scopes = [...new Set(g.flatMap((x) => String(x.scope || '').split(/\s+/).filter(Boolean)))].sort();++ assert.deepStrictEqual(scopes, ['sources.item.read', 'sources.source.read']);assert.ok(t.sources.calls.some((c) => /category=trade/.test(c.url) && /include_removed=1/.test(c.url)));});
@@ -49,7 +49,8 @@ Other tables in the same database:
@@ -49,7 +49,8 @@ Other tables in the same database:
`trade_context_reviews` (a person's review of an AI draft) and `trade_index_revisions` (the
`trade_context_reviews` (a person's review of an AI draft) and `trade_index_revisions` (the
Search document sequencer).
Search document sequencer).
- `trade_source_status`: freshness of each source (Sources' health plus Trade's own clock).
- `trade_source_status`: freshness of each source (Sources' health plus Trade's own clock).
-- `trade_sync_state`: the Sources cursor, the last error and counters of what was skipped and why.
+- `trade_ingest_cursor`: the change cursor of the Sources feed (`openvibe-publishing/ingest`).
+- `trade_sync_state`: the last sync error and counters of what was skipped and why.
- `event_outbox` (SDK outbox) and `idempotency_receipts` (SDK inbox for signed webhook deliveries).
- `event_outbox` (SDK outbox) and `idempotency_receipts` (SDK inbox for signed webhook deliveries).
## Does not own
## Does not own
@@ -222,7 +223,7 @@ services need to call Trade, is under [Grants the Network must hold](#grants-the
@@ -222,7 +223,7 @@ services need to call Trade, is under [Grants the Network must hold](#grants-the
- **PostgreSQL 18 and Valkey 9** (OpenVibe.Host `roles/data/`, ADR-035): every read and write is async through
- **PostgreSQL 18 and Valkey 9** (OpenVibe.Host `roles/data/`, ADR-035): every read and write is async through
`openvibe-sdk/db`; Valkey holds the per-actor limit counters (optional: without `VALKEY_URL` they count per process).
`openvibe-sdk/db`; Valkey holds the per-actor limit counters (optional: without `VALKEY_URL` they count per process).
- **Packages** (pinned by release tarball): `openvibe-contracts` v0.76.0, `openvibe-publishing`
- **Packages** (pinned by release tarball): `openvibe-contracts` v0.76.0, `openvibe-publishing`
- v1.0.0 (revisions, authorship, seo, index-hooks, ssr), `openvibe-shared` v2.2.0 (Frame, app icon,
+ v1.1.0 (revisions, authorship, seo, index-hooks, ssr, ingest, publication), `openvibe-shared` v2.2.0 (Frame, app icon,
footer, legal, release, metrics, ready), `openvibe-sdk` v0.20.0 (events outbox and inbox, webhook
footer, legal, release, metrics, ready), `openvibe-sdk` v0.20.0 (events outbox and inbox, webhook
signatures v2, service tokens, per-actor limits).
signatures v2, service tokens, per-actor limits).
- **OpenVibe.Network:** SSO (OAuth client `trade`, redirect `https://openvibe.trade/auth/callback`),
- **OpenVibe.Network:** SSO (OAuth client `trade`, redirect `https://openvibe.trade/auth/callback`),
@@ -11,7 +11,7 @@
@@ -11,7 +11,7 @@
"contracts": "openvibe-contracts v0.79.0",
"contracts": "openvibe-contracts v0.79.0",
"packages": [
"packages": [
"openvibe-contracts v0.79.0",
"openvibe-contracts v0.79.0",
- "openvibe-publishing v1.0.0",
+ "openvibe-publishing v1.1.0",
"openvibe-shared v2.2.0",
"openvibe-shared v2.2.0",
"openvibe-sdk v0.25.0"
"openvibe-sdk v0.25.0"
],
],
@@ -36,7 +36,7 @@
@@ -36,7 +36,7 @@
"deployed": true,
"deployed": true,
"deployment": "Deployed with `sudo ovhost deploy trade`: unit openvibe-trade, 127.0.0.1:4860 loopback only; on 2026-09-23 /api/ready answered 200 with status degraded [freshness], which is honest: no feed is configured; Network OAuth client trade and its grants exist; no Events subscription; production database held 0 instruments and 0 observations. Deployed internally, not launched: openvibe.trade still serves the OpenVibe.Sites placeholder until a person verifies the SEC XBRL terms and enables sec-xbrl-filings in Sources, a licensed market data feed exists, and editors add instruments",
"deployment": "Deployed with `sudo ovhost deploy trade`: unit openvibe-trade, 127.0.0.1:4860 loopback only; on 2026-09-23 /api/ready answered 200 with status degraded [freshness], which is honest: no feed is configured; Network OAuth client trade and its grants exist; no Events subscription; production database held 0 instruments and 0 observations. Deployed internally, not launched: openvibe.trade still serves the OpenVibe.Sites placeholder until a person verifies the SEC XBRL terms and enables sec-xbrl-filings in Sources, a licensed market data feed exists, and editors add instruments",
"placeholderInSites": true,
"placeholderInSites": true,
- "updated": "2026-09-28",
+ "updated": "2026-09-29",
"features": [
"features": [
"informational only: a route-inventory test refuses any execution, custody or advice endpoint",
"informational only: a route-inventory test refuses any execution, custody or advice endpoint",
"instruments with exact, deterministic resolution (CIK, ticker, name)",
"instruments with exact, deterministic resolution (CIK, ticker, name)",
new file mode 100644
new file mode 100644
@@ -0,0 +1,15 @@
@@ -0,0 +1,15 @@
+-- phase: expand
+-- openvibe-publishing/ingest (prefix trade): the change cursor of the Sources feed, replacing the
+-- cursor Trade kept in trade_sync_state before the chassis (T9). Additive.
+CREATE TABLE IF NOT EXISTS trade_ingest_cursor (
+ name text COLLATE "C" NOT NULL,
+ cursor bigint NOT NULL,
+ updated_at bigint NOT NULL,
+ PRIMARY KEY (name)
+);
+
+-- One-time continuity: seed the chassis cursor from the row Trade already had, so the feed resumes
+-- where it was instead of restarting at zero. A fresh database has no such row and starts at 0.
+INSERT INTO trade_ingest_cursor (name, cursor, updated_at)
+SELECT 'sources.trade', cursor, 0 FROM trade_sync_state WHERE name = 'sources.trade'
+ON CONFLICT (name) DO NOTHING;
@@ -16,7 +16,7 @@
@@ -16,7 +16,7 @@
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",
- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",
+ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.1.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.25.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.25.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.2.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.2.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -1163,9 +1163,9 @@
@@ -1163,9 +1163,9 @@
}
}
},
},
"node_modules/openvibe-publishing": {
"node_modules/openvibe-publishing": {
- "version": "1.0.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",
- "integrity": "sha512-g4mU0PPtI4y38NcsdppEbrXTrCYp0rPmIYWyxLxh4GpEYhn48jMWq2+cMQA2uwZCeeZlB3UiXVrQNQyQLiliwg==",
+ "version": "1.1.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.1.0",
+ "integrity": "sha512-bJsVl82hDivazoiM/np9ckKJE4V5F4OxyIiDwB6xuhK8M6P42sGU+V88rFCkyiI4BVQiKv9xs4GVx8Fj3pyjsg==",
"license": "MIT",
"license": "MIT",
"engines": {
"engines": {
"node": ">=20"
"node": ">=20"
@@ -28,7 +28,7 @@
@@ -28,7 +28,7 @@
"helmet": "^8.1.0",
"helmet": "^8.1.0",
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.79.0",
- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",
+ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.1.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.25.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.25.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.2.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v2.2.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -9,7 +9,7 @@
@@ -9,7 +9,7 @@
* Editor (http/editor.js) /editor/…
* Editor (http/editor.js) /editor/…
* Discovery (http/discovery.js) robots, llms, sitemaps, feeds
* Discovery (http/discovery.js) robots, llms, sitemaps, feeds
* API (http/api.js) /api/v1/…
* API (http/api.js) /api/v1/…
- * Webhook (events/webhook.js) POST /internal/events (signed OpenVibe.Events deliveries)
+ * Webhook (events/consumer.js) POST /internal/events (signed OpenVibe.Events deliveries)
* Machine /api/health, /api/ready, /release.json, /metrics
* Machine /api/health, /api/ready, /release.json, /metrics
*
*
* Information only (ADR-025): no route takes an order, holds value, runs escrow, lists goods
* Information only (ADR-025): no route takes an order, holds value, runs escrow, lists goods
@@ -26,14 +26,15 @@ const { createSsoClient } = require('openvibe-sdk/sso');
@@ -26,14 +26,15 @@ const { createSsoClient } = require('openvibe-sdk/sso');
const { jwksClient } = require('openvibe-sdk/auth');
const { jwksClient } = require('openvibe-sdk/auth');
const { createServiceOutbox } = require('openvibe-sdk/events');
const { createServiceOutbox } = require('openvibe-sdk/events');
const { createIndexNow } = require('openvibe-shared/indexnow');
const { createIndexNow } = require('openvibe-shared/indexnow');
+const { createSourcesClient } = require('openvibe-publishing/ingest');
const configLib = require('./config');
const configLib = require('./config');
const { openStore } = require('./db');
const { openStore } = require('./db');
const { createViewerResolver } = require('./auth/viewer');
const { createViewerResolver } = require('./auth/viewer');
-const { createWebhook } = require('./events/webhook');
+const { createWebhook } = require('./events/consumer');
const { createUrls } = require('./domain/urls');
const { createUrls } = require('./domain/urls');
const { createInstruments } = require('./domain/instruments');
const { createInstruments } = require('./domain/instruments');
-const { createFreshness } = require('./domain/freshness');
+const { createFreshness } = require('./domain/source-status');
const { createObservations } = require('./domain/observations');
const { createObservations } = require('./domain/observations');
const { createDocuments } = require('./domain/documents');
const { createDocuments } = require('./domain/documents');
const { createAlerts } = require('./domain/alerts');
const { createAlerts } = require('./domain/alerts');
@@ -41,8 +42,7 @@ const { createWatchlists } = require('./domain/watchlists');
@@ -41,8 +42,7 @@ const { createWatchlists } = require('./domain/watchlists');
const { createContext } = require('./domain/context');
const { createContext } = require('./domain/context');
const { createIndexing } = require('./domain/indexing');
const { createIndexing } = require('./domain/indexing');
const { createReading } = require('./domain/reading');
const { createReading } = require('./domain/reading');
-const { createSync } = require('./domain/sync');
-const { createSourcesClient } = require('./clients/sources');
+const { createIngest } = require('./domain/ingest');
const { createCommon } = require('./http/common');
const { createCommon } = require('./http/common');
const { createPages } = require('./http/pages');
const { createPages } = require('./http/pages');
const { createPrivate } = require('./http/private');
const { createPrivate } = require('./http/private');
@@ -92,8 +92,8 @@ async function createApp(opts = {}) {
@@ -92,8 +92,8 @@ async function createApp(opts = {}) {
ctx.context = createContext({ store, config, ctx });
ctx.context = createContext({ store, config, ctx });
ctx.indexing = createIndexing({ store, config, ctx, indexnow });
ctx.indexing = createIndexing({ store, config, ctx, indexnow });
ctx.reading = createReading({ store, ctx });
ctx.reading = createReading({ store, ctx });
- ctx.sources = opts.sourcesClient || createSourcesClient({ config, fetchImpl });
- ctx.sync = createSync({ store, config, ctx, sources: ctx.sources, log });
+ ctx.sources = opts.sourcesClient || createSourcesClient({ config, fetchImpl, timeoutMs: 15_000 });
+ ctx.sync = createIngest({ store, config, ctx, sources: ctx.sources, log });
// Sign-in with OpenVibe.Network (openvibe-sdk/sso): routes, offline session verification through the
// Sign-in with OpenVibe.Network (openvibe-sdk/sso): routes, offline session verification through the
// shared JWKS client, state + PKCE. ctx.jwks is the same client viewer.js verifies service tokens with.
// shared JWKS client, state + PKCE. ctx.jwks is the same client viewer.js verifies service tokens with.
ctx.jwksUrl = `${String(config.networkInternalUrl || config.networkUrl).replace(/\/+$/, '')}/api/.well-known/jwks`;
ctx.jwksUrl = `${String(config.networkInternalUrl || config.networkUrl).replace(/\/+$/, '')}/api/.well-known/jwks`;
@@ -163,7 +163,7 @@ async function createApp(opts = {}) {
@@ -163,7 +163,7 @@ async function createApp(opts = {}) {
app.use(machine);
app.use(machine);
// ── Signed OpenVibe.Events deliveries (before any body parser) ──
// ── Signed OpenVibe.Events deliveries (before any body parser) ──
- app.use(createWebhook({ store, config, sync: ctx.sync, log }).router);
+ app.use(createWebhook({ store, config, ingest: ctx.sync, log }).router);
// ── Sign-in (OAuth2 client of OpenVibe.Network) ─────────
// ── Sign-in (OAuth2 client of OpenVibe.Network) ─────────
app.use('/auth/', rateLimit({ windowMs: 15 * 60_000, max: 60, standardHeaders: true, legacyHeaders: false }));
app.use('/auth/', rateLimit({ windowMs: 15 * 60_000, max: 60, standardHeaders: true, legacyHeaders: false }));
deleted file mode 100644
deleted file mode 100644
@@ -1,42 +0,0 @@
@@ -1,42 +0,0 @@
-'use strict';
-
-/**
- * OpenVibe.Sources, read with Trade's service token (audience openvibe.sources):
- * sources.item.read GET /api/v1/items?category=trade&after=<change_seq>&include_removed=1
- * sources.source.read GET /api/v1/sources (health and staleness of every source)
- * Off until OV_OAUTH_CLIENT_SECRET is set. A failure is an error for the caller to record, never an
- * empty answer.
- */
-const { serviceAuth } = require('openvibe-contracts');
-
-function createSourcesClient({ config, fetchImpl = globalThis.fetch }) {
- const enabled = Boolean(config.oauth.clientSecret && config.sources.internalUrl);
- const tokens = enabled ? serviceAuth.createTokenClient({
- tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,
- audience: 'openvibe.sources', scope: 'sources.item.read sources.source.read', fetchImpl,
- }) : null;
-
- async function get(path) {
- if (!enabled) throw new Error('Sources client disabled (OV_OAUTH_CLIENT_SECRET unset)');
- const res = await fetchImpl(`${config.sources.internalUrl}${path}`, {
- headers: { Accept: 'application/json', ...(await tokens.authHeaders()) },
- signal: AbortSignal.timeout(15_000),
- });
- if (res.status === 401 && tokens.invalidate) tokens.invalidate();
- const data = await res.json().catch(() => null);
- if (!res.ok || !data) {
- const err = new Error(`Sources answered ${res.status}${data && data.code ? ` (${data.code})` : ''} for ${path.split('?')[0]}`);
- err.status = res.status;
- throw err;
- }
- return data;
- }
-
- return {
- enabled,
- items: async ({ after = 0, limit = 200 } = {}) => await get(`/api/v1/items?category=${encodeURIComponent(config.sources.category)}&after=${after}&limit=${limit}&include_removed=1`),
- sources: async () => await get('/api/v1/sources'),
- };
-}
-
-module.exports = { createSourcesClient };
@@ -23,7 +23,8 @@
@@ -23,7 +23,8 @@
*
*
* Also here: trade_context_drafts / trade_context_revision_purges / trade_context_reviews (package
* Also here: trade_context_drafts / trade_context_revision_purges / trade_context_reviews (package
* companions), trade_index_revisions (Search document sequencer), trade_source_status (freshness
* companions), trade_index_revisions (Search document sequencer), trade_source_status (freshness
- * per source), trade_sync_state (the Sources cursor), event_outbox (SDK outbox) and
+ * per source), trade_ingest_cursor (the Sources change cursor, openvibe-publishing/ingest),
+ * trade_sync_state (the last sync error and counters), event_outbox (SDK outbox) and
* idempotency_receipts (SDK inbox for signed webhook deliveries).
* idempotency_receipts (SDK inbox for signed webhook deliveries).
*/
*/
const fs = require('fs');
const fs = require('fs');
@@ -21,7 +21,7 @@
@@ -21,7 +21,7 @@
* timestamps and source), never rendered notification text: Network renders the notification.
* timestamps and source), never rendered notification text: Network renders the notification.
*/
*/
const { ApiError } = require('../http/errors');
const { ApiError } = require('../http/errors');
-const { newId, iso, parseDecimal, invalid, str, json } = require('./util');
+const { newId, iso, parseDecimal, invalid, str, json } = require('./helpers');
const { formType } = require('./mapping');
const { formType } = require('./mapping');
const KINDS = ['threshold', 'filing_type', 'new_document'];
const KINDS = ['threshold', 'filing_type', 'new_document'];
@@ -20,7 +20,7 @@
@@ -20,7 +20,7 @@
const authorship = require('openvibe-publishing/authorship');
const authorship = require('openvibe-publishing/authorship');
const ssr = require('openvibe-publishing/ssr');
const ssr = require('openvibe-publishing/ssr');
const { ApiError } = require('../http/errors');
const { ApiError } = require('../http/errors');
-const { iso, invalid } = require('./util');
+const { iso, invalid } = require('./helpers');
const WORKFLOW = 'trade.summarize_market_context';
const WORKFLOW = 'trade.summarize_market_context';
const ADVICE_RE = /\b(you should (buy|sell|hold|invest|short)|we recommend|i recommend|recommend(s|ed)? (buying|selling|holding)|strong (buy|sell)|(buy|sell|hold|outperform|underperform) rating|price target|guaranteed (return|profit)s?)\b/i;
const ADVICE_RE = /\b(you should (buy|sell|hold|invest|short)|we recommend|i recommend|recommend(s|ed)? (buying|selling|holding)|strong (buy|sell)|(buy|sell|hold|outperform|underperform) rating|price target|guaranteed (return|profit)s?)\b/i;
@@ -9,7 +9,7 @@
@@ -9,7 +9,7 @@
* A newly seen document is what new_document and filing_type alert rules trigger on (once per rule
* A newly seen document is what new_document and filing_type alert rules trigger on (once per rule
* and document: alerts.onDocument). Item revisions update the row and never trigger again.
* and document: alerts.onDocument). Item revisions update the row and never trigger again.
*/
*/
-const { newId, iso } = require('./util');
+const { newId, iso } = require('./helpers');
function createDocuments({ store, ctx }) {
function createDocuments({ store, ctx }) {
const { db } = store;
const { db } = store;
deleted file mode 100644
deleted file mode 100644
@@ -1,119 +0,0 @@
@@ -1,119 +0,0 @@
-'use strict';
-
-/**
- * Freshness per source: is what Trade shows from a source still current?
- *
- * A source is FRESH at time `now` only when its last successful fetch is known and younger than its
- * staleness window (Sources' stale_after_sec, else TRADE_DEFAULT_STALE_AFTER_SEC). Everything
- * else is stale:
- * - the last success is older than the window → stale since last success + window
- * - Sources never reported a success → stale since Trade first saw the source
- * - Trade has never heard of the source at all → freshness unknown (shown as stale)
- * Sources' own health status (failing, disabled, …) is shown next to it, but the verdict is
- * computed with Trade's clock, so an outage of Sources or of the sync makes data go stale on its own
- * rather than look current forever. Nothing is ever replaced by an invented value: a stale value is
- * shown with its timestamps and a "stale since" label, and a missing one is shown as missing.
- *
- * `trade.source.stale` and `trade.source.recovered` are emitted only on a transition (evaluate()),
- * so a replay or a second tick changes nothing.
- */
-const { iso } = require('./util');
-
-function createFreshness({ store, config, outbox }) {
- const { db } = store;
- const def = config.freshness.defaultStaleAfterSec;
- const q = {
- get: db.prepare('SELECT * FROM trade_source_status WHERE source_key = ?'),
- all: db.prepare('SELECT * FROM trade_source_status ORDER BY source_key'),
- ensure: db.prepare(`INSERT INTO trade_source_status (source_key, stale, stale_since, evaluated_at) VALUES (?, 1, ?, NULL) ON CONFLICT DO NOTHING`),
- report: db.prepare(`UPDATE trade_source_status SET name = COALESCE(@name, name), upstream_status = @status,
- last_success_at = CASE WHEN @last::bigint IS NULL THEN last_success_at WHEN last_success_at IS NULL OR @last > last_success_at THEN @last ELSE last_success_at END,
- stale_after_sec = COALESCE(@window, stale_after_sec), reported_at = @now,
- terms_note = COALESCE(@terms, terms_note), license_note = COALESCE(@license, license_note)
- WHERE source_key = @key`),
- retrieval: db.prepare(`UPDATE trade_source_status SET last_success_at = @at WHERE source_key = @key AND (last_success_at IS NULL OR last_success_at < @at)`),
- mark: db.prepare('UPDATE trade_source_status SET stale = @stale, stale_since = @since, evaluated_at = @now WHERE source_key = @key'),
- };
-
- /** Pure: the verdict for one status row at `now`. row may be null (unknown source). */
- function verdict(row, now) {
- if (!row) return { known: false, stale: true, staleSince: null, window: def, lastSuccessAt: null };
- const window = row.stale_after_sec || def;
- if (row.last_success_at == null) return { known: true, stale: true, staleSince: row.stale_since, window, lastSuccessAt: null };
- const until = row.last_success_at + window * 1000;
- return { known: true, stale: now > until, staleSince: now > until ? until : null, window, lastSuccessAt: row.last_success_at };
- }
-
- async function view(key, now = store.now()) {
- const row = await q.get.get(key) || null;
- const v = verdict(row, now);
- return {
- key,
- name: row ? row.name : null,
- known: v.known,
- status: !v.known ? 'unknown' : v.stale ? 'stale' : 'fresh',
- stale: v.stale,
- stale_since: iso(v.staleSince),
- last_success_at: iso(v.lastSuccessAt),
- stale_after_sec: v.window,
- upstream_status: row ? row.upstream_status : null,
- reported_at: row ? iso(row.reported_at) : null,
- terms_note: row ? row.terms_note : null,
- license_note: row ? row.license_note : null,
- };
- }
-
- async function ensure(key) { await q.ensure.run(key, store.now()); }
-
- /** Emit on a transition only. Inside the caller's transaction (or its own). */
- async function evaluate(key) {
- return await store.tx(async () => {
- const row = await q.get.get(key);
- if (!row) return null;
- const v = verdict(row, store.now());
- const first = row.evaluated_at == null;
- const was = Boolean(row.stale);
- const since = v.stale ? (v.staleSince != null ? v.staleSince : (row.stale_since || store.now())) : null;
- await q.mark.run({ key, stale: v.stale ? 1 : 0, since, now: store.now() });
- // First sight: a fresh source is not a recovery; a stale one is reported once.
- if (first ? !v.stale : v.stale === was) return null;
- const envelope = await outbox.emit({
- event_type: v.stale ? 'trade.source.stale' : 'trade.source.recovered',
- actor: { type: 'service', id: 'trade' }, visibility: 'public', priority: 'low',
- subject: { type: 'source', id: key },
- payload: {
- source_key: key, stale: v.stale, stale_since: iso(since), last_success_at: iso(v.lastSuccessAt),
- stale_after_sec: v.window, upstream_status: row.upstream_status || null, evaluated_at: iso(store.now()),
- },
- });
- return envelope;
- });
- }
-
- return {
- verdict,
- view,
- ensure,
- evaluate,
- async evaluateAll() { return (await Promise.all((await q.all.all()).map(async (r) => await evaluate(r.source_key)))).filter(Boolean); },
- async all(now = store.now()) { return (await Promise.all((await q.all.all()).map(async (r) => await view(r.source_key, now)))); },
-
- /** What Sources says about a source (its health block). Evaluates the transition. */
- async report(key, { name = null, status = null, lastSuccessAt = null, staleAfterSec = null, termsNote = null, licenseNote = null } = {}) {
- return await store.tx(async () => {
- await ensure(key);
- await q.report.run({ key, name, status, last: lastSuccessAt, window: staleAfterSec, now: store.now(), terms: termsNote, license: licenseNote });
- return await evaluate(key);
- });
- },
-
- /** A datum retrieved at `at` proves a successful fetch at that time. */
- async noteRetrieval(key, at) {
- await ensure(key);
- await q.retrieval.run({ key, at });
- return await evaluate(key);
- },
- };
-}
-
-module.exports = { createFreshness };
new file mode 100644
new file mode 100644
@@ -0,0 +1,16 @@
@@ -0,0 +1,16 @@
+'use strict';
+
+/**
+ * Trade's small shared helpers. The generic normalisers (folds, keys, amounts, instants, text and
+ * decimal parsing, ids) come from the ingest chassis (openvibe-publishing/ingest); only the two
+ * things the chassis does not own live here — id generation and the request-invalid error.
+ * Nothing here invents a value: a date the caller did not state stays null, never "now".
+ */
+const { normalize } = require('openvibe-publishing/ingest');
+const { ids } = require('openvibe-contracts');
+const { ApiError } = require('../http/errors');
+
+const newId = (prefix, now) => `${prefix}_${ids.ulid(now)}`;
+const invalid = (detail, code = 'request.invalid') => new ApiError(422, code, detail);
+
+module.exports = { ...normalize, newId, invalid };
@@ -2,8 +2,13 @@
@@ -2,8 +2,13 @@
/**
/**
* Indexability and OpenVibe.Search for instrument pages (openvibe-publishing/seo gate and
* Indexability and OpenVibe.Search for instrument pages (openvibe-publishing/seo gate and
- * index-hooks). The only public, indexable objects Trade has are instrument pages; watchlists,
- * alert rules and deliveries are private and never reach this module.
+ * index-hooks), on the shared publication glue (openvibe-publishing/publication).
+ *
+ * The gate facts, the document body and the canonical path are Trade's; the mechanics — the
+ * sequencer revision, the two events, the outbox write and the IndexNow ping — are createPublication,
+ * which stamps and enqueues on the caller's transaction handle so the document, its event and
+ * Trade's own change commit or roll back together. The only public, indexable objects Trade has are
+ * instrument pages; watchlists, alert rules and deliveries are private and never reach this module.
*
*
* Gate facts for an instrument page:
* Gate facts for an instrument page:
* state active → published, archived → unpublished
* state active → published, archived → unpublished
@@ -22,6 +27,7 @@ const seo = require('openvibe-publishing/seo');
@@ -22,6 +27,7 @@ const seo = require('openvibe-publishing/seo');
const hooks = require('openvibe-publishing/index-hooks');
const hooks = require('openvibe-publishing/index-hooks');
const authorship = require('openvibe-publishing/authorship');
const authorship = require('openvibe-publishing/authorship');
const ssr = require('openvibe-publishing/ssr');
const ssr = require('openvibe-publishing/ssr');
+const { createPublication } = require('openvibe-publishing/publication');
const OWNER = 'trade';
const OWNER = 'trade';
@@ -78,22 +84,17 @@ function createIndexing({ store, config, ctx, indexnow }) {
@@ -78,22 +84,17 @@ function createIndexing({ store, config, ctx, indexnow }) {
});
});
}
}
- /** Stamp and enqueue the Search document when it changed. Inside the caller's transaction. */
+ const publication = createPublication({
+ owner: OWNER, sequencer: store.sequencer, outbox: ctx.outbox, baseUrl: config.baseUrl, indexnow, now: store.now,
+ decide: (instrument, now) => decide(instrument, now),
+ document: (instrument, decision) => document(instrument, decision),
+ page: (instrument) => ctx.urls.instrument(instrument),
+ });
+
+ /** Stamp and enqueue the Search document when it changed, on the caller's handle (ambient tx). */
async function refresh(instrument, { traceparent } = {}) {
async function refresh(instrument, { traceparent } = {}) {
if (!instrument) return null;
if (!instrument) return null;
- const decision = await decide(instrument);
- const doc = await document(instrument, decision);
- const prev = await store.sequencer.current(OWNER, 'instrument', instrument.id);
- if (doc.deleted && prev == null) return null;
- const stamped = await store.sequencer.stamp(store.db, doc);
- if (prev != null && stamped.revision === prev) return null;
- // IndexNow: tell the engines a public, indexable instrument page appeared or changed — or
- // that a page which was in the index left it (retracted, archived, stale price). Never for a
- // page that was never indexable (a draft, private or noindex from the start).
- if (indexnow && indexnow.enabled && (decision.indexable || prev != null)) {
- indexnow.pingSoon([ctx.urls.instrument(instrument), ctx.urls.abs('/sitemap.xml')]);
- }
- return await ctx.outbox.emit(hooks.indexEvent({ document: stamped, now: store.now() }), { traceparent });
+ return await publication.sync(db, instrument, { traceparent });
}
}
return {
return {
new file mode 100644
new file mode 100644
@@ -0,0 +1,138 @@
@@ -0,0 +1,138 @@
+'use strict';
+
+/**
+ * Pull the trade category from OpenVibe.Sources and record what it states.
+ *
+ * The Sources client, the change cursor and the per-page pull loop are the ingest chassis
+ * (openvibe-publishing/ingest): one transaction per page (the page's writes and the cursor advance
+ * commit together), one savepoint per item, so a crash replays the page and a bad item is isolated.
+ * This module keeps only Trade's own half — item mapping, instrument resolution, observations,
+ * documents, freshness reporting and the trade_sync_state bookkeeping.
+ *
+ * 1. GET /api/v1/sources every trade source's health → freshness (report)
+ * 2. GET /api/v1/items?category=trade&after=<cursor>&include_removed=1, in change order:
+ * removed item → the document is hidden (the row stays)
+ * observation item → an observation (domain/observations.js; idempotent per item revision)
+ * filing / keyed item → a document (domain/documents.js; new → document alerts)
+ * no instrument matches → skipped (counted by reason; nothing is created for it)
+ *
+ * A failed call is recorded in trade_sync_state (last_error) and changes nothing else; freshness
+ * then decays on Trade's own clock, so pages show the data as stale instead of pretending.
+ */
+const { createChangeCursor, pullChanges, normalize } = require('openvibe-publishing/ingest');
+const { mapItem } = require('./mapping');
+const { parseTime, iso, json } = normalize;
+
+const PREFIX = 'trade';
+const CURSOR = 'sources.trade';
+
+/** The chassis outcome for a domain counter key: a removal is a tombstone, everything else applied. */
+const outcomeOf = (key) => (key === 'documents_removed' || key === 'removed_unknown' ? 'removed' : 'applied');
+
+function createIngest({ store, config, ctx, sources, log = console }) {
+ const { db } = store;
+ const cursor = createChangeCursor(db, { prefix: PREFIX, now: store.now });
+ const q = {
+ get: db.prepare('SELECT * FROM trade_sync_state WHERE name = ?'),
+ ensure: db.prepare("INSERT INTO trade_sync_state (name, cursor, counts) VALUES (?, 0, '{}') ON CONFLICT DO NOTHING"),
+ save: db.prepare('UPDATE trade_sync_state SET counts = @counts, last_ok_at = @ok, last_error = NULL WHERE name = @name'),
+ fail: db.prepare('UPDATE trade_sync_state SET last_error = ?, last_error_at = ? WHERE name = ?'),
+ };
+ let ensured = false; // the state row, written on the first run (not at construction: the factory stays synchronous)
+ const ensureState = async () => { if (!ensured) { await q.ensure.run(CURSOR); ensured = true; } };
+ let running = null;
+
+ async function reportHealth(key, h, extra = {}) {
+ if (!h || typeof h !== 'object') return;
+ await ctx.freshness.report(key, {
+ name: extra.name || null,
+ status: h.status || null,
+ lastSuccessAt: parseTime(h.last_success_at),
+ staleAfterSec: Number.isInteger(h.stale_after_sec) ? h.stale_after_sec : null,
+ termsNote: extra.terms_note || null,
+ licenseNote: extra.license_note || null,
+ });
+ }
+
+ async function resolveKeys(keys) {
+ if (keys.cik) {
+ const r = await ctx.instruments.resolve(keys.cik, { kind: 'cik' });
+ if (r.status === 'resolved') return r.instrument;
+ }
+ if (keys.symbol) {
+ const r = await ctx.instruments.resolve(keys.symbol, { kind: 'ticker' });
+ if (r.status === 'resolved') return r.instrument;
+ }
+ return null;
+ }
+
+ /** Apply one item. → a counter name. Inside the page transaction. */
+ async function applyItem(item) {
+ if (item.removed) return await ctx.documents.remove(item.id, { at: parseTime(item.removed.at), reason: item.removed.reason }) ? 'documents_removed' : 'removed_unknown';
+ const m = mapItem(item);
+ if (m.type === 'skip') return `skipped_${m.reason}`;
+ const instrument = await resolveKeys(m.keys);
+ if (!instrument) return 'skipped_no_matching_instrument';
+ if (instrument.status !== 'active') return 'skipped_archived_instrument';
+ if (m.type === 'observation') {
+ try {
+ const r = await ctx.observations.record(m.observation, instrument, { recordedBy: 'svc:trade' });
+ return r.created ? 'observations_created' : 'observations_unchanged';
+ } catch (err) {
+ if (err && (err.status === 409 || err.status === 422)) return `skipped_${String(err.code || 'invalid').replace(/\W+/g, '_')}`;
+ throw err;
+ }
+ }
+ const r = await ctx.documents.upsert(m.document, instrument);
+ return r.created ? 'documents_created' : r.changed ? 'documents_updated' : 'documents_unchanged';
+ }
+
+ async function runOnce() {
+ if (!sources.enabled) return { ok: false, skipped: 'disabled' };
+ await ensureState();
+ const summary = { pages: 0, counts: {} };
+ const counts = {};
+ try {
+ const list = await sources.listSources();
+ for (const s of (list.sources || [])) {
+ if (s && s.category === config.sources.category && s.key) await reportHealth(s.key, s.health, s);
+ }
+ const pull = await pullChanges({
+ db, cursor, source: sources, name: CURSOR,
+ maxPages: config.sources.maxPagesPerRun, pageSize: config.sources.pageLimit,
+ apply: async (item) => {
+ const key = await applyItem(item);
+ counts[key] = (counts[key] || 0) + 1;
+ return outcomeOf(key);
+ },
+ });
+ summary.pages = pull.pages;
+ summary.counts = counts;
+ await q.save.run({ name: CURSOR, counts: JSON.stringify(counts), ok: store.now() });
+ ctx.outbox.kick();
+ return { ok: true, ...summary };
+ } catch (err) {
+ await q.fail.run(String(err.message).slice(0, 500), store.now(), CURSOR);
+ log.warn('[Trade] Sources sync failed:', err.message);
+ return { ok: false, error: err.message, ...summary };
+ }
+ }
+
+ return {
+ CURSOR,
+ enabled: sources.enabled,
+ applyItem,
+ /** One run at a time; concurrent callers share it. */
+ run() {
+ if (!running) running = runOnce().finally(() => { running = null; });
+ return running;
+ },
+ async state() {
+ await ensureState();
+ const s = await q.get.get(CURSOR);
+ return { enabled: sources.enabled, cursor: await cursor.get(CURSOR), last_ok_at: iso(s.last_ok_at), last_error: s.last_error, last_error_at: iso(s.last_error_at), counts: json(s.counts, {}) };
+ },
+ };
+}
+
+module.exports = { createIngest, CURSOR };
@@ -20,7 +20,7 @@
@@ -20,7 +20,7 @@
* same way. A ticker or CIK names exactly one instrument (a unique index enforces it).
* same way. A ticker or CIK names exactly one instrument (a unique index enforces it).
*/
*/
const { ApiError } = require('../http/errors');
const { ApiError } = require('../http/errors');
-const { newId, invalid, str } = require('./util');
+const { newId, invalid, str } = require('./helpers');
const KINDS = ['equity', 'fund', 'index', 'currency', 'commodity', 'crypto', 'other'];
const KINDS = ['equity', 'fund', 'index', 'currency', 'commodity', 'crypto', 'other'];
const SYMBOL_RE = /^[A-Z0-9][A-Z0-9.-]{0,15}$/;
const SYMBOL_RE = /^[A-Z0-9][A-Z0-9.-]{0,15}$/;
@@ -16,7 +16,7 @@
@@ -16,7 +16,7 @@
* (kind 'record' for record items, else 'article').
* (kind 'record' for record items, else 'article').
* skip everything else, with a reason (counted in the sync state, never guessed at).
* skip everything else, with a reason (counted in the sync state, never guessed at).
*/
*/
-const { parseTime, parseDecimal } = require('./util');
+const { parseTime, parseDecimal } = require('./helpers');
const SEC_PATH_RE = /^https?:\/\/(?:www\.)?sec\.gov\/Archives\/edgar\/data\/(\d{1,10})\/(\d{18})(?:\/|$)/i;
const SEC_PATH_RE = /^https?:\/\/(?:www\.)?sec\.gov\/Archives\/edgar\/data\/(\d{1,10})\/(\d{18})(?:\/|$)/i;
const FORM_RE = /^[A-Z0-9][A-Z0-9 \-/.]{0,19}$/;
const FORM_RE = /^[A-Z0-9][A-Z0-9 \-/.]{0,19}$/;
@@ -18,7 +18,7 @@
@@ -18,7 +18,7 @@
* instrument's Search document (the gate's stale_price rule looks at monetary observations).
* instrument's Search document (the gate's stale_price rule looks at monetary observations).
*/
*/
const { ApiError } = require('../http/errors');
const { ApiError } = require('../http/errors');
-const { newId, iso, parseTime, parseDecimal, invalid, str, httpUrl } = require('./util');
+const { newId, iso, parseTime, parseDecimal, invalid, str, httpUrl } = require('./helpers');
const METRIC_RE = /^[A-Za-z][A-Za-z0-9_.:-]{0,79}$/;
const METRIC_RE = /^[A-Za-z][A-Za-z0-9_.:-]{0,79}$/;
const UNIT_RE = /^[A-Za-z][A-Za-z0-9/%._-]{0,19}$/;
const UNIT_RE = /^[A-Za-z][A-Za-z0-9/%._-]{0,19}$/;
@@ -5,7 +5,7 @@
@@ -5,7 +5,7 @@
* machine-readable representation always describe the same data (§32.5: no divergence).
* machine-readable representation always describe the same data (§32.5: no divergence).
*/
*/
const seo = require('openvibe-publishing/seo');
const seo = require('openvibe-publishing/seo');
-const { iso } = require('./util');
+const { iso } = require('./helpers');
const { DISCLAIMER } = require('./alerts');
const { DISCLAIMER } = require('./alerts');
function createReading({ store, ctx }) {
function createReading({ store, ctx }) {
new file mode 100644
new file mode 100644
@@ -0,0 +1,106 @@
@@ -0,0 +1,106 @@
+'use strict';
+
+/**
+ * Freshness per source: is what Trade shows from a source still current?
+ *
+ * The staleness verdict itself is the ingest chassis's generic rule (openvibe-publishing/ingest,
+ * freshness.verdict/view) with Trade's window as the parameter; this module keeps only what is
+ * Trade's own — the trade_source_status projection and the trade.source.stale|recovered transitions.
+ *
+ * A source is FRESH at time `now` only when its last successful fetch is known and younger than its
+ * staleness window (Sources' stale_after_sec, else TRADE_DEFAULT_STALE_AFTER_SEC). Everything
+ * else is stale:
+ * - the last success is older than the window → stale since last success + window
+ * - Sources never reported a success → stale since Trade first saw the source
+ * - Trade has never heard of the source at all → freshness unknown (shown as stale)
+ * Sources' own health status (failing, disabled, …) is shown next to it, but the verdict is
+ * computed with Trade's clock, so an outage of Sources or of the sync makes data go stale on its own
+ * rather than look current forever. Nothing is ever replaced by an invented value: a stale value is
+ * shown with its timestamps and a "stale since" label, and a missing one is shown as missing.
+ *
+ * `trade.source.stale` and `trade.source.recovered` are emitted only on a transition (evaluate()),
+ * so a replay or a second tick changes nothing.
+ */
+const { freshness: fresh, normalize } = require('openvibe-publishing/ingest');
+const { iso } = normalize;
+
+function createFreshness({ store, config, outbox }) {
+ const { db } = store;
+ const def = config.freshness.defaultStaleAfterSec;
+ const q = {
+ get: db.prepare('SELECT * FROM trade_source_status WHERE source_key = ?'),
+ all: db.prepare('SELECT * FROM trade_source_status ORDER BY source_key'),
+ ensure: db.prepare(`INSERT INTO trade_source_status (source_key, stale, stale_since, evaluated_at) VALUES (?, 1, ?, NULL) ON CONFLICT DO NOTHING`),
+ report: db.prepare(`UPDATE trade_source_status SET name = COALESCE(@name, name), upstream_status = @status,
+ last_success_at = CASE WHEN @last::bigint IS NULL THEN last_success_at WHEN last_success_at IS NULL OR @last > last_success_at THEN @last ELSE last_success_at END,
+ stale_after_sec = COALESCE(@window, stale_after_sec), reported_at = @now,
+ terms_note = COALESCE(@terms, terms_note), license_note = COALESCE(@license, license_note)
+ WHERE source_key = @key`),
+ retrieval: db.prepare(`UPDATE trade_source_status SET last_success_at = @at WHERE source_key = @key AND (last_success_at IS NULL OR last_success_at < @at)`),
+ mark: db.prepare('UPDATE trade_source_status SET stale = @stale, stale_since = @since, evaluated_at = @now WHERE source_key = @key'),
+ };
+
+ /** Pure: the verdict for one status row at `now`. row may be null (unknown source). */
+ function verdict(row, now) {
+ return fresh.verdict(row, now, def);
+ }
+
+ async function view(key, now = store.now()) {
+ const row = await q.get.get(key) || null;
+ return fresh.view(row, key, now, def);
+ }
+
+ async function ensure(key) { await q.ensure.run(key, store.now()); }
+
+ /** Emit on a transition only. Inside the caller's transaction (or its own). */
+ async function evaluate(key) {
+ return await store.tx(async () => {
+ const row = await q.get.get(key);
+ if (!row) return null;
+ const v = verdict(row, store.now());
+ const first = row.evaluated_at == null;
+ const was = Boolean(row.stale);
+ const since = v.stale ? (v.staleSince != null ? v.staleSince : (row.stale_since || store.now())) : null;
+ await q.mark.run({ key, stale: v.stale ? 1 : 0, since, now: store.now() });
+ // First sight: a fresh source is not a recovery; a stale one is reported once.
+ if (first ? !v.stale : v.stale === was) return null;
+ const envelope = await outbox.emit({
+ event_type: v.stale ? 'trade.source.stale' : 'trade.source.recovered',
+ actor: { type: 'service', id: 'trade' }, visibility: 'public', priority: 'low',
+ subject: { type: 'source', id: key },
+ payload: {
+ source_key: key, stale: v.stale, stale_since: iso(since), last_success_at: iso(v.lastSuccessAt),
+ stale_after_sec: v.window, upstream_status: row.upstream_status || null, evaluated_at: iso(store.now()),
+ },
+ });
+ return envelope;
+ });
+ }
+
+ return {
+ verdict,
+ view,
+ ensure,
+ evaluate,
+ async evaluateAll() { return (await Promise.all((await q.all.all()).map(async (r) => await evaluate(r.source_key)))).filter(Boolean); },
+ async all(now = store.now()) { return (await Promise.all((await q.all.all()).map(async (r) => await view(r.source_key, now)))); },
+
+ /** What Sources says about a source (its health block). Evaluates the transition. */
+ async report(key, { name = null, status = null, lastSuccessAt = null, staleAfterSec = null, termsNote = null, licenseNote = null } = {}) {
+ return await store.tx(async () => {
+ await ensure(key);
+ await q.report.run({ key, name, status, last: lastSuccessAt, window: staleAfterSec, now: store.now(), terms: termsNote, license: licenseNote });
+ return await evaluate(key);
+ });
+ },
+
+ /** A datum retrieved at `at` proves a successful fetch at that time. */
+ async noteRetrieval(key, at) {
+ await ensure(key);
+ await q.retrieval.run({ key, at });
+ return await evaluate(key);
+ },
+ };
+}
+
+module.exports = { createFreshness };
deleted file mode 100644
deleted file mode 100644
@@ -1,136 +0,0 @@
@@ -1,136 +0,0 @@
-'use strict';
-
-/**
- * Pull the trade category from OpenVibe.Sources and record what it states.
- *
- * 1. GET /api/v1/sources every trade source's health → freshness (report)
- * 2. GET /api/v1/items?category=trade&after=<cursor>&include_removed=1, in change order:
- * removed item → the document is hidden (the row stays)
- * observation item → an observation (domain/observations.js; idempotent per item revision)
- * filing / keyed item → a document (domain/documents.js; new → document alerts)
- * no instrument matches → skipped (counted by reason; nothing is created for it)
- * Each page is applied in one transaction together with the cursor, so a crash replays the
- * page and every effect is idempotent (UNIQUE keys on observations, documents and deliveries).
- * The page's per-source health block → freshness as well.
- *
- * A failed call is recorded in trade_sync_state (last_error) and changes nothing else; freshness
- * then decays on Trade's own clock, so pages show the data as stale instead of pretending.
- */
-const { mapItem } = require('./mapping');
-const { parseTime, iso, json } = require('./util');
-
-const CURSOR = 'sources.trade';
-
-function createSync({ store, config, ctx, sources, log = console }) {
- const { db } = store;
- const q = {
- get: db.prepare('SELECT * FROM trade_sync_state WHERE name = ?'),
- ensure: db.prepare("INSERT INTO trade_sync_state (name, cursor, counts) VALUES (?, 0, '{}') ON CONFLICT DO NOTHING"),
- advance: db.prepare('UPDATE trade_sync_state SET cursor = @cursor, counts = @counts WHERE name = @name'),
- ok: db.prepare('UPDATE trade_sync_state SET last_ok_at = ?, last_error = NULL WHERE name = ?'),
- fail: db.prepare('UPDATE trade_sync_state SET last_error = ?, last_error_at = ? WHERE name = ?'),
- };
- let ensured = false; // the cursor row, written on the first run (not at construction: the factory stays synchronous)
- const ensureCursor = async () => { if (!ensured) { await q.ensure.run(CURSOR); ensured = true; } };
- let running = null;
-
- async function reportHealth(key, h, extra = {}) {
- if (!h || typeof h !== 'object') return;
- await ctx.freshness.report(key, {
- name: extra.name || null,
- status: h.status || null,
- lastSuccessAt: parseTime(h.last_success_at),
- staleAfterSec: Number.isInteger(h.stale_after_sec) ? h.stale_after_sec : null,
- termsNote: extra.terms_note || null,
- licenseNote: extra.license_note || null,
- });
- }
-
- async function resolveKeys(keys) {
- if (keys.cik) {
- const r = await ctx.instruments.resolve(keys.cik, { kind: 'cik' });
- if (r.status === 'resolved') return r.instrument;
- }
- if (keys.symbol) {
- const r = await ctx.instruments.resolve(keys.symbol, { kind: 'ticker' });
- if (r.status === 'resolved') return r.instrument;
- }
- return null;
- }
-
- /** Apply one item. → a counter name. Inside the page transaction. */
- async function applyItem(item) {
- if (item.removed) return await ctx.documents.remove(item.id, { at: parseTime(item.removed.at), reason: item.removed.reason }) ? 'documents_removed' : 'removed_unknown';
- const m = mapItem(item);
- if (m.type === 'skip') return `skipped_${m.reason}`;
- const instrument = await resolveKeys(m.keys);
- if (!instrument) return 'skipped_no_matching_instrument';
- if (instrument.status !== 'active') return 'skipped_archived_instrument';
- if (m.type === 'observation') {
- try {
- const r = await ctx.observations.record(m.observation, instrument, { recordedBy: 'svc:trade' });
- return r.created ? 'observations_created' : 'observations_unchanged';
- } catch (err) {
- if (err && (err.status === 409 || err.status === 422)) return `skipped_${String(err.code || 'invalid').replace(/\W+/g, '_')}`;
- throw err;
- }
- }
- const r = await ctx.documents.upsert(m.document, instrument);
- return r.created ? 'documents_created' : r.changed ? 'documents_updated' : 'documents_unchanged';
- }
-
- async function runOnce() {
- if (!sources.enabled) return { ok: false, skipped: 'disabled' };
- await ensureCursor();
- const summary = { pages: 0, counts: {} };
- try {
- const list = await sources.sources();
- for (const s of (list.sources || [])) {
- if (s && s.category === config.sources.category && s.key) await reportHealth(s.key, s.health, s);
- }
- for (let page = 0; page < config.sources.maxPagesPerRun; page++) {
- const state = await q.get.get(CURSOR);
- const data = await sources.items({ after: state.cursor, limit: config.sources.pageLimit });
- const items = Array.isArray(data.items) ? data.items : [];
- await store.tx(async () => {
- const counts = json((await q.get.get(CURSOR)).counts, {});
- for (const item of items) {
- const k = await applyItem(item);
- counts[k] = (counts[k] || 0) + 1;
- summary.counts[k] = (summary.counts[k] || 0) + 1;
- }
- for (const [key, h] of Object.entries(data.sources || {})) await reportHealth(key, h);
- const next = Number.isInteger(data.next_after) ? data.next_after : state.cursor;
- await q.advance.run({ name: CURSOR, cursor: Math.max(next, state.cursor), counts: JSON.stringify(counts) });
- });
- summary.pages++;
- if (!data.more || !items.length) break;
- }
- await q.ok.run(store.now(), CURSOR);
- ctx.outbox.kick();
- return { ok: true, ...summary };
- } catch (err) {
- await q.fail.run(String(err.message).slice(0, 500), store.now(), CURSOR);
- log.warn('[Trade] Sources sync failed:', err.message);
- return { ok: false, error: err.message, ...summary };
- }
- }
-
- return {
- CURSOR,
- enabled: sources.enabled,
- applyItem,
- /** One run at a time; concurrent callers share it. */
- run() {
- if (!running) running = runOnce().finally(() => { running = null; });
- return running;
- },
- async state() {
- await ensureCursor();
- const s = await q.get.get(CURSOR);
- return { enabled: sources.enabled, cursor: s.cursor, last_ok_at: iso(s.last_ok_at), last_error: s.last_error, last_error_at: iso(s.last_error_at), counts: json(s.counts, {}) };
- },
- };
-}
-
-module.exports = { createSync };
deleted file mode 100644
deleted file mode 100644
@@ -1,68 +0,0 @@
@@ -1,68 +0,0 @@
-'use strict';
-
-/** Small shared helpers: ids, times, decimals, validation errors. Nothing here invents a value. */
-const { ids } = require('openvibe-contracts');
-const { ApiError } = require('../http/errors');
-
-const newId = (prefix, now) => `${prefix}_${ids.ulid(now)}`;
-
-/** epoch ms → ISO string, or null. */
-const iso = (ms) => (ms == null ? null : new Date(ms).toISOString());
-
-/**
- * A date the caller STATES: an ISO 8601 date-time with a zone (…Z or ±hh:mm), or epoch ms as a
- * number. Anything else (including a missing value) is null — never "now".
- */
-function parseTime(v) {
- if (v == null || v === '') return null;
- if (typeof v === 'number') return Number.isFinite(v) && v > 0 ? Math.floor(v) : null;
- const s = String(v).trim();
- if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:?\d{2})$/i.test(s)) return null;
- const t = Date.parse(s);
- return Number.isFinite(t) ? t : null;
-}
-
-const DECIMAL_RE = /^-?(\d{1,20})(\.\d{1,12})?([eE][-+]?\d{1,3})?$/;
-
-/** A decimal as stated: → { text, num } or null. Numbers are kept as their shortest string form. */
-function parseDecimal(v) {
- if (typeof v === 'number') {
- if (!Number.isFinite(v)) return null;
- return { text: String(v), num: v };
- }
- if (typeof v !== 'string') return null;
- const s = v.trim();
- if (!DECIMAL_RE.test(s)) return null;
- const num = Number(s);
- return Number.isFinite(num) ? { text: s, num } : null;
-}
-
-const invalid = (detail, code = 'request.invalid') => new ApiError(422, code, detail);
-
-function str(v, max, name, { required = false } = {}) {
- if (v == null || v === '') {
- if (required) throw invalid(`${name} is required`);
- return null;
- }
- if (typeof v !== 'string') throw invalid(`${name} must be a string`);
- const s = v.trim();
- if (!s && required) throw invalid(`${name} is required`);
- if (s.length > max) throw invalid(`${name} is longer than ${max} characters`);
- return s || null;
-}
-
-function httpUrl(v, name) {
- if (v == null || v === '') return null;
- let u;
- try { u = new URL(String(v)); } catch { throw invalid(`${name} must be an absolute URL`); }
- if (u.protocol !== 'https:' && u.protocol !== 'http:') throw invalid(`${name} must be http(s)`);
- return u.toString();
-}
-
-/** Parse JSON stored by this service; a broken value is an empty default, never a crash. */
-function json(text, def) {
- if (text == null) return def;
- try { return JSON.parse(text); } catch { return def; }
-}
-
-module.exports = { newId, iso, parseTime, parseDecimal, invalid, str, httpUrl, json };
@@ -7,7 +7,7 @@
@@ -7,7 +7,7 @@
* an event, and every response that shows one is Cache-Control: private, no-store.
* an event, and every response that shows one is Cache-Control: private, no-store.
*/
*/
const { ApiError } = require('../http/errors');
const { ApiError } = require('../http/errors');
-const { newId, iso, str } = require('./util');
+const { newId, iso, str } = require('./helpers');
function createWatchlists({ store, config, ctx }) {
function createWatchlists({ store, config, ctx }) {
const { db } = store;
const { db } = store;
new file mode 100644
new file mode 100644
@@ -0,0 +1,41 @@
@@ -0,0 +1,41 @@
+'use strict';
+
+/**
+ * POST /internal/events — signed OpenVibe.Events webhook deliveries (subscription topic sources.*).
+ *
+ * The signature check, the ±300 s window and the exactly-once inbox are the ingest chassis
+ * (openvibe-publishing/ingest, createEventConsumer); this route only maps its result onto HTTP and
+ * wakes the Sources ingest. A webhook is a wake-up, not durable truth: a sources.item.* or
+ * sources.fetch.failed event makes the cursor-based ingest run now, and the ingest reads the items
+ * themselves from the Sources API. The route answers 404 while no secret is configured.
+ */
+const express = require('express');
+const { createEventConsumer } = require('openvibe-publishing/ingest');
+const { http } = require('openvibe-contracts');
+
+const WAKE_RE = /^sources\.(item\.(created|updated|removed)|fetch\.failed)$/;
+
+function createWebhook({ store, config, ingest, log = console }) {
+ const router = express.Router();
+ const consumer = createEventConsumer({ db: store.db, secrets: config.events.webhookSecret, consumer: 'sources', now: store.now });
+
+ router.post('/internal/events', express.raw({ type: '*/*', limit: '1mb' }), async function receiveEventsDelivery(req, res, next) {
+ if (!config.events.webhookSecret) return http.sendProblem(res, 404, 'route.not_found', { detail: 'Not found', ctx: req.ov });
+ const raw = Buffer.isBuffer(req.body) ? req.body : Buffer.from('');
+ let r;
+ try {
+ r = await consumer.apply(raw, req.headers, async (e) => {
+ const wake = WAKE_RE.test(e.event_type);
+ if (wake) ingest.run().catch((err) => log.warn('[Trade] ingest after webhook failed:', err.message));
+ return wake;
+ });
+ } catch (err) { return next(err); }
+ if (r.status === 401) return http.sendProblem(res, 401, 'webhook.signature_invalid', { detail: 'The delivery signature does not verify', ctx: req.ov });
+ if (r.status !== 200) return http.sendProblem(res, 400, 'webhook.malformed', { detail: 'Not an event envelope', ctx: req.ov });
+ res.status(200).json({ accepted: true, duplicate: r.duplicate, sync: r.duplicate ? false : Boolean(r.outcome) });
+ });
+
+ return { router, consumer };
+}
+
+module.exports = { createWebhook };
deleted file mode 100644
deleted file mode 100644
@@ -1,39 +0,0 @@
@@ -1,39 +0,0 @@
-'use strict';
-
-/**
- * POST /internal/events — signed OpenVibe.Events webhook deliveries (subscription topic sources.*).
- *
- * The body is verified against X-OpenVibe-Signature (HMAC-SHA256 of the raw body with
- * TRADE_EVENTS_WEBHOOK_SECRET) before anything is read; a bad or missing signature is 401 and the
- * route answers 404 while no secret is configured. Each event is recorded once in the SDK inbox
- * (idempotency_receipts, consumer 'sources'); a webhook is a wake-up, not durable truth: a
- * sources.item.* or sources.fetch.failed event makes the cursor-based sync run now, and the sync
- * reads the items themselves from the Sources API.
- */
-const express = require('express');
-const { parseDelivery, createPgInbox } = require('openvibe-sdk/events');
-const { http } = require('openvibe-contracts');
-
-function createWebhook({ store, config, sync, log = console }) {
- const router = express.Router();
- const inbox = createPgInbox(store.db, { now: store.now }); // idempotency_receipts: migrations/0001_initial.sql
-
- router.post('/internal/events', express.raw({ type: '*/*', limit: '1mb' }), async function receiveEventsDelivery(req, res, next) {
- if (!config.events.webhookSecret) return http.sendProblem(res, 404, 'route.not_found', { detail: 'Not found', ctx: req.ov });
- const raw = Buffer.isBuffer(req.body) ? req.body : Buffer.from('');
- // Signature v2 only: HMAC over "<t>.<raw body>" with t within ±300 s; a v1-only (v2 stripped) or stale delivery is refused.
- const delivery = parseDelivery(raw, req.headers, config.events.webhookSecret, { requireV2: true });
- if (!delivery) return http.sendProblem(res, 401, 'webhook.signature_invalid', { detail: 'The delivery signature does not verify', ctx: req.ov });
- const e = delivery.event;
- if (typeof e.event_id !== 'string' || typeof e.event_type !== 'string') return http.sendProblem(res, 400, 'webhook.malformed', { detail: 'Not an event envelope', ctx: req.ov });
- let r;
- try { r = await inbox.once('sources', e.event_id, async () => true); } catch (err) { return next(err); }
- const wake = !r.duplicate && /^sources\.(item\.(created|updated|removed)|fetch\.failed)$/.test(e.event_type);
- if (wake) sync.run().catch((err) => log.warn('[Trade] sync after webhook failed:', err.message));
- res.status(200).json({ accepted: true, duplicate: r.duplicate, sync: wake });
- });
-
- return { router, inbox };
-}
-
-module.exports = { createWebhook };
@@ -3,7 +3,7 @@
@@ -3,7 +3,7 @@
* Background work in the Trade process (two timers):
* Background work in the Trade process (two timers):
*
*
* Sources sync every TRADE_SYNC_INTERVAL_MS (and early when a signed webhook arrives):
* Sources sync every TRADE_SYNC_INTERVAL_MS (and early when a signed webhook arrives):
- * domain/sync.js pulls the trade category; idempotent per item revision
+ * domain/ingest.js pulls the trade category; idempotent per item revision
* freshness every TRADE_FRESHNESS_INTERVAL_MS: re-evaluate every source on Trade's clock
* freshness every TRADE_FRESHNESS_INTERVAL_MS: re-evaluate every source on Trade's clock
* (trade.source.stale / recovered on transitions only) and re-send any instrument
* (trade.source.stale / recovered on transitions only) and re-send any instrument
* Search document whose gate decision changed (e.g. a price observation aged
* Search document whose gate decision changed (e.g. a price observation aged
@@ -94,7 +94,7 @@ const MIN = 60e3;
@@ -94,7 +94,7 @@ const MIN = 60e3;
assert.strictEqual((await bobs()).length, 1, 'new_document fired, filing_type(8-K) did not for a 10-Q');
assert.strictEqual((await bobs()).length, 1, 'new_document fired, filing_type(8-K) did not for a 10-Q');
t.sources.putItem({ id: item.id, canonical_url: url, title: 'ACME CORP (amended title)', summary: '10-Q', published_at: iso(t.T0), retrieved_at: iso(t.clock.now()) });
t.sources.putItem({ id: item.id, canonical_url: url, title: 'ACME CORP (amended title)', summary: '10-Q', published_at: iso(t.T0), retrieved_at: iso(t.clock.now()) });
await t.ctx.sync.run();
await t.ctx.sync.run();
- await t.ctx.store.db.prepare("UPDATE trade_sync_state SET cursor = 0 WHERE name = 'sources.trade'").run();
+ await t.ctx.store.db.prepare("UPDATE trade_ingest_cursor SET cursor = 0 WHERE name = 'sources.trade'").run();
await t.ctx.sync.run();
await t.ctx.sync.run();
assert.strictEqual((await bobs()).length, 1, 'item revision and page replay: no second delivery');
assert.strictEqual((await bobs()).length, 1, 'item revision and page replay: no second delivery');
assert.strictEqual((await t.get('/i/ACME.json')).json().documents[0].title, 'ACME CORP (amended title)');
assert.strictEqual((await t.get('/i/ACME.json')).json().documents[0].title, 'ACME CORP (amended title)');
new file mode 100644
new file mode 100644
@@ -0,0 +1,149 @@
@@ -0,0 +1,149 @@
+'use strict';
+/**
+ * T9 J6 — Trade on the openvibe-publishing 1.1.0 chassis.
+ *
+ * 1. the local copies the brief lists (domain/sync.js, domain/util.js, domain/freshness.js,
+ * events/webhook.js, clients/sources.js) are no longer required by any source or test file: the
+ * Sources client, the change cursor/pull loop, the signed event consumer, the normalisers, the
+ * freshness verdict and the publication glue now come from openvibe-publishing/ingest and
+ * /publication. (`rm` is denied in this harness, so the files are kept on disk but unused.)
+ * 2. a dry-run ingest from a captured Sources fixture produces the same domain rows as the golden
+ * captured from the pre-conversion code.
+ * 3. the Search document the publication glue emits validates against search.index-document@1 and
+ * is public and indexable.
+ * 4. the outbox row and the state change share one transaction: rolled back together, committed
+ * together.
+ */
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const contracts = require('openvibe-contracts');
+const { boot, check, done } = require('./helpers/boot');
+
+const ROOT = path.join(__dirname, '..');
+const REPLACED = ['server/domain/sync.js', 'server/domain/util.js', 'server/domain/freshness.js', 'server/events/webhook.js', 'server/clients/sources.js'];
+const WORDS = (n) => Array.from({ length: n }, (_, i) => `word${i}`).join(' ');
+const isoOf = (v) => (v == null ? null : new Date(Number(v)).toISOString());
+
+/** The golden captured from the pre-conversion code (id fields normalised away). */
+const GOLDEN = {
+ run: { ok: true, counts: { documents_created: 1, observations_created: 1, skipped_no_observation_time: 1 } },
+ state: { enabled: true, cursor: 3, counts: { documents_created: 1, observations_created: 1, skipped_no_observation_time: 1 } },
+ docs: [{
+ kind: 'filing', form_type: '10-Q', title: 'APPLE INC (0000320193) (Filer)', filer_name: 'APPLE INC',
+ cik: '0000320193', accession: '0000320193-26-000105',
+ published_at: '2026-09-22T11:30:00.000Z', retrieved_at: '2026-09-22T11:55:00.000Z', removed_reason: null,
+ }],
+ obs: [{
+ metric: 'price.close', value: '231.40', unit: 'USD', currency: 'USD',
+ observed_at: '2026-09-21T20:00:00.000Z', retrieved_at: '2026-09-22T11:55:00.000Z', source_key: 'test-quotes',
+ }],
+};
+
+function jsFiles(dir) {
+ const out = [];
+ for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+ if (e.name === 'node_modules' || e.name === 'data') continue;
+ const p = path.join(dir, e.name);
+ if (e.isDirectory()) out.push(...jsFiles(p));
+ else if (e.name.endsWith('.js')) out.push(p);
+ }
+ return out;
+}
+
+(async () => {
+ await check('the replaced local copies are unrequired; the chassis is used instead', async () => {
+ const gone = new Set(REPLACED.map((p) => path.join(ROOT, p)));
+ const requirers = [];
+ for (const f of jsFiles(path.join(ROOT, 'server')).concat(jsFiles(path.join(ROOT, 'test')))) {
+ if (gone.has(f)) continue;
+ const src = fs.readFileSync(f, 'utf8');
+ for (const m of src.matchAll(/require\(\s*['"]([^'"]+)['"]\s*\)/g)) {
+ if (!m[1].startsWith('.')) continue;
+ const target = path.resolve(path.dirname(f), m[1]);
+ if (gone.has(target) || gone.has(`${target}.js`)) requirers.push(`${path.relative(ROOT, f)} → ${m[1]}`);
+ }
+ }
+ assert.deepStrictEqual(requirers, [], `still required: ${requirers.join(', ')}`);
+ // The chassis entry points are what the app requires now.
+ const app = fs.readFileSync(path.join(ROOT, 'server', 'app.js'), 'utf8');
+ assert.ok(app.includes("require('openvibe-publishing/ingest')"), 'the Sources client comes from the chassis');
+ assert.ok(app.includes("require('./domain/ingest')"), 'the ingest composition');
+ assert.ok(app.includes("require('./events/consumer')"), 'the signed event consumer route');
+ });
+
+ await check('a dry-run ingest from the captured Sources fixture reproduces the golden domain rows', async () => {
+ const t = await boot();
+ const ret = t.iso(t.T0 - 5 * 60e3);
+ await t.instrument({ symbol: 'AAPL', name: 'Apple Inc.', cik: '320193', exchange: 'NASDAQ' });
+ t.sources.setSource('sec-xbrl-filings', { name: 'SEC EDGAR: latest XBRL filings', lastSuccessAt: ret, staleAfterSec: 2700 });
+ t.sources.putItem({
+ canonical_url: 'https://www.sec.gov/Archives/edgar/data/320193/000032019326000105/0000320193-26-000105-index.htm',
+ title: 'APPLE INC (0000320193) (Filer)', summary: '10-Q', published_at: '2026-09-22T11:30:00.000Z', retrieved_at: ret,
+ });
+ t.sources.putItem({
+ source_key: 'test-quotes', kind: 'observation', canonical_url: 'https://quotes.example/aapl', title: 'AAPL close',
+ retrieved_at: ret, fields: { symbol: 'AAPL', metric: 'price.close', value: '231.40', unit: 'USD', currency: 'USD', observed_at: '2026-09-21T20:00:00Z' },
+ });
+ t.sources.putItem({
+ source_key: 'test-quotes', kind: 'observation', canonical_url: 'https://quotes.example/aapl2', title: 'AAPL open (no time)',
+ retrieved_at: ret, fields: { symbol: 'AAPL', metric: 'price.open', value: '229.00', unit: 'USD', currency: 'USD' },
+ });
+ const r = await t.ctx.sync.run();
+ const state = await t.ctx.sync.state();
+ const docs = (await t.ctx.store.db.prepare(
+ 'SELECT kind, form_type, title, filer_name, cik, accession, published_at, retrieved_at, removed_reason FROM trade_source_documents ORDER BY source_item_id').all())
+ .map((d) => ({ ...d, published_at: isoOf(d.published_at), retrieved_at: isoOf(d.retrieved_at) }));
+ const obs = (await t.ctx.store.db.prepare(
+ 'SELECT metric, value, unit, currency, observed_at, retrieved_at, source_key FROM trade_market_observations ORDER BY metric').all())
+ .map((o) => ({ ...o, observed_at: isoOf(o.observed_at), retrieved_at: isoOf(o.retrieved_at) }));
+ assert.deepStrictEqual({ run: { ok: r.ok, counts: r.counts }, state: { enabled: state.enabled, cursor: state.cursor, counts: state.counts }, docs, obs }, GOLDEN);
+ await t.close();
+ });
+
+ await check('the emitted Search document validates against search.index-document@1 (public, indexable)', async () => {
+ const t = await boot();
+ const inst = await t.instrument({ symbol: 'CHS', name: 'Chassis Corp' });
+ const rev = await t.ctx.context.propose({ kind: 'user', subject: t.editor.subject, editor: true }, inst, { body: WORDS(80) });
+ await t.ctx.context.publish(inst, rev.revision.number);
+ const upserted = await t.events('trade.index_document.upserted');
+ assert.strictEqual(upserted.length, 1, 'exactly one upserted document');
+ const doc = upserted[0].payload;
+ const v = contracts.validate('search.index-document@1', doc);
+ assert.ok(v.valid, JSON.stringify(v.errors));
+ assert.strictEqual(doc.owner, 'trade');
+ assert.strictEqual(doc.type, 'instrument');
+ assert.strictEqual(doc.visibility, 'public');
+ assert.strictEqual(doc.indexability.decision, 'index');
+ assert.strictEqual(doc.canonical_url, 'https://openvibe.trade/i/CHS');
+ assert.ok(Array.isArray(doc.provenance), 'provenance is a list of references');
+ await t.close();
+ });
+
+ await check('the outbox row and the state change share one transaction', async () => {
+ const t = await boot();
+ const { db } = t.ctx.store;
+ const inst = await t.instrument({ symbol: 'TXN', name: 'Txn Corp' });
+ const rev = await t.ctx.context.propose({ kind: 'user', subject: t.editor.subject, editor: true }, inst, { body: WORDS(80) });
+ await t.ctx.context.publish(inst, rev.revision.number);
+ const before = (await t.events('trade.index_document.upserted')).length;
+
+ await assert.rejects(t.ctx.store.tx(async () => {
+ await db.prepare("UPDATE trade_instruments SET name = 'Txn Changed' WHERE id = ?").run(inst.id);
+ await t.ctx.indexing.refresh(await t.ctx.instruments.get(inst.id));
+ throw new Error('boom');
+ }));
+ assert.strictEqual((await db.prepare('SELECT name FROM trade_instruments WHERE id = ?').get(inst.id)).name, 'Txn Corp', 'the state change rolled back');
+ assert.strictEqual((await t.events('trade.index_document.upserted')).length, before, 'the outbox row did not survive the rollback');
+
+ await t.ctx.store.tx(async () => {
+ await db.prepare("UPDATE trade_instruments SET name = 'Txn Changed' WHERE id = ?").run(inst.id);
+ await t.ctx.indexing.refresh(await t.ctx.instruments.get(inst.id));
+ });
+ assert.strictEqual((await db.prepare('SELECT name FROM trade_instruments WHERE id = ?').get(inst.id)).name, 'Txn Changed');
+ assert.strictEqual((await t.events('trade.index_document.upserted')).length, before + 1, 'committed together');
+ await t.close();
+ });
+
+ done();
+})().catch((err) => { console.error(err); process.exit(1); });
@@ -64,7 +64,7 @@ const { mapItem } = require('../server/domain/mapping');
@@ -64,7 +64,7 @@ const { mapItem } = require('../server/domain/mapping');
const docs = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM trade_source_documents').get()).n;
const docs = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM trade_source_documents').get()).n;
const obs = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM trade_market_observations').get()).n;
const obs = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM trade_market_observations').get()).n;
const evts = (await t.events()).length;
const evts = (await t.events()).length;
- await t.ctx.store.db.prepare("UPDATE trade_sync_state SET cursor = 0 WHERE name = 'sources.trade'").run();
+ await t.ctx.store.db.prepare("UPDATE trade_ingest_cursor SET cursor = 0 WHERE name = 'sources.trade'").run();
const r = await t.ctx.sync.run();
const r = await t.ctx.sync.run();
assert.strictEqual(r.ok, true);
assert.strictEqual(r.ok, true);
assert.strictEqual((await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM trade_source_documents').get()).n, docs);
assert.strictEqual((await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM trade_source_documents').get()).n, docs);
@@ -136,7 +136,8 @@ const { mapItem } = require('../server/domain/mapping');
@@ -136,7 +136,8 @@ const { mapItem } = require('../server/domain/mapping');
await check('Trade asks Sources with its own service token for exactly sources.item.read and sources.source.read', async () => {
await check('Trade asks Sources with its own service token for exactly sources.item.read and sources.source.read', async () => {
const g = t.network.grants.filter((x) => x.audience === 'openvibe.sources');
const g = t.network.grants.filter((x) => x.audience === 'openvibe.sources');
assert.ok(g.length >= 1);
assert.ok(g.length >= 1);
- assert.deepStrictEqual(g[0].scope.split(' ').sort(), ['sources.item.read', 'sources.source.read']);
+ const scopes = [...new Set(g.flatMap((x) => String(x.scope || '').split(/\s+/).filter(Boolean)))].sort();
+ assert.deepStrictEqual(scopes, ['sources.item.read', 'sources.source.read']);
assert.ok(t.sources.calls.some((c) => /category=trade/.test(c.url) && /include_removed=1/.test(c.url)));
assert.ok(t.sources.calls.some((c) => /category=trade/.test(c.url) && /include_removed=1/.test(c.url)));
});
});