openvibers/openvibe.sources Commits
account-data: a merchant's own claim, with the guard on every person column (#146)
Changes
@@@ -576,6 +576,6 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [<!-- versions:start -->- openvibe-contracts: v0.224.0--- openvibe-sdk: v0.67.0++- openvibe-sdk: v0.74.0- openvibe-shared: v4.48.0<!-- versions:end -->@@@ -8,7 +8,7 @@"port": 4720,"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, the merchant opt-in pages, health and JSON 404s; the JSON API is loopback-only; README \"Public host\")","contracts": "openvibe-contracts v0.224.0",-- "sdk": "openvibe-sdk v0.67.0",++ "sdk": "openvibe-sdk v0.74.0","shared": "openvibe-shared v4.48.0","capabilities": ["sources.source.read",nnew file mode 100644@@@ -0,0 +1,13 @@++-- phase: expand++-- OpenVibe.Sources: the receipts of the ADR-033 account export and deletion deliveries this service applied++-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.++-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.++++CREATE TABLE IF NOT EXISTS account_data_events (++ id TEXT PRIMARY KEY,++ kind TEXT NOT NULL,++ subject TEXT NOT NULL,++ outcome JSONB,++ applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),++ sent_at TIMESTAMPTZ++);@@@ -13,7 +13,7 @@"express": "^4.21.2","fast-xml-parser": "^4.5.3","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.224.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0","pg": "^8.23.0"},@@@ -1060,9 +1060,9 @@}},"node_modules/openvibe-sdk": {-- "version": "0.67.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",-- "integrity": "sha512-s7NzjaNcowVcOIiup2z0ze+xsq/qnr++3pHSI8JzirqJPQ5JsBSUF5RkjOHyfk9bRKlmhY0fsNS6x4dsYylyqw==",++ "version": "0.74.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",++ "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==","license": "MIT","bin": {"openvibe-download": "bin/openvibe-download.js"@@@ -31,7 +31,7 @@"express": "^4.21.2","fast-xml-parser": "^4.5.3","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.224.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0","pg": "^8.23.0"},nnew file mode 100644@@@ -0,0 +1,47 @@++'use strict';++/**++ * Account export and deletion → Sources (ADR-033; openvibe-sdk/account-data).++ *++ * Sources keeps the outside web's records: registries, items, fetch runs and robots policies. None of them names a++ * Network person; a source is keyed by its host and the item by its URL.++ *++ * The one row that does is the merchant opt-in (merchant_sources): the signed-in merchant's own claim, with the++ * challenge they proved, their name and the permission note that names them. Their claim goes with the account, and++ * the source it created stops being fetched, because the permission it rests on is gone. The note goes too: it holds++ * the person's name in free text, which a kept row may not (ADR-033).++ *++ * So a person's export part carries their merchant claims (never the challenge token) and their deletion confirms++ * what it erased. Telemetry samples that name a person are covered by the SDK itself (telemetry.json, then subject NULL).++ */++const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');++++const TABLES = [];++++// The merchant's own claim. The challenge token is never exported; their name and subject are in Network's part.++const CLAIM_COLUMNS = ['id', 'store_name', 'domain', 'method', 'listing_url', 'listing_kind', 'meta_url', 'status', 'source_key',++ 'terms_note', 'verified_at', 'verified_by', 'verification_detail', 'last_checked_at', 'check_count', 'created_at', 'withdrawn_at'];++++/** The claims the person made, without the challenge that proves them. */++async function extraExport(db, subject) {++ const claims = await db.many(`SELECT ${CLAIM_COLUMNS.join(', ')} FROM merchant_sources WHERE owner_sub = $1 ORDER BY created_at DESC LIMIT 5000`, [subject]);++ return claims.length ? [{ name: 'merchant-sources.json', content: claims }] : [];++}++++/** The claim goes, and the source it created stops being fetched (its note named the person). */++async function extraErase(t, subjects, counts) {++ const rows = await t.many('SELECT id, source_key FROM merchant_sources WHERE owner_sub = ANY($1::text[])', [subjects]);++ if (!rows.length) return;++ const keys = rows.map((r) => r.source_key).filter(Boolean);++ if (keys.length) {++ counts.add(counts.retained, 'sources_disabled', await t.exec(++ `UPDATE sources SET enabled = 0, terms_note = 'Merchant opt-in withdrawn: the merchant closed their account.', updated_at = $2 WHERE key = ANY($1::text[])`,++ [keys, Date.now()]));++ }++ counts.add(counts.erased, 'merchant_sources', await t.exec('DELETE FROM merchant_sources WHERE owner_sub = ANY($1::text[])', [subjects]));++}++++function create({ db, log = console } = {}) {++ return createAccountData({ db, service: 'sources', tables: TABLES, extraExport, extraErase, log });++}++++module.exports = { create, TABLES, TOPICS, extraExport, extraErase };@@@ -69,7 +69,7 @@ Source code: <a href="https://github.com/OpenVibers/OpenVibe.Sources">OpenVibers</html>`;--function createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log = console, limitsNow = null, merchant = null, merchantAuth = null }) {++function createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log = console, limitsNow = null, merchant = null, merchantAuth = null, accountData = null, accountSend = null }) {const app = express();app.disable('x-powered-by');app.set('trust proxy', 'loopback');@@@ -80,6 +80,9 @@ function createApp({ config, db, registry, items, ingest, scheduler, auth, outboapp.use(require('openvibe-sdk/service').telemetryMiddleware); // request telemetry: a no-op until server/index.js calls telemetry.init()registerSourcesGauges(metrics.registry, { db, sources: registry, ingest, outbox, now });app.use(http.middleware());++ // The account export/deletion consumer (ADR-033), signed and loopback only. No body parser before it: the v2++ // signature covers the raw body.++ if (accountData) app.post('/internal/events', accountData.consumer({ secrets: [config.events.secret].filter((s) => s.length >= 32), send: accountSend }));// One W3C trace across services (openvibe-shared/trace): calls made while serving a request carry its traceparent.require('openvibe-shared/trace').install(app);app.use((req, res, next) => {@@@ -85,6 +85,9 @@ function load(env = process.env) {events: {url: strip(env.EVENTS_URL || ''),relayIntervalMs: int(env.EVENTS_RELAY_INTERVAL_MS, 2000),++ // The account-data consumer (POST /internal/events): the signing secret Network's deliveries use. Unset:++ // the route is off and Network's account export/deletion does not wait for Sources.++ secret: strip(env.SOURCES_EVENTS_SECRET || ''),},// Ingestion@@@ -28,6 +28,8 @@ const { createAuth } = require('./auth');const { createMerchant } = require('./merchant');const { createMerchantAuth } = require('./merchant-auth');const { createApp } = require('./app');++const accountDataLib = require('./account-data');++const { createNetworkSender, startSubscriptions } = require('openvibe-sdk/account-data');async function start({ config, db: givenDb = null, now = () => Date.now(), fetchImpl = globalThis.fetch, lookupImpl, merchantResolver = null, log = console, listen = true, limitsNow = null } = {}) {config = config || load();@@@ -58,7 +60,9 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetchconst auth = createAuth({ config, log, jwks: keys });const merchant = createMerchant({ db, config, registry, fetcher, resolver: merchantResolver, now, log });const merchantAuth = createMerchantAuth({ config, auth, log, fetchImpl });-- const app = createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log, limitsNow, merchant, merchantAuth });++ const accountData = accountDataLib.create({ db, log });++ const accountSend = config.oauth.clientSecret ? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl }) : null;++ const app = createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log, limitsNow, merchant, merchantAuth, accountData, accountSend });// One JWKS client for the process (the SDK shares it with verifyUserToken): refresh in the background// on an unref'd timer, keeping the last good keys through a Network outage.keys.start();@@@ -79,12 +83,19 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetchpruneTimer.unref?.();let server = null;++ let accountSubscription = null;if (listen) {server = await new Promise((resolve, reject) => {const s = app.listen(config.port, config.host, () => resolve(s));s.on('error', reject);});log.log(`[sources] listening on http://${config.host}:${server.address().port}`);++ if (config.events.secret.length >= 32 && config.events.url && config.oauth.clientSecret) {++ accountSubscription = startSubscriptions({++ eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.callbackPort || config.port}/internal/events`, secret: config.events.secret,++ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl, log,++ });++ }}async function close() {@@@ -92,6 +103,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetchif (merchantTimer) clearInterval(merchantTimer);if (merchantKick) clearTimeout(merchantKick);keys.stop();++ if (accountSubscription) accountSubscription.stop();await scheduler.stop();await outbox.stop();if (server) {nnew file mode 100644@@@ -0,0 +1,55 @@++'use strict';++/**++ * ADR-033: Sources' part of an account export and deletion. Sources holds no person's rows (a source is keyed by its++ * host, an item by its URL), so the part is empty and a deletion confirms nothing erased; the one exception is the++ * merchant opt-in, which is the merchant's own claim and goes with their account. The guard still runs, so a later++ * table with a person column fails here until it is answered for.++ */++const assert = require('assert');++const { uncovered } = require('openvibe-sdk/account-data');++const { testDb } = require('./db');++const { TABLES, create } = require('../server/account-data');++++const A = 'usr_01JZ0000000000000000000AAA';++const B = 'usr_01JZ0000000000000000000BBB';++++(async () => {++ const d = await testDb();++ try {++ const accountData = create({ db: d.db });++ const missing = await uncovered(d.db, {++ tables: [...TABLES, 'merchant_sources'], // the merchant's claim: exported and erased by this module++ exempt: { account_data_events: 'the ledger of requests itself: which export and deletion Sources applied' },++ });++ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);++++ const part = await accountData.exportPart(A);++ assert.deepStrictEqual(part.files, [], 'Sources has no person rows to export');++ const counts = await accountData.erase([A]);++ assert.deepStrictEqual(counts, { erased: {}, retained: {} }, 'Sources has no person rows to erase');++++ // The merchant opt-in: their claim, with the challenge they proved and the note that names them.++ for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {++ await d.db.exec(`INSERT INTO merchant_sources (id, owner_sub, owner_name, store_name, domain, method, token, listing_url, listing_kind, status, source_key, terms_note, created_at, updated_at)++ VALUES ($1, $2, $3, 'a store', $4, 'dns', 'challenge', 'https://example.test/sitemap.xml', 'sitemap', 'verified', $5, $6, 1, 1)`,++ [`msrc_${name}`, subject, name, `${name}.example.test`, `merchant-${name}-${name}`, `Merchant opt-in: ${subject} (${name}) gave OpenVibe explicit permission`]);++ }++ await d.db.exec("INSERT INTO sources (key, name, type, category, endpoints, terms_note, min_interval_ms, poll_interval_sec, stale_after_sec, max_items, enabled, review_required, sensitivity, default_indexability, created_at, updated_at) VALUES ('merchant-ada-ada', 'a store', 'sitemap', 'deals', '[]', 'Merchant opt-in: usr_01JZ0000000000000000000AAA (ada) gave OpenVibe explicit permission', 60, 900, 86400, 100, 1, 0, 'none', 'noindex', 1, 1)");++++ const withClaim = await accountData.exportPart(A);++ const file = withClaim.files.find((f) => f.name === 'merchant-sources.json');++ assert.ok(file, 'the merchant claim is in the part');++ assert.strictEqual(file.content.length, 1);++ assert.strictEqual(file.content[0].id, 'msrc_ada');++ assert.ok(!JSON.stringify(withClaim).includes('challenge'), 'the challenge is never exported');++ assert.ok(!JSON.stringify(withClaim).includes(B), 'nobody else\'s rows');++++ const erased = await accountData.erase([A]);++ assert.strictEqual(erased.erased.merchant_sources, 1);++ assert.strictEqual(Number(await d.db.value('SELECT count(*)::int FROM merchant_sources WHERE owner_sub = $1', [A])), 0, 'their claim is gone');++ assert.strictEqual(Number(await d.db.value('SELECT count(*)::int FROM merchant_sources WHERE owner_sub = $1', [B])), 1, 'Bo\'s claim stays');++ assert.strictEqual(Number(await d.db.value('SELECT enabled::int FROM sources WHERE key = $1', ['merchant-ada-ada'])), 0, 'the source it created stops being fetched');++ assert.ok(!String(await d.db.value('SELECT terms_note FROM sources WHERE key = $1', ['merchant-ada-ada'])).includes(A), 'the note no longer names them');++ console.log(' ✓ empty part, empty erase, the merchant claim, and the guard');++ } finally { await d.close(); }++})();
@@ -576,6 +576,6 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [
@@ -576,6 +576,6 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [
<!-- versions:start -->
<!-- versions:start -->
- openvibe-contracts: v0.224.0
- openvibe-contracts: v0.224.0
-- openvibe-sdk: v0.67.0
+- openvibe-sdk: v0.74.0
- openvibe-shared: v4.48.0
- openvibe-shared: v4.48.0
<!-- versions:end -->
<!-- versions:end -->
@@ -8,7 +8,7 @@
@@ -8,7 +8,7 @@
"port": 4720,
"port": 4720,
"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, the merchant opt-in pages, health and JSON 404s; the JSON API is loopback-only; README \"Public host\")",
"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, the merchant opt-in pages, health and JSON 404s; the JSON API is loopback-only; README \"Public host\")",
"contracts": "openvibe-contracts v0.224.0",
"contracts": "openvibe-contracts v0.224.0",
- "sdk": "openvibe-sdk v0.67.0",
+ "sdk": "openvibe-sdk v0.74.0",
"shared": "openvibe-shared v4.48.0",
"shared": "openvibe-shared v4.48.0",
"capabilities": [
"capabilities": [
"sources.source.read",
"sources.source.read",
new file mode 100644
new file mode 100644
@@ -0,0 +1,13 @@
@@ -0,0 +1,13 @@
+-- phase: expand
+-- OpenVibe.Sources: the receipts of the ADR-033 account export and deletion deliveries this service applied
+-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.
+-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.
+
+CREATE TABLE IF NOT EXISTS account_data_events (
+ id TEXT PRIMARY KEY,
+ kind TEXT NOT NULL,
+ subject TEXT NOT NULL,
+ outcome JSONB,
+ applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ sent_at TIMESTAMPTZ
+);
@@ -13,7 +13,7 @@
@@ -13,7 +13,7 @@
"express": "^4.21.2",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"fast-xml-parser": "^4.5.3",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.224.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.224.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
},
},
@@ -1060,9 +1060,9 @@
@@ -1060,9 +1060,9 @@
}
}
},
},
"node_modules/openvibe-sdk": {
"node_modules/openvibe-sdk": {
- "version": "0.67.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
- "integrity": "sha512-s7NzjaNcowVcOIiup2z0ze+xsq/qnr++3pHSI8JzirqJPQ5JsBSUF5RkjOHyfk9bRKlmhY0fsNS6x4dsYylyqw==",
+ "version": "0.74.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
+ "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==",
"license": "MIT",
"license": "MIT",
"bin": {
"bin": {
"openvibe-download": "bin/openvibe-download.js"
"openvibe-download": "bin/openvibe-download.js"
@@ -31,7 +31,7 @@
@@ -31,7 +31,7 @@
"express": "^4.21.2",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"fast-xml-parser": "^4.5.3",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.224.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.224.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
},
},
new file mode 100644
new file mode 100644
@@ -0,0 +1,47 @@
@@ -0,0 +1,47 @@
+'use strict';
+/**
+ * Account export and deletion → Sources (ADR-033; openvibe-sdk/account-data).
+ *
+ * Sources keeps the outside web's records: registries, items, fetch runs and robots policies. None of them names a
+ * Network person; a source is keyed by its host and the item by its URL.
+ *
+ * The one row that does is the merchant opt-in (merchant_sources): the signed-in merchant's own claim, with the
+ * challenge they proved, their name and the permission note that names them. Their claim goes with the account, and
+ * the source it created stops being fetched, because the permission it rests on is gone. The note goes too: it holds
+ * the person's name in free text, which a kept row may not (ADR-033).
+ *
+ * So a person's export part carries their merchant claims (never the challenge token) and their deletion confirms
+ * what it erased. Telemetry samples that name a person are covered by the SDK itself (telemetry.json, then subject NULL).
+ */
+const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
+
+const TABLES = [];
+
+// The merchant's own claim. The challenge token is never exported; their name and subject are in Network's part.
+const CLAIM_COLUMNS = ['id', 'store_name', 'domain', 'method', 'listing_url', 'listing_kind', 'meta_url', 'status', 'source_key',
+ 'terms_note', 'verified_at', 'verified_by', 'verification_detail', 'last_checked_at', 'check_count', 'created_at', 'withdrawn_at'];
+
+/** The claims the person made, without the challenge that proves them. */
+async function extraExport(db, subject) {
+ const claims = await db.many(`SELECT ${CLAIM_COLUMNS.join(', ')} FROM merchant_sources WHERE owner_sub = $1 ORDER BY created_at DESC LIMIT 5000`, [subject]);
+ return claims.length ? [{ name: 'merchant-sources.json', content: claims }] : [];
+}
+
+/** The claim goes, and the source it created stops being fetched (its note named the person). */
+async function extraErase(t, subjects, counts) {
+ const rows = await t.many('SELECT id, source_key FROM merchant_sources WHERE owner_sub = ANY($1::text[])', [subjects]);
+ if (!rows.length) return;
+ const keys = rows.map((r) => r.source_key).filter(Boolean);
+ if (keys.length) {
+ counts.add(counts.retained, 'sources_disabled', await t.exec(
+ `UPDATE sources SET enabled = 0, terms_note = 'Merchant opt-in withdrawn: the merchant closed their account.', updated_at = $2 WHERE key = ANY($1::text[])`,
+ [keys, Date.now()]));
+ }
+ counts.add(counts.erased, 'merchant_sources', await t.exec('DELETE FROM merchant_sources WHERE owner_sub = ANY($1::text[])', [subjects]));
+}
+
+function create({ db, log = console } = {}) {
+ return createAccountData({ db, service: 'sources', tables: TABLES, extraExport, extraErase, log });
+}
+
+module.exports = { create, TABLES, TOPICS, extraExport, extraErase };
@@ -69,7 +69,7 @@ Source code: <a href="https://github.com/OpenVibers/OpenVibe.Sources">OpenVibers
@@ -69,7 +69,7 @@ Source code: <a href="https://github.com/OpenVibers/OpenVibe.Sources">OpenVibers
</html>
</html>
`;
`;
-function createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log = console, limitsNow = null, merchant = null, merchantAuth = null }) {
+function createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log = console, limitsNow = null, merchant = null, merchantAuth = null, accountData = null, accountSend = null }) {
const app = express();
const app = express();
app.disable('x-powered-by');
app.disable('x-powered-by');
app.set('trust proxy', 'loopback');
app.set('trust proxy', 'loopback');
@@ -80,6 +80,9 @@ function createApp({ config, db, registry, items, ingest, scheduler, auth, outbo
@@ -80,6 +80,9 @@ function createApp({ config, db, registry, items, ingest, scheduler, auth, outbo
app.use(require('openvibe-sdk/service').telemetryMiddleware); // request telemetry: a no-op until server/index.js calls telemetry.init()
app.use(require('openvibe-sdk/service').telemetryMiddleware); // request telemetry: a no-op until server/index.js calls telemetry.init()
registerSourcesGauges(metrics.registry, { db, sources: registry, ingest, outbox, now });
registerSourcesGauges(metrics.registry, { db, sources: registry, ingest, outbox, now });
app.use(http.middleware());
app.use(http.middleware());
+ // The account export/deletion consumer (ADR-033), signed and loopback only. No body parser before it: the v2
+ // signature covers the raw body.
+ if (accountData) app.post('/internal/events', accountData.consumer({ secrets: [config.events.secret].filter((s) => s.length >= 32), send: accountSend }));
// One W3C trace across services (openvibe-shared/trace): calls made while serving a request carry its traceparent.
// One W3C trace across services (openvibe-shared/trace): calls made while serving a request carry its traceparent.
require('openvibe-shared/trace').install(app);
require('openvibe-shared/trace').install(app);
app.use((req, res, next) => {
app.use((req, res, next) => {
@@ -85,6 +85,9 @@ function load(env = process.env) {
@@ -85,6 +85,9 @@ function load(env = process.env) {
events: {
events: {
url: strip(env.EVENTS_URL || ''),
url: strip(env.EVENTS_URL || ''),
relayIntervalMs: int(env.EVENTS_RELAY_INTERVAL_MS, 2000),
relayIntervalMs: int(env.EVENTS_RELAY_INTERVAL_MS, 2000),
+ // The account-data consumer (POST /internal/events): the signing secret Network's deliveries use. Unset:
+ // the route is off and Network's account export/deletion does not wait for Sources.
+ secret: strip(env.SOURCES_EVENTS_SECRET || ''),
},
},
// Ingestion
// Ingestion
@@ -28,6 +28,8 @@ const { createAuth } = require('./auth');
@@ -28,6 +28,8 @@ const { createAuth } = require('./auth');
const { createMerchant } = require('./merchant');
const { createMerchant } = require('./merchant');
const { createMerchantAuth } = require('./merchant-auth');
const { createMerchantAuth } = require('./merchant-auth');
const { createApp } = require('./app');
const { createApp } = require('./app');
+const accountDataLib = require('./account-data');
+const { createNetworkSender, startSubscriptions } = require('openvibe-sdk/account-data');
async function start({ config, db: givenDb = null, now = () => Date.now(), fetchImpl = globalThis.fetch, lookupImpl, merchantResolver = null, log = console, listen = true, limitsNow = null } = {}) {
async function start({ config, db: givenDb = null, now = () => Date.now(), fetchImpl = globalThis.fetch, lookupImpl, merchantResolver = null, log = console, listen = true, limitsNow = null } = {}) {
config = config || load();
config = config || load();
@@ -58,7 +60,9 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
@@ -58,7 +60,9 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
const auth = createAuth({ config, log, jwks: keys });
const auth = createAuth({ config, log, jwks: keys });
const merchant = createMerchant({ db, config, registry, fetcher, resolver: merchantResolver, now, log });
const merchant = createMerchant({ db, config, registry, fetcher, resolver: merchantResolver, now, log });
const merchantAuth = createMerchantAuth({ config, auth, log, fetchImpl });
const merchantAuth = createMerchantAuth({ config, auth, log, fetchImpl });
- const app = createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log, limitsNow, merchant, merchantAuth });
+ const accountData = accountDataLib.create({ db, log });
+ const accountSend = config.oauth.clientSecret ? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl }) : null;
+ const app = createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log, limitsNow, merchant, merchantAuth, accountData, accountSend });
// One JWKS client for the process (the SDK shares it with verifyUserToken): refresh in the background
// One JWKS client for the process (the SDK shares it with verifyUserToken): refresh in the background
// on an unref'd timer, keeping the last good keys through a Network outage.
// on an unref'd timer, keeping the last good keys through a Network outage.
keys.start();
keys.start();
@@ -79,12 +83,19 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
@@ -79,12 +83,19 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
pruneTimer.unref?.();
pruneTimer.unref?.();
let server = null;
let server = null;
+ let accountSubscription = null;
if (listen) {
if (listen) {
server = await new Promise((resolve, reject) => {
server = await new Promise((resolve, reject) => {
const s = app.listen(config.port, config.host, () => resolve(s));
const s = app.listen(config.port, config.host, () => resolve(s));
s.on('error', reject);
s.on('error', reject);
});
});
log.log(`[sources] listening on http://${config.host}:${server.address().port}`);
log.log(`[sources] listening on http://${config.host}:${server.address().port}`);
+ if (config.events.secret.length >= 32 && config.events.url && config.oauth.clientSecret) {
+ accountSubscription = startSubscriptions({
+ eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.callbackPort || config.port}/internal/events`, secret: config.events.secret,
+ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl, log,
+ });
+ }
}
}
async function close() {
async function close() {
@@ -92,6 +103,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
@@ -92,6 +103,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
if (merchantTimer) clearInterval(merchantTimer);
if (merchantTimer) clearInterval(merchantTimer);
if (merchantKick) clearTimeout(merchantKick);
if (merchantKick) clearTimeout(merchantKick);
keys.stop();
keys.stop();
+ if (accountSubscription) accountSubscription.stop();
await scheduler.stop();
await scheduler.stop();
await outbox.stop();
await outbox.stop();
if (server) {
if (server) {
new file mode 100644
new file mode 100644
@@ -0,0 +1,55 @@
@@ -0,0 +1,55 @@
+'use strict';
+/**
+ * ADR-033: Sources' part of an account export and deletion. Sources holds no person's rows (a source is keyed by its
+ * host, an item by its URL), so the part is empty and a deletion confirms nothing erased; the one exception is the
+ * merchant opt-in, which is the merchant's own claim and goes with their account. The guard still runs, so a later
+ * table with a person column fails here until it is answered for.
+ */
+const assert = require('assert');
+const { uncovered } = require('openvibe-sdk/account-data');
+const { testDb } = require('./db');
+const { TABLES, create } = require('../server/account-data');
+
+const A = 'usr_01JZ0000000000000000000AAA';
+const B = 'usr_01JZ0000000000000000000BBB';
+
+(async () => {
+ const d = await testDb();
+ try {
+ const accountData = create({ db: d.db });
+ const missing = await uncovered(d.db, {
+ tables: [...TABLES, 'merchant_sources'], // the merchant's claim: exported and erased by this module
+ exempt: { account_data_events: 'the ledger of requests itself: which export and deletion Sources applied' },
+ });
+ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
+
+ const part = await accountData.exportPart(A);
+ assert.deepStrictEqual(part.files, [], 'Sources has no person rows to export');
+ const counts = await accountData.erase([A]);
+ assert.deepStrictEqual(counts, { erased: {}, retained: {} }, 'Sources has no person rows to erase');
+
+ // The merchant opt-in: their claim, with the challenge they proved and the note that names them.
+ for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {
+ await d.db.exec(`INSERT INTO merchant_sources (id, owner_sub, owner_name, store_name, domain, method, token, listing_url, listing_kind, status, source_key, terms_note, created_at, updated_at)
+ VALUES ($1, $2, $3, 'a store', $4, 'dns', 'challenge', 'https://example.test/sitemap.xml', 'sitemap', 'verified', $5, $6, 1, 1)`,
+ [`msrc_${name}`, subject, name, `${name}.example.test`, `merchant-${name}-${name}`, `Merchant opt-in: ${subject} (${name}) gave OpenVibe explicit permission`]);
+ }
+ await d.db.exec("INSERT INTO sources (key, name, type, category, endpoints, terms_note, min_interval_ms, poll_interval_sec, stale_after_sec, max_items, enabled, review_required, sensitivity, default_indexability, created_at, updated_at) VALUES ('merchant-ada-ada', 'a store', 'sitemap', 'deals', '[]', 'Merchant opt-in: usr_01JZ0000000000000000000AAA (ada) gave OpenVibe explicit permission', 60, 900, 86400, 100, 1, 0, 'none', 'noindex', 1, 1)");
+
+ const withClaim = await accountData.exportPart(A);
+ const file = withClaim.files.find((f) => f.name === 'merchant-sources.json');
+ assert.ok(file, 'the merchant claim is in the part');
+ assert.strictEqual(file.content.length, 1);
+ assert.strictEqual(file.content[0].id, 'msrc_ada');
+ assert.ok(!JSON.stringify(withClaim).includes('challenge'), 'the challenge is never exported');
+ assert.ok(!JSON.stringify(withClaim).includes(B), 'nobody else\'s rows');
+
+ const erased = await accountData.erase([A]);
+ assert.strictEqual(erased.erased.merchant_sources, 1);
+ assert.strictEqual(Number(await d.db.value('SELECT count(*)::int FROM merchant_sources WHERE owner_sub = $1', [A])), 0, 'their claim is gone');
+ assert.strictEqual(Number(await d.db.value('SELECT count(*)::int FROM merchant_sources WHERE owner_sub = $1', [B])), 1, 'Bo\'s claim stays');
+ assert.strictEqual(Number(await d.db.value('SELECT enabled::int FROM sources WHERE key = $1', ['merchant-ada-ada'])), 0, 'the source it created stops being fetched');
+ assert.ok(!String(await d.db.value('SELECT terms_note FROM sources WHERE key = $1', ['merchant-ada-ada'])).includes(A), 'the note no longer names them');
+ console.log(' ✓ empty part, empty erase, the merchant claim, and the guard');
+ } finally { await d.close(); }
+})();