openvibers/openvibe.sources Commits

account-data: a merchant's own claim, with the guard on every person column (#146)

OP OpenVibers committed 14 days ago · f34bd751a4 · 10 files, +142 −9

* account-data: a merchant's own claim, with the guard on every person column

* account-data: the receipts table comes from a migration, not from the runtime role

The boot-time ensureSchema() ran DDL as the serving role, which may not change
the schema; the table is now migration NNNN, like every other table here.

* ci: run the checks for this branch

* Merge remote-tracking branch 'origin/main' into adr033-account-data

# Conflicts:
#	README.md
#	STATUS.json
#	package-lock.json
#	package.json

Changes

ddiff --git a/README.md b/README.md
iindex c63e07a..9586c2f 100644
---- a/README.md
++++ b/README.md
@@@ -576,6 +576,6 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [
<!-- versions:start -->
- openvibe-contracts: v0.224.0
--- openvibe-sdk: v0.67.0
++- openvibe-sdk: v0.74.0
- openvibe-shared: v4.48.0
<!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex 9307d10..790aed5 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -8,7 +8,7 @@
"port": 4720,
"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, the merchant opt-in pages, health and JSON 404s; the JSON API is loopback-only; README \"Public host\")",
"contracts": "openvibe-contracts v0.224.0",
-- "sdk": "openvibe-sdk v0.67.0",
++ "sdk": "openvibe-sdk v0.74.0",
"shared": "openvibe-shared v4.48.0",
"capabilities": [
"sources.source.read",
ddiff --git a/migrations/0008_account_data.sql b/migrations/0008_account_data.sql
nnew file mode 100644
iindex 0000000..0cc7a49
---- /dev/null
++++ b/migrations/0008_account_data.sql
@@@ -0,0 +1,13 @@
++-- phase: expand
++-- OpenVibe.Sources: the receipts of the ADR-033 account export and deletion deliveries this service applied
++-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.
++-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.
++
++CREATE TABLE IF NOT EXISTS account_data_events (
++ id TEXT PRIMARY KEY,
++ kind TEXT NOT NULL,
++ subject TEXT NOT NULL,
++ outcome JSONB,
++ applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),
++ sent_at TIMESTAMPTZ
++);
ddiff --git a/package-lock.json b/package-lock.json
iindex b67bec6..41f9045 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -13,7 +13,7 @@
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.224.0",
-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
},
@@@ -1060,9 +1060,9 @@
}
},
"node_modules/openvibe-sdk": {
-- "version": "0.67.0",
-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
-- "integrity": "sha512-s7NzjaNcowVcOIiup2z0ze+xsq/qnr++3pHSI8JzirqJPQ5JsBSUF5RkjOHyfk9bRKlmhY0fsNS6x4dsYylyqw==",
++ "version": "0.74.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
++ "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==",
"license": "MIT",
"bin": {
"openvibe-download": "bin/openvibe-download.js"
ddiff --git a/package.json b/package.json
iindex d0e6d2f..2157f89 100644
---- a/package.json
++++ b/package.json
@@@ -31,7 +31,7 @@
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.224.0",
-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
},
ddiff --git a/server/account-data.js b/server/account-data.js
nnew file mode 100644
iindex 0000000..d3ca61f
---- /dev/null
++++ b/server/account-data.js
@@@ -0,0 +1,47 @@
++'use strict';
++/**
++ * Account export and deletion → Sources (ADR-033; openvibe-sdk/account-data).
++ *
++ * Sources keeps the outside web's records: registries, items, fetch runs and robots policies. None of them names a
++ * Network person; a source is keyed by its host and the item by its URL.
++ *
++ * The one row that does is the merchant opt-in (merchant_sources): the signed-in merchant's own claim, with the
++ * challenge they proved, their name and the permission note that names them. Their claim goes with the account, and
++ * the source it created stops being fetched, because the permission it rests on is gone. The note goes too: it holds
++ * the person's name in free text, which a kept row may not (ADR-033).
++ *
++ * So a person's export part carries their merchant claims (never the challenge token) and their deletion confirms
++ * what it erased. Telemetry samples that name a person are covered by the SDK itself (telemetry.json, then subject NULL).
++ */
++const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
++
++const TABLES = [];
++
++// The merchant's own claim. The challenge token is never exported; their name and subject are in Network's part.
++const CLAIM_COLUMNS = ['id', 'store_name', 'domain', 'method', 'listing_url', 'listing_kind', 'meta_url', 'status', 'source_key',
++ 'terms_note', 'verified_at', 'verified_by', 'verification_detail', 'last_checked_at', 'check_count', 'created_at', 'withdrawn_at'];
++
++/** The claims the person made, without the challenge that proves them. */
++async function extraExport(db, subject) {
++ const claims = await db.many(`SELECT ${CLAIM_COLUMNS.join(', ')} FROM merchant_sources WHERE owner_sub = $1 ORDER BY created_at DESC LIMIT 5000`, [subject]);
++ return claims.length ? [{ name: 'merchant-sources.json', content: claims }] : [];
++}
++
++/** The claim goes, and the source it created stops being fetched (its note named the person). */
++async function extraErase(t, subjects, counts) {
++ const rows = await t.many('SELECT id, source_key FROM merchant_sources WHERE owner_sub = ANY($1::text[])', [subjects]);
++ if (!rows.length) return;
++ const keys = rows.map((r) => r.source_key).filter(Boolean);
++ if (keys.length) {
++ counts.add(counts.retained, 'sources_disabled', await t.exec(
++ `UPDATE sources SET enabled = 0, terms_note = 'Merchant opt-in withdrawn: the merchant closed their account.', updated_at = $2 WHERE key = ANY($1::text[])`,
++ [keys, Date.now()]));
++ }
++ counts.add(counts.erased, 'merchant_sources', await t.exec('DELETE FROM merchant_sources WHERE owner_sub = ANY($1::text[])', [subjects]));
++}
++
++function create({ db, log = console } = {}) {
++ return createAccountData({ db, service: 'sources', tables: TABLES, extraExport, extraErase, log });
++}
++
++module.exports = { create, TABLES, TOPICS, extraExport, extraErase };
ddiff --git a/server/app.js b/server/app.js
iindex a79e165..60479df 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -69,7 +69,7 @@ Source code: <a href="https://github.com/OpenVibers/OpenVibe.Sources">OpenVibers
</html>
`;
--function createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log = console, limitsNow = null, merchant = null, merchantAuth = null }) {
++function createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log = console, limitsNow = null, merchant = null, merchantAuth = null, accountData = null, accountSend = null }) {
const app = express();
app.disable('x-powered-by');
app.set('trust proxy', 'loopback');
@@@ -80,6 +80,9 @@ function createApp({ config, db, registry, items, ingest, scheduler, auth, outbo
app.use(require('openvibe-sdk/service').telemetryMiddleware); // request telemetry: a no-op until server/index.js calls telemetry.init()
registerSourcesGauges(metrics.registry, { db, sources: registry, ingest, outbox, now });
app.use(http.middleware());
++ // The account export/deletion consumer (ADR-033), signed and loopback only. No body parser before it: the v2
++ // signature covers the raw body.
++ if (accountData) app.post('/internal/events', accountData.consumer({ secrets: [config.events.secret].filter((s) => s.length >= 32), send: accountSend }));
// One W3C trace across services (openvibe-shared/trace): calls made while serving a request carry its traceparent.
require('openvibe-shared/trace').install(app);
app.use((req, res, next) => {
ddiff --git a/server/config.js b/server/config.js
iindex ed98b08..8ac285e 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -85,6 +85,9 @@ function load(env = process.env) {
events: {
url: strip(env.EVENTS_URL || ''),
relayIntervalMs: int(env.EVENTS_RELAY_INTERVAL_MS, 2000),
++ // The account-data consumer (POST /internal/events): the signing secret Network's deliveries use. Unset:
++ // the route is off and Network's account export/deletion does not wait for Sources.
++ secret: strip(env.SOURCES_EVENTS_SECRET || ''),
},
// Ingestion
ddiff --git a/server/index.js b/server/index.js
iindex 9c93cd8..e8c01b0 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -28,6 +28,8 @@ const { createAuth } = require('./auth');
const { createMerchant } = require('./merchant');
const { createMerchantAuth } = require('./merchant-auth');
const { createApp } = require('./app');
++const accountDataLib = require('./account-data');
++const { createNetworkSender, startSubscriptions } = require('openvibe-sdk/account-data');
async function start({ config, db: givenDb = null, now = () => Date.now(), fetchImpl = globalThis.fetch, lookupImpl, merchantResolver = null, log = console, listen = true, limitsNow = null } = {}) {
config = config || load();
@@@ -58,7 +60,9 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
const auth = createAuth({ config, log, jwks: keys });
const merchant = createMerchant({ db, config, registry, fetcher, resolver: merchantResolver, now, log });
const merchantAuth = createMerchantAuth({ config, auth, log, fetchImpl });
-- const app = createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log, limitsNow, merchant, merchantAuth });
++ const accountData = accountDataLib.create({ db, log });
++ const accountSend = config.oauth.clientSecret ? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl }) : null;
++ const app = createApp({ config, db, registry, items, ingest, scheduler, auth, outbox, now, log, limitsNow, merchant, merchantAuth, accountData, accountSend });
// One JWKS client for the process (the SDK shares it with verifyUserToken): refresh in the background
// on an unref'd timer, keeping the last good keys through a Network outage.
keys.start();
@@@ -79,12 +83,19 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
pruneTimer.unref?.();
let server = null;
++ let accountSubscription = null;
if (listen) {
server = await new Promise((resolve, reject) => {
const s = app.listen(config.port, config.host, () => resolve(s));
s.on('error', reject);
});
log.log(`[sources] listening on http://${config.host}:${server.address().port}`);
++ if (config.events.secret.length >= 32 && config.events.url && config.oauth.clientSecret) {
++ accountSubscription = startSubscriptions({
++ eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.callbackPort || config.port}/internal/events`, secret: config.events.secret,
++ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl, log,
++ });
++ }
}
async function close() {
@@@ -92,6 +103,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
if (merchantTimer) clearInterval(merchantTimer);
if (merchantKick) clearTimeout(merchantKick);
keys.stop();
++ if (accountSubscription) accountSubscription.stop();
await scheduler.stop();
await outbox.stop();
if (server) {
ddiff --git a/test/account-data.test.js b/test/account-data.test.js
nnew file mode 100644
iindex 0000000..b213745
---- /dev/null
++++ b/test/account-data.test.js
@@@ -0,0 +1,55 @@
++'use strict';
++/**
++ * ADR-033: Sources' part of an account export and deletion. Sources holds no person's rows (a source is keyed by its
++ * host, an item by its URL), so the part is empty and a deletion confirms nothing erased; the one exception is the
++ * merchant opt-in, which is the merchant's own claim and goes with their account. The guard still runs, so a later
++ * table with a person column fails here until it is answered for.
++ */
++const assert = require('assert');
++const { uncovered } = require('openvibe-sdk/account-data');
++const { testDb } = require('./db');
++const { TABLES, create } = require('../server/account-data');
++
++const A = 'usr_01JZ0000000000000000000AAA';
++const B = 'usr_01JZ0000000000000000000BBB';
++
++(async () => {
++ const d = await testDb();
++ try {
++ const accountData = create({ db: d.db });
++ const missing = await uncovered(d.db, {
++ tables: [...TABLES, 'merchant_sources'], // the merchant's claim: exported and erased by this module
++ exempt: { account_data_events: 'the ledger of requests itself: which export and deletion Sources applied' },
++ });
++ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
++
++ const part = await accountData.exportPart(A);
++ assert.deepStrictEqual(part.files, [], 'Sources has no person rows to export');
++ const counts = await accountData.erase([A]);
++ assert.deepStrictEqual(counts, { erased: {}, retained: {} }, 'Sources has no person rows to erase');
++
++ // The merchant opt-in: their claim, with the challenge they proved and the note that names them.
++ for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {
++ await d.db.exec(`INSERT INTO merchant_sources (id, owner_sub, owner_name, store_name, domain, method, token, listing_url, listing_kind, status, source_key, terms_note, created_at, updated_at)
++ VALUES ($1, $2, $3, 'a store', $4, 'dns', 'challenge', 'https://example.test/sitemap.xml', 'sitemap', 'verified', $5, $6, 1, 1)`,
++ [`msrc_${name}`, subject, name, `${name}.example.test`, `merchant-${name}-${name}`, `Merchant opt-in: ${subject} (${name}) gave OpenVibe explicit permission`]);
++ }
++ await d.db.exec("INSERT INTO sources (key, name, type, category, endpoints, terms_note, min_interval_ms, poll_interval_sec, stale_after_sec, max_items, enabled, review_required, sensitivity, default_indexability, created_at, updated_at) VALUES ('merchant-ada-ada', 'a store', 'sitemap', 'deals', '[]', 'Merchant opt-in: usr_01JZ0000000000000000000AAA (ada) gave OpenVibe explicit permission', 60, 900, 86400, 100, 1, 0, 'none', 'noindex', 1, 1)");
++
++ const withClaim = await accountData.exportPart(A);
++ const file = withClaim.files.find((f) => f.name === 'merchant-sources.json');
++ assert.ok(file, 'the merchant claim is in the part');
++ assert.strictEqual(file.content.length, 1);
++ assert.strictEqual(file.content[0].id, 'msrc_ada');
++ assert.ok(!JSON.stringify(withClaim).includes('challenge'), 'the challenge is never exported');
++ assert.ok(!JSON.stringify(withClaim).includes(B), 'nobody else\'s rows');
++
++ const erased = await accountData.erase([A]);
++ assert.strictEqual(erased.erased.merchant_sources, 1);
++ assert.strictEqual(Number(await d.db.value('SELECT count(*)::int FROM merchant_sources WHERE owner_sub = $1', [A])), 0, 'their claim is gone');
++ assert.strictEqual(Number(await d.db.value('SELECT count(*)::int FROM merchant_sources WHERE owner_sub = $1', [B])), 1, 'Bo\'s claim stays');
++ assert.strictEqual(Number(await d.db.value('SELECT enabled::int FROM sources WHERE key = $1', ['merchant-ada-ada'])), 0, 'the source it created stops being fetched');
++ assert.ok(!String(await d.db.value('SELECT terms_note FROM sources WHERE key = $1', ['merchant-ada-ada'])).includes(A), 'the note no longer names them');
++ console.log(' ✓ empty part, empty erase, the merchant claim, and the guard');
++ } finally { await d.close(); }
++})();