openvibers/openvibe.search Commits
account-data: a person's saved searches, with the guard on every person column (#145)
Changes
@@@ -484,6 +484,6 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [<!-- versions:start -->- openvibe-contracts: v0.225.0--- openvibe-sdk: v0.67.0++- openvibe-sdk: v0.74.0- openvibe-shared: v4.48.0<!-- versions:end -->@@@ -8,7 +8,7 @@"port": 4710,"domain": "search.openvibe.network (vhost in deploy/nginx; the service manifest records the search page and query API as public since 2026-09-23)","contracts": "openvibe-contracts v0.225.0",-- "sdk": "openvibe-sdk v0.67.0",++ "sdk": "openvibe-sdk v0.74.0","shared": "openvibe-shared v4.48.0","capabilities": ["search.document.write",nnew file mode 100644@@@ -0,0 +1,13 @@++-- phase: expand++-- OpenVibe.Search: the receipts of the ADR-033 account export and deletion deliveries this service applied++-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.++-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.++++CREATE TABLE IF NOT EXISTS account_data_events (++ id TEXT PRIMARY KEY,++ kind TEXT NOT NULL,++ subject TEXT NOT NULL,++ outcome JSONB,++ applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),++ sent_at TIMESTAMPTZ++);@@@ -14,7 +14,7 @@"dotenv": "^16.4.7","express": "^4.21.2","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.225.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0","pg": "^8.23.0"},@@@ -1041,9 +1041,9 @@}},"node_modules/openvibe-sdk": {-- "version": "0.67.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",-- "integrity": "sha512-s7NzjaNcowVcOIiup2z0ze+xsq/qnr++3pHSI8JzirqJPQ5JsBSUF5RkjOHyfk9bRKlmhY0fsNS6x4dsYylyqw==",++ "version": "0.74.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",++ "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==","license": "MIT","bin": {"openvibe-download": "bin/openvibe-download.js"@@@ -26,7 +26,7 @@"dotenv": "^16.4.7","express": "^4.21.2","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.225.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0","pg": "^8.23.0"},nnew file mode 100644@@@ -0,0 +1,23 @@++'use strict';++/**++ * Account export and deletion → Search (ADR-033; openvibe-sdk/account-data).++ *++ * What Search keeps of a person's own is their saved searches (saved_searches.subject = usr_…): the query, its++ * filters and when it last ran. They are exported and deleted with the account.++ *++ * The index itself is not a person's: documents, ingest_rejections, removals and search_embeddings are keyed by the++ * publishing owner (a service), and a document is erased when its owner sends index_document.deleted. A person's++ * deletion does not silently remove published pages: their owner does that. Telemetry samples that name a person are++ * covered by the SDK itself (telemetry.json, then subject NULL).++ */++const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');++++const TABLES = [++ { table: 'saved_searches', subject: 'subject', file: 'saved-searches.json', columns: ['id', 'name', 'q', 'filters', 'created_at', 'updated_at', 'last_run_at'], order: 'updated_at' },++];++++function create({ db, log = console } = {}) {++ return createAccountData({ db, service: 'search', tables: TABLES, log });++}++++module.exports = { create, TABLES, TOPICS };@@@ -20,6 +20,7 @@ const crypto = require('crypto');const express = require('express');const { http } = require('openvibe-contracts');const { validate } = require('../document');++const { TOPICS: ACCOUNT_TOPICS } = require('openvibe-sdk/account-data');const CONSUMER = 'search-index';const TYPE_RE = /^([a-z][a-z0-9-]{1,39})\.index_document\.(upserted|deleted)$/;@@@ -119,7 +120,7 @@ function documentFromEvent(event, owners) {return { doc };}--function webhookRouter({ config, db, store, outbox, log = console, now }) {++function webhookRouter({ config, db, store, outbox, accountData = null, accountSend = null, log = console, now }) {const router = express.Router();const inbox = createInbox(db, { now });@@@ -142,6 +143,7 @@ function webhookRouter({ config, db, store, outbox, log = console, now }) {}const r = await inbox.once(event.event_id, async () => {++ if (accountData && ACCOUNT_TOPICS.includes(event.event_type)) return accountData.apply(event, { send: accountSend });const parsed = documentFromEvent(event, config.events.owners);if (parsed.ignore) return 'ignored';if (parsed.reject) {@@@ -23,7 +23,7 @@ const { createIndexNow } = require('openvibe-shared/indexnow');const openapi = require('./openapi');const pkg = require('../package.json');--function createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher: givenSearcher = null, notifier = null, vectorStore = null, embeddings = null, fetchImpl = globalThis.fetch, log = console, now }) {++function createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher: givenSearcher = null, notifier = null, vectorStore = null, embeddings = null, accountData = null, accountSend = null, fetchImpl = globalThis.fetch, log = console, now }) {const app = express();app.disable('x-powered-by');app.set('trust proxy', 'loopback');@@@ -58,7 +58,7 @@ function createApp({ config, db, store, engine, auth, outbox, purges, purger, sa// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).app.use('/shared', require('openvibe-shared/serve').handler());-- app.use(webhookRouter({ config, db, store, outbox, log, now }));++ app.use(webhookRouter({ config, db, store, outbox, accountData, accountSend, log, now }));app.use('/api', express.json({ limit: config.maxBodyBytes, type: ['application/json', 'application/*+json'] }));@@@ -23,6 +23,8 @@ const { createSearcher } = require('./api/query');const { createNetworkPush } = require('./network-push');const { createSavedNotifier } = require('./saved-notifier');const { createApp } = require('./app');++const accountDataLib = require('./account-data');++const { createNetworkSender, startSubscriptions } = require('openvibe-sdk/account-data');const { createPgVectorStore } = require('./fabric/vector-store');const { createEmbeddingClient } = require('./fabric/embed');const { createEmbedder } = require('./fabric/embedder');@@@ -62,7 +64,9 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetchconfig, saved, searcher, now, log,network: networkPush || createNetworkPush({ config, fetchImpl }),});-- const app = createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher, notifier, fetchImpl, log, now, vectorStore, embeddings: embeddingClient });++ const accountData = accountDataLib.create({ db, log });++ const accountSend = config.oauth.clientSecret ? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret }) : null;++ const app = createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher, notifier, fetchImpl, log, now, vectorStore, embeddings: embeddingClient, accountData, accountSend });// The background refresher is not started under the test runtime (the tests stub the fetch and load// the keys once below); a real boot refreshes on an unref'd timer. keyLoaded settles the first load.@@@ -80,16 +84,24 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetchcachePruneTimer.unref?.();let server = null;++ let accountSubscription = null;if (listen) {server = await new Promise((resolve, reject) => {const s = app.listen(config.port, config.host, () => resolve(s));s.on('error', reject);});log.log(`[search] listening on http://${config.host}:${server.address().port} (engine ${engine.name})`);++ if (config.events.webhookSecrets.length && config.events.url && config.oauth.clientSecret) {++ accountSubscription = startSubscriptions({++ eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.port}/internal/events`, secret: config.events.webhookSecrets[0],++ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log,++ });++ }}async function close() {clearInterval(pruneTimer);++ if (accountSubscription) accountSubscription.stop();clearInterval(cachePruneTimer);jwks.stop();await outbox.stop();nnew file mode 100644@@@ -0,0 +1,93 @@++'use strict';++/**++ * ADR-033: Search's part of an account export and deletion, through the signed POST /internal/events route with a++ * stand-in Network. Ada's saved searches are exported and erased; Bo's stay; the index rows (keyed by their publishing++ * owner, not a person) are untouched. The guard fails when a new table with a person column is neither covered nor++ * exempted with a reason.++ */++const assert = require('assert');++const http = require('http');++const { uncovered } = require('openvibe-sdk/account-data');++const { signDeliveryHeaders } = require('openvibe-sdk/events');++const { boot, suite } = require('./helpers');++const { TABLES } = require('../server/account-data');++++const t = suite('account-data');++const SECRET = `whsec_${'fixture'.repeat(6)}`;++const A = 'usr_01JZ0000000000000000000AAA';++const B = 'usr_01JZ0000000000000000000BBB';++const EXP = 'exp_01JZ0000000000000000000EXP';++const DEL = 'del_01JZ0000000000000000000DEX';++++async function startNetworkStub() {++ const calls = [];++ const server = http.createServer((req, res) => {++ const chunks = [];++ req.on('data', (c) => chunks.push(c));++ req.on('end', () => {++ const json = (status, obj) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(obj)); };++ if (req.url === '/oauth/token') return json(200, { access_token: 'tok_search', token_type: 'Bearer', expires_in: 300 });++ calls.push({ url: req.url, body: JSON.parse(Buffer.concat(chunks).toString() || 'null') });++ return json(201, {});++ });++ });++ await new Promise((r) => server.listen(0, '127.0.0.1', r));++ return { url: `http://127.0.0.1:${server.address().port}`, calls, close: () => new Promise((r) => server.close(r)) };++}++++const ev = (id, type, payload) => ({ event_id: id, event_type: type, source: 'network', version: 1, timestamp: new Date().toISOString(), payload });++const body = (event) => JSON.stringify({ event, seq: 1 });++++t('every table that names a person is covered or exempted with a reason', async () => {++ const stub = await startNetworkStub();++ const h = await boot({ env: { SEARCH_EVENTS_SECRET: SECRET, OV_NETWORK_INTERNAL_URL: stub.url, OV_OAUTH_CLIENT_SECRET: 'search-secret' } });++ try {++ const missing = await uncovered(h.db, {++ tables: TABLES,++ exempt: {++ account_data_events: 'the ledger of requests itself: which export and deletion Search applied',++ documents: 'the index is keyed by the publishing owner (a service), not a person',++ ingest_rejections: 'the index intake is keyed by the publishing owner, not a person',++ removals: 'the index removals are keyed by the publishing owner, not a person',++ search_embeddings: 'the embeddings belong to an indexed document, not a person',++ },++ });++ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);++ } finally { await h.stop(); await stub.close(); }++});++++t('an export carries the person\'s saved searches; a deletion erases them and confirms once', async () => {++ const stub = await startNetworkStub();++ const h = await boot({ env: { SEARCH_EVENTS_SECRET: SECRET, OV_NETWORK_INTERNAL_URL: stub.url, OV_OAUTH_CLIENT_SECRET: 'search-secret' } });++ try {++ for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {++ await h.db.exec("INSERT INTO saved_searches (id, subject, name, q, filters, query_hash, created_at, updated_at) VALUES ($1, $2, $3, 'x', '{}', $4, 1, 1)",++ [`svs_${name}`, subject, `search-${name}`, `h-${name}`]);++ }++ const deliver = async (event) => {++ const raw = body(event);++ const res = await fetch(`${h.base}/internal/events`, { method: 'POST', body: raw, headers: { 'Content-Type': 'application/json', ...signDeliveryHeaders(raw, SECRET) } });++ return { status: res.status, json: await res.json().catch(() => null) };++ };++++ const r = await deliver(ev('evt_01JZ0000000000000000000E01', 'network.account.export_requested', { export_id: EXP, subject: A }));++ assert.strictEqual(r.status, 200, JSON.stringify(r.json));++ assert.strictEqual(r.json.outcome, 'exported');++ const part = stub.calls.find((c) => c.url === `/internal/account-exports/${EXP}/parts`);++ assert.ok(part, 'the part was pushed to Network');++ const file = part.body.files.find((f) => f.name === 'saved-searches.json');++ assert.ok(file, 'saved-searches.json is in the part');++ assert.deepStrictEqual(file.content.map((x) => x.name), ['search-ada']);++++ const d = await deliver(ev('evt_01JZ0000000000000000000D01', 'network.account.deleted', { deletion_id: DEL, subject: A, aliases: [] }));++ assert.strictEqual(d.status, 200, JSON.stringify(d.json));++ assert.strictEqual(d.json.outcome, 'erased');++ assert.strictEqual(Number(await h.db.value('SELECT count(*)::int FROM saved_searches WHERE subject = $1', [A])), 0);++ assert.strictEqual(Number(await h.db.value('SELECT count(*)::int FROM saved_searches WHERE subject = $1', [B])), 1);++ const confirmation = stub.calls.find((c) => c.url === `/internal/account-deletions/${DEL}/confirmations`);++ assert.ok(confirmation, 'the confirmation was sent');++ assert.strictEqual(confirmation.body.erased.saved_searches, 1);++ } finally { await h.stop(); await stub.close(); }++});++++t.run();
@@ -484,6 +484,6 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [
@@ -484,6 +484,6 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [
<!-- versions:start -->
<!-- versions:start -->
- openvibe-contracts: v0.225.0
- openvibe-contracts: v0.225.0
-- openvibe-sdk: v0.67.0
+- openvibe-sdk: v0.74.0
- openvibe-shared: v4.48.0
- openvibe-shared: v4.48.0
<!-- versions:end -->
<!-- versions:end -->
@@ -8,7 +8,7 @@
@@ -8,7 +8,7 @@
"port": 4710,
"port": 4710,
"domain": "search.openvibe.network (vhost in deploy/nginx; the service manifest records the search page and query API as public since 2026-09-23)",
"domain": "search.openvibe.network (vhost in deploy/nginx; the service manifest records the search page and query API as public since 2026-09-23)",
"contracts": "openvibe-contracts v0.225.0",
"contracts": "openvibe-contracts v0.225.0",
- "sdk": "openvibe-sdk v0.67.0",
+ "sdk": "openvibe-sdk v0.74.0",
"shared": "openvibe-shared v4.48.0",
"shared": "openvibe-shared v4.48.0",
"capabilities": [
"capabilities": [
"search.document.write",
"search.document.write",
new file mode 100644
new file mode 100644
@@ -0,0 +1,13 @@
@@ -0,0 +1,13 @@
+-- phase: expand
+-- OpenVibe.Search: the receipts of the ADR-033 account export and deletion deliveries this service applied
+-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.
+-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.
+
+CREATE TABLE IF NOT EXISTS account_data_events (
+ id TEXT PRIMARY KEY,
+ kind TEXT NOT NULL,
+ subject TEXT NOT NULL,
+ outcome JSONB,
+ applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ sent_at TIMESTAMPTZ
+);
@@ -14,7 +14,7 @@
@@ -14,7 +14,7 @@
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.225.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.225.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
},
},
@@ -1041,9 +1041,9 @@
@@ -1041,9 +1041,9 @@
}
}
},
},
"node_modules/openvibe-sdk": {
"node_modules/openvibe-sdk": {
- "version": "0.67.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
- "integrity": "sha512-s7NzjaNcowVcOIiup2z0ze+xsq/qnr++3pHSI8JzirqJPQ5JsBSUF5RkjOHyfk9bRKlmhY0fsNS6x4dsYylyqw==",
+ "version": "0.74.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
+ "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==",
"license": "MIT",
"license": "MIT",
"bin": {
"bin": {
"openvibe-download": "bin/openvibe-download.js"
"openvibe-download": "bin/openvibe-download.js"
@@ -26,7 +26,7 @@
@@ -26,7 +26,7 @@
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.225.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.225.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
},
},
new file mode 100644
new file mode 100644
@@ -0,0 +1,23 @@
@@ -0,0 +1,23 @@
+'use strict';
+/**
+ * Account export and deletion → Search (ADR-033; openvibe-sdk/account-data).
+ *
+ * What Search keeps of a person's own is their saved searches (saved_searches.subject = usr_…): the query, its
+ * filters and when it last ran. They are exported and deleted with the account.
+ *
+ * The index itself is not a person's: documents, ingest_rejections, removals and search_embeddings are keyed by the
+ * publishing owner (a service), and a document is erased when its owner sends index_document.deleted. A person's
+ * deletion does not silently remove published pages: their owner does that. Telemetry samples that name a person are
+ * covered by the SDK itself (telemetry.json, then subject NULL).
+ */
+const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
+
+const TABLES = [
+ { table: 'saved_searches', subject: 'subject', file: 'saved-searches.json', columns: ['id', 'name', 'q', 'filters', 'created_at', 'updated_at', 'last_run_at'], order: 'updated_at' },
+];
+
+function create({ db, log = console } = {}) {
+ return createAccountData({ db, service: 'search', tables: TABLES, log });
+}
+
+module.exports = { create, TABLES, TOPICS };
@@ -20,6 +20,7 @@ const crypto = require('crypto');
@@ -20,6 +20,7 @@ const crypto = require('crypto');
const express = require('express');
const express = require('express');
const { http } = require('openvibe-contracts');
const { http } = require('openvibe-contracts');
const { validate } = require('../document');
const { validate } = require('../document');
+const { TOPICS: ACCOUNT_TOPICS } = require('openvibe-sdk/account-data');
const CONSUMER = 'search-index';
const CONSUMER = 'search-index';
const TYPE_RE = /^([a-z][a-z0-9-]{1,39})\.index_document\.(upserted|deleted)$/;
const TYPE_RE = /^([a-z][a-z0-9-]{1,39})\.index_document\.(upserted|deleted)$/;
@@ -119,7 +120,7 @@ function documentFromEvent(event, owners) {
@@ -119,7 +120,7 @@ function documentFromEvent(event, owners) {
return { doc };
return { doc };
}
}
-function webhookRouter({ config, db, store, outbox, log = console, now }) {
+function webhookRouter({ config, db, store, outbox, accountData = null, accountSend = null, log = console, now }) {
const router = express.Router();
const router = express.Router();
const inbox = createInbox(db, { now });
const inbox = createInbox(db, { now });
@@ -142,6 +143,7 @@ function webhookRouter({ config, db, store, outbox, log = console, now }) {
@@ -142,6 +143,7 @@ function webhookRouter({ config, db, store, outbox, log = console, now }) {
}
}
const r = await inbox.once(event.event_id, async () => {
const r = await inbox.once(event.event_id, async () => {
+ if (accountData && ACCOUNT_TOPICS.includes(event.event_type)) return accountData.apply(event, { send: accountSend });
const parsed = documentFromEvent(event, config.events.owners);
const parsed = documentFromEvent(event, config.events.owners);
if (parsed.ignore) return 'ignored';
if (parsed.ignore) return 'ignored';
if (parsed.reject) {
if (parsed.reject) {
@@ -23,7 +23,7 @@ const { createIndexNow } = require('openvibe-shared/indexnow');
@@ -23,7 +23,7 @@ const { createIndexNow } = require('openvibe-shared/indexnow');
const openapi = require('./openapi');
const openapi = require('./openapi');
const pkg = require('../package.json');
const pkg = require('../package.json');
-function createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher: givenSearcher = null, notifier = null, vectorStore = null, embeddings = null, fetchImpl = globalThis.fetch, log = console, now }) {
+function createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher: givenSearcher = null, notifier = null, vectorStore = null, embeddings = null, accountData = null, accountSend = null, fetchImpl = globalThis.fetch, log = console, now }) {
const app = express();
const app = express();
app.disable('x-powered-by');
app.disable('x-powered-by');
app.set('trust proxy', 'loopback');
app.set('trust proxy', 'loopback');
@@ -58,7 +58,7 @@ function createApp({ config, db, store, engine, auth, outbox, purges, purger, sa
@@ -58,7 +58,7 @@ function createApp({ config, db, store, engine, auth, outbox, purges, purger, sa
// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).
// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).
app.use('/shared', require('openvibe-shared/serve').handler());
app.use('/shared', require('openvibe-shared/serve').handler());
- app.use(webhookRouter({ config, db, store, outbox, log, now }));
+ app.use(webhookRouter({ config, db, store, outbox, accountData, accountSend, log, now }));
app.use('/api', express.json({ limit: config.maxBodyBytes, type: ['application/json', 'application/*+json'] }));
app.use('/api', express.json({ limit: config.maxBodyBytes, type: ['application/json', 'application/*+json'] }));
@@ -23,6 +23,8 @@ const { createSearcher } = require('./api/query');
@@ -23,6 +23,8 @@ const { createSearcher } = require('./api/query');
const { createNetworkPush } = require('./network-push');
const { createNetworkPush } = require('./network-push');
const { createSavedNotifier } = require('./saved-notifier');
const { createSavedNotifier } = require('./saved-notifier');
const { createApp } = require('./app');
const { createApp } = require('./app');
+const accountDataLib = require('./account-data');
+const { createNetworkSender, startSubscriptions } = require('openvibe-sdk/account-data');
const { createPgVectorStore } = require('./fabric/vector-store');
const { createPgVectorStore } = require('./fabric/vector-store');
const { createEmbeddingClient } = require('./fabric/embed');
const { createEmbeddingClient } = require('./fabric/embed');
const { createEmbedder } = require('./fabric/embedder');
const { createEmbedder } = require('./fabric/embedder');
@@ -62,7 +64,9 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
@@ -62,7 +64,9 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
config, saved, searcher, now, log,
config, saved, searcher, now, log,
network: networkPush || createNetworkPush({ config, fetchImpl }),
network: networkPush || createNetworkPush({ config, fetchImpl }),
});
});
- const app = createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher, notifier, fetchImpl, log, now, vectorStore, embeddings: embeddingClient });
+ const accountData = accountDataLib.create({ db, log });
+ const accountSend = config.oauth.clientSecret ? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret }) : null;
+ const app = createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher, notifier, fetchImpl, log, now, vectorStore, embeddings: embeddingClient, accountData, accountSend });
// The background refresher is not started under the test runtime (the tests stub the fetch and load
// The background refresher is not started under the test runtime (the tests stub the fetch and load
// the keys once below); a real boot refreshes on an unref'd timer. keyLoaded settles the first load.
// the keys once below); a real boot refreshes on an unref'd timer. keyLoaded settles the first load.
@@ -80,16 +84,24 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
@@ -80,16 +84,24 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
cachePruneTimer.unref?.();
cachePruneTimer.unref?.();
let server = null;
let server = null;
+ let accountSubscription = null;
if (listen) {
if (listen) {
server = await new Promise((resolve, reject) => {
server = await new Promise((resolve, reject) => {
const s = app.listen(config.port, config.host, () => resolve(s));
const s = app.listen(config.port, config.host, () => resolve(s));
s.on('error', reject);
s.on('error', reject);
});
});
log.log(`[search] listening on http://${config.host}:${server.address().port} (engine ${engine.name})`);
log.log(`[search] listening on http://${config.host}:${server.address().port} (engine ${engine.name})`);
+ if (config.events.webhookSecrets.length && config.events.url && config.oauth.clientSecret) {
+ accountSubscription = startSubscriptions({
+ eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.port}/internal/events`, secret: config.events.webhookSecrets[0],
+ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log,
+ });
+ }
}
}
async function close() {
async function close() {
clearInterval(pruneTimer);
clearInterval(pruneTimer);
+ if (accountSubscription) accountSubscription.stop();
clearInterval(cachePruneTimer);
clearInterval(cachePruneTimer);
jwks.stop();
jwks.stop();
await outbox.stop();
await outbox.stop();
new file mode 100644
new file mode 100644
@@ -0,0 +1,93 @@
@@ -0,0 +1,93 @@
+'use strict';
+/**
+ * ADR-033: Search's part of an account export and deletion, through the signed POST /internal/events route with a
+ * stand-in Network. Ada's saved searches are exported and erased; Bo's stay; the index rows (keyed by their publishing
+ * owner, not a person) are untouched. The guard fails when a new table with a person column is neither covered nor
+ * exempted with a reason.
+ */
+const assert = require('assert');
+const http = require('http');
+const { uncovered } = require('openvibe-sdk/account-data');
+const { signDeliveryHeaders } = require('openvibe-sdk/events');
+const { boot, suite } = require('./helpers');
+const { TABLES } = require('../server/account-data');
+
+const t = suite('account-data');
+const SECRET = `whsec_${'fixture'.repeat(6)}`;
+const A = 'usr_01JZ0000000000000000000AAA';
+const B = 'usr_01JZ0000000000000000000BBB';
+const EXP = 'exp_01JZ0000000000000000000EXP';
+const DEL = 'del_01JZ0000000000000000000DEX';
+
+async function startNetworkStub() {
+ const calls = [];
+ const server = http.createServer((req, res) => {
+ const chunks = [];
+ req.on('data', (c) => chunks.push(c));
+ req.on('end', () => {
+ const json = (status, obj) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(obj)); };
+ if (req.url === '/oauth/token') return json(200, { access_token: 'tok_search', token_type: 'Bearer', expires_in: 300 });
+ calls.push({ url: req.url, body: JSON.parse(Buffer.concat(chunks).toString() || 'null') });
+ return json(201, {});
+ });
+ });
+ await new Promise((r) => server.listen(0, '127.0.0.1', r));
+ return { url: `http://127.0.0.1:${server.address().port}`, calls, close: () => new Promise((r) => server.close(r)) };
+}
+
+const ev = (id, type, payload) => ({ event_id: id, event_type: type, source: 'network', version: 1, timestamp: new Date().toISOString(), payload });
+const body = (event) => JSON.stringify({ event, seq: 1 });
+
+t('every table that names a person is covered or exempted with a reason', async () => {
+ const stub = await startNetworkStub();
+ const h = await boot({ env: { SEARCH_EVENTS_SECRET: SECRET, OV_NETWORK_INTERNAL_URL: stub.url, OV_OAUTH_CLIENT_SECRET: 'search-secret' } });
+ try {
+ const missing = await uncovered(h.db, {
+ tables: TABLES,
+ exempt: {
+ account_data_events: 'the ledger of requests itself: which export and deletion Search applied',
+ documents: 'the index is keyed by the publishing owner (a service), not a person',
+ ingest_rejections: 'the index intake is keyed by the publishing owner, not a person',
+ removals: 'the index removals are keyed by the publishing owner, not a person',
+ search_embeddings: 'the embeddings belong to an indexed document, not a person',
+ },
+ });
+ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
+ } finally { await h.stop(); await stub.close(); }
+});
+
+t('an export carries the person\'s saved searches; a deletion erases them and confirms once', async () => {
+ const stub = await startNetworkStub();
+ const h = await boot({ env: { SEARCH_EVENTS_SECRET: SECRET, OV_NETWORK_INTERNAL_URL: stub.url, OV_OAUTH_CLIENT_SECRET: 'search-secret' } });
+ try {
+ for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {
+ await h.db.exec("INSERT INTO saved_searches (id, subject, name, q, filters, query_hash, created_at, updated_at) VALUES ($1, $2, $3, 'x', '{}', $4, 1, 1)",
+ [`svs_${name}`, subject, `search-${name}`, `h-${name}`]);
+ }
+ const deliver = async (event) => {
+ const raw = body(event);
+ const res = await fetch(`${h.base}/internal/events`, { method: 'POST', body: raw, headers: { 'Content-Type': 'application/json', ...signDeliveryHeaders(raw, SECRET) } });
+ return { status: res.status, json: await res.json().catch(() => null) };
+ };
+
+ const r = await deliver(ev('evt_01JZ0000000000000000000E01', 'network.account.export_requested', { export_id: EXP, subject: A }));
+ assert.strictEqual(r.status, 200, JSON.stringify(r.json));
+ assert.strictEqual(r.json.outcome, 'exported');
+ const part = stub.calls.find((c) => c.url === `/internal/account-exports/${EXP}/parts`);
+ assert.ok(part, 'the part was pushed to Network');
+ const file = part.body.files.find((f) => f.name === 'saved-searches.json');
+ assert.ok(file, 'saved-searches.json is in the part');
+ assert.deepStrictEqual(file.content.map((x) => x.name), ['search-ada']);
+
+ const d = await deliver(ev('evt_01JZ0000000000000000000D01', 'network.account.deleted', { deletion_id: DEL, subject: A, aliases: [] }));
+ assert.strictEqual(d.status, 200, JSON.stringify(d.json));
+ assert.strictEqual(d.json.outcome, 'erased');
+ assert.strictEqual(Number(await h.db.value('SELECT count(*)::int FROM saved_searches WHERE subject = $1', [A])), 0);
+ assert.strictEqual(Number(await h.db.value('SELECT count(*)::int FROM saved_searches WHERE subject = $1', [B])), 1);
+ const confirmation = stub.calls.find((c) => c.url === `/internal/account-deletions/${DEL}/confirmations`);
+ assert.ok(confirmation, 'the confirmation was sent');
+ assert.strictEqual(confirmation.body.erased.saved_searches, 1);
+ } finally { await h.stop(); await stub.close(); }
+});
+
+t.run();