openvibers/openvibe.search Commits

account-data: a person's saved searches, with the guard on every person column (#145)

OP OpenVibers committed 14 days ago · df04306373 · 10 files, +154 −11

* account-data: a person's saved searches, with the guard on every person column

* account-data: the receipts table comes from a migration, not from the runtime role

The boot-time ensureSchema() ran DDL as the serving role, which may not change
the schema; the table is now migration NNNN, like every other table here.

Changes

ddiff --git a/README.md b/README.md
iindex f35ba7d..4ad7a8f 100644
---- a/README.md
++++ b/README.md
@@@ -484,6 +484,6 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [
<!-- versions:start -->
- openvibe-contracts: v0.225.0
--- openvibe-sdk: v0.67.0
++- openvibe-sdk: v0.74.0
- openvibe-shared: v4.48.0
<!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex fae1572..b8ad344 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -8,7 +8,7 @@
"port": 4710,
"domain": "search.openvibe.network (vhost in deploy/nginx; the service manifest records the search page and query API as public since 2026-09-23)",
"contracts": "openvibe-contracts v0.225.0",
-- "sdk": "openvibe-sdk v0.67.0",
++ "sdk": "openvibe-sdk v0.74.0",
"shared": "openvibe-shared v4.48.0",
"capabilities": [
"search.document.write",
ddiff --git a/migrations/0007_account_data.sql b/migrations/0007_account_data.sql
nnew file mode 100644
iindex 0000000..b62fb89
---- /dev/null
++++ b/migrations/0007_account_data.sql
@@@ -0,0 +1,13 @@
++-- phase: expand
++-- OpenVibe.Search: the receipts of the ADR-033 account export and deletion deliveries this service applied
++-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.
++-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.
++
++CREATE TABLE IF NOT EXISTS account_data_events (
++ id TEXT PRIMARY KEY,
++ kind TEXT NOT NULL,
++ subject TEXT NOT NULL,
++ outcome JSONB,
++ applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),
++ sent_at TIMESTAMPTZ
++);
ddiff --git a/package-lock.json b/package-lock.json
iindex 49ef9e9..b5486ad 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -14,7 +14,7 @@
"dotenv": "^16.4.7",
"express": "^4.21.2",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.225.0",
-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
},
@@@ -1041,9 +1041,9 @@
}
},
"node_modules/openvibe-sdk": {
-- "version": "0.67.0",
-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
-- "integrity": "sha512-s7NzjaNcowVcOIiup2z0ze+xsq/qnr++3pHSI8JzirqJPQ5JsBSUF5RkjOHyfk9bRKlmhY0fsNS6x4dsYylyqw==",
++ "version": "0.74.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
++ "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==",
"license": "MIT",
"bin": {
"openvibe-download": "bin/openvibe-download.js"
ddiff --git a/package.json b/package.json
iindex 09dbe49..128c4e6 100644
---- a/package.json
++++ b/package.json
@@@ -26,7 +26,7 @@
"dotenv": "^16.4.7",
"express": "^4.21.2",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.225.0",
-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.67.0",
++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
},
ddiff --git a/server/account-data.js b/server/account-data.js
nnew file mode 100644
iindex 0000000..8694472
---- /dev/null
++++ b/server/account-data.js
@@@ -0,0 +1,23 @@
++'use strict';
++/**
++ * Account export and deletion → Search (ADR-033; openvibe-sdk/account-data).
++ *
++ * What Search keeps of a person's own is their saved searches (saved_searches.subject = usr_…): the query, its
++ * filters and when it last ran. They are exported and deleted with the account.
++ *
++ * The index itself is not a person's: documents, ingest_rejections, removals and search_embeddings are keyed by the
++ * publishing owner (a service), and a document is erased when its owner sends index_document.deleted. A person's
++ * deletion does not silently remove published pages: their owner does that. Telemetry samples that name a person are
++ * covered by the SDK itself (telemetry.json, then subject NULL).
++ */
++const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
++
++const TABLES = [
++ { table: 'saved_searches', subject: 'subject', file: 'saved-searches.json', columns: ['id', 'name', 'q', 'filters', 'created_at', 'updated_at', 'last_run_at'], order: 'updated_at' },
++];
++
++function create({ db, log = console } = {}) {
++ return createAccountData({ db, service: 'search', tables: TABLES, log });
++}
++
++module.exports = { create, TABLES, TOPICS };
ddiff --git a/server/api/webhook.js b/server/api/webhook.js
iindex 2c92574..f293e4b 100644
---- a/server/api/webhook.js
++++ b/server/api/webhook.js
@@@ -20,6 +20,7 @@ const crypto = require('crypto');
const express = require('express');
const { http } = require('openvibe-contracts');
const { validate } = require('../document');
++const { TOPICS: ACCOUNT_TOPICS } = require('openvibe-sdk/account-data');
const CONSUMER = 'search-index';
const TYPE_RE = /^([a-z][a-z0-9-]{1,39})\.index_document\.(upserted|deleted)$/;
@@@ -119,7 +120,7 @@ function documentFromEvent(event, owners) {
return { doc };
}
--function webhookRouter({ config, db, store, outbox, log = console, now }) {
++function webhookRouter({ config, db, store, outbox, accountData = null, accountSend = null, log = console, now }) {
const router = express.Router();
const inbox = createInbox(db, { now });
@@@ -142,6 +143,7 @@ function webhookRouter({ config, db, store, outbox, log = console, now }) {
}
const r = await inbox.once(event.event_id, async () => {
++ if (accountData && ACCOUNT_TOPICS.includes(event.event_type)) return accountData.apply(event, { send: accountSend });
const parsed = documentFromEvent(event, config.events.owners);
if (parsed.ignore) return 'ignored';
if (parsed.reject) {
ddiff --git a/server/app.js b/server/app.js
iindex 17d9f79..1a4b93f 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -23,7 +23,7 @@ const { createIndexNow } = require('openvibe-shared/indexnow');
const openapi = require('./openapi');
const pkg = require('../package.json');
--function createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher: givenSearcher = null, notifier = null, vectorStore = null, embeddings = null, fetchImpl = globalThis.fetch, log = console, now }) {
++function createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher: givenSearcher = null, notifier = null, vectorStore = null, embeddings = null, accountData = null, accountSend = null, fetchImpl = globalThis.fetch, log = console, now }) {
const app = express();
app.disable('x-powered-by');
app.set('trust proxy', 'loopback');
@@@ -58,7 +58,7 @@ function createApp({ config, db, store, engine, auth, outbox, purges, purger, sa
// This site's own pinned copy of the OpenVibe Frame's browser files (openvibe-shared/serve).
app.use('/shared', require('openvibe-shared/serve').handler());
-- app.use(webhookRouter({ config, db, store, outbox, log, now }));
++ app.use(webhookRouter({ config, db, store, outbox, accountData, accountSend, log, now }));
app.use('/api', express.json({ limit: config.maxBodyBytes, type: ['application/json', 'application/*+json'] }));
ddiff --git a/server/index.js b/server/index.js
iindex 4c489af..17ffedd 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -23,6 +23,8 @@ const { createSearcher } = require('./api/query');
const { createNetworkPush } = require('./network-push');
const { createSavedNotifier } = require('./saved-notifier');
const { createApp } = require('./app');
++const accountDataLib = require('./account-data');
++const { createNetworkSender, startSubscriptions } = require('openvibe-sdk/account-data');
const { createPgVectorStore } = require('./fabric/vector-store');
const { createEmbeddingClient } = require('./fabric/embed');
const { createEmbedder } = require('./fabric/embedder');
@@@ -62,7 +64,9 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
config, saved, searcher, now, log,
network: networkPush || createNetworkPush({ config, fetchImpl }),
});
-- const app = createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher, notifier, fetchImpl, log, now, vectorStore, embeddings: embeddingClient });
++ const accountData = accountDataLib.create({ db, log });
++ const accountSend = config.oauth.clientSecret ? createNetworkSender({ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret }) : null;
++ const app = createApp({ config, db, store, engine, auth, outbox, purges, purger, saved, searcher, notifier, fetchImpl, log, now, vectorStore, embeddings: embeddingClient, accountData, accountSend });
// The background refresher is not started under the test runtime (the tests stub the fetch and load
// the keys once below); a real boot refreshes on an unref'd timer. keyLoaded settles the first load.
@@@ -80,16 +84,24 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
cachePruneTimer.unref?.();
let server = null;
++ let accountSubscription = null;
if (listen) {
server = await new Promise((resolve, reject) => {
const s = app.listen(config.port, config.host, () => resolve(s));
s.on('error', reject);
});
log.log(`[search] listening on http://${config.host}:${server.address().port} (engine ${engine.name})`);
++ if (config.events.webhookSecrets.length && config.events.url && config.oauth.clientSecret) {
++ accountSubscription = startSubscriptions({
++ eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.port}/internal/events`, secret: config.events.webhookSecrets[0],
++ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log,
++ });
++ }
}
async function close() {
clearInterval(pruneTimer);
++ if (accountSubscription) accountSubscription.stop();
clearInterval(cachePruneTimer);
jwks.stop();
await outbox.stop();
ddiff --git a/test/account-data.test.js b/test/account-data.test.js
nnew file mode 100644
iindex 0000000..aa83198
---- /dev/null
++++ b/test/account-data.test.js
@@@ -0,0 +1,93 @@
++'use strict';
++/**
++ * ADR-033: Search's part of an account export and deletion, through the signed POST /internal/events route with a
++ * stand-in Network. Ada's saved searches are exported and erased; Bo's stay; the index rows (keyed by their publishing
++ * owner, not a person) are untouched. The guard fails when a new table with a person column is neither covered nor
++ * exempted with a reason.
++ */
++const assert = require('assert');
++const http = require('http');
++const { uncovered } = require('openvibe-sdk/account-data');
++const { signDeliveryHeaders } = require('openvibe-sdk/events');
++const { boot, suite } = require('./helpers');
++const { TABLES } = require('../server/account-data');
++
++const t = suite('account-data');
++const SECRET = `whsec_${'fixture'.repeat(6)}`;
++const A = 'usr_01JZ0000000000000000000AAA';
++const B = 'usr_01JZ0000000000000000000BBB';
++const EXP = 'exp_01JZ0000000000000000000EXP';
++const DEL = 'del_01JZ0000000000000000000DEX';
++
++async function startNetworkStub() {
++ const calls = [];
++ const server = http.createServer((req, res) => {
++ const chunks = [];
++ req.on('data', (c) => chunks.push(c));
++ req.on('end', () => {
++ const json = (status, obj) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(obj)); };
++ if (req.url === '/oauth/token') return json(200, { access_token: 'tok_search', token_type: 'Bearer', expires_in: 300 });
++ calls.push({ url: req.url, body: JSON.parse(Buffer.concat(chunks).toString() || 'null') });
++ return json(201, {});
++ });
++ });
++ await new Promise((r) => server.listen(0, '127.0.0.1', r));
++ return { url: `http://127.0.0.1:${server.address().port}`, calls, close: () => new Promise((r) => server.close(r)) };
++}
++
++const ev = (id, type, payload) => ({ event_id: id, event_type: type, source: 'network', version: 1, timestamp: new Date().toISOString(), payload });
++const body = (event) => JSON.stringify({ event, seq: 1 });
++
++t('every table that names a person is covered or exempted with a reason', async () => {
++ const stub = await startNetworkStub();
++ const h = await boot({ env: { SEARCH_EVENTS_SECRET: SECRET, OV_NETWORK_INTERNAL_URL: stub.url, OV_OAUTH_CLIENT_SECRET: 'search-secret' } });
++ try {
++ const missing = await uncovered(h.db, {
++ tables: TABLES,
++ exempt: {
++ account_data_events: 'the ledger of requests itself: which export and deletion Search applied',
++ documents: 'the index is keyed by the publishing owner (a service), not a person',
++ ingest_rejections: 'the index intake is keyed by the publishing owner, not a person',
++ removals: 'the index removals are keyed by the publishing owner, not a person',
++ search_embeddings: 'the embeddings belong to an indexed document, not a person',
++ },
++ });
++ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
++ } finally { await h.stop(); await stub.close(); }
++});
++
++t('an export carries the person\'s saved searches; a deletion erases them and confirms once', async () => {
++ const stub = await startNetworkStub();
++ const h = await boot({ env: { SEARCH_EVENTS_SECRET: SECRET, OV_NETWORK_INTERNAL_URL: stub.url, OV_OAUTH_CLIENT_SECRET: 'search-secret' } });
++ try {
++ for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {
++ await h.db.exec("INSERT INTO saved_searches (id, subject, name, q, filters, query_hash, created_at, updated_at) VALUES ($1, $2, $3, 'x', '{}', $4, 1, 1)",
++ [`svs_${name}`, subject, `search-${name}`, `h-${name}`]);
++ }
++ const deliver = async (event) => {
++ const raw = body(event);
++ const res = await fetch(`${h.base}/internal/events`, { method: 'POST', body: raw, headers: { 'Content-Type': 'application/json', ...signDeliveryHeaders(raw, SECRET) } });
++ return { status: res.status, json: await res.json().catch(() => null) };
++ };
++
++ const r = await deliver(ev('evt_01JZ0000000000000000000E01', 'network.account.export_requested', { export_id: EXP, subject: A }));
++ assert.strictEqual(r.status, 200, JSON.stringify(r.json));
++ assert.strictEqual(r.json.outcome, 'exported');
++ const part = stub.calls.find((c) => c.url === `/internal/account-exports/${EXP}/parts`);
++ assert.ok(part, 'the part was pushed to Network');
++ const file = part.body.files.find((f) => f.name === 'saved-searches.json');
++ assert.ok(file, 'saved-searches.json is in the part');
++ assert.deepStrictEqual(file.content.map((x) => x.name), ['search-ada']);
++
++ const d = await deliver(ev('evt_01JZ0000000000000000000D01', 'network.account.deleted', { deletion_id: DEL, subject: A, aliases: [] }));
++ assert.strictEqual(d.status, 200, JSON.stringify(d.json));
++ assert.strictEqual(d.json.outcome, 'erased');
++ assert.strictEqual(Number(await h.db.value('SELECT count(*)::int FROM saved_searches WHERE subject = $1', [A])), 0);
++ assert.strictEqual(Number(await h.db.value('SELECT count(*)::int FROM saved_searches WHERE subject = $1', [B])), 1);
++ const confirmation = stub.calls.find((c) => c.url === `/internal/account-deletions/${DEL}/confirmations`);
++ assert.ok(confirmation, 'the confirmation was sent');
++ assert.strictEqual(confirmation.body.erased.saved_searches, 1);
++ } finally { await h.stop(); await stub.close(); }
++});
++
++t.run();