openvibers/openvibe.news Commits
account-data: a person's drafts and profile, with the guard on every person column (#147)
Changes
@@@ -448,7 +448,7 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [<!-- versions:start -->- openvibe-contracts: v0.222.0--- openvibe-sdk: v0.73.0++- openvibe-sdk: v0.74.0- openvibe-shared: v4.48.0- openvibe-publishing: v1.4.1<!-- versions:end -->@@@ -8,13 +8,13 @@"port": 4820,"domain": "openvibe.news","contracts": "openvibe-contracts v0.222.0",-- "sdk": "openvibe-sdk v0.73.0",++ "sdk": "openvibe-sdk v0.74.0","shared": "openvibe-shared v4.48.0","packages": ["openvibe-publishing v1.4.1","openvibe-contracts v0.222.0","openvibe-shared v4.48.0",-- "openvibe-sdk v0.73.0"++ "openvibe-sdk v0.74.0"],"capabilities": ["news.story.create",nnew file mode 100644@@@ -0,0 +1,13 @@++-- phase: expand++-- OpenVibe.News: the receipts of the ADR-033 account export and deletion deliveries this service applied++-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.++-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.++++CREATE TABLE IF NOT EXISTS account_data_events (++ id TEXT PRIMARY KEY,++ kind TEXT NOT NULL,++ subject TEXT NOT NULL,++ outcome JSONB,++ applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),++ sent_at TIMESTAMPTZ++);@@@ -17,7 +17,7 @@"iovalkey": "^0.4.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0","openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0","pg": "^8.23.0"},@@@ -1182,9 +1182,9 @@}},"node_modules/openvibe-sdk": {-- "version": "0.73.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",-- "integrity": "sha512-XClxoVm9M6RA8eXlGgYZ0u6q9n5h79mCXqAAEg9fX21KgcqP69RmG+cX+6wrOf7TaEj7eSBWMxHjiIeIa9N7Dg==",++ "version": "0.74.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",++ "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==","license": "MIT","bin": {"openvibe-download": "bin/openvibe-download.js"@@@ -31,7 +31,7 @@"iovalkey": "^0.4.0","openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0","openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0","pg": "^8.23.0"},nnew file mode 100644@@@ -0,0 +1,33 @@++'use strict';++/**++ * Account export and deletion → News (ADR-033; openvibe-sdk/account-data).++ *++ * A person's private working rows go: their story drafts (news_story_drafts.owner) and the cached profile++ * (subject_projections). Their editorial contribution to a story — the story, a perspective, a timeline entry, a++ * cluster, an editorial flag — stays: it is the publication's record, and a byline is part of what readers are owed;++ * those rows are exported and kept, with created_by as the editorial attribution. Telemetry samples that name a++ * person are covered by the SDK itself.++ *++ * news_index_revisions.owner is the index document's owning service, not a person.++ */++const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');++++const editorial = (why) => ({ keep: why });++++const TABLES = [++ { table: 'news_story_drafts', subject: 'owner', file: 'story-drafts.json', order: 'updated_at', columns: ['entity_id', 'base_revision', 'content', 'fields', 'created_at', 'updated_at'] },++ { table: 'subject_projections', subject: 'subject', file: 'profile.json', order: 'refreshed_at' },++ { table: 'news_stories', subject: 'created_by', file: 'stories.json', order: 'first_published_at', columns: ['id', 'slug', 'working_headline', 'topic_id', 'state', 'first_published_at', 'published_at'], erase: editorial('published journalism keeps its editor byline as the publication record') },++ { table: 'news_perspectives', subject: 'created_by', file: 'perspectives.json', order: 'created_at', columns: ['id', 'story_id', 'label', 'description', 'position', 'created_at'], erase: editorial('a published perspective is part of the story') },++ { table: 'news_timeline_entries', subject: 'created_by', file: 'timeline-entries.json', order: 'created_at', columns: ['id', 'story_id', 'occurred_on', 'text', 'created_at'], erase: editorial('a published timeline entry is part of the story') },++ { table: 'news_story_clusters', subject: 'created_by', file: 'story-clusters.json', order: 'window_start', columns: ['id', 'label', 'status', 'window_start', 'window_end'], erase: editorial('a story cluster is the newsroom\'s record') },++ { table: 'news_editorial_flags', subject: 'created_by', file: 'editorial-flags.json', order: 'created_at', columns: ['id', 'story_id', 'kind', 'status', 'note', 'created_at'], erase: editorial('a correction, update or retraction is the record readers rely on') },++ // The edit history of a story: the publication's record of how the story was written, with its byline.++ { table: 'news_story_revisions', subject: 'author', file: 'story-revisions.json', order: 'created_at', columns: ['id', 'entity_id', 'number', 'kind', 'message', 'created_at'], erase: editorial('a published story keeps its edit history and byline as the publication record') },++];++++function create({ db, log = console } = {}) {++ return createAccountData({ db, service: 'news', tables: TABLES, log });++}++++module.exports = { create, TABLES, TOPICS };@@@ -40,6 +40,8 @@ const { createEditorRoutes } = require('./http/editor');const { createApi } = require('./http/api');const { createDiscoveryRoutes } = require('./http/discovery');const { createEvents } = require('./http/events');++const accountDataLib = require('./account-data');++const { createNetworkSender } = require('openvibe-sdk/account-data');const { createActorLimits } = require('./http/actor-limits');const { createNewsReadiness } = require('./observability');const { createWorker } = require('./worker');@@@ -58,6 +60,10 @@ async function createApp(opts = {}) {const fetchImpl = opts.fetchImpl || globalThis.fetch;// PostgreSQL (ADR-035): opened and migrated here unless the caller (a test, a script) hands in a store.const store = opts.store || await openStore(config, { now: opts.now, log });++ const accountData = opts.accountData || accountDataLib.create({ db: store.db, log });++ const accountSend = opts.accountSend || (config.oauth.clientSecret ? createNetworkSender({++ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl,++ }) : null);const outbox = createServiceOutbox({db: store.db, source: 'news',@@@ -101,7 +107,7 @@ async function createApp(opts = {}) {const briefs = createBriefs({ store, ai: config.briefs.enabled ? ai : null, log });const worker = createWorker({ config, ingest, outbox, briefs, log });-- const ctx = { config, store, outbox, publication, clusters, sources, ai, community, discussion, ingest, stories, topics, people, reading, auth, viewers, worker, indexnow, briefs };++ const ctx = { config, store, accountData, accountSend, outbox, publication, clusters, sources, ai, community, discussion, ingest, stories, topics, people, reading, auth, viewers, worker, indexnow, briefs };const app = express();app.disable('x-powered-by');@@@ -158,7 +164,7 @@ async function createApp(opts = {}) {if (indexnow.enabled) app.use(indexnow.keyFile);// ── Events webhook (raw body; before any other body parser) ─-- app.use(createEvents({ config, store, ingest, sources, log }));++ app.use(createEvents({ config, store, ingest, sources, log , accountData, accountSend }));app.use(cookieParser());@@@ -17,11 +17,12 @@const express = require('express');const { http } = require('openvibe-contracts');const { createEventConsumer } = require('openvibe-publishing/ingest');++const { TOPICS: ACCOUNT_TOPICS } = require('openvibe-sdk/account-data');const CONSUMER = 'news-sources';const EVT_RE = /^evt_[0-9A-HJKMNP-TV-Z]{26}$/;--function createEvents({ config, store, ingest, sources, log = console }) {++function createEvents({ config, store, ingest, sources, accountData = null, accountSend = null, log = console }) {const router = express.Router();const consumer = createEventConsumer({ db: store.db, secrets: config.events.webhookSecrets, consumer: CONSUMER, now: store.now });const { inbox, verify } = consumer;@@@ -40,6 +41,15 @@ function createEvents({ config, store, ingest, sources, log = console }) {}const attempt = Number(req.get('x-openvibe-delivery-attempt')) || 1;const type = String(event.event_type || '');++ if (accountData && ACCOUNT_TOPICS.includes(type)) {++ try {++ const outcome = await accountData.apply(event, { send: accountSend });++ return res.status(200).json({ event_id: event.event_id, outcome });++ } catch (err) {++ log.warn('[News] account-data delivery failed:', err && err.message ? err.message : err);++ return http.sendProblem(res, 503, 'news.account_retry', { detail: 'not applied; Events will retry', ctx });++ }++ }const p = event.payload && typeof event.payload === 'object' ? event.payload : {};const fromSources = event.source === 'sources';@@@ -8,6 +8,7 @@const { createApp } = require('./app');const { gracefulStop } = require('openvibe-sdk/service');const { startTelemetryRetention } = require('openvibe-sdk/telemetry');++const { startSubscriptions } = require('openvibe-sdk/account-data');/*** The process stop (openvibe-sdk/service, docs/service.md's 5 s family): the Sources pull worker stops@@@ -15,11 +16,11 @@ const { startTelemetryRetention } = require('openvibe-sdk/telemetry');* closes; past the deadline the process exits 0, as the hand-rolled 5 s timer did. Exported so a test* can inject `exit` and `signals: false`.*/--function createLifecycle({ server, ctx, exit, signals }) {++function createLifecycle({ server, ctx, exit, signals, extra = [] }) {return gracefulStop({name: 'News', server, deadlineExitCode: 0, exit, signals,stop: [() => ctx.worker.stop()],-- close: [() => ctx.outbox.stop(), async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); }, () => ctx.store.close()],++ close: [...extra, () => ctx.outbox.stop(), async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); }, () => ctx.store.close()],});}@@@ -41,7 +42,14 @@ async function start() {ctx.telemetryTick = setInterval(() => { ctx.telemetry.flush().catch(() => {}); }, 60_000);ctx.telemetryTick.unref();-- createLifecycle({ server, ctx });++ let accountSubscription = null;++ if (config.events.webhookSecrets.length && config.events.url && config.oauth.clientSecret) {++ accountSubscription = startSubscriptions({++ eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.port}/internal/events`, secret: config.events.webhookSecrets[0],++ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log: console,++ });++ }++ createLifecycle({ server, ctx, extra: [() => { if (accountSubscription) accountSubscription.stop(); }] });return { server, ctx };}nnew file mode 100644@@@ -0,0 +1,67 @@++'use strict';++/**++ * ADR-033: News' part of an account export and deletion, through the signed POST /internal/events route. A person's++ * drafts and cached profile are exported and erased; the story they opened and its byline stay as the publication's++ * record; Bo's rows stay. The guard fails on a new table with a person column that is neither covered nor exempted.++ */++const assert = require('assert');++const { uncovered } = require('openvibe-sdk/account-data');++const { signDeliveryHeaders } = require('openvibe-sdk/events');++const { boot, check, done } = require('./helpers/boot');++const { TABLES } = require('../server/account-data');++++const SECRET = `whsec_${'fixture'.repeat(6)}`;++const A = 'usr_01JZ0000000000000000000AAA';++const B = 'usr_01JZ0000000000000000000BBB';++++(async () => {++ const calls = [];++ const accountSend = async (path, body) => { calls.push({ path, body }); return { ok: true, status: 201 }; };++ const t = await boot({ env: { NEWS_EVENTS_SECRET: SECRET }, accountSend });++ const db = t.ctx.store.db;++ const deliver = async (event) => {++ const raw = JSON.stringify({ event, seq: 1 });++ const res = await fetch(`${t.base}/internal/events`, { method: 'POST', body: raw, headers: { 'content-type': 'application/json', ...signDeliveryHeaders(raw, SECRET) } });++ return { status: res.status, json: await res.json().catch(() => null) };++ };++ const ev = (id, type, payload) => ({ event_id: id, event_type: type, source: 'network', version: 1, timestamp: new Date().toISOString(), payload });++ try {++ await check('every table that names a person is covered or exempted with a reason', async () => {++ const missing = await uncovered(db, {++ tables: TABLES,++ exempt: {++ account_data_events: 'the ledger of requests itself: which export and deletion News applied',++ news_index_revisions: "owner is the search-index document's owner (type/id), not a person; the row holds a revision and hash",++ },++ });++ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);++ });++++ await check('an export carries drafts and profile; a deletion erases them and keeps the editorial record with its byline', async () => {++ for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {++ await db.exec("INSERT INTO news_stories (id, slug, working_headline, state, created_by, created_at, updated_at) VALUES ($1, $2, $3, 'published', $4, 1, 1)", [`sty_${name}`, `story-${name}`, `headline ${name}`, subject]);++ await db.exec("INSERT INTO news_story_drafts (entity_id, owner, base_revision, content, fields, meta, created_at, updated_at) VALUES ($1, $2, 0, 'draft', '{}', '{}', 1, 1)", [`sty_${name}`, subject]);++ await db.exec('INSERT INTO subject_projections (subject, username, display_name, refreshed_at) VALUES ($1, $2, $2, 1)', [subject, name]);++ }++ const exp = await deliver(ev('evt_01JZ0000000000000000000E01', 'network.account.export_requested', { export_id: 'exp_01JZ0000000000000000000EXP', subject: A }));++ assert.deepStrictEqual([exp.status, exp.json.outcome], [200, 'exported']);++ const part = calls.find((c) => c.path === '/internal/account-exports/exp_01JZ0000000000000000000EXP/parts');++ assert.ok(part, 'the part was pushed');++ const files = Object.fromEntries(part.body.files.map((f) => [f.name, f.content]));++ assert.strictEqual(files['story-drafts.json'].length, 1);++ assert.strictEqual(files['stories.json'].length, 1);++ assert.strictEqual(files['profile.json'].length, 1);++ assert.ok(!JSON.stringify(part.body).includes(B), 'nobody else\'s rows');++++ const del = await deliver(ev('evt_01JZ0000000000000000000D01', 'network.account.deleted', { deletion_id: 'del_01JZ0000000000000000000DEX', subject: A, aliases: [] }));++ assert.deepStrictEqual([del.status, del.json.outcome], [200, 'erased']);++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM news_story_drafts WHERE owner = $1', [A])), 0);++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM subject_projections WHERE subject = $1', [A])), 0);++ assert.strictEqual(await db.value('SELECT created_by FROM news_stories WHERE id = $1', ['sty_ada']), A, 'the published story keeps its byline');++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM news_story_drafts WHERE owner = $1', [B])), 1);++ const confirmation = calls.find((c) => c.path === '/internal/account-deletions/del_01JZ0000000000000000000DEX/confirmations');++ assert.ok(confirmation, 'the confirmation was sent');++ });++ } finally { await t.close(); }++ done();++})().catch((e) => { console.error(e); process.exit(1); });@@@ -56,7 +56,7 @@ async function boot(opts = {}) {let built = null;async function start() {const config = configLib.load(env);-- built = await createApp({ config, telemetrySink: opts.telemetrySink, store: createStore(testdb.db, { now: clock.now }), now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow, indexnow: opts.indexnow });++ built = await createApp({ config, telemetrySink: opts.telemetrySink, store: createStore(testdb.db, { now: clock.now }), now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow, indexnow: opts.indexnow, accountData: opts.accountData, accountSend: opts.accountSend });await require('openvibe-sdk/auth').jwksClient(`${config.networkInternalUrl}/api/.well-known/jwks`).keys();server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });t.base = `http://127.0.0.1:${server.address().port}`;
@@ -448,7 +448,7 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [
@@ -448,7 +448,7 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [
<!-- versions:start -->
<!-- versions:start -->
- openvibe-contracts: v0.222.0
- openvibe-contracts: v0.222.0
-- openvibe-sdk: v0.73.0
+- openvibe-sdk: v0.74.0
- openvibe-shared: v4.48.0
- openvibe-shared: v4.48.0
- openvibe-publishing: v1.4.1
- openvibe-publishing: v1.4.1
<!-- versions:end -->
<!-- versions:end -->
@@ -8,13 +8,13 @@
@@ -8,13 +8,13 @@
"port": 4820,
"port": 4820,
"domain": "openvibe.news",
"domain": "openvibe.news",
"contracts": "openvibe-contracts v0.222.0",
"contracts": "openvibe-contracts v0.222.0",
- "sdk": "openvibe-sdk v0.73.0",
+ "sdk": "openvibe-sdk v0.74.0",
"shared": "openvibe-shared v4.48.0",
"shared": "openvibe-shared v4.48.0",
"packages": [
"packages": [
"openvibe-publishing v1.4.1",
"openvibe-publishing v1.4.1",
"openvibe-contracts v0.222.0",
"openvibe-contracts v0.222.0",
"openvibe-shared v4.48.0",
"openvibe-shared v4.48.0",
- "openvibe-sdk v0.73.0"
+ "openvibe-sdk v0.74.0"
],
],
"capabilities": [
"capabilities": [
"news.story.create",
"news.story.create",
new file mode 100644
new file mode 100644
@@ -0,0 +1,13 @@
@@ -0,0 +1,13 @@
+-- phase: expand
+-- OpenVibe.News: the receipts of the ADR-033 account export and deletion deliveries this service applied
+-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.
+-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.
+
+CREATE TABLE IF NOT EXISTS account_data_events (
+ id TEXT PRIMARY KEY,
+ kind TEXT NOT NULL,
+ subject TEXT NOT NULL,
+ outcome JSONB,
+ applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ sent_at TIMESTAMPTZ
+);
@@ -17,7 +17,7 @@
@@ -17,7 +17,7 @@
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
},
},
@@ -1182,9 +1182,9 @@
@@ -1182,9 +1182,9 @@
}
}
},
},
"node_modules/openvibe-sdk": {
"node_modules/openvibe-sdk": {
- "version": "0.73.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
- "integrity": "sha512-XClxoVm9M6RA8eXlGgYZ0u6q9n5h79mCXqAAEg9fX21KgcqP69RmG+cX+6wrOf7TaEj7eSBWMxHjiIeIa9N7Dg==",
+ "version": "0.74.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
+ "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==",
"license": "MIT",
"license": "MIT",
"bin": {
"bin": {
"openvibe-download": "bin/openvibe-download.js"
"openvibe-download": "bin/openvibe-download.js"
@@ -31,7 +31,7 @@
@@ -31,7 +31,7 @@
"iovalkey": "^0.4.0",
"iovalkey": "^0.4.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
},
},
new file mode 100644
new file mode 100644
@@ -0,0 +1,33 @@
@@ -0,0 +1,33 @@
+'use strict';
+/**
+ * Account export and deletion → News (ADR-033; openvibe-sdk/account-data).
+ *
+ * A person's private working rows go: their story drafts (news_story_drafts.owner) and the cached profile
+ * (subject_projections). Their editorial contribution to a story — the story, a perspective, a timeline entry, a
+ * cluster, an editorial flag — stays: it is the publication's record, and a byline is part of what readers are owed;
+ * those rows are exported and kept, with created_by as the editorial attribution. Telemetry samples that name a
+ * person are covered by the SDK itself.
+ *
+ * news_index_revisions.owner is the index document's owning service, not a person.
+ */
+const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
+
+const editorial = (why) => ({ keep: why });
+
+const TABLES = [
+ { table: 'news_story_drafts', subject: 'owner', file: 'story-drafts.json', order: 'updated_at', columns: ['entity_id', 'base_revision', 'content', 'fields', 'created_at', 'updated_at'] },
+ { table: 'subject_projections', subject: 'subject', file: 'profile.json', order: 'refreshed_at' },
+ { table: 'news_stories', subject: 'created_by', file: 'stories.json', order: 'first_published_at', columns: ['id', 'slug', 'working_headline', 'topic_id', 'state', 'first_published_at', 'published_at'], erase: editorial('published journalism keeps its editor byline as the publication record') },
+ { table: 'news_perspectives', subject: 'created_by', file: 'perspectives.json', order: 'created_at', columns: ['id', 'story_id', 'label', 'description', 'position', 'created_at'], erase: editorial('a published perspective is part of the story') },
+ { table: 'news_timeline_entries', subject: 'created_by', file: 'timeline-entries.json', order: 'created_at', columns: ['id', 'story_id', 'occurred_on', 'text', 'created_at'], erase: editorial('a published timeline entry is part of the story') },
+ { table: 'news_story_clusters', subject: 'created_by', file: 'story-clusters.json', order: 'window_start', columns: ['id', 'label', 'status', 'window_start', 'window_end'], erase: editorial('a story cluster is the newsroom\'s record') },
+ { table: 'news_editorial_flags', subject: 'created_by', file: 'editorial-flags.json', order: 'created_at', columns: ['id', 'story_id', 'kind', 'status', 'note', 'created_at'], erase: editorial('a correction, update or retraction is the record readers rely on') },
+ // The edit history of a story: the publication's record of how the story was written, with its byline.
+ { table: 'news_story_revisions', subject: 'author', file: 'story-revisions.json', order: 'created_at', columns: ['id', 'entity_id', 'number', 'kind', 'message', 'created_at'], erase: editorial('a published story keeps its edit history and byline as the publication record') },
+];
+
+function create({ db, log = console } = {}) {
+ return createAccountData({ db, service: 'news', tables: TABLES, log });
+}
+
+module.exports = { create, TABLES, TOPICS };
@@ -40,6 +40,8 @@ const { createEditorRoutes } = require('./http/editor');
@@ -40,6 +40,8 @@ const { createEditorRoutes } = require('./http/editor');
const { createApi } = require('./http/api');
const { createApi } = require('./http/api');
const { createDiscoveryRoutes } = require('./http/discovery');
const { createDiscoveryRoutes } = require('./http/discovery');
const { createEvents } = require('./http/events');
const { createEvents } = require('./http/events');
+const accountDataLib = require('./account-data');
+const { createNetworkSender } = require('openvibe-sdk/account-data');
const { createActorLimits } = require('./http/actor-limits');
const { createActorLimits } = require('./http/actor-limits');
const { createNewsReadiness } = require('./observability');
const { createNewsReadiness } = require('./observability');
const { createWorker } = require('./worker');
const { createWorker } = require('./worker');
@@ -58,6 +60,10 @@ async function createApp(opts = {}) {
@@ -58,6 +60,10 @@ async function createApp(opts = {}) {
const fetchImpl = opts.fetchImpl || globalThis.fetch;
const fetchImpl = opts.fetchImpl || globalThis.fetch;
// PostgreSQL (ADR-035): opened and migrated here unless the caller (a test, a script) hands in a store.
// PostgreSQL (ADR-035): opened and migrated here unless the caller (a test, a script) hands in a store.
const store = opts.store || await openStore(config, { now: opts.now, log });
const store = opts.store || await openStore(config, { now: opts.now, log });
+ const accountData = opts.accountData || accountDataLib.create({ db: store.db, log });
+ const accountSend = opts.accountSend || (config.oauth.clientSecret ? createNetworkSender({
+ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl,
+ }) : null);
const outbox = createServiceOutbox({
const outbox = createServiceOutbox({
db: store.db, source: 'news',
db: store.db, source: 'news',
@@ -101,7 +107,7 @@ async function createApp(opts = {}) {
@@ -101,7 +107,7 @@ async function createApp(opts = {}) {
const briefs = createBriefs({ store, ai: config.briefs.enabled ? ai : null, log });
const briefs = createBriefs({ store, ai: config.briefs.enabled ? ai : null, log });
const worker = createWorker({ config, ingest, outbox, briefs, log });
const worker = createWorker({ config, ingest, outbox, briefs, log });
- const ctx = { config, store, outbox, publication, clusters, sources, ai, community, discussion, ingest, stories, topics, people, reading, auth, viewers, worker, indexnow, briefs };
+ const ctx = { config, store, accountData, accountSend, outbox, publication, clusters, sources, ai, community, discussion, ingest, stories, topics, people, reading, auth, viewers, worker, indexnow, briefs };
const app = express();
const app = express();
app.disable('x-powered-by');
app.disable('x-powered-by');
@@ -158,7 +164,7 @@ async function createApp(opts = {}) {
@@ -158,7 +164,7 @@ async function createApp(opts = {}) {
if (indexnow.enabled) app.use(indexnow.keyFile);
if (indexnow.enabled) app.use(indexnow.keyFile);
// ── Events webhook (raw body; before any other body parser) ─
// ── Events webhook (raw body; before any other body parser) ─
- app.use(createEvents({ config, store, ingest, sources, log }));
+ app.use(createEvents({ config, store, ingest, sources, log , accountData, accountSend }));
app.use(cookieParser());
app.use(cookieParser());
@@ -17,11 +17,12 @@
@@ -17,11 +17,12 @@
const express = require('express');
const express = require('express');
const { http } = require('openvibe-contracts');
const { http } = require('openvibe-contracts');
const { createEventConsumer } = require('openvibe-publishing/ingest');
const { createEventConsumer } = require('openvibe-publishing/ingest');
+const { TOPICS: ACCOUNT_TOPICS } = require('openvibe-sdk/account-data');
const CONSUMER = 'news-sources';
const CONSUMER = 'news-sources';
const EVT_RE = /^evt_[0-9A-HJKMNP-TV-Z]{26}$/;
const EVT_RE = /^evt_[0-9A-HJKMNP-TV-Z]{26}$/;
-function createEvents({ config, store, ingest, sources, log = console }) {
+function createEvents({ config, store, ingest, sources, accountData = null, accountSend = null, log = console }) {
const router = express.Router();
const router = express.Router();
const consumer = createEventConsumer({ db: store.db, secrets: config.events.webhookSecrets, consumer: CONSUMER, now: store.now });
const consumer = createEventConsumer({ db: store.db, secrets: config.events.webhookSecrets, consumer: CONSUMER, now: store.now });
const { inbox, verify } = consumer;
const { inbox, verify } = consumer;
@@ -40,6 +41,15 @@ function createEvents({ config, store, ingest, sources, log = console }) {
@@ -40,6 +41,15 @@ function createEvents({ config, store, ingest, sources, log = console }) {
}
}
const attempt = Number(req.get('x-openvibe-delivery-attempt')) || 1;
const attempt = Number(req.get('x-openvibe-delivery-attempt')) || 1;
const type = String(event.event_type || '');
const type = String(event.event_type || '');
+ if (accountData && ACCOUNT_TOPICS.includes(type)) {
+ try {
+ const outcome = await accountData.apply(event, { send: accountSend });
+ return res.status(200).json({ event_id: event.event_id, outcome });
+ } catch (err) {
+ log.warn('[News] account-data delivery failed:', err && err.message ? err.message : err);
+ return http.sendProblem(res, 503, 'news.account_retry', { detail: 'not applied; Events will retry', ctx });
+ }
+ }
const p = event.payload && typeof event.payload === 'object' ? event.payload : {};
const p = event.payload && typeof event.payload === 'object' ? event.payload : {};
const fromSources = event.source === 'sources';
const fromSources = event.source === 'sources';
@@ -8,6 +8,7 @@
@@ -8,6 +8,7 @@
const { createApp } = require('./app');
const { createApp } = require('./app');
const { gracefulStop } = require('openvibe-sdk/service');
const { gracefulStop } = require('openvibe-sdk/service');
const { startTelemetryRetention } = require('openvibe-sdk/telemetry');
const { startTelemetryRetention } = require('openvibe-sdk/telemetry');
+const { startSubscriptions } = require('openvibe-sdk/account-data');
/**
/**
* The process stop (openvibe-sdk/service, docs/service.md's 5 s family): the Sources pull worker stops
* The process stop (openvibe-sdk/service, docs/service.md's 5 s family): the Sources pull worker stops
@@ -15,11 +16,11 @@ const { startTelemetryRetention } = require('openvibe-sdk/telemetry');
@@ -15,11 +16,11 @@ const { startTelemetryRetention } = require('openvibe-sdk/telemetry');
* closes; past the deadline the process exits 0, as the hand-rolled 5 s timer did. Exported so a test
* closes; past the deadline the process exits 0, as the hand-rolled 5 s timer did. Exported so a test
* can inject `exit` and `signals: false`.
* can inject `exit` and `signals: false`.
*/
*/
-function createLifecycle({ server, ctx, exit, signals }) {
+function createLifecycle({ server, ctx, exit, signals, extra = [] }) {
return gracefulStop({
return gracefulStop({
name: 'News', server, deadlineExitCode: 0, exit, signals,
name: 'News', server, deadlineExitCode: 0, exit, signals,
stop: [() => ctx.worker.stop()],
stop: [() => ctx.worker.stop()],
- close: [() => ctx.outbox.stop(), async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); }, () => ctx.store.close()],
+ close: [...extra, () => ctx.outbox.stop(), async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); }, () => ctx.store.close()],
});
});
}
}
@@ -41,7 +42,14 @@ async function start() {
@@ -41,7 +42,14 @@ async function start() {
ctx.telemetryTick = setInterval(() => { ctx.telemetry.flush().catch(() => {}); }, 60_000);
ctx.telemetryTick = setInterval(() => { ctx.telemetry.flush().catch(() => {}); }, 60_000);
ctx.telemetryTick.unref();
ctx.telemetryTick.unref();
- createLifecycle({ server, ctx });
+ let accountSubscription = null;
+ if (config.events.webhookSecrets.length && config.events.url && config.oauth.clientSecret) {
+ accountSubscription = startSubscriptions({
+ eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.port}/internal/events`, secret: config.events.webhookSecrets[0],
+ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log: console,
+ });
+ }
+ createLifecycle({ server, ctx, extra: [() => { if (accountSubscription) accountSubscription.stop(); }] });
return { server, ctx };
return { server, ctx };
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,67 @@
@@ -0,0 +1,67 @@
+'use strict';
+/**
+ * ADR-033: News' part of an account export and deletion, through the signed POST /internal/events route. A person's
+ * drafts and cached profile are exported and erased; the story they opened and its byline stay as the publication's
+ * record; Bo's rows stay. The guard fails on a new table with a person column that is neither covered nor exempted.
+ */
+const assert = require('assert');
+const { uncovered } = require('openvibe-sdk/account-data');
+const { signDeliveryHeaders } = require('openvibe-sdk/events');
+const { boot, check, done } = require('./helpers/boot');
+const { TABLES } = require('../server/account-data');
+
+const SECRET = `whsec_${'fixture'.repeat(6)}`;
+const A = 'usr_01JZ0000000000000000000AAA';
+const B = 'usr_01JZ0000000000000000000BBB';
+
+(async () => {
+ const calls = [];
+ const accountSend = async (path, body) => { calls.push({ path, body }); return { ok: true, status: 201 }; };
+ const t = await boot({ env: { NEWS_EVENTS_SECRET: SECRET }, accountSend });
+ const db = t.ctx.store.db;
+ const deliver = async (event) => {
+ const raw = JSON.stringify({ event, seq: 1 });
+ const res = await fetch(`${t.base}/internal/events`, { method: 'POST', body: raw, headers: { 'content-type': 'application/json', ...signDeliveryHeaders(raw, SECRET) } });
+ return { status: res.status, json: await res.json().catch(() => null) };
+ };
+ const ev = (id, type, payload) => ({ event_id: id, event_type: type, source: 'network', version: 1, timestamp: new Date().toISOString(), payload });
+ try {
+ await check('every table that names a person is covered or exempted with a reason', async () => {
+ const missing = await uncovered(db, {
+ tables: TABLES,
+ exempt: {
+ account_data_events: 'the ledger of requests itself: which export and deletion News applied',
+ news_index_revisions: "owner is the search-index document's owner (type/id), not a person; the row holds a revision and hash",
+ },
+ });
+ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
+ });
+
+ await check('an export carries drafts and profile; a deletion erases them and keeps the editorial record with its byline', async () => {
+ for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {
+ await db.exec("INSERT INTO news_stories (id, slug, working_headline, state, created_by, created_at, updated_at) VALUES ($1, $2, $3, 'published', $4, 1, 1)", [`sty_${name}`, `story-${name}`, `headline ${name}`, subject]);
+ await db.exec("INSERT INTO news_story_drafts (entity_id, owner, base_revision, content, fields, meta, created_at, updated_at) VALUES ($1, $2, 0, 'draft', '{}', '{}', 1, 1)", [`sty_${name}`, subject]);
+ await db.exec('INSERT INTO subject_projections (subject, username, display_name, refreshed_at) VALUES ($1, $2, $2, 1)', [subject, name]);
+ }
+ const exp = await deliver(ev('evt_01JZ0000000000000000000E01', 'network.account.export_requested', { export_id: 'exp_01JZ0000000000000000000EXP', subject: A }));
+ assert.deepStrictEqual([exp.status, exp.json.outcome], [200, 'exported']);
+ const part = calls.find((c) => c.path === '/internal/account-exports/exp_01JZ0000000000000000000EXP/parts');
+ assert.ok(part, 'the part was pushed');
+ const files = Object.fromEntries(part.body.files.map((f) => [f.name, f.content]));
+ assert.strictEqual(files['story-drafts.json'].length, 1);
+ assert.strictEqual(files['stories.json'].length, 1);
+ assert.strictEqual(files['profile.json'].length, 1);
+ assert.ok(!JSON.stringify(part.body).includes(B), 'nobody else\'s rows');
+
+ const del = await deliver(ev('evt_01JZ0000000000000000000D01', 'network.account.deleted', { deletion_id: 'del_01JZ0000000000000000000DEX', subject: A, aliases: [] }));
+ assert.deepStrictEqual([del.status, del.json.outcome], [200, 'erased']);
+ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM news_story_drafts WHERE owner = $1', [A])), 0);
+ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM subject_projections WHERE subject = $1', [A])), 0);
+ assert.strictEqual(await db.value('SELECT created_by FROM news_stories WHERE id = $1', ['sty_ada']), A, 'the published story keeps its byline');
+ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM news_story_drafts WHERE owner = $1', [B])), 1);
+ const confirmation = calls.find((c) => c.path === '/internal/account-deletions/del_01JZ0000000000000000000DEX/confirmations');
+ assert.ok(confirmation, 'the confirmation was sent');
+ });
+ } finally { await t.close(); }
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });
@@ -56,7 +56,7 @@ async function boot(opts = {}) {
@@ -56,7 +56,7 @@ async function boot(opts = {}) {
let built = null;
let built = null;
async function start() {
async function start() {
const config = configLib.load(env);
const config = configLib.load(env);
- built = await createApp({ config, telemetrySink: opts.telemetrySink, store: createStore(testdb.db, { now: clock.now }), now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow, indexnow: opts.indexnow });
+ built = await createApp({ config, telemetrySink: opts.telemetrySink, store: createStore(testdb.db, { now: clock.now }), now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow, indexnow: opts.indexnow, accountData: opts.accountData, accountSend: opts.accountSend });
await require('openvibe-sdk/auth').jwksClient(`${config.networkInternalUrl}/api/.well-known/jwks`).keys();
await require('openvibe-sdk/auth').jwksClient(`${config.networkInternalUrl}/api/.well-known/jwks`).keys();
server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
t.base = `http://127.0.0.1:${server.address().port}`;
t.base = `http://127.0.0.1:${server.address().port}`;