openvibers/openvibe.news Commits

account-data: a person's drafts and profile, with the guard on every person column (#147)

OP OpenVibers committed 14 days ago · 98370ae0fe · 11 files, +152 −15

* account-data: a person's drafts and profile, with the guard on every person column

* account-data: the receipts table comes from a migration, not from the runtime role

The boot-time ensureSchema() ran DDL as the serving role, which may not change
the schema; the table is now migration NNNN, like every other table here.

* ci: run the checks for this branch

* Merge remote-tracking branch 'origin/main' into adr033-account-data

# Conflicts:
#	README.md
#	STATUS.json
#	package-lock.json
#	package.json

Changes

ddiff --git a/README.md b/README.md
iindex 4fdb9bd..faf0a3f 100644
---- a/README.md
++++ b/README.md
@@@ -448,7 +448,7 @@ Part of the [OpenVibe network](https://openvibe.network). Built in the open by [
<!-- versions:start -->
- openvibe-contracts: v0.222.0
--- openvibe-sdk: v0.73.0
++- openvibe-sdk: v0.74.0
- openvibe-shared: v4.48.0
- openvibe-publishing: v1.4.1
<!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex 7936b1c..d0b9424 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -8,13 +8,13 @@
"port": 4820,
"domain": "openvibe.news",
"contracts": "openvibe-contracts v0.222.0",
-- "sdk": "openvibe-sdk v0.73.0",
++ "sdk": "openvibe-sdk v0.74.0",
"shared": "openvibe-shared v4.48.0",
"packages": [
"openvibe-publishing v1.4.1",
"openvibe-contracts v0.222.0",
"openvibe-shared v4.48.0",
-- "openvibe-sdk v0.73.0"
++ "openvibe-sdk v0.74.0"
],
"capabilities": [
"news.story.create",
ddiff --git a/migrations/0005_account_data.sql b/migrations/0005_account_data.sql
nnew file mode 100644
iindex 0000000..8d42579
---- /dev/null
++++ b/migrations/0005_account_data.sql
@@@ -0,0 +1,13 @@
++-- phase: expand
++-- OpenVibe.News: the receipts of the ADR-033 account export and deletion deliveries this service applied
++-- (openvibe-sdk/account-data's ACCOUNT_DATA_SCHEMA), so a redelivered export or deletion changes nothing.
++-- The row holds an export_id or deletion_id, the subject and the counts sent to Network; it is not a secret.
++
++CREATE TABLE IF NOT EXISTS account_data_events (
++ id TEXT PRIMARY KEY,
++ kind TEXT NOT NULL,
++ subject TEXT NOT NULL,
++ outcome JSONB,
++ applied_at TIMESTAMPTZ NOT NULL DEFAULT now(),
++ sent_at TIMESTAMPTZ
++);
ddiff --git a/package-lock.json b/package-lock.json
iindex 5c5daa1..8ef8c9d 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -17,7 +17,7 @@
"iovalkey": "^0.4.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",
-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
},
@@@ -1182,9 +1182,9 @@
}
},
"node_modules/openvibe-sdk": {
-- "version": "0.73.0",
-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
-- "integrity": "sha512-XClxoVm9M6RA8eXlGgYZ0u6q9n5h79mCXqAAEg9fX21KgcqP69RmG+cX+6wrOf7TaEj7eSBWMxHjiIeIa9N7Dg==",
++ "version": "0.74.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
++ "integrity": "sha512-DO9KVPnUriY/I4lX6ZEJtXaUvRDbJyqnqJxvlpl+zkNGFILpmuBXVJogNOCfK071Qci1gg7C4Iw1Xrseklcajg==",
"license": "MIT",
"bin": {
"openvibe-download": "bin/openvibe-download.js"
ddiff --git a/package.json b/package.json
iindex 1fea3a4..2554c4b 100644
---- a/package.json
++++ b/package.json
@@@ -31,7 +31,7 @@
"iovalkey": "^0.4.0",
"openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.222.0",
"openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.4.1",
-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.73.0",
++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.74.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.48.0",
"pg": "^8.23.0"
},
ddiff --git a/server/account-data.js b/server/account-data.js
nnew file mode 100644
iindex 0000000..1ff5e34
---- /dev/null
++++ b/server/account-data.js
@@@ -0,0 +1,33 @@
++'use strict';
++/**
++ * Account export and deletion → News (ADR-033; openvibe-sdk/account-data).
++ *
++ * A person's private working rows go: their story drafts (news_story_drafts.owner) and the cached profile
++ * (subject_projections). Their editorial contribution to a story — the story, a perspective, a timeline entry, a
++ * cluster, an editorial flag — stays: it is the publication's record, and a byline is part of what readers are owed;
++ * those rows are exported and kept, with created_by as the editorial attribution. Telemetry samples that name a
++ * person are covered by the SDK itself.
++ *
++ * news_index_revisions.owner is the index document's owning service, not a person.
++ */
++const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
++
++const editorial = (why) => ({ keep: why });
++
++const TABLES = [
++ { table: 'news_story_drafts', subject: 'owner', file: 'story-drafts.json', order: 'updated_at', columns: ['entity_id', 'base_revision', 'content', 'fields', 'created_at', 'updated_at'] },
++ { table: 'subject_projections', subject: 'subject', file: 'profile.json', order: 'refreshed_at' },
++ { table: 'news_stories', subject: 'created_by', file: 'stories.json', order: 'first_published_at', columns: ['id', 'slug', 'working_headline', 'topic_id', 'state', 'first_published_at', 'published_at'], erase: editorial('published journalism keeps its editor byline as the publication record') },
++ { table: 'news_perspectives', subject: 'created_by', file: 'perspectives.json', order: 'created_at', columns: ['id', 'story_id', 'label', 'description', 'position', 'created_at'], erase: editorial('a published perspective is part of the story') },
++ { table: 'news_timeline_entries', subject: 'created_by', file: 'timeline-entries.json', order: 'created_at', columns: ['id', 'story_id', 'occurred_on', 'text', 'created_at'], erase: editorial('a published timeline entry is part of the story') },
++ { table: 'news_story_clusters', subject: 'created_by', file: 'story-clusters.json', order: 'window_start', columns: ['id', 'label', 'status', 'window_start', 'window_end'], erase: editorial('a story cluster is the newsroom\'s record') },
++ { table: 'news_editorial_flags', subject: 'created_by', file: 'editorial-flags.json', order: 'created_at', columns: ['id', 'story_id', 'kind', 'status', 'note', 'created_at'], erase: editorial('a correction, update or retraction is the record readers rely on') },
++ // The edit history of a story: the publication's record of how the story was written, with its byline.
++ { table: 'news_story_revisions', subject: 'author', file: 'story-revisions.json', order: 'created_at', columns: ['id', 'entity_id', 'number', 'kind', 'message', 'created_at'], erase: editorial('a published story keeps its edit history and byline as the publication record') },
++];
++
++function create({ db, log = console } = {}) {
++ return createAccountData({ db, service: 'news', tables: TABLES, log });
++}
++
++module.exports = { create, TABLES, TOPICS };
ddiff --git a/server/app.js b/server/app.js
iindex 0204467..06c5333 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -40,6 +40,8 @@ const { createEditorRoutes } = require('./http/editor');
const { createApi } = require('./http/api');
const { createDiscoveryRoutes } = require('./http/discovery');
const { createEvents } = require('./http/events');
++const accountDataLib = require('./account-data');
++const { createNetworkSender } = require('openvibe-sdk/account-data');
const { createActorLimits } = require('./http/actor-limits');
const { createNewsReadiness } = require('./observability');
const { createWorker } = require('./worker');
@@@ -58,6 +60,10 @@ async function createApp(opts = {}) {
const fetchImpl = opts.fetchImpl || globalThis.fetch;
// PostgreSQL (ADR-035): opened and migrated here unless the caller (a test, a script) hands in a store.
const store = opts.store || await openStore(config, { now: opts.now, log });
++ const accountData = opts.accountData || accountDataLib.create({ db: store.db, log });
++ const accountSend = opts.accountSend || (config.oauth.clientSecret ? createNetworkSender({
++ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, fetch: fetchImpl,
++ }) : null);
const outbox = createServiceOutbox({
db: store.db, source: 'news',
@@@ -101,7 +107,7 @@ async function createApp(opts = {}) {
const briefs = createBriefs({ store, ai: config.briefs.enabled ? ai : null, log });
const worker = createWorker({ config, ingest, outbox, briefs, log });
-- const ctx = { config, store, outbox, publication, clusters, sources, ai, community, discussion, ingest, stories, topics, people, reading, auth, viewers, worker, indexnow, briefs };
++ const ctx = { config, store, accountData, accountSend, outbox, publication, clusters, sources, ai, community, discussion, ingest, stories, topics, people, reading, auth, viewers, worker, indexnow, briefs };
const app = express();
app.disable('x-powered-by');
@@@ -158,7 +164,7 @@ async function createApp(opts = {}) {
if (indexnow.enabled) app.use(indexnow.keyFile);
// ── Events webhook (raw body; before any other body parser) ─
-- app.use(createEvents({ config, store, ingest, sources, log }));
++ app.use(createEvents({ config, store, ingest, sources, log , accountData, accountSend }));
app.use(cookieParser());
ddiff --git a/server/http/events.js b/server/http/events.js
iindex ad868fe..88bb7d3 100644
---- a/server/http/events.js
++++ b/server/http/events.js
@@@ -17,11 +17,12 @@
const express = require('express');
const { http } = require('openvibe-contracts');
const { createEventConsumer } = require('openvibe-publishing/ingest');
++const { TOPICS: ACCOUNT_TOPICS } = require('openvibe-sdk/account-data');
const CONSUMER = 'news-sources';
const EVT_RE = /^evt_[0-9A-HJKMNP-TV-Z]{26}$/;
--function createEvents({ config, store, ingest, sources, log = console }) {
++function createEvents({ config, store, ingest, sources, accountData = null, accountSend = null, log = console }) {
const router = express.Router();
const consumer = createEventConsumer({ db: store.db, secrets: config.events.webhookSecrets, consumer: CONSUMER, now: store.now });
const { inbox, verify } = consumer;
@@@ -40,6 +41,15 @@ function createEvents({ config, store, ingest, sources, log = console }) {
}
const attempt = Number(req.get('x-openvibe-delivery-attempt')) || 1;
const type = String(event.event_type || '');
++ if (accountData && ACCOUNT_TOPICS.includes(type)) {
++ try {
++ const outcome = await accountData.apply(event, { send: accountSend });
++ return res.status(200).json({ event_id: event.event_id, outcome });
++ } catch (err) {
++ log.warn('[News] account-data delivery failed:', err && err.message ? err.message : err);
++ return http.sendProblem(res, 503, 'news.account_retry', { detail: 'not applied; Events will retry', ctx });
++ }
++ }
const p = event.payload && typeof event.payload === 'object' ? event.payload : {};
const fromSources = event.source === 'sources';
ddiff --git a/server/index.js b/server/index.js
iindex 461cbea..37b9fe9 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -8,6 +8,7 @@
const { createApp } = require('./app');
const { gracefulStop } = require('openvibe-sdk/service');
const { startTelemetryRetention } = require('openvibe-sdk/telemetry');
++const { startSubscriptions } = require('openvibe-sdk/account-data');
/**
* The process stop (openvibe-sdk/service, docs/service.md's 5 s family): the Sources pull worker stops
@@@ -15,11 +16,11 @@ const { startTelemetryRetention } = require('openvibe-sdk/telemetry');
* closes; past the deadline the process exits 0, as the hand-rolled 5 s timer did. Exported so a test
* can inject `exit` and `signals: false`.
*/
--function createLifecycle({ server, ctx, exit, signals }) {
++function createLifecycle({ server, ctx, exit, signals, extra = [] }) {
return gracefulStop({
name: 'News', server, deadlineExitCode: 0, exit, signals,
stop: [() => ctx.worker.stop()],
-- close: [() => ctx.outbox.stop(), async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); }, () => ctx.store.close()],
++ close: [...extra, () => ctx.outbox.stop(), async () => { clearInterval(ctx.telemetryTick); if (ctx.retention) await ctx.retention.stop(); if (ctx.telemetry) await ctx.telemetry.stop(); }, () => ctx.store.close()],
});
}
@@@ -41,7 +42,14 @@ async function start() {
ctx.telemetryTick = setInterval(() => { ctx.telemetry.flush().catch(() => {}); }, 60_000);
ctx.telemetryTick.unref();
-- createLifecycle({ server, ctx });
++ let accountSubscription = null;
++ if (config.events.webhookSecrets.length && config.events.url && config.oauth.clientSecret) {
++ accountSubscription = startSubscriptions({
++ eventsUrl: config.events.url, endpoint: `http://127.0.0.1:${config.port}/internal/events`, secret: config.events.webhookSecrets[0],
++ networkInternalUrl: config.networkInternalUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret, log: console,
++ });
++ }
++ createLifecycle({ server, ctx, extra: [() => { if (accountSubscription) accountSubscription.stop(); }] });
return { server, ctx };
}
ddiff --git a/test/account-data.test.js b/test/account-data.test.js
nnew file mode 100644
iindex 0000000..caf7729
---- /dev/null
++++ b/test/account-data.test.js
@@@ -0,0 +1,67 @@
++'use strict';
++/**
++ * ADR-033: News' part of an account export and deletion, through the signed POST /internal/events route. A person's
++ * drafts and cached profile are exported and erased; the story they opened and its byline stay as the publication's
++ * record; Bo's rows stay. The guard fails on a new table with a person column that is neither covered nor exempted.
++ */
++const assert = require('assert');
++const { uncovered } = require('openvibe-sdk/account-data');
++const { signDeliveryHeaders } = require('openvibe-sdk/events');
++const { boot, check, done } = require('./helpers/boot');
++const { TABLES } = require('../server/account-data');
++
++const SECRET = `whsec_${'fixture'.repeat(6)}`;
++const A = 'usr_01JZ0000000000000000000AAA';
++const B = 'usr_01JZ0000000000000000000BBB';
++
++(async () => {
++ const calls = [];
++ const accountSend = async (path, body) => { calls.push({ path, body }); return { ok: true, status: 201 }; };
++ const t = await boot({ env: { NEWS_EVENTS_SECRET: SECRET }, accountSend });
++ const db = t.ctx.store.db;
++ const deliver = async (event) => {
++ const raw = JSON.stringify({ event, seq: 1 });
++ const res = await fetch(`${t.base}/internal/events`, { method: 'POST', body: raw, headers: { 'content-type': 'application/json', ...signDeliveryHeaders(raw, SECRET) } });
++ return { status: res.status, json: await res.json().catch(() => null) };
++ };
++ const ev = (id, type, payload) => ({ event_id: id, event_type: type, source: 'network', version: 1, timestamp: new Date().toISOString(), payload });
++ try {
++ await check('every table that names a person is covered or exempted with a reason', async () => {
++ const missing = await uncovered(db, {
++ tables: TABLES,
++ exempt: {
++ account_data_events: 'the ledger of requests itself: which export and deletion News applied',
++ news_index_revisions: "owner is the search-index document's owner (type/id), not a person; the row holds a revision and hash",
++ },
++ });
++ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
++ });
++
++ await check('an export carries drafts and profile; a deletion erases them and keeps the editorial record with its byline', async () => {
++ for (const [subject, name] of [[A, 'ada'], [B, 'bo']]) {
++ await db.exec("INSERT INTO news_stories (id, slug, working_headline, state, created_by, created_at, updated_at) VALUES ($1, $2, $3, 'published', $4, 1, 1)", [`sty_${name}`, `story-${name}`, `headline ${name}`, subject]);
++ await db.exec("INSERT INTO news_story_drafts (entity_id, owner, base_revision, content, fields, meta, created_at, updated_at) VALUES ($1, $2, 0, 'draft', '{}', '{}', 1, 1)", [`sty_${name}`, subject]);
++ await db.exec('INSERT INTO subject_projections (subject, username, display_name, refreshed_at) VALUES ($1, $2, $2, 1)', [subject, name]);
++ }
++ const exp = await deliver(ev('evt_01JZ0000000000000000000E01', 'network.account.export_requested', { export_id: 'exp_01JZ0000000000000000000EXP', subject: A }));
++ assert.deepStrictEqual([exp.status, exp.json.outcome], [200, 'exported']);
++ const part = calls.find((c) => c.path === '/internal/account-exports/exp_01JZ0000000000000000000EXP/parts');
++ assert.ok(part, 'the part was pushed');
++ const files = Object.fromEntries(part.body.files.map((f) => [f.name, f.content]));
++ assert.strictEqual(files['story-drafts.json'].length, 1);
++ assert.strictEqual(files['stories.json'].length, 1);
++ assert.strictEqual(files['profile.json'].length, 1);
++ assert.ok(!JSON.stringify(part.body).includes(B), 'nobody else\'s rows');
++
++ const del = await deliver(ev('evt_01JZ0000000000000000000D01', 'network.account.deleted', { deletion_id: 'del_01JZ0000000000000000000DEX', subject: A, aliases: [] }));
++ assert.deepStrictEqual([del.status, del.json.outcome], [200, 'erased']);
++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM news_story_drafts WHERE owner = $1', [A])), 0);
++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM subject_projections WHERE subject = $1', [A])), 0);
++ assert.strictEqual(await db.value('SELECT created_by FROM news_stories WHERE id = $1', ['sty_ada']), A, 'the published story keeps its byline');
++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM news_story_drafts WHERE owner = $1', [B])), 1);
++ const confirmation = calls.find((c) => c.path === '/internal/account-deletions/del_01JZ0000000000000000000DEX/confirmations');
++ assert.ok(confirmation, 'the confirmation was sent');
++ });
++ } finally { await t.close(); }
++ done();
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/test/helpers/boot.js b/test/helpers/boot.js
iindex d1f64c1..6415ade 100644
---- a/test/helpers/boot.js
++++ b/test/helpers/boot.js
@@@ -56,7 +56,7 @@ async function boot(opts = {}) {
let built = null;
async function start() {
const config = configLib.load(env);
-- built = await createApp({ config, telemetrySink: opts.telemetrySink, store: createStore(testdb.db, { now: clock.now }), now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow, indexnow: opts.indexnow });
++ built = await createApp({ config, telemetrySink: opts.telemetrySink, store: createStore(testdb.db, { now: clock.now }), now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow, indexnow: opts.indexnow, accountData: opts.accountData, accountSend: opts.accountSend });
await require('openvibe-sdk/auth').jwksClient(`${config.networkInternalUrl}/api/.well-known/jwks`).keys();
server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
t.base = `http://127.0.0.1:${server.address().port}`;