openvibers/openvibe.bench Commits

account-data: guard every Bench table, and the runs, watches and benchmarks a person owns (#36)

OP OpenVibers committed 14 days ago · e3b1139125 · 2 files, +127 −8

Changes

ddiff --git a/server/identity/account-data.js b/server/identity/account-data.js
iindex 11317fc..c26bb35 100644
---- a/server/identity/account-data.js
++++ b/server/identity/account-data.js
@@@ -1,27 +1,84 @@
'use strict';
/**
-- * Account export and deletion → Bench (ADR-033; openvibe-sdk/account-data). The rows a person owns are the tasks
-- * and rounds they started, the judgements they cast and the private evals they own. Ratings are aggregate and name
-- * no person, so they stay; a public dataset judgement never carries a voter id.
++ * Account export and deletion → Bench (ADR-033; openvibe-sdk/account-data). What Bench holds about a person:
++ *
++ * theirs, and gone the tasks and rounds they started, the private evals they own, the runs they
++ * submitted, the pairs they have seen and the benchmarks they watch
++ * theirs, and kept a judgement they cast stays (it sits under somebody else's round, so the round and
++ * the ratings keep their place) and loses the person; a benchmark other people ran
++ * stays as a tombstone, so their runs keep theirs
++ * not theirs the ratings (an aggregate over the models: the subject column names a model, not a
++ * person), the entries (a model's output) and the receipts of the requests themselves
++ *
++ * A benchmark and its frozen versions are the person's, but other people's runs hang under them, so what happens
++ * to them depends on what else is there: a benchmark nobody else ran goes with its versions, one others ran stays
++ * as a tombstone. extraErase decides that, and GUARDED names the tables the guard needs to know about.
*/
const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
++// The tombstone a row keeps when it stays without its person: the column holds an id (usr_…, prj_…, a service),
++// so the tombstone is a marker, not a null.
++const TOMBSTONE = 'deleted';
++
const TABLES = [
{ table: 'bench_judgements', subject: 'subject', file: 'bench_judgements.json', erase: { anonymize: {} } },
++ // The pairs they have seen go before the rounds: a seen pair hangs under a round, so the round's own deletion
++ // would take it and the count would miss it. The rounds go before the tasks, which they point at.
++ { table: 'bench_battle_seen', subject: 'subject', file: 'bench_battle_seen.json', erase: 'delete' },
{ table: 'bench_rounds', subject: 'created_by', file: 'bench_rounds.json', erase: 'delete' },
{ table: 'bench_tasks', subject: 'created_by', file: 'bench_tasks.json', erase: 'delete' },
{ table: 'bench_evals', subject: 'owner', file: 'bench_evals.json', erase: 'delete' },
++ // The runs they submitted and the benchmarks they watch: theirs, so they go.
++ { table: 'bench_runs', subject: 'created_by', file: 'bench_runs.json', erase: 'delete' },
++ { table: 'bench_watches', subject: 'subject', file: 'bench_watches.json', erase: 'delete' },
];
++// The tables the map cannot say on its own: a benchmark and its versions are theirs, but other people's runs hang
++// under them. extraErase decides; the guard needs the names.
++const GUARDED = ['bench_benchmarks', 'bench_benchmark_versions'];
++
++/** The benchmarks of the person: what they own and what they wrote, with their versions. */
++async function extraExport(db, subject) {
++ const files = [];
++ const benchmarks = await db.many(`SELECT id, owner, title, description, kind, tags, visibility, parent, latest_version, draft, created_by, created_at, updated_at
++ FROM bench_benchmarks WHERE owner = $1 OR created_by = $1 ORDER BY created_at DESC LIMIT 5000`, [subject]);
++ if (benchmarks.length) files.push({ name: 'benchmarks.json', content: benchmarks });
++ const versions = await db.many(`SELECT benchmark_id, version, definition, hash, created_by, created_at FROM bench_benchmark_versions
++ WHERE created_by = $1 OR benchmark_id IN (SELECT id FROM bench_benchmarks WHERE owner = $1) ORDER BY created_at DESC LIMIT 5000`, [subject]);
++ if (versions.length) files.push({ name: 'benchmark_versions.json', content: versions });
++ return files;
++}
++
++async function extraErase(t, subjects, counts) {
++ // The benchmarks the person owns. One nobody else ran goes with its versions; one others ran stays as a
++ // tombstone, so their runs keep their place.
++ const ids = (await t.many('SELECT id FROM bench_benchmarks WHERE owner = ANY($1::text[])', [subjects])).map((r) => r.id);
++ const kept = [];
++ for (const id of ids) {
++ const runs = Number(await t.value('SELECT count(*)::int FROM bench_runs WHERE benchmark_id = $1', [id])) || 0;
++ if (runs) { kept.push(id); continue; }
++ counts.add(counts.erased, 'bench_benchmarks', await t.exec('DELETE FROM bench_benchmarks WHERE id = $1', [id]));
++ }
++ if (kept.length) {
++ counts.add(counts.retained, 'tombstones', await t.exec('UPDATE bench_benchmarks SET owner = $2, created_by = $2 WHERE id = ANY($1::text[])', [kept, TOMBSTONE]));
++ }
++ // The benchmarks of somebody else that the person wrote: the definition stays, the person leaves it.
++ counts.add(counts.retained, 'tombstones', await t.exec('UPDATE bench_benchmarks SET created_by = $2 WHERE created_by = ANY($1::text[])', [subjects, TOMBSTONE]));
++ // The versions: those of the benchmarks that stay lose their author, those of somebody else's benchmark stay.
++ counts.add(counts.retained, 'tombstones', await t.exec('UPDATE bench_benchmark_versions SET created_by = $2 WHERE created_by = ANY($1::text[])', [subjects, TOMBSTONE]));
++}
++
function create({ db, log = console } = {}) {
return createAccountData({
db,
service: 'bench',
tables: TABLES,
-- note: 'Bench exports your tasks, rounds, judgements and private evals; the public dataset is anonymous and a rating never names a person.',
++ extraExport,
++ extraErase,
++ note: 'Bench exports your tasks, rounds, judgements, private evals, runs, watched benchmarks and the pairs you have seen; the public dataset is anonymous and a rating never names a person.',
log,
});
}
--module.exports = { create, TABLES, TOPICS };
++module.exports = { create, TABLES, GUARDED, TOPICS };
ddiff --git a/test/account-data.test.js b/test/account-data.test.js
iindex 7a73ce6..e71fb01 100644
---- a/test/account-data.test.js
++++ b/test/account-data.test.js
@@@ -2,11 +2,15 @@
const assert = require('assert');
const { signDeliveryHeaders } = require('openvibe-sdk/events');
++const { uncovered } = require('openvibe-sdk/account-data');
const { boot, check, done } = require('./helpers/boot');
++const accountDataLib = require('../server/identity/account-data');
const SECRET = 'whsec_fixturefixturefixturefixturefixturefixture';
const EXPORT_ID = 'exp_01M624SX2FFR7RMSXMFJ2GM16F';
const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
++// The ids the schema checks: a ULID (26 characters of Crockford's alphabet) behind the table's prefix.
++const ulid = (prefix, tail = 'AAA') => `${prefix}_01JZ${'0'.repeat(19)}${tail}`;
(async () => {
const sent = [];
@@@ -16,18 +20,32 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
};
const t = await boot({ env: { BENCH_EVENTS_SECRET: SECRET, BENCH_ROUNDS_PER_DAY: '100', BENCH_ROUNDS_DAILY: '1000' }, accountSend });
const alice = t.network.addUser('alice');
++ const carol = t.network.addUser('carol');
++ const db = t.ctx.s.db;
let taskId;
let roundId;
let judgementId;
++ let sharedBenchmark;
++ let lonelyBenchmark;
const signedEvent = (event) => {
const raw = JSON.stringify({ event });
return { raw, headers: { 'Content-Type': 'application/json', ...signDeliveryHeaders(Buffer.from(raw), SECRET) } };
};
++ await check('the guard: every table with a person column is covered or exempted with a reason', async () => {
++ const missing = await uncovered(db, {
++ tables: [...accountDataLib.TABLES, ...accountDataLib.GUARDED],
++ exempt: {
++ account_data_events: 'the ledger of the requests themselves: which export and deletion Bench applied',
++ bench_ratings: 'the ratings are an aggregate over the models: the subject column names a model, not a person',
++ },
++ });
++ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
++ });
++
await check('a person leaves tasks, rounds and a judgement', async () => {
-- const carol = t.network.addUser('carol');
const other = await t.get('/api/v1/kinds/design.page/tasks', { as: carol, json: { prompt: 'A page for a human.', public: true } });
const otherRound = await t.get(`/api/v1/tasks/${other.json().task.id}/rounds`, { as: carol, json: {} });
await t.tick();
@@@ -45,6 +63,30 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
judgementId = vote.json().judgement.id;
});
++ await check('a person owns benchmarks, runs, watches and the pairs they have seen', async () => {
++ // A benchmark other people ran: it stays, without them, when they go.
++ sharedBenchmark = ulid('bmk', 'AAA');
++ lonelyBenchmark = ulid('bmk', 'AAB');
++ for (const [id, owner] of [[sharedBenchmark, alice.subject], [lonelyBenchmark, alice.subject]]) {
++ await db.exec(`INSERT INTO bench_benchmarks (id, owner, title, kind, visibility, created_by, created_at, updated_at)
++ VALUES ($1, $2, 'A benchmark', 'design.page', 'unlisted', $2, now(), now())`, [id, owner]);
++ await db.exec(`INSERT INTO bench_benchmark_versions (benchmark_id, version, definition, hash, created_by, created_at)
++ VALUES ($1, 1, '{"kind":"design.page"}', $2, $3, now())`, [id, 'a'.repeat(64), owner]);
++ }
++ // Her run on the benchmark, and Carol's run on the same one.
++ await db.exec(`INSERT INTO bench_runs (id, benchmark_id, benchmark_version, model, provider, provenance, created_by, created_at)
++ VALUES ($1, $2, 1, 'model-a', 'openai', 'submitted', $3, now())`, [ulid('brn', 'AAA'), sharedBenchmark, alice.subject]);
++ await db.exec(`INSERT INTO bench_runs (id, benchmark_id, benchmark_version, model, provider, provenance, created_by, created_at)
++ VALUES ($1, $2, 1, 'model-b', 'openai', 'submitted', $3, now())`, [ulid('brn', 'AAB'), sharedBenchmark, carol.subject]);
++ // A pair she has seen.
++ await db.exec(`INSERT INTO bench_battle_seen (id, subject, pair_key, round_id, kind_id, a_entry, b_entry, at)
++ VALUES ($1, $2, 'a|b', $3, 'design.page', 'bte_1', 'bte_2', now())`, [ulid('bbs', 'AAA'), alice.subject, roundId]);
++ // The watches table arrives with the watches; when it is there, her watch is hers.
++ if (await db.value("SELECT to_regclass('bench_watches') IS NOT NULL")) {
++ await db.exec(`INSERT INTO bench_watches (id, subject, benchmark_id, created_at) VALUES ($1, $2, $3, now())`, [ulid('bwa', 'AAA'), alice.subject, lonelyBenchmark]);
++ }
++ });
++
await check('account export sends the person\'s rows to Network', async () => {
const ev = { event_id: 'evt_export_1', event_type: 'network.account.export_requested', source: 'network', payload: { export_id: EXPORT_ID, subject: alice.subject } };
const r = await t.get('/internal/events', signedEvent(ev));
@@@ -56,6 +98,13 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
assert.ok(part.body.files.some((f) => f.name === 'bench_tasks.json' && f.content.length === 1));
assert.ok(part.body.files.some((f) => f.name === 'bench_rounds.json' && f.content.length === 1));
assert.ok(part.body.files.some((f) => f.name === 'bench_judgements.json' && f.content.length === 1));
++ assert.ok(part.body.files.some((f) => f.name === 'bench_runs.json' && f.content.length === 1));
++ assert.ok(part.body.files.some((f) => f.name === 'bench_battle_seen.json' && f.content.length === 1));
++ const benchmarks = part.body.files.find((f) => f.name === 'benchmarks.json');
++ assert.ok(benchmarks && benchmarks.content.length === 2, 'her benchmarks are in the part');
++ const versions = part.body.files.find((f) => f.name === 'benchmark_versions.json');
++ assert.ok(versions && versions.content.length === 2, 'the versions of her benchmarks are in the part');
++ assert.ok(!JSON.stringify(part.body).includes(carol.subject), 'nobody else\'s rows');
});
await check('account deletion removes tasks and rounds and anonymises votes', async () => {
@@@ -68,14 +117,27 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
assert.strictEqual(confirm.body.subject, alice.subject);
assert.strictEqual(confirm.body.erased.bench_tasks, 1);
assert.strictEqual(confirm.body.erased.bench_rounds, 1);
-- assert.strictEqual(confirm.body.retained.tombstones, 1);
++ assert.strictEqual(confirm.body.erased.bench_runs, 1, 'her run is gone');
++ assert.strictEqual(confirm.body.erased.bench_benchmarks, 1, 'the benchmark nobody else ran is gone');
++ assert.strictEqual(confirm.body.erased.bench_battle_seen, 1, 'the pairs she had seen are gone');
++ // Three rows stay without her: her judgement, the benchmark other people ran and its frozen version.
++ assert.strictEqual(confirm.body.retained.tombstones, 3, 'her judgement, the benchmark others ran and its version stay');
const task = await t.get(`/api/v1/tasks/${taskId}`);
assert.strictEqual(task.status, 404, task.text);
const round = await t.get(`/api/v1/rounds/${roundId}`);
assert.strictEqual(round.status, 404, round.text);
-- const j = await t.ctx.s.db.maybe('SELECT subject FROM bench_judgements WHERE id = $1', [judgementId]);
++ const j = await db.maybe('SELECT subject FROM bench_judgements WHERE id = $1', [judgementId]);
assert.strictEqual(j.subject, null);
++ // The benchmark other people ran stays, without her; their run stays too.
++ const kept = await db.maybe('SELECT owner, created_by FROM bench_benchmarks WHERE id = $1', [sharedBenchmark]);
++ assert.strictEqual(kept.owner, 'deleted');
++ assert.strictEqual(kept.created_by, 'deleted');
++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM bench_runs WHERE benchmark_id = $1', [sharedBenchmark])), 1, 'Carol\'s run is still there');
++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM bench_benchmarks WHERE id = $1', [lonelyBenchmark])), 0, 'the benchmark nobody ran is gone');
++ if (await db.value("SELECT to_regclass('bench_watches') IS NOT NULL")) {
++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM bench_watches WHERE subject = $1', [alice.subject])), 0, 'her watch is gone');
++ }
});
await check('stop', async () => { await t.close(); });