openvibers/openvibe.bench Commits
account-data: guard every Bench table, and the runs, watches and benchmarks a person owns (#36)
Changes
@@@ -1,27 +1,84 @@'use strict';/**-- * Account export and deletion → Bench (ADR-033; openvibe-sdk/account-data). The rows a person owns are the tasks-- * and rounds they started, the judgements they cast and the private evals they own. Ratings are aggregate and name-- * no person, so they stay; a public dataset judgement never carries a voter id.++ * Account export and deletion → Bench (ADR-033; openvibe-sdk/account-data). What Bench holds about a person:++ *++ * theirs, and gone the tasks and rounds they started, the private evals they own, the runs they++ * submitted, the pairs they have seen and the benchmarks they watch++ * theirs, and kept a judgement they cast stays (it sits under somebody else's round, so the round and++ * the ratings keep their place) and loses the person; a benchmark other people ran++ * stays as a tombstone, so their runs keep theirs++ * not theirs the ratings (an aggregate over the models: the subject column names a model, not a++ * person), the entries (a model's output) and the receipts of the requests themselves++ *++ * A benchmark and its frozen versions are the person's, but other people's runs hang under them, so what happens++ * to them depends on what else is there: a benchmark nobody else ran goes with its versions, one others ran stays++ * as a tombstone. extraErase decides that, and GUARDED names the tables the guard needs to know about.*/const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');++// The tombstone a row keeps when it stays without its person: the column holds an id (usr_…, prj_…, a service),++// so the tombstone is a marker, not a null.++const TOMBSTONE = 'deleted';++const TABLES = [{ table: 'bench_judgements', subject: 'subject', file: 'bench_judgements.json', erase: { anonymize: {} } },++ // The pairs they have seen go before the rounds: a seen pair hangs under a round, so the round's own deletion++ // would take it and the count would miss it. The rounds go before the tasks, which they point at.++ { table: 'bench_battle_seen', subject: 'subject', file: 'bench_battle_seen.json', erase: 'delete' },{ table: 'bench_rounds', subject: 'created_by', file: 'bench_rounds.json', erase: 'delete' },{ table: 'bench_tasks', subject: 'created_by', file: 'bench_tasks.json', erase: 'delete' },{ table: 'bench_evals', subject: 'owner', file: 'bench_evals.json', erase: 'delete' },++ // The runs they submitted and the benchmarks they watch: theirs, so they go.++ { table: 'bench_runs', subject: 'created_by', file: 'bench_runs.json', erase: 'delete' },++ { table: 'bench_watches', subject: 'subject', file: 'bench_watches.json', erase: 'delete' },];++// The tables the map cannot say on its own: a benchmark and its versions are theirs, but other people's runs hang++// under them. extraErase decides; the guard needs the names.++const GUARDED = ['bench_benchmarks', 'bench_benchmark_versions'];++++/** The benchmarks of the person: what they own and what they wrote, with their versions. */++async function extraExport(db, subject) {++ const files = [];++ const benchmarks = await db.many(`SELECT id, owner, title, description, kind, tags, visibility, parent, latest_version, draft, created_by, created_at, updated_at++ FROM bench_benchmarks WHERE owner = $1 OR created_by = $1 ORDER BY created_at DESC LIMIT 5000`, [subject]);++ if (benchmarks.length) files.push({ name: 'benchmarks.json', content: benchmarks });++ const versions = await db.many(`SELECT benchmark_id, version, definition, hash, created_by, created_at FROM bench_benchmark_versions++ WHERE created_by = $1 OR benchmark_id IN (SELECT id FROM bench_benchmarks WHERE owner = $1) ORDER BY created_at DESC LIMIT 5000`, [subject]);++ if (versions.length) files.push({ name: 'benchmark_versions.json', content: versions });++ return files;++}++++async function extraErase(t, subjects, counts) {++ // The benchmarks the person owns. One nobody else ran goes with its versions; one others ran stays as a++ // tombstone, so their runs keep their place.++ const ids = (await t.many('SELECT id FROM bench_benchmarks WHERE owner = ANY($1::text[])', [subjects])).map((r) => r.id);++ const kept = [];++ for (const id of ids) {++ const runs = Number(await t.value('SELECT count(*)::int FROM bench_runs WHERE benchmark_id = $1', [id])) || 0;++ if (runs) { kept.push(id); continue; }++ counts.add(counts.erased, 'bench_benchmarks', await t.exec('DELETE FROM bench_benchmarks WHERE id = $1', [id]));++ }++ if (kept.length) {++ counts.add(counts.retained, 'tombstones', await t.exec('UPDATE bench_benchmarks SET owner = $2, created_by = $2 WHERE id = ANY($1::text[])', [kept, TOMBSTONE]));++ }++ // The benchmarks of somebody else that the person wrote: the definition stays, the person leaves it.++ counts.add(counts.retained, 'tombstones', await t.exec('UPDATE bench_benchmarks SET created_by = $2 WHERE created_by = ANY($1::text[])', [subjects, TOMBSTONE]));++ // The versions: those of the benchmarks that stay lose their author, those of somebody else's benchmark stay.++ counts.add(counts.retained, 'tombstones', await t.exec('UPDATE bench_benchmark_versions SET created_by = $2 WHERE created_by = ANY($1::text[])', [subjects, TOMBSTONE]));++}++function create({ db, log = console } = {}) {return createAccountData({db,service: 'bench',tables: TABLES,-- note: 'Bench exports your tasks, rounds, judgements and private evals; the public dataset is anonymous and a rating never names a person.',++ extraExport,++ extraErase,++ note: 'Bench exports your tasks, rounds, judgements, private evals, runs, watched benchmarks and the pairs you have seen; the public dataset is anonymous and a rating never names a person.',log,});}--module.exports = { create, TABLES, TOPICS };++module.exports = { create, TABLES, GUARDED, TOPICS };@@@ -2,11 +2,15 @@const assert = require('assert');const { signDeliveryHeaders } = require('openvibe-sdk/events');++const { uncovered } = require('openvibe-sdk/account-data');const { boot, check, done } = require('./helpers/boot');++const accountDataLib = require('../server/identity/account-data');const SECRET = 'whsec_fixturefixturefixturefixturefixturefixture';const EXPORT_ID = 'exp_01M624SX2FFR7RMSXMFJ2GM16F';const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';++// The ids the schema checks: a ULID (26 characters of Crockford's alphabet) behind the table's prefix.++const ulid = (prefix, tail = 'AAA') => `${prefix}_01JZ${'0'.repeat(19)}${tail}`;(async () => {const sent = [];@@@ -16,18 +20,32 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';};const t = await boot({ env: { BENCH_EVENTS_SECRET: SECRET, BENCH_ROUNDS_PER_DAY: '100', BENCH_ROUNDS_DAILY: '1000' }, accountSend });const alice = t.network.addUser('alice');++ const carol = t.network.addUser('carol');++ const db = t.ctx.s.db;let taskId;let roundId;let judgementId;++ let sharedBenchmark;++ let lonelyBenchmark;const signedEvent = (event) => {const raw = JSON.stringify({ event });return { raw, headers: { 'Content-Type': 'application/json', ...signDeliveryHeaders(Buffer.from(raw), SECRET) } };};++ await check('the guard: every table with a person column is covered or exempted with a reason', async () => {++ const missing = await uncovered(db, {++ tables: [...accountDataLib.TABLES, ...accountDataLib.GUARDED],++ exempt: {++ account_data_events: 'the ledger of the requests themselves: which export and deletion Bench applied',++ bench_ratings: 'the ratings are an aggregate over the models: the subject column names a model, not a person',++ },++ });++ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);++ });++await check('a person leaves tasks, rounds and a judgement', async () => {-- const carol = t.network.addUser('carol');const other = await t.get('/api/v1/kinds/design.page/tasks', { as: carol, json: { prompt: 'A page for a human.', public: true } });const otherRound = await t.get(`/api/v1/tasks/${other.json().task.id}/rounds`, { as: carol, json: {} });await t.tick();@@@ -45,6 +63,30 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';judgementId = vote.json().judgement.id;});++ await check('a person owns benchmarks, runs, watches and the pairs they have seen', async () => {++ // A benchmark other people ran: it stays, without them, when they go.++ sharedBenchmark = ulid('bmk', 'AAA');++ lonelyBenchmark = ulid('bmk', 'AAB');++ for (const [id, owner] of [[sharedBenchmark, alice.subject], [lonelyBenchmark, alice.subject]]) {++ await db.exec(`INSERT INTO bench_benchmarks (id, owner, title, kind, visibility, created_by, created_at, updated_at)++ VALUES ($1, $2, 'A benchmark', 'design.page', 'unlisted', $2, now(), now())`, [id, owner]);++ await db.exec(`INSERT INTO bench_benchmark_versions (benchmark_id, version, definition, hash, created_by, created_at)++ VALUES ($1, 1, '{"kind":"design.page"}', $2, $3, now())`, [id, 'a'.repeat(64), owner]);++ }++ // Her run on the benchmark, and Carol's run on the same one.++ await db.exec(`INSERT INTO bench_runs (id, benchmark_id, benchmark_version, model, provider, provenance, created_by, created_at)++ VALUES ($1, $2, 1, 'model-a', 'openai', 'submitted', $3, now())`, [ulid('brn', 'AAA'), sharedBenchmark, alice.subject]);++ await db.exec(`INSERT INTO bench_runs (id, benchmark_id, benchmark_version, model, provider, provenance, created_by, created_at)++ VALUES ($1, $2, 1, 'model-b', 'openai', 'submitted', $3, now())`, [ulid('brn', 'AAB'), sharedBenchmark, carol.subject]);++ // A pair she has seen.++ await db.exec(`INSERT INTO bench_battle_seen (id, subject, pair_key, round_id, kind_id, a_entry, b_entry, at)++ VALUES ($1, $2, 'a|b', $3, 'design.page', 'bte_1', 'bte_2', now())`, [ulid('bbs', 'AAA'), alice.subject, roundId]);++ // The watches table arrives with the watches; when it is there, her watch is hers.++ if (await db.value("SELECT to_regclass('bench_watches') IS NOT NULL")) {++ await db.exec(`INSERT INTO bench_watches (id, subject, benchmark_id, created_at) VALUES ($1, $2, $3, now())`, [ulid('bwa', 'AAA'), alice.subject, lonelyBenchmark]);++ }++ });++await check('account export sends the person\'s rows to Network', async () => {const ev = { event_id: 'evt_export_1', event_type: 'network.account.export_requested', source: 'network', payload: { export_id: EXPORT_ID, subject: alice.subject } };const r = await t.get('/internal/events', signedEvent(ev));@@@ -56,6 +98,13 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';assert.ok(part.body.files.some((f) => f.name === 'bench_tasks.json' && f.content.length === 1));assert.ok(part.body.files.some((f) => f.name === 'bench_rounds.json' && f.content.length === 1));assert.ok(part.body.files.some((f) => f.name === 'bench_judgements.json' && f.content.length === 1));++ assert.ok(part.body.files.some((f) => f.name === 'bench_runs.json' && f.content.length === 1));++ assert.ok(part.body.files.some((f) => f.name === 'bench_battle_seen.json' && f.content.length === 1));++ const benchmarks = part.body.files.find((f) => f.name === 'benchmarks.json');++ assert.ok(benchmarks && benchmarks.content.length === 2, 'her benchmarks are in the part');++ const versions = part.body.files.find((f) => f.name === 'benchmark_versions.json');++ assert.ok(versions && versions.content.length === 2, 'the versions of her benchmarks are in the part');++ assert.ok(!JSON.stringify(part.body).includes(carol.subject), 'nobody else\'s rows');});await check('account deletion removes tasks and rounds and anonymises votes', async () => {@@@ -68,14 +117,27 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';assert.strictEqual(confirm.body.subject, alice.subject);assert.strictEqual(confirm.body.erased.bench_tasks, 1);assert.strictEqual(confirm.body.erased.bench_rounds, 1);-- assert.strictEqual(confirm.body.retained.tombstones, 1);++ assert.strictEqual(confirm.body.erased.bench_runs, 1, 'her run is gone');++ assert.strictEqual(confirm.body.erased.bench_benchmarks, 1, 'the benchmark nobody else ran is gone');++ assert.strictEqual(confirm.body.erased.bench_battle_seen, 1, 'the pairs she had seen are gone');++ // Three rows stay without her: her judgement, the benchmark other people ran and its frozen version.++ assert.strictEqual(confirm.body.retained.tombstones, 3, 'her judgement, the benchmark others ran and its version stay');const task = await t.get(`/api/v1/tasks/${taskId}`);assert.strictEqual(task.status, 404, task.text);const round = await t.get(`/api/v1/rounds/${roundId}`);assert.strictEqual(round.status, 404, round.text);-- const j = await t.ctx.s.db.maybe('SELECT subject FROM bench_judgements WHERE id = $1', [judgementId]);++ const j = await db.maybe('SELECT subject FROM bench_judgements WHERE id = $1', [judgementId]);assert.strictEqual(j.subject, null);++ // The benchmark other people ran stays, without her; their run stays too.++ const kept = await db.maybe('SELECT owner, created_by FROM bench_benchmarks WHERE id = $1', [sharedBenchmark]);++ assert.strictEqual(kept.owner, 'deleted');++ assert.strictEqual(kept.created_by, 'deleted');++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM bench_runs WHERE benchmark_id = $1', [sharedBenchmark])), 1, 'Carol\'s run is still there');++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM bench_benchmarks WHERE id = $1', [lonelyBenchmark])), 0, 'the benchmark nobody ran is gone');++ if (await db.value("SELECT to_regclass('bench_watches') IS NOT NULL")) {++ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM bench_watches WHERE subject = $1', [alice.subject])), 0, 'her watch is gone');++ }});await check('stop', async () => { await t.close(); });
@@ -1,27 +1,84 @@
@@ -1,27 +1,84 @@
'use strict';
'use strict';
/**
/**
- * Account export and deletion → Bench (ADR-033; openvibe-sdk/account-data). The rows a person owns are the tasks
- * and rounds they started, the judgements they cast and the private evals they own. Ratings are aggregate and name
- * no person, so they stay; a public dataset judgement never carries a voter id.
+ * Account export and deletion → Bench (ADR-033; openvibe-sdk/account-data). What Bench holds about a person:
+ *
+ * theirs, and gone the tasks and rounds they started, the private evals they own, the runs they
+ * submitted, the pairs they have seen and the benchmarks they watch
+ * theirs, and kept a judgement they cast stays (it sits under somebody else's round, so the round and
+ * the ratings keep their place) and loses the person; a benchmark other people ran
+ * stays as a tombstone, so their runs keep theirs
+ * not theirs the ratings (an aggregate over the models: the subject column names a model, not a
+ * person), the entries (a model's output) and the receipts of the requests themselves
+ *
+ * A benchmark and its frozen versions are the person's, but other people's runs hang under them, so what happens
+ * to them depends on what else is there: a benchmark nobody else ran goes with its versions, one others ran stays
+ * as a tombstone. extraErase decides that, and GUARDED names the tables the guard needs to know about.
*/
*/
const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
+// The tombstone a row keeps when it stays without its person: the column holds an id (usr_…, prj_…, a service),
+// so the tombstone is a marker, not a null.
+const TOMBSTONE = 'deleted';
+
const TABLES = [
const TABLES = [
{ table: 'bench_judgements', subject: 'subject', file: 'bench_judgements.json', erase: { anonymize: {} } },
{ table: 'bench_judgements', subject: 'subject', file: 'bench_judgements.json', erase: { anonymize: {} } },
+ // The pairs they have seen go before the rounds: a seen pair hangs under a round, so the round's own deletion
+ // would take it and the count would miss it. The rounds go before the tasks, which they point at.
+ { table: 'bench_battle_seen', subject: 'subject', file: 'bench_battle_seen.json', erase: 'delete' },
{ table: 'bench_rounds', subject: 'created_by', file: 'bench_rounds.json', erase: 'delete' },
{ table: 'bench_rounds', subject: 'created_by', file: 'bench_rounds.json', erase: 'delete' },
{ table: 'bench_tasks', subject: 'created_by', file: 'bench_tasks.json', erase: 'delete' },
{ table: 'bench_tasks', subject: 'created_by', file: 'bench_tasks.json', erase: 'delete' },
{ table: 'bench_evals', subject: 'owner', file: 'bench_evals.json', erase: 'delete' },
{ table: 'bench_evals', subject: 'owner', file: 'bench_evals.json', erase: 'delete' },
+ // The runs they submitted and the benchmarks they watch: theirs, so they go.
+ { table: 'bench_runs', subject: 'created_by', file: 'bench_runs.json', erase: 'delete' },
+ { table: 'bench_watches', subject: 'subject', file: 'bench_watches.json', erase: 'delete' },
];
];
+// The tables the map cannot say on its own: a benchmark and its versions are theirs, but other people's runs hang
+// under them. extraErase decides; the guard needs the names.
+const GUARDED = ['bench_benchmarks', 'bench_benchmark_versions'];
+
+/** The benchmarks of the person: what they own and what they wrote, with their versions. */
+async function extraExport(db, subject) {
+ const files = [];
+ const benchmarks = await db.many(`SELECT id, owner, title, description, kind, tags, visibility, parent, latest_version, draft, created_by, created_at, updated_at
+ FROM bench_benchmarks WHERE owner = $1 OR created_by = $1 ORDER BY created_at DESC LIMIT 5000`, [subject]);
+ if (benchmarks.length) files.push({ name: 'benchmarks.json', content: benchmarks });
+ const versions = await db.many(`SELECT benchmark_id, version, definition, hash, created_by, created_at FROM bench_benchmark_versions
+ WHERE created_by = $1 OR benchmark_id IN (SELECT id FROM bench_benchmarks WHERE owner = $1) ORDER BY created_at DESC LIMIT 5000`, [subject]);
+ if (versions.length) files.push({ name: 'benchmark_versions.json', content: versions });
+ return files;
+}
+
+async function extraErase(t, subjects, counts) {
+ // The benchmarks the person owns. One nobody else ran goes with its versions; one others ran stays as a
+ // tombstone, so their runs keep their place.
+ const ids = (await t.many('SELECT id FROM bench_benchmarks WHERE owner = ANY($1::text[])', [subjects])).map((r) => r.id);
+ const kept = [];
+ for (const id of ids) {
+ const runs = Number(await t.value('SELECT count(*)::int FROM bench_runs WHERE benchmark_id = $1', [id])) || 0;
+ if (runs) { kept.push(id); continue; }
+ counts.add(counts.erased, 'bench_benchmarks', await t.exec('DELETE FROM bench_benchmarks WHERE id = $1', [id]));
+ }
+ if (kept.length) {
+ counts.add(counts.retained, 'tombstones', await t.exec('UPDATE bench_benchmarks SET owner = $2, created_by = $2 WHERE id = ANY($1::text[])', [kept, TOMBSTONE]));
+ }
+ // The benchmarks of somebody else that the person wrote: the definition stays, the person leaves it.
+ counts.add(counts.retained, 'tombstones', await t.exec('UPDATE bench_benchmarks SET created_by = $2 WHERE created_by = ANY($1::text[])', [subjects, TOMBSTONE]));
+ // The versions: those of the benchmarks that stay lose their author, those of somebody else's benchmark stay.
+ counts.add(counts.retained, 'tombstones', await t.exec('UPDATE bench_benchmark_versions SET created_by = $2 WHERE created_by = ANY($1::text[])', [subjects, TOMBSTONE]));
+}
+
function create({ db, log = console } = {}) {
function create({ db, log = console } = {}) {
return createAccountData({
return createAccountData({
db,
db,
service: 'bench',
service: 'bench',
tables: TABLES,
tables: TABLES,
- note: 'Bench exports your tasks, rounds, judgements and private evals; the public dataset is anonymous and a rating never names a person.',
+ extraExport,
+ extraErase,
+ note: 'Bench exports your tasks, rounds, judgements, private evals, runs, watched benchmarks and the pairs you have seen; the public dataset is anonymous and a rating never names a person.',
log,
log,
});
});
}
}
-module.exports = { create, TABLES, TOPICS };
+module.exports = { create, TABLES, GUARDED, TOPICS };
@@ -2,11 +2,15 @@
@@ -2,11 +2,15 @@
const assert = require('assert');
const assert = require('assert');
const { signDeliveryHeaders } = require('openvibe-sdk/events');
const { signDeliveryHeaders } = require('openvibe-sdk/events');
+const { uncovered } = require('openvibe-sdk/account-data');
const { boot, check, done } = require('./helpers/boot');
const { boot, check, done } = require('./helpers/boot');
+const accountDataLib = require('../server/identity/account-data');
const SECRET = 'whsec_fixturefixturefixturefixturefixturefixture';
const SECRET = 'whsec_fixturefixturefixturefixturefixturefixture';
const EXPORT_ID = 'exp_01M624SX2FFR7RMSXMFJ2GM16F';
const EXPORT_ID = 'exp_01M624SX2FFR7RMSXMFJ2GM16F';
const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
+// The ids the schema checks: a ULID (26 characters of Crockford's alphabet) behind the table's prefix.
+const ulid = (prefix, tail = 'AAA') => `${prefix}_01JZ${'0'.repeat(19)}${tail}`;
(async () => {
(async () => {
const sent = [];
const sent = [];
@@ -16,18 +20,32 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
@@ -16,18 +20,32 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
};
};
const t = await boot({ env: { BENCH_EVENTS_SECRET: SECRET, BENCH_ROUNDS_PER_DAY: '100', BENCH_ROUNDS_DAILY: '1000' }, accountSend });
const t = await boot({ env: { BENCH_EVENTS_SECRET: SECRET, BENCH_ROUNDS_PER_DAY: '100', BENCH_ROUNDS_DAILY: '1000' }, accountSend });
const alice = t.network.addUser('alice');
const alice = t.network.addUser('alice');
+ const carol = t.network.addUser('carol');
+ const db = t.ctx.s.db;
let taskId;
let taskId;
let roundId;
let roundId;
let judgementId;
let judgementId;
+ let sharedBenchmark;
+ let lonelyBenchmark;
const signedEvent = (event) => {
const signedEvent = (event) => {
const raw = JSON.stringify({ event });
const raw = JSON.stringify({ event });
return { raw, headers: { 'Content-Type': 'application/json', ...signDeliveryHeaders(Buffer.from(raw), SECRET) } };
return { raw, headers: { 'Content-Type': 'application/json', ...signDeliveryHeaders(Buffer.from(raw), SECRET) } };
};
};
+ await check('the guard: every table with a person column is covered or exempted with a reason', async () => {
+ const missing = await uncovered(db, {
+ tables: [...accountDataLib.TABLES, ...accountDataLib.GUARDED],
+ exempt: {
+ account_data_events: 'the ledger of the requests themselves: which export and deletion Bench applied',
+ bench_ratings: 'the ratings are an aggregate over the models: the subject column names a model, not a person',
+ },
+ });
+ assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
+ });
+
await check('a person leaves tasks, rounds and a judgement', async () => {
await check('a person leaves tasks, rounds and a judgement', async () => {
- const carol = t.network.addUser('carol');
const other = await t.get('/api/v1/kinds/design.page/tasks', { as: carol, json: { prompt: 'A page for a human.', public: true } });
const other = await t.get('/api/v1/kinds/design.page/tasks', { as: carol, json: { prompt: 'A page for a human.', public: true } });
const otherRound = await t.get(`/api/v1/tasks/${other.json().task.id}/rounds`, { as: carol, json: {} });
const otherRound = await t.get(`/api/v1/tasks/${other.json().task.id}/rounds`, { as: carol, json: {} });
await t.tick();
await t.tick();
@@ -45,6 +63,30 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
@@ -45,6 +63,30 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
judgementId = vote.json().judgement.id;
judgementId = vote.json().judgement.id;
});
});
+ await check('a person owns benchmarks, runs, watches and the pairs they have seen', async () => {
+ // A benchmark other people ran: it stays, without them, when they go.
+ sharedBenchmark = ulid('bmk', 'AAA');
+ lonelyBenchmark = ulid('bmk', 'AAB');
+ for (const [id, owner] of [[sharedBenchmark, alice.subject], [lonelyBenchmark, alice.subject]]) {
+ await db.exec(`INSERT INTO bench_benchmarks (id, owner, title, kind, visibility, created_by, created_at, updated_at)
+ VALUES ($1, $2, 'A benchmark', 'design.page', 'unlisted', $2, now(), now())`, [id, owner]);
+ await db.exec(`INSERT INTO bench_benchmark_versions (benchmark_id, version, definition, hash, created_by, created_at)
+ VALUES ($1, 1, '{"kind":"design.page"}', $2, $3, now())`, [id, 'a'.repeat(64), owner]);
+ }
+ // Her run on the benchmark, and Carol's run on the same one.
+ await db.exec(`INSERT INTO bench_runs (id, benchmark_id, benchmark_version, model, provider, provenance, created_by, created_at)
+ VALUES ($1, $2, 1, 'model-a', 'openai', 'submitted', $3, now())`, [ulid('brn', 'AAA'), sharedBenchmark, alice.subject]);
+ await db.exec(`INSERT INTO bench_runs (id, benchmark_id, benchmark_version, model, provider, provenance, created_by, created_at)
+ VALUES ($1, $2, 1, 'model-b', 'openai', 'submitted', $3, now())`, [ulid('brn', 'AAB'), sharedBenchmark, carol.subject]);
+ // A pair she has seen.
+ await db.exec(`INSERT INTO bench_battle_seen (id, subject, pair_key, round_id, kind_id, a_entry, b_entry, at)
+ VALUES ($1, $2, 'a|b', $3, 'design.page', 'bte_1', 'bte_2', now())`, [ulid('bbs', 'AAA'), alice.subject, roundId]);
+ // The watches table arrives with the watches; when it is there, her watch is hers.
+ if (await db.value("SELECT to_regclass('bench_watches') IS NOT NULL")) {
+ await db.exec(`INSERT INTO bench_watches (id, subject, benchmark_id, created_at) VALUES ($1, $2, $3, now())`, [ulid('bwa', 'AAA'), alice.subject, lonelyBenchmark]);
+ }
+ });
+
await check('account export sends the person\'s rows to Network', async () => {
await check('account export sends the person\'s rows to Network', async () => {
const ev = { event_id: 'evt_export_1', event_type: 'network.account.export_requested', source: 'network', payload: { export_id: EXPORT_ID, subject: alice.subject } };
const ev = { event_id: 'evt_export_1', event_type: 'network.account.export_requested', source: 'network', payload: { export_id: EXPORT_ID, subject: alice.subject } };
const r = await t.get('/internal/events', signedEvent(ev));
const r = await t.get('/internal/events', signedEvent(ev));
@@ -56,6 +98,13 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
@@ -56,6 +98,13 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
assert.ok(part.body.files.some((f) => f.name === 'bench_tasks.json' && f.content.length === 1));
assert.ok(part.body.files.some((f) => f.name === 'bench_tasks.json' && f.content.length === 1));
assert.ok(part.body.files.some((f) => f.name === 'bench_rounds.json' && f.content.length === 1));
assert.ok(part.body.files.some((f) => f.name === 'bench_rounds.json' && f.content.length === 1));
assert.ok(part.body.files.some((f) => f.name === 'bench_judgements.json' && f.content.length === 1));
assert.ok(part.body.files.some((f) => f.name === 'bench_judgements.json' && f.content.length === 1));
+ assert.ok(part.body.files.some((f) => f.name === 'bench_runs.json' && f.content.length === 1));
+ assert.ok(part.body.files.some((f) => f.name === 'bench_battle_seen.json' && f.content.length === 1));
+ const benchmarks = part.body.files.find((f) => f.name === 'benchmarks.json');
+ assert.ok(benchmarks && benchmarks.content.length === 2, 'her benchmarks are in the part');
+ const versions = part.body.files.find((f) => f.name === 'benchmark_versions.json');
+ assert.ok(versions && versions.content.length === 2, 'the versions of her benchmarks are in the part');
+ assert.ok(!JSON.stringify(part.body).includes(carol.subject), 'nobody else\'s rows');
});
});
await check('account deletion removes tasks and rounds and anonymises votes', async () => {
await check('account deletion removes tasks and rounds and anonymises votes', async () => {
@@ -68,14 +117,27 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
@@ -68,14 +117,27 @@ const DELETION_ID = 'del_01M624SX2FFR7RMSXMFJ2GM16F';
assert.strictEqual(confirm.body.subject, alice.subject);
assert.strictEqual(confirm.body.subject, alice.subject);
assert.strictEqual(confirm.body.erased.bench_tasks, 1);
assert.strictEqual(confirm.body.erased.bench_tasks, 1);
assert.strictEqual(confirm.body.erased.bench_rounds, 1);
assert.strictEqual(confirm.body.erased.bench_rounds, 1);
- assert.strictEqual(confirm.body.retained.tombstones, 1);
+ assert.strictEqual(confirm.body.erased.bench_runs, 1, 'her run is gone');
+ assert.strictEqual(confirm.body.erased.bench_benchmarks, 1, 'the benchmark nobody else ran is gone');
+ assert.strictEqual(confirm.body.erased.bench_battle_seen, 1, 'the pairs she had seen are gone');
+ // Three rows stay without her: her judgement, the benchmark other people ran and its frozen version.
+ assert.strictEqual(confirm.body.retained.tombstones, 3, 'her judgement, the benchmark others ran and its version stay');
const task = await t.get(`/api/v1/tasks/${taskId}`);
const task = await t.get(`/api/v1/tasks/${taskId}`);
assert.strictEqual(task.status, 404, task.text);
assert.strictEqual(task.status, 404, task.text);
const round = await t.get(`/api/v1/rounds/${roundId}`);
const round = await t.get(`/api/v1/rounds/${roundId}`);
assert.strictEqual(round.status, 404, round.text);
assert.strictEqual(round.status, 404, round.text);
- const j = await t.ctx.s.db.maybe('SELECT subject FROM bench_judgements WHERE id = $1', [judgementId]);
+ const j = await db.maybe('SELECT subject FROM bench_judgements WHERE id = $1', [judgementId]);
assert.strictEqual(j.subject, null);
assert.strictEqual(j.subject, null);
+ // The benchmark other people ran stays, without her; their run stays too.
+ const kept = await db.maybe('SELECT owner, created_by FROM bench_benchmarks WHERE id = $1', [sharedBenchmark]);
+ assert.strictEqual(kept.owner, 'deleted');
+ assert.strictEqual(kept.created_by, 'deleted');
+ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM bench_runs WHERE benchmark_id = $1', [sharedBenchmark])), 1, 'Carol\'s run is still there');
+ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM bench_benchmarks WHERE id = $1', [lonelyBenchmark])), 0, 'the benchmark nobody ran is gone');
+ if (await db.value("SELECT to_regclass('bench_watches') IS NOT NULL")) {
+ assert.strictEqual(Number(await db.value('SELECT count(*)::int FROM bench_watches WHERE subject = $1', [alice.subject])), 0, 'her watch is gone');
+ }
});
});
await check('stop', async () => { await t.close(); });
await check('stop', async () => { await t.close(); });