openvibers/openvibe.codes Commits

Account export and deletion cover everything Codes keeps about a person (#179)

OP OpenVibers committed 14 days ago · f3e13630e4 · 2 files, +91 −21

Only run_tasks and the session lines were covered. Everything stages C2–C8 and C4b added was left in place when a
person asked to leave, and missing from their export:
- attempts (with their cost and tokens);
- the proof (screenshots, diff, review);
- run tokens (ids and expiry);
- workspace settings;
- the events sent about them;
- telemetry that names them.

Now:
- Export carries attempts.json, proof.json, settings.json, run-tokens.json (no jti), events-sent.json and
  telemetry.json beside run-tasks.json and session-lines.json.
- Deletion erases everything task-keyed (session lines, attempts, proof, the events inbox) before the tasks, plus the
  settings, run tokens and outbox rows, in one transaction.
- Telemetry samples stay as numbers with the person's id removed.

The test seeds every table for two people and checks that only the first's rows leave. It fails on the old module.

Changes

ddiff --git a/server/account-data.js b/server/account-data.js
iindex 12297f9..b39ed38 100644
---- a/server/account-data.js
++++ b/server/account-data.js
@@@ -1,42 +1,81 @@
'use strict';
/**
-- * Account export and deletion → Codes (ADR-033; openvibe-sdk/account-data). What Codes holds about a person is the
-- * coding tasks they started on their own machine (run_tasks, plan T16) and the session lines of those tasks
-- * (codes_session_lines): what they asked, the Run job, its result and its session. The export carries the tasks whole
-- * and their sessions as session-lines.json; a deletion erases the session lines and then the tasks, in one
-- * transaction. The Run jobs themselves are OpenVibe.Run's records (usage billed to the project) and are Run's to
-- * answer for.
++ * Account export and deletion → Codes (ADR-033; openvibe-sdk/account-data).
++ *
++ * What Codes holds about a person:
++ * - the coding tasks they started (run_tasks, plan T16), with their session lines (codes_session_lines);
++ * - each task's attempts (codes_attempts, C2), the task's proof (codes_task_proof, C8) and the events Codes took in
++ * about it (codes_events_inbox);
++ * - the run tokens minted for their attempts (codes_run_tokens, C3: ids and expiry, never the token);
++ * - their workspace choices (codes_user_settings, C4b);
++ * - the events Codes sent about them (codes_outbox);
++ * - telemetry samples that name them.
++ *
++ * Export carries the tasks whole, and the task-keyed rows as session-lines.json, attempts.json and proof.json.
++ * Deletion erases everything task-keyed and then the tasks, in one transaction. Telemetry samples stay as numbers with
++ * the person's id removed. The Run jobs themselves are OpenVibe.Run's records (usage billed to the project) and are
++ * Run's to answer for.
*/
const { createAccountData, TOPICS } = require('openvibe-sdk/account-data');
const SESSION_LINES_MAX = 20000;
++const TASK_ROWS_MAX = 20000;
const TABLES = [
{
table: 'run_tasks', subject: 'subject', file: 'run-tasks.json', order: 'created_at',
columns: ['id', 'project_id', 'job_id', 'repo', 'ref', 'prompt', 'agent', 'push', 'pull_request', 'state', 'result', 'error', 'started_at', 'finished_at', 'created_at', 'updated_at'],
-- // Erased by extraErase below, after the session lines that point at these task ids (the SDK deletes the
-- // table's rows before calling it, so a default delete here would lose the ids first).
-- erase: { keep: 'erased with its codes_session_lines by the service extraErase' },
++ // Erased by extraErase below, after the rows that point at these task ids (the SDK deletes the table's rows
++ // before calling it, so a default delete here would lose the ids first).
++ erase: { keep: 'erased with its task-keyed rows by the service extraErase' },
},
++ { table: 'codes_user_settings', subject: 'subject', file: 'settings.json', order: 'updated_at' },
++ // The token itself is never stored; its jti is an internal handle and stays out of the export.
++ { table: 'codes_run_tokens', subject: 'subject', file: 'run-tokens.json', order: 'created_at', columns: ['id', 'task_id', 'attempt_id', 'provider', 'expires_at', 'created_at'] },
++ { table: 'codes_outbox', subject: 'subject', file: 'events-sent.json', order: 'created_at', columns: ['event_id', 'topic', 'created_at', 'published_at'] },
++ {
++ table: 'telemetry_samples', subject: 'subject', file: 'telemetry.json', order: 'at',
++ columns: ['at', 'operation', 'resource', 'status', 'latency_ms'],
++ // The sample stays (latency and error rates are the service's), without the person in it.
++ erase: { anonymize: {} },
++ },
++];
++
++// The rows that hang under a person's tasks: exported under the task, erased before it.
++const TASK_KEYED = [
++ { table: 'codes_session_lines', file: 'session-lines.json', columns: 'l.task_id, l.seq, l.at, l.kind, l.body', order: 'l.seq ASC', kind: 'session_lines', max: SESSION_LINES_MAX },
++ { table: 'codes_attempts', file: 'attempts.json', columns: 'l.id, l.task_id, l.n, l.agent, l.model, l.machine, l.state, l.reason, l.why, l.kind, l.cost_usd, l.tokens, l.steps, l.started_at, l.finished_at, l.created_at', order: 'l.n ASC', kind: 'attempts', max: TASK_ROWS_MAX },
++ { table: 'codes_task_proof', file: 'proof.json', columns: 'l.task_id, l.attempt_n, l.n, l.kind, l.label, l.ok, l.skipped, l.summary, l.family, l.detail, l.created_at', order: 'l.n ASC', kind: 'proof', max: TASK_ROWS_MAX },
++ { table: 'codes_events_inbox', file: null, kind: 'events_received' },
];
++async function exists(q, table) {
++ return Boolean(await q.maybe('SELECT 1 FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = $1', [table]));
++}
++
function create({ db, log = console }) {
return createAccountData({
db, service: 'codes', tables: TABLES, log,
-- // The sessions of their tasks, line by line, in task and stream order (at most SESSION_LINES_MAX, oldest
-- // tasks first out).
++ // The task-keyed rows of their tasks, newest tasks first (at most max rows each).
extraExport: async (d, subject) => {
-- const lines = await d.many(
-- `SELECT l.task_id, l.seq, l.at, l.kind, l.body FROM codes_session_lines l JOIN run_tasks r ON r.id = l.task_id
-- WHERE r.subject = $1 ORDER BY r.created_at DESC, l.seq ASC LIMIT ${SESSION_LINES_MAX}`, [subject]);
-- return lines.length ? [{ name: 'session-lines.json', content: lines }] : [];
++ const files = [];
++ for (const k of TASK_KEYED) {
++ if (!k.file || !(await exists(d, k.table))) continue;
++ const rows = await d.many(
++ `SELECT ${k.columns} FROM ${k.table} l JOIN run_tasks r ON r.id = l.task_id
++ WHERE r.subject = $1 ORDER BY r.created_at DESC, ${k.order} LIMIT ${k.max}`, [subject]);
++ if (rows.length) files.push({ name: k.file, content: rows });
++ }
++ return files;
},
extraErase: async (t, subjects, { erased, retained, add }) => {
const rows = await t.many('SELECT id FROM run_tasks WHERE subject = ANY($1::text[])', [subjects]);
if (rows.length) {
const ids = rows.map((r) => r.id);
-- add(erased, 'session_lines', await t.exec('DELETE FROM codes_session_lines WHERE task_id = ANY($1::text[])', [ids]));
++ for (const k of TASK_KEYED) {
++ if (!(await exists(t, k.table))) continue;
++ add(erased, k.kind, await t.exec(`DELETE FROM ${k.table} WHERE task_id = ANY($1::text[])`, [ids]));
++ }
}
delete retained.run_tasks;
add(erased, 'run_tasks', await t.exec('DELETE FROM run_tasks WHERE subject = ANY($1::text[])', [subjects]));
@@@ -44,4 +83,4 @@ function create({ db, log = console }) {
});
}
--module.exports = { create, TABLES, TOPICS };
++module.exports = { create, TABLES, TASK_KEYED, TOPICS };
ddiff --git a/test/account-data.test.js b/test/account-data.test.js
iindex 20544ce..0721c05 100644
---- a/test/account-data.test.js
++++ b/test/account-data.test.js
@@@ -36,10 +36,25 @@ async function startNetworkStub() {
const db = t.ctx.store.db;
const ada = ids.newId('user');
const bo = ids.newId('user');
-- const task = (subject, prompt) => db.exec(`INSERT INTO run_tasks (id, subject, project_id, job_id, repo, prompt, state, result, created_at, updated_at)
-- VALUES ($1, $2, $3, $4, 'https://github.com/a/b', $5, 'succeeded', '{"ok":true}'::jsonb, 1, 2)`, [ids.newId('task').replace(/^[a-z]+_/, 'tsk_'), subject, ids.newId('project'), `job_${ids.ulid()}`, prompt]);
-- await task(ada, 'Ada’s task');
-- await task(bo, 'Bo’s task');
++ const task = async (subject, prompt) => {
++ const id = ids.newId('task').replace(/^[a-z]+_/, 'tsk_');
++ await db.exec(`INSERT INTO run_tasks (id, subject, project_id, job_id, repo, prompt, state, result, created_at, updated_at)
++ VALUES ($1, $2, $3, $4, 'https://github.com/a/b', $5, 'succeeded', '{"ok":true}'::jsonb, 1, 2)`, [id, subject, ids.newId('project'), `job_${ids.ulid()}`, prompt]);
++ return id;
++ };
++ const adaTask = await task(ada, 'Ada’s task');
++ const boTask = await task(bo, 'Bo’s task');
++ // Everything later stages keep about a person or their tasks (C2 attempts, C3 run tokens, C8 proof, C4b settings,
++ // the outbox, telemetry): each must leave with the person, and only theirs.
++ for (const [subject, taskId] of [[ada, adaTask], [bo, boTask]]) {
++ await db.exec("INSERT INTO codes_attempts (id, task_id, n, state, agent) VALUES ($1, $2, 1, 'done', 'codex')", [`att_${ids.ulid()}`, taskId]);
++ await db.exec("INSERT INTO codes_task_proof (id, task_id, attempt_n, n, kind, label, ok) VALUES ($1, $2, 1, 1, 'tests', 'tests', true)", [`prf_${ids.ulid()}`, taskId]);
++ await db.exec("INSERT INTO codes_run_tokens (id, task_id, subject, provider, jti, expires_at) VALUES ($1, $2, $3, 'openai', $4, now())", [`rtk_${ids.ulid()}`, taskId, subject, `jti_${ids.ulid()}`]);
++ await db.exec("INSERT INTO codes_user_settings (subject, default_agent) VALUES ($1, 'codex')", [subject]);
++ await db.exec("INSERT INTO codes_outbox (event_id, topic, subject, payload) VALUES ($1, 'codes.task.updated', $2, '{}'::jsonb)", [`evt_${ids.ulid()}`, subject]);
++ await db.exec("INSERT INTO telemetry_samples (at, service, operation, subject) VALUES (now(), 'codes', 'task.start', $1)", [subject]);
++ await db.exec("INSERT INTO codes_events_inbox (event_id, event_type, task_id) VALUES ($1, 'run.job.finished', $2)", [`evt_${ids.ulid()}`, taskId]);
++ }
const deliver = async (event, { secret = SECRET } = {}) => {
const body = JSON.stringify({ event, seq: 1 });
const res = await fetch(`${t.base}/internal/events`, { method: 'POST', body, headers: { 'Content-Type': 'application/json', ...(secret ? signDeliveryHeaders(body, secret) : {}) } });
@@@ -64,6 +79,13 @@ async function startNetworkStub() {
assert.strictEqual(file.content.length, 1);
assert.strictEqual(file.content[0].prompt, 'Ada’s task');
assert.deepStrictEqual(file.content[0].result, { ok: true });
++ for (const name of ['attempts.json', 'proof.json', 'settings.json', 'run-tokens.json']) {
++ const f = part.body.files.find((x) => x.name === name);
++ assert.ok(f, `${name} is in the export`);
++ assert.strictEqual(f.content.length, 1, `${name} holds Ada’s row only`);
++ }
++ const tok = part.body.files.find((x) => x.name === 'run-tokens.json').content[0];
++ assert.ok(!('jti' in tok), 'a run token’s jti stays out of the export');
});
await check('a deletion erases them, keeps everyone else’s, and confirms once', async () => {
@@@ -76,6 +98,15 @@ async function startNetworkStub() {
const confirmations = stub.calls.filter((c) => c.url === `/internal/account-deletions/${deletionId}/confirmations`);
assert.strictEqual(confirmations.length, 1);
assert.strictEqual(confirmations[0].body.erased.run_tasks, 1);
++ for (const [table, key, who] of [
++ ['codes_attempts', 'task_id', [boTask]], ['codes_task_proof', 'task_id', [boTask]], ['codes_events_inbox', 'task_id', [boTask]],
++ ['codes_run_tokens', 'subject', [bo]], ['codes_user_settings', 'subject', [bo]], ['codes_outbox', 'subject', [bo]],
++ ]) {
++ const left = (await db.many(`SELECT ${key} FROM ${table}`)).map((r) => r[key]);
++ assert.deepStrictEqual(left, who, `${table}: only Bo’s row is left`);
++ }
++ const samples = await db.many('SELECT subject FROM telemetry_samples ORDER BY subject NULLS FIRST');
++ assert.deepStrictEqual(samples.map((r) => r.subject), [null, bo], 'Ada’s telemetry sample stays, without her');
});
} finally {
await t.close();