openvibers/openvibe.quest Commits

Leave test accounts out of Quest's public counts (#284)

OP OpenVibers committed 1 days ago · 39b8636645 · 8 files, +166 −7

* Leave test accounts out of Quest's public counts

* Exempt the test-account filter cache from account export

* Sync the test-account list before Quest counts its first page

Changes

ddiff --git a/migrations/0016_test_subjects.sql b/migrations/0016_test_subjects.sql
nnew file mode 100644
iindex 0000000..138b0ca
---- /dev/null
++++ b/migrations/0016_test_subjects.sql
@@@ -0,0 +1,8 @@
++-- phase: expand
++-- Test accounts (probe and manual test accounts, OpenVibe.Network users.is_test). The subject ids are read from the
++-- identity authority on a timer (server/identity/test-subjects.js) and kept here, so the public counts can leave them
++-- out in SQL without asking Network on every page read. Additive: an older release never reads this table.
++CREATE TABLE IF NOT EXISTS test_subject (
++ subject text COLLATE "C" PRIMARY KEY,
++ seen_at text COLLATE "C" NOT NULL DEFAULT (to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS'))
++);
ddiff --git a/server/app.js b/server/app.js
iindex 14e7499..de1ce21 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -30,6 +30,7 @@ const { createPrincipal } = require('./http/principal');
const { createApi } = require('./http/api');
const { createPageRoutes } = require('./http/pages');
const { createCoins } = require('./quests/coins');
++const { createTestSubjects } = require('./identity/test-subjects');
const { createItems } = require('./quests/items');
const { createDaily } = require('./quests/daily');
const { createGlyphDraftClient } = require('./quests/glyph-drafts');
@@@ -60,6 +61,7 @@ async function createApp(opts = {}) {
const sso = createSso({ config, keys, fetchImpl, now: s.now, log });
const principal = createPrincipal({ config, keys });
const coins = createCoins({ config, s, fetchImpl, log });
++ const testSubjects = createTestSubjects({ config, s, fetchImpl, log, now: s.now });
const levels = require('./quests/levels').createLevels({ config, s, fetchImpl });
const items = createItems({ config, s, fetchImpl, log });
// Account export and deletion (ADR-033, ./identity/account-data.js): the three tables that hold a person's rows.
@@@ -78,7 +80,7 @@ async function createApp(opts = {}) {
const eventsConsumer = createEventsConsumer({ db: s.db, s, secrets: config.events.secrets, coins, items, accountData, accountSend, appQuests, daily, now: s.now, log });
// The quests in OpenVibe.Search (./search-index.js), swept on its own timer once server/index.js starts it.
const searchIndex = opts.searchIndex || createSearchIndex({ config, s, log });
-- const ctx = { config, s, keys, sso, principal, levels, coins, items, daily, glyphs, eventsConsumer, accountData, accountSend, searchIndex, appQuests, log };
++ const ctx = { config, s, keys, sso, principal, levels, coins, testSubjects, items, daily, glyphs, eventsConsumer, accountData, accountSend, searchIndex, appQuests, log };
const app = express();
// GET /openapi.json (SEO-3): this service's public API as OpenAPI 3.1, generated from the contracts
ddiff --git a/server/identity/test-subjects.js b/server/identity/test-subjects.js
nnew file mode 100644
iindex 0000000..0a1b15b
---- /dev/null
++++ b/server/identity/test-subjects.js
@@@ -0,0 +1,75 @@
++'use strict';
++
++/**
++ * Test accounts for Quest's public numbers (OpenVibe.Network users.is_test).
++ *
++ * Quest shows counts, never people, but a probe account that finished a quest still moved "quests finished", "people
++ * with badges" and the "badges people earn most" list. The identity authority lists its test accounts at
++ * GET /internal/identity/test-subjects (identity.subject.resolve) and this module keeps their subject ids in
++ * test_subject, so the counts can exclude them in SQL (server/quests/store.js). A failed refresh keeps the last good
++ * list: a Network outage never makes a test account public again. Their own signed-in quest log and badges still work,
++ * because nothing filters a subject reading its own rows.
++ */
++const { serviceAuth } = require('openvibe-contracts');
++
++const AUDIENCE = 'openvibe.network';
++const SCOPES = 'identity.subject.resolve';
++const TIMEOUT_MS = 5000;
++const REFRESH_MS = 60 * 60 * 1000;
++const SUBJECT_RE = /^usr_[0-9A-HJKMNP-TV-Z]{26}$/;
++
++function createTestSubjects({ config, s, fetchImpl = globalThis.fetch, log = console, now = () => Date.now() } = {}) {
++ const base = String(config.networkInternalUrl || '').replace(/\/+$/, '');
++ const tokens = serviceAuth.createTokenClient({
++ tokenUrl: `${base}/oauth/token`,
++ clientId: config.oauth.clientId,
++ clientSecret: config.oauth.clientSecret,
++ audience: AUDIENCE,
++ scope: SCOPES,
++ fetchImpl,
++ });
++ let timer = null;
++ const stats = { subjects: 0, refreshed_at: null, failures: 0 };
++
++ /** Replace the list with the Network's answer. Throws with a short, safe reason; never logs a token. */
++ async function refresh() {
++ const res = await fetchImpl(`${base}/internal/identity/test-subjects`, {
++ headers: { Accept: 'application/json', ...(await tokens.authHeaders()) },
++ signal: AbortSignal.timeout(TIMEOUT_MS),
++ });
++ const body = await res.json().catch(() => null);
++ if (!res.ok || !body || !Array.isArray(body.subjects)) {
++ throw new Error(`Network ${res.status}${body && (body.code || body.error) ? `: ${body.code || body.error}` : ''}`);
++ }
++ const subjects = [...new Set(body.subjects.map((p) => p && p.subject && p.subject.id).filter((id) => SUBJECT_RE.test(String(id))))];
++ await s.db.tx(async () => {
++ for (const subject of subjects) {
++ await s.db.exec(`INSERT INTO test_subject (subject, seen_at) VALUES ($1, $2)
++ ON CONFLICT (subject) DO UPDATE SET seen_at = excluded.seen_at`, [subject, s.iso()]);
++ }
++ // A subject no longer on the list is no longer a test account (it was unmarked); remove it after the upsert.
++ if (subjects.length) await s.db.exec('DELETE FROM test_subject WHERE subject <> ALL($1::text[])', [subjects]);
++ else await s.db.exec('DELETE FROM test_subject');
++ });
++ stats.subjects = subjects.length;
++ stats.refreshed_at = new Date(now()).toISOString();
++ return subjects;
++ }
++
++ /** Refresh once now (unless the caller already awaited one), then hourly; a failure keeps the last good list. */
++ function start({ refreshNow = true } = {}) {
++ if (timer) return timer;
++ if (refreshNow) refresh().catch((err) => { stats.failures++; log.warn && log.warn('[Test accounts] refresh failed:', err.message); });
++ timer = setInterval(() => {
++ refresh().catch((err) => { stats.failures++; log.warn && log.warn('[Test accounts] refresh failed:', err.message); });
++ }, REFRESH_MS);
++ if (timer.unref) timer.unref();
++ return timer;
++ }
++
++ function stop() { if (timer) clearInterval(timer); timer = null; }
++
++ return { refresh, start, stop, status: () => ({ ...stats }) };
++}
++
++module.exports = { createTestSubjects, REFRESH_MS };
ddiff --git a/server/index.js b/server/index.js
iindex 9eecc62..8dac7e6 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -24,6 +24,9 @@ function createLifecycle({ server, ctx, exit, signals, timers = [], extra = [] }
async function start() {
const { app, ctx } = await createApp();
const { config } = ctx;
++ // Test accounts (Network users.is_test): the first list is awaited before the listener opens, so the first public
++ // page already counts without probes; a failure keeps the last good list and the hourly timer still runs.
++ await ctx.testSubjects.refresh().catch((err) => console.warn('[Test accounts] refresh failed:', err.message));
// Socket activation (2026-10-31): systemd's .socket unit holds the listener across restarts, so a deploy
// queues requests in the kernel instead of answering 502. Without the socket unit, the usual listen.
@@@ -53,6 +56,8 @@ async function start() {
ctx.levels.retryPending().catch(() => {});
const coinsTimer = ctx.coins.start();
if (coinsTimer) timers.push(coinsTimer);
++ const testSubjectsTimer = ctx.testSubjects.start({ refreshNow: false });
++ if (testSubjectsTimer) timers.push(testSubjectsTimer);
// Item rewards owed (a failed grant, or a completion from before items) are granted by the same kind of timer.
const itemsTimer = ctx.items.start();
if (itemsTimer) timers.push(itemsTimer);
@@@ -60,7 +65,7 @@ async function start() {
// App quests: each project with an active production quest gets its app.<project_key>.* subscription.
ctx.appQuests.setSubscriber(subscriptions ? subscriptions.subscribe : null);
ctx.appQuests.load().catch((err) => console.warn('[Quest] app quests not loaded:', err.message));
-- const extra = [stopDaily, () => { if (subscriptions) subscriptions.stop(); }, () => ctx.coins.stop(), () => ctx.items.stop()];
++ const extra = [stopDaily, () => { if (subscriptions) subscriptions.stop(); }, () => ctx.coins.stop(), () => ctx.testSubjects.stop(), () => ctx.items.stop()];
createLifecycle({ server, ctx, timers, extra });
return { server, ctx };
}
ddiff --git a/server/quests/store.js b/server/quests/store.js
iindex 7cd06fc..e90a41e 100644
---- a/server/quests/store.js
++++ b/server/quests/store.js
@@@ -43,7 +43,9 @@ const completionsFor = (db, subject) => db.many(
'SELECT * FROM quest_completions WHERE subject = $1 ORDER BY completed_at DESC, id DESC', [subject]);
/** Every completion for a quest — a completion count for the pages, never a list of people. */
--const completionCount = async (db, questId) => Number(await db.value('SELECT count(*) FROM quest_completions WHERE quest_id = $1', [questId]));
++const completionCount = async (db, questId) => Number(await db.value(
++ `SELECT count(*) FROM quest_completions c
++ WHERE c.quest_id = $1 AND NOT EXISTS (SELECT 1 FROM test_subject t WHERE t.subject = c.subject)`, [questId]));
/**
* Record a completion. Returns the row when it was inserted now, or null when this person already completed the
@@@ -126,7 +128,8 @@ async function totals(db, since) {
`SELECT count(*)::int AS completions, count(DISTINCT subject)::int AS people,
COALESCE(sum(coins) FILTER (WHERE coins_state = 'credited'), 0)::int AS coins,
count(*) FILTER (WHERE completed_at >= $1::text)::int AS recent
-- FROM quest_completions`,
++ FROM quest_completions c
++ WHERE NOT EXISTS (SELECT 1 FROM test_subject t WHERE t.subject = c.subject)`,
[since]);
return {
completions: Number(row && row.completions) || 0,
@@@ -137,7 +140,9 @@ async function totals(db, since) {
}
/** Completions per quest (each is that quest's badge, awarded once): [{ quest_id, n }], counts and never people. */
--const completionsByQuest = (db) => db.many('SELECT quest_id, count(*)::int AS n FROM quest_completions GROUP BY quest_id');
++const completionsByQuest = (db) => db.many(`SELECT c.quest_id, count(*)::int AS n FROM quest_completions c
++ WHERE NOT EXISTS (SELECT 1 FROM test_subject t WHERE t.subject = c.subject)
++ GROUP BY c.quest_id`);
/** How many people have a badge — the count a quest page can show without naming anyone. */
const badgeHolders = async (db, badgeId) => Number(await db.value('SELECT count(*) FROM quest_badges WHERE badge_id = $1', [badgeId]));
ddiff --git a/test/account-data.test.js b/test/account-data.test.js
iindex 275d7f3..21d1253 100644
---- a/test/account-data.test.js
++++ b/test/account-data.test.js
@@@ -65,7 +65,7 @@ const rowsFor = async (t, table, subject) => Number(await t.ctx.s.db.value(`SELE
try {
const missing = await uncovered(d.db, {
tables: TABLES,
-- exempt: {"account_data_events":"the ledger of requests itself: which export and deletion quest applied","quest_index_revisions":"owner is the search-index document's owner (type/id), not a person; the row holds a revision and hash"},
++ exempt: {"account_data_events":"the ledger of requests itself: which export and deletion quest applied","quest_index_revisions":"owner is the search-index document's owner (type/id), not a person; the row holds a revision and hash","test_subject":"the identity authority's test-account subject list, a filter cache: not a person's own data, never exported"},
});
assert.deepStrictEqual(missing, [], `a table with a person column is neither covered nor exempted: ${missing.join(', ')}`);
} finally { await d.close(); }
ddiff --git a/test/helpers/mocks.js b/test/helpers/mocks.js
iindex d38bd0a..c166661 100644
---- a/test/helpers/mocks.js
++++ b/test/helpers/mocks.js
@@@ -61,14 +61,16 @@ async function startNetwork({ sandboxAudiences = ['openvibe.events', 'openvibe.m
grants: new Map(), quotas: new Map(), usage: new Map(), audit: [], codes: new Map(), refresh: new Map(),
catalog: null, down: false, requests: [], tokenRequests: [], exportTokens: [], exportTtl: 300,
coins: { balances: new Map(), byKey: new Map(), credits: [], down: false },
++ testSubjects: new Set(),
};
let issuer = null;
let nextUserId = 1;
const problem = (json, status, code, detail) => json(status, { type: `https://openvibe.network/problems/${code}`, title: 'Error', status, code, detail, error: detail, request_id: `req_${crypto.randomBytes(6).toString('hex')}` }, { 'Content-Type': 'application/problem+json' });
-- function addUser(username, { role = 'user' } = {}) {
++ function addUser(username, { role = 'user', is_test = false } = {}) {
const u = { id: nextUserId++, subject: ids.newId('user'), username, display_name: username, role };
++ if (is_test) st.testSubjects.add(u.subject);
st.users.set(u.id, u); st.byUsername.set(username, u);
return u;
}
@@@ -326,6 +328,13 @@ async function startNetwork({ sandboxAudiences = ['openvibe.events', 'openvibe.m
if (!u) return problem(json, 404, 'identity.subject_not_found', 'no subject for that id');
return json(200, { subject: { type: 'user', id: u.subject }, network_user_id: u.id, username: u.username, display_name: u.display_name, banned: false });
}
++ if (url.pathname === '/internal/identity/test-subjects' && req.method === 'GET') {
++ const claims = serviceFrom(req);
++ if (!claims || !hasCap(claims, 'identity.subject.resolve')) return problem(json, 403, 'capability.denied', 'identity.subject.resolve is required');
++ const subjects = [...st.users.values()].filter((u) => st.testSubjects.has(u.subject))
++ .map((u) => ({ subject: { type: 'user', id: u.subject }, network_user_id: u.id, username: u.username, display_name: u.display_name, avatar_url: null, banned: false, is_test: true }));
++ return json(200, { subjects });
++ }
if (url.pathname === '/internal/coins/credit' && req.method === 'POST') {
const claims = serviceFrom(req);
if (!claims || !hasCap(claims, 'network.coins.credit')) return problem(json, 403, 'capability.denied', 'network.coins.credit is required');
ddiff --git a/test/test-subjects.test.js b/test/test-subjects.test.js
nnew file mode 100644
iindex 0000000..a0964a2
---- /dev/null
++++ b/test/test-subjects.test.js
@@@ -0,0 +1,55 @@
++'use strict';
++/**
++ * Test accounts in Quest's public numbers (server/identity/test-subjects.js, server/quests/store.js).
++ *
++ * The identity authority lists the accounts marked is_test (the probes and older manual test accounts); Quest keeps
++ * their subject ids in test_subject and every public count (quests finished, people with badges, badges people earn
++ * most) leaves them out. The account's own rows are untouched: its quest log and badges still answer for itself.
++ */
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++const catalog = require('../server/quests/catalog');
++const store = require('../server/quests/store');
++
++(async () => {
++ const t = await boot();
++ const normal = t.network.addUser('normal_person');
++ const probe = t.network.addUser('ovprobe_merge_keep', { is_test: true });
++ const quest = catalog.all()[0];
++ const insert = async (id, subject, at) => await t.ctx.s.db.exec(
++ `INSERT INTO quest_completions (id, subject, quest_id, completed_at, coins, coins_state) VALUES ($1, $2, $3, $4, 0, 'skipped')`,
++ [id, subject, quest.id, at]);
++ await insert('qcp_normal', normal.subject, t.ctx.s.iso());
++ await insert('qcp_probe', probe.subject, t.ctx.s.iso());
++ const since = new Date(Date.now() - 7 * 86400000).toISOString();
++
++ await check('before the identity list arrives, both count (the filter is data, not a guess)', async () => {
++ const totals = await store.totals(t.ctx.s.db, since);
++ assert.deepStrictEqual([totals.completions, totals.people, totals.recent], [2, 2, 2]);
++ });
++
++ await check('the Network list marks the test account; every public count leaves it out', async () => {
++ const subjects = await t.ctx.testSubjects.refresh();
++ assert.deepStrictEqual(subjects, [probe.subject]);
++ const totals = await store.totals(t.ctx.s.db, since);
++ assert.deepStrictEqual([totals.completions, totals.people, totals.recent], [1, 1, 1], 'the public figures count one person');
++ const byQuest = await store.completionsByQuest(t.ctx.s.db);
++ assert.deepStrictEqual(byQuest, [{ quest_id: quest.id, n: 1 }], 'the badge list counts one holder');
++ assert.strictEqual(await store.completionCount(t.ctx.s.db, quest.id), 1);
++ });
++
++ await check('the test account\'s own rows still answer for itself', async () => {
++ const mine = await store.completionsFor(t.ctx.s.db, probe.subject);
++ assert.strictEqual(mine.length, 1, 'the probe still sees its own completion');
++ assert.strictEqual((await store.badgesOf(t.ctx.s.db, probe.subject)).length, 0);
++ });
++
++ await check('a subject the Network unmarks is no longer hidden', async () => {
++ t.network.state.testSubjects.clear();
++ assert.deepStrictEqual(await t.ctx.testSubjects.refresh(), []);
++ assert.strictEqual((await store.totals(t.ctx.s.db, since)).people, 2);
++ });
++
++ await t.close();
++ done();
++})().catch((err) => { console.error(err); process.exit(1); });