openvibers/openvibe.community Commits

Leave test accounts out of Pulse and the public feed (#383)

OP OpenVibers committed 2 days ago · 1b9145c839 · 10 files, +96 −11

* Leave test accounts out of Pulse and the public feed

* Classify the test-account list call in the SSRF inventory

Changes

ddiff --git a/migrations/0036_test_accounts.sql b/migrations/0036_test_accounts.sql
nnew file mode 100644
iindex 0000000..8a6725b
---- /dev/null
++++ b/migrations/0036_test_accounts.sql
@@@ -0,0 +1,5 @@
++-- phase: expand
++-- Test accounts (probe and manual test accounts, OpenVibe.Network users.is_test): the projection cache keeps the
++-- flag next to the name, and Pulse's read excludes any item whose actor is a test account. Additive: an older
++-- release never reads the column.
++ALTER TABLE subject_projection ADD COLUMN IF NOT EXISTS is_test bigint NOT NULL DEFAULT 0;
ddiff --git a/server/app.js b/server/app.js
iindex 55a7bdd..46b1f53 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -154,7 +154,7 @@ async function createApp(opts = {}) {
// Public activity without an Events topic (Today's calendar, Network's open stats, Quest badges): a due-based
// job that fills Pulse, started by server/index.js and never called while a feed request is being answered.
const pulseSources = require('./pulse/sources').createPulseSources({
-- db, config, fetchImpl: opts.sourcesFetch || opts.fetchImpl || globalThis.fetch, ...(opts.pulseSources || {}),
++ db, config, fetchImpl: opts.sourcesFetch || opts.fetchImpl || globalThis.fetch, testSubjects: () => pulse.testSubjectIds(), ...(opts.pulseSources || {}),
});
app.locals.pulseSources = pulseSources;
const relay = opts.relay || createDiscordRelay({
ddiff --git a/server/identity/network.js b/server/identity/network.js
iindex 30aeefa..079c433 100644
---- a/server/identity/network.js
++++ b/server/identity/network.js
@@@ -70,9 +70,30 @@ function createNetworkIdentity({ config, db, fetchImpl = globalThis.fetch } = {}
await store.upsertProjection(db, {
subject_id: p.subject.id, username: p.username || null, display_name: p.display_name || p.username || null,
avatar_url: absAvatar(p.avatar_url), profile_color: p.profile_color || null,
++ is_test: p.is_test ? 1 : 0,
});
}
++ /**
++ * Every account the Network marks is_test (identity.subject.resolve, GET /internal/identity/test-subjects).
++ * Remembered in the projection cache with the flag, so Pulse can leave their activity out at read time.
++ */
++ async function testSubjects(retried = false) {
++ const res = await fetchImpl(`${base}/internal/identity/test-subjects`, {
++ headers: { Accept: 'application/json', ...(await tokens.authHeaders()) },
++ signal: AbortSignal.timeout(8000),
++ });
++ if (res.status === 401 && !retried) { tokens.invalidate(); return await testSubjects(true); }
++ const data = await res.json().catch(() => null);
++ if (!res.ok || !data || !Array.isArray(data.subjects)) {
++ const err = new Error(`test-subjects ${res.status}: ${(data && (data.detail || data.error)) || 'bad response'}`);
++ err.status = res.status;
++ throw err;
++ }
++ for (const p of data.subjects) await remember(p);
++ return data.subjects;
++ }
++
/** Subject ids → projections (and cache them). */
async function resolveSubjects(subjectIds) {
const map = await resolveChunks(subjectIds, (chunk) => ({ subject_ids: chunk }));
@@@ -158,7 +179,7 @@ function createNetworkIdentity({ config, db, fetchImpl = globalThis.fetch } = {}
return map;
}
-- return { resolveSubjects, resolveLegacy, subjectForUsername, rememberClaims, projections, tokens };
++ return { resolveSubjects, resolveLegacy, subjectForUsername, rememberClaims, projections, testSubjects, tokens };
}
module.exports = { createNetworkIdentity, PROJECTION_TTL_MS };
ddiff --git a/server/pastes/store.js b/server/pastes/store.js
iindex 37bb5ae..8572e2e 100644
---- a/server/pastes/store.js
++++ b/server/pastes/store.js
@@@ -448,17 +448,19 @@ async function getProjections(db, subjectIds) {
/** Upsert what we were told about a subject. Fields left undefined keep their stored value. */
async function upsertProjection(db, p) {
if (!p || !p.subject_id) return;
-- await db.prepare(`INSERT INTO subject_projection (subject_id, username, display_name, avatar_url, profile_color, refreshed_at)
-- VALUES (@subject_id, @username, @display_name, @avatar_url, @profile_color, ov_now())
++ await db.prepare(`INSERT INTO subject_projection (subject_id, username, display_name, avatar_url, profile_color, is_test, refreshed_at)
++ VALUES (@subject_id, @username, @display_name, @avatar_url, @profile_color, @is_test, ov_now())
ON CONFLICT(subject_id) DO UPDATE SET
username = COALESCE(excluded.username, subject_projection.username),
display_name = COALESCE(excluded.display_name, subject_projection.display_name),
avatar_url = COALESCE(excluded.avatar_url, subject_projection.avatar_url),
profile_color = COALESCE(excluded.profile_color, subject_projection.profile_color),
++ is_test = GREATEST(subject_projection.is_test, excluded.is_test),
refreshed_at = ov_now()`)
.run({
subject_id: p.subject_id, username: p.username ?? null, display_name: p.display_name ?? null,
avatar_url: p.avatar_url ?? null, profile_color: p.profile_color ?? null,
++ is_test: p.is_test ? 1 : 0,
});
}
ddiff --git a/server/pulse/service.js b/server/pulse/service.js
iindex 7c8d3d4..f988b67 100644
---- a/server/pulse/service.js
++++ b/server/pulse/service.js
@@@ -32,6 +32,21 @@ function createPulse({ db, network = null, config = {}, fetchImpl = globalThis.f
const authors = createAuthors({ db, network });
const base = (config.baseUrl || '').replace(/\/$/, '');
let originsCache = { at: 0, list: null };
++ // The probe and test accounts, from the identity authority (GET /internal/identity/test-subjects). Cached a
++ // little; the projection cache also carries the flag, and list() filters on both so a new probe cannot stay.
++ let testCache = { at: 0, ids: [] };
++ async function testSubjectIds() {
++ if (testCache.at && Date.now() - testCache.at < 10 * 60_000) return testCache.ids;
++ if (!network || typeof network.testSubjects !== 'function') { testCache = { at: Date.now(), ids: [] }; return testCache.ids; }
++ try {
++ const list = await network.testSubjects();
++ testCache = { at: Date.now(), ids: list.map((p) => p && p.subject && p.subject.id).filter(Boolean) };
++ } catch (err) {
++ console.warn('[Pulse] test-account list failed:', err.message);
++ testCache = { at: Date.now(), ids: testCache.ids };
++ }
++ return testCache.ids;
++ }
const followsCache = new Map();
const appWrites = new Map();
@@@ -113,6 +128,9 @@ function createPulse({ db, network = null, config = {}, fetchImpl = globalThis.f
actor, origin: 'user', at: thread.created_at, text, pictureUrl });
},
++ /** The test-account subject ids, so the Pulse sources skip them too. */
++ testSubjectIds,
++
// ── hooks for Community's own writes ─────────────────
async pasteCreated(p) {
if (!p || p.visibility !== 'public' || p.origin !== 'user' || Number(p.burn_after_read) || Number(p.is_nsfw)) return;
@@@ -256,10 +274,16 @@ function createPulse({ db, network = null, config = {}, fetchImpl = globalThis.f
}
const followed = viewer && viewer.kind === 'user' && viewer.token ? await followedSubjects(viewer) : [];
const hour = mine ? (before ? before[0] : sqlTime(Date.now() - 60 * 60_000)) : null;
-- const { rows, hasMore } = await store.listItems(db, { origin, before: before && (mine ? before.slice(1) : [0, ...before]), limit, followed, viewerSubject: viewer && viewer.kind === 'user' ? viewer.subject : null, hour });
++ const excluded = await testSubjectIds();
++ const { rows, hasMore } = await store.listItems(db, { origin, before: before && (mine ? before.slice(1) : [0, ...before]), limit, followed, viewerSubject: viewer && viewer.kind === 'user' ? viewer.subject : null, hour, excludeSubjects: excluded });
const projections = await authors.projectionsFor(rows.map((r) => r.actor_subject));
++ // The projection is authority too: a test account resolved now is left out even if the cached list was
++ // fetched before it existed (the list refreshes on a timer).
++ const meSubject = viewer && viewer.kind === 'user' ? viewer.subject : null;
++ const visible = rows.filter((r) => !(r.actor_subject && r.actor_subject !== meSubject
++ && projections.get(r.actor_subject) && projections.get(r.actor_subject).is_test));
const last = rows[rows.length - 1];
-- return { items: rows.map((r) => shape(r, projections)), next_cursor: hasMore && last
++ return { items: visible.map((r) => shape(r, projections)), next_cursor: hasMore && last
? encodeCursor(mine ? [hour, last.feed_priority, last.occurred_at, last.id] : [last.occurred_at, last.id]) : null,
// What this reader may do here: the menu's Feed shows its composer only when this says so (openvibe-shared
// ov-home.js), so it never offers a post the status route would refuse.
ddiff --git a/server/pulse/sources.js b/server/pulse/sources.js
iindex 6e88e1e..2dc24ad 100644
---- a/server/pulse/sources.js
++++ b/server/pulse/sources.js
@@@ -32,7 +32,7 @@ const sqlFrom = (v, fallback) => {
};
const dayOf = (ms) => new Date(ms).toISOString().slice(0, 10);
--function createPulseSources({ db, config = {}, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console } = {}) {
++function createPulseSources({ db, config = {}, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console, testSubjects = null } = {}) {
const todayApi = String(config.todayInternalUrl || config.todayUrl || 'https://openvibe.today').replace(/\/+$/, '');
const todaySite = String(config.todayUrl || 'https://openvibe.today').replace(/\/+$/, '');
const networkApi = String(config.networkInternalUrl || config.networkUrl || 'https://openvibe.network').replace(/\/+$/, '');
@@@ -121,12 +121,15 @@ function createPulseSources({ db, config = {}, fetchImpl = globalThis.fetch, now
WHERE i.actor_subject IS NOT NULL AND i.origin = 'user' AND i.occurred_at >= ?
AND (q.subject IS NULL OR q.checked_at < ?)
ORDER BY i.actor_subject LIMIT 25`).all(since, checkedBefore);
++ // A test account never gets a quest-badge item; the identity list says which subjects are test accounts.
++ let excluded = new Set();
++ if (testSubjects) { try { excluded = new Set(await testSubjects()); } catch { /* the list is optional */ } }
let n = 0;
const MAX_CREATED = 25;
for (const p of people) {
if (n >= MAX_CREATED) break;
const who = subject(p.subject);
-- if (!who) continue;
++ if (!who || excluded.has(who)) continue;
try {
const res = await fetchImpl(`${questApi}/api/v1/profiles/${encodeURIComponent(who)}/badges`, { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(8000) });
if (res.ok) {
ddiff --git a/server/pulse/store.js b/server/pulse/store.js
iindex ee64ed6..91c9ef4 100644
---- a/server/pulse/store.js
++++ b/server/pulse/store.js
@@@ -112,13 +112,18 @@ async function removeItem(db, service, type, id) {
* again within six hours, so one line and its newest time remain.
* opts.origin 'user' | 'ai' | 'system'
* opts.before [occurred_at, id] of the last item already seen
++ * opts.excludeSubjects test accounts (Network's is_test): their items never reach a public read
* → { rows, hasMore }
*/
--async function listItems(db, { origin = null, before = null, limit = 30, followed = [], viewerSubject = null, hour = null } = {}) {
++async function listItems(db, { origin = null, before = null, limit = 30, followed = [], viewerSubject = null, hour = null, excludeSubjects = [] } = {}) {
const priority = "CASE WHEN @hour::text IS NOT NULL AND i.occurred_at >= @hour AND jsonb_exists(@followed::jsonb, i.actor_subject) THEN 1 ELSE 0 END";
const rows = await db.prepare(`
SELECT i.*, ${priority} AS feed_priority FROM pulse_items i
WHERE (@origin::text IS NULL OR i.origin = @origin)
++ AND (@exclude::text[] IS NULL OR i.actor_subject IS NULL OR i.actor_subject = @viewerSubject::text
++ OR NOT (i.actor_subject = ANY(@exclude::text[])))
++ AND (i.actor_subject = @viewerSubject::text
++ OR NOT EXISTS (SELECT 1 FROM subject_projection sp WHERE sp.subject_id = i.actor_subject AND sp.is_test = 1))
AND (i.visibility = 'public' OR (i.visibility = 'followers' AND i.actor_subject IS NOT NULL
AND (i.actor_subject = @viewerSubject::text OR jsonb_exists(@followed::jsonb, i.actor_subject))))
AND (i.source_service <> 'today' OR i.source_type <> 'event' OR (i.starts_at IS NOT NULL AND i.starts_at > to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS')))
@@@ -147,6 +152,7 @@ async function listItems(db, { origin = null, before = null, limit = 30, followe
OR (i.source_type = 'submission' AND EXISTS (SELECT 1 FROM submissions su WHERE su.slug = i.source_id AND su.status = 'accepted')))
ORDER BY feed_priority DESC, i.occurred_at DESC, i.id DESC
LIMIT @limit`).all({ origin, followed: JSON.stringify(followed), viewerSubject, hour,
++ exclude: Array.isArray(excludeSubjects) && excludeSubjects.length ? excludeSubjects : null,
priority: before ? before[0] : null, at: before ? before[1] : null, id: before ? before[2] : null, limit: limit + 1 });
const hasMore = rows.length > limit;
if (hasMore) rows.pop();
ddiff --git a/test/helpers/mock-network.js b/test/helpers/mock-network.js
iindex eb529dd..32ecb27 100644
---- a/test/helpers/mock-network.js
++++ b/test/helpers/mock-network.js
@@@ -35,7 +35,7 @@ function start({ clientSecret = 'shh' } = {}) {
directory.users.push(user);
return user;
}
-- const projection = (u) => (u ? { subject: { type: 'user', id: u.subject_id }, network_user_id: u.network_user_id, username: u.username, display_name: u.display_name, avatar_url: u.avatar_url, banned: false } : null);
++ const projection = (u) => (u ? { subject: { type: 'user', id: u.subject_id }, network_user_id: u.network_user_id, username: u.username, display_name: u.display_name, avatar_url: u.avatar_url, banned: !!u.banned, is_test: !!u.is_test } : null);
const server = http.createServer((req, res) => {
const chunks = [];
@@@ -91,6 +91,15 @@ function start({ clientSecret = 'shh' } = {}) {
const found = lockerKeys[subject];
return found ? json(200, found) : json(404, { code: 'locker.no_keys', error: 'no keys' });
}
++ if (req.url === '/internal/identity/test-subjects' && req.method === 'GET') {
++ const auth = String(req.headers.authorization || '');
++ const v = serviceAuth.verifyServiceToken(auth.slice(7), { publicKey: publicPem, issuer, audience: 'openvibe.network' });
++ if (!v.ok) return json(401, { code: v.code, error: v.reason });
++ if (!(v.claims.cap || []).includes('identity.subject.resolve')) return json(403, { code: 'capability.denied', error: 'not granted' });
++ resolveCalls.push({ test_subjects: true });
++ if (directory.down) return json(503, { error: 'down' });
++ return json(200, { subjects: directory.users.filter((u) => u.is_test).map(projection) });
++ }
if (req.url === '/internal/identity/resolve-batch' && req.method === 'POST') {
const auth = String(req.headers.authorization || '');
const v = serviceAuth.verifyServiceToken(auth.slice(7), { publicKey: publicPem, issuer, audience: 'openvibe.network' });
ddiff --git a/test/pulse.test.js b/test/pulse.test.js
iindex d865496..15f03f9 100644
---- a/test/pulse.test.js
++++ b/test/pulse.test.js
@@@ -217,6 +217,21 @@ const { boot, check, done } = require('./helpers/app');
}
});
++ await check('a test account (Network is_test) never reaches the public Pulse or feed', async () => {
++ const probe = net.addUser({ network_user_id: 30, username: 'ovprobe_merge_keep', display_name: 'ovprobe_merge_keep', is_test: true });
++ const asProbe = { kind: 'user', subject: probe.subject_id, staff: false, origin: 'user' };
++ const slug = (await pastes.createText(asProbe, { title: 'Probe paste', content: 'probe' })).slug;
++ assert.ok(await bySource('paste', slug), 'the activity is recorded, so this is a read-time filter and not a lost write');
++ for (const path of ['/api/v1/pulse?limit=100', '/api/v1/feed?limit=100']) {
++ const body = (await t.get(path)).text;
++ assert.ok(!body.includes('ovprobe_merge_keep'), `${path} leaves the test account out`);
++ assert.ok(!body.includes('Probe paste'), `${path} leaves its item out`);
++ }
++ // A test account's own signed-in read still works: the filter is for public surfaces, not for the account itself.
++ const own = (await t.get('/api/v1/pulse?for=me&limit=100', { cookies: [`ov_token=${net.sign({ id: 30, subject_id: probe.subject_id, username: 'ovprobe_merge_keep', role: 'user' })}`] })).json();
++ assert.ok(own.items.some((i) => i.title === 'Probe paste'), 'the probe can still read its own activity as itself');
++ });
++
await t.close();
done();
})();
ddiff --git a/test/security-ssrf.test.js b/test/security-ssrf.test.js
iindex d870fc1..b31be8b 100644
---- a/test/security-ssrf.test.js
++++ b/test/security-ssrf.test.js
@@@ -27,7 +27,7 @@ const INVENTORY = {
'server/chat-rooms.js': [1, 'OV_CHAT_INTERNAL_URL /api/chat/rooms/<slug>/attachments; the slug must match ROOM_SLUG and is encoded'],
'server/events.js': [3, 'service token (Network) and the outbox publisher: EVENTS_URL'],
'server/forum/api.js': [1, 'Media composer attachment keep at the configured mediaUrl origin and a fixed path; the attachment id is validated as att_ plus a ULID'],
-- 'server/identity/network.js': [3, 'Network /internal/identity/resolve-batch and /internal/identity/resolve?username= with a service token (configured base, never a user URL)'],
++ 'server/identity/network.js': [4, 'Network /internal/identity/resolve-batch, /internal/identity/resolve?username= and /internal/identity/test-subjects with a service token (configured base, never a user URL)'],
'server/render/pages.js': [1, 'not a call: the home page shows a fetch() sample as text (never executed)'],
'server/identity/account-data.js': [2, 'Network /oauth/token and /internal/account-exports and /internal/account-deletions (fixed paths) with a service token'],
'server/identity/profile-module.js': [3, 'Network modules API (community.profile) with a service token'],