openvibers/openvibe.sources Commits
A failed fetch's event no longer carries the endpoint's credential (roadmap WS-R task 5: internal-secret leak; low). sources.fetch.failed named the endpoint as registered, so a feed whose URL carries a provider key (?api_key=…) sent that key to OpenVibe.Events and every first-party subscriber's logs. Secret-looking query parameters (key, api_key, token, access_token, secret, signature, password, …) are now redacted in the event; the run records, readable only with sources.source.read, keep the endpoint as it is.
Changes
@@@ -61,12 +61,23 @@ function createIngest({ db, config, registry, items, robots, fetcher, spacer, ou};}++ // An endpoint may carry a provider credential in its query string (?api_key=…). Events travel to++ // other services and their logs, so the event names the endpoint with such values redacted.++ const SECRET_PARAM = /^(api[_-]?key|apikey|key|token|access[_-]?token|auth|secret|client[_-]?secret|password|passwd|sig|signature)$/i;++ function redactedUrl(u) {++ try {++ const url = new URL(u);++ for (const k of [...url.searchParams.keys()]) if (SECRET_PARAM.test(k)) url.searchParams.set(k, 'redacted');++ return url.href;++ } catch { return null; }++ }++function failedEvent(source, run, failures) {outbox.enqueue({event_type: 'sources.fetch.failed',subject: { type: 'source', id: source.key },payload: {-- source_key: source.key, category: source.category, run_id: run.id, endpoint_url: run.endpoint_url,++ source_key: source.key, category: source.category, run_id: run.id, endpoint_url: redactedUrl(run.endpoint_url),state: run.state, error_code: run.error_code, http_status: run.http_status, consecutive_failures: failures,},});nnew file mode 100644@@@ -0,0 +1,87 @@++'use strict';++/**++ * Sources' secrets never leave in a response or an event, and its API answers nobody without a++ * valid service token (roadmap WS-R task 5: internal-secret leak; the private-bypass class for a++ * service whose every read is capability-gated). Sources boots with a sentinel as its Network OAuth++ * client secret (it mints its service token with it), a source whose endpoint carries a sentinel++ * credential in its query string (as a feed's API key would), and a failing fetch of it. Then every++ * route the booted app has (listed from Express's router stack) is requested anonymously, with a++ * garbage token, with another service's audience, with an expired token and with a read token,++ * with real and nonsense ids, plus the probes and every write route with a broken body. No body or++ * header may carry the client secret; only a caller holding sources.source.read may see a source's++ * endpoints (its admin); every /api/v1 route refuses the others; the outbox carries neither.++ *++ * node test/security-secrets.test.js++ */++const assert = require('assert');++const { boot, site, sourceDef, rss, suite, serviceToken, request } = require('./helpers');++++const t = suite('security-secrets');++const SECRET = 'sentinel-not-a-secret-sources-oauth-client';++const FEED_KEY = 'sentinel-not-a-secret-feed-key';++let svc, web;++++function listRoutes(app) {++ const out = [];++ const mount = (layer) => {++ if (!layer.regexp || layer.regexp.fast_slash) return '';++ const src = layer.regexp.source.replace(/^\^/, '').replace(/\\\/\?\(\?=\\\/\|\$\)$/i, '').replace(/\\\//g, '/');++ return /[\\^$()|[\]*+?]/.test(src) ? null : src;++ };++ const walk = (stack, prefix) => {++ for (const layer of stack) {++ if (layer.route) for (const p of [].concat(layer.route.path)) { if (typeof p === 'string') out.push({ path: prefix + p, methods: Object.keys(layer.route.methods) }); }++ else if (layer.handle && Array.isArray(layer.handle.stack)) { const m = mount(layer); if (m !== null) walk(layer.handle.stack, prefix + m); }++ }++ };++ walk(app._router.stack, '');++ return out;++}++const fill = (p, v) => p.replace(/:([A-Za-z0-9_]+)/g, () => encodeURIComponent(v));++++t('boot with a sentinel client secret and a source whose endpoint carries a key', async () => {++ web = await site({ '/robots.txt': () => ({ status: 404 }), '/feed.xml': () => ({ status: 500, body: 'upstream broke' }) });++ svc = await boot({ env: { OV_OAUTH_CLIENT_SECRET: SECRET, OV_OAUTH_CLIENT_ID: 'sources', OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9' } });++ svc.registry.create(sourceDef({ key: 'keyed', endpoints: [`${web.origin}/feed.xml?api_key=${FEED_KEY}`] }), 'test');++ await svc.ingest.run('keyed', { trigger: 'manual' });++});++++t('every route, as every kind of caller, with real and nonsense ids: the client secret never appears; the feed key only to a source reader', async () => {++ const now = Math.floor(Date.now() / 1000);++ const callers = {++ anonymous: null,++ garbage: 'not-a-token',++ 'other audience': serviceToken('news', ['sources.*'], { aud: 'openvibe.elsewhere' }),++ expired: serviceToken('news', ['sources.*'], { exp: now - 60 }),++ 'item reader': serviceToken('deals', ['sources.item.read']),++ 'source reader': serviceToken('news', ['sources.source.read', 'sources.item.read']),++ };++ const found = [];++ const refusedApi = [];++ for (const route of listRoutes(svc.app)) {++ for (const method of route.methods) {++ for (const id of ['keyed', '1', 'nope', "'\"<x>", 'x'.repeat(300)]) {++ const p = fill(route.path, id);++ for (const [who, token] of Object.entries(callers)) {++ const r = await request(svc.base, method.toUpperCase(), `${p}${method === 'get' ? '?q=x&limit=-1' : ''}`, { token, ...(method === 'get' ? {} : { headers: { 'content-type': 'application/json' } }) });++ const text = r.text + JSON.stringify([...r.headers.entries()]);++ if (text.includes(SECRET)) found.push(`${who}: ${method} ${p} → ${r.status} carries the client secret`);++ if (text.includes(FEED_KEY) && who !== 'source reader') found.push(`${who}: ${method} ${p} → ${r.status} carries the feed key`);++ if (p.startsWith('/api/v1/') && !/^\/api\/v1\/health/.test(p) && ['anonymous', 'garbage', 'other audience', 'expired'].includes(who) && r.status < 400) refusedApi.push(`${who}: ${method} ${p} → ${r.status}`);++ }++ }++ }++ }++ assert.deepStrictEqual(found, []);++ assert.deepStrictEqual(refusedApi, [], 'an /api/v1 route answered without a valid service token');++});++++t('the outbox carries neither', () => {++ const rows = JSON.stringify(svc.outbox.all ? svc.outbox.all() : svc.db.prepare('SELECT * FROM event_outbox').all());++ assert.ok(!rows.includes(SECRET), 'client secret in the outbox');++ assert.ok(!rows.includes(FEED_KEY), 'feed key in the outbox');++});++++t('done', async () => { await svc.stop(); await web.close(); });++++t.run();
@@ -61,12 +61,23 @@ function createIngest({ db, config, registry, items, robots, fetcher, spacer, ou
@@ -61,12 +61,23 @@ function createIngest({ db, config, registry, items, robots, fetcher, spacer, ou
};
};
}
}
+ // An endpoint may carry a provider credential in its query string (?api_key=…). Events travel to
+ // other services and their logs, so the event names the endpoint with such values redacted.
+ const SECRET_PARAM = /^(api[_-]?key|apikey|key|token|access[_-]?token|auth|secret|client[_-]?secret|password|passwd|sig|signature)$/i;
+ function redactedUrl(u) {
+ try {
+ const url = new URL(u);
+ for (const k of [...url.searchParams.keys()]) if (SECRET_PARAM.test(k)) url.searchParams.set(k, 'redacted');
+ return url.href;
+ } catch { return null; }
+ }
+
function failedEvent(source, run, failures) {
function failedEvent(source, run, failures) {
outbox.enqueue({
outbox.enqueue({
event_type: 'sources.fetch.failed',
event_type: 'sources.fetch.failed',
subject: { type: 'source', id: source.key },
subject: { type: 'source', id: source.key },
payload: {
payload: {
- source_key: source.key, category: source.category, run_id: run.id, endpoint_url: run.endpoint_url,
+ source_key: source.key, category: source.category, run_id: run.id, endpoint_url: redactedUrl(run.endpoint_url),
state: run.state, error_code: run.error_code, http_status: run.http_status, consecutive_failures: failures,
state: run.state, error_code: run.error_code, http_status: run.http_status, consecutive_failures: failures,
},
},
});
});
new file mode 100644
new file mode 100644
@@ -0,0 +1,87 @@
@@ -0,0 +1,87 @@
+'use strict';
+/**
+ * Sources' secrets never leave in a response or an event, and its API answers nobody without a
+ * valid service token (roadmap WS-R task 5: internal-secret leak; the private-bypass class for a
+ * service whose every read is capability-gated). Sources boots with a sentinel as its Network OAuth
+ * client secret (it mints its service token with it), a source whose endpoint carries a sentinel
+ * credential in its query string (as a feed's API key would), and a failing fetch of it. Then every
+ * route the booted app has (listed from Express's router stack) is requested anonymously, with a
+ * garbage token, with another service's audience, with an expired token and with a read token,
+ * with real and nonsense ids, plus the probes and every write route with a broken body. No body or
+ * header may carry the client secret; only a caller holding sources.source.read may see a source's
+ * endpoints (its admin); every /api/v1 route refuses the others; the outbox carries neither.
+ *
+ * node test/security-secrets.test.js
+ */
+const assert = require('assert');
+const { boot, site, sourceDef, rss, suite, serviceToken, request } = require('./helpers');
+
+const t = suite('security-secrets');
+const SECRET = 'sentinel-not-a-secret-sources-oauth-client';
+const FEED_KEY = 'sentinel-not-a-secret-feed-key';
+let svc, web;
+
+function listRoutes(app) {
+ const out = [];
+ const mount = (layer) => {
+ if (!layer.regexp || layer.regexp.fast_slash) return '';
+ const src = layer.regexp.source.replace(/^\^/, '').replace(/\\\/\?\(\?=\\\/\|\$\)$/i, '').replace(/\\\//g, '/');
+ return /[\\^$()|[\]*+?]/.test(src) ? null : src;
+ };
+ const walk = (stack, prefix) => {
+ for (const layer of stack) {
+ if (layer.route) for (const p of [].concat(layer.route.path)) { if (typeof p === 'string') out.push({ path: prefix + p, methods: Object.keys(layer.route.methods) }); }
+ else if (layer.handle && Array.isArray(layer.handle.stack)) { const m = mount(layer); if (m !== null) walk(layer.handle.stack, prefix + m); }
+ }
+ };
+ walk(app._router.stack, '');
+ return out;
+}
+const fill = (p, v) => p.replace(/:([A-Za-z0-9_]+)/g, () => encodeURIComponent(v));
+
+t('boot with a sentinel client secret and a source whose endpoint carries a key', async () => {
+ web = await site({ '/robots.txt': () => ({ status: 404 }), '/feed.xml': () => ({ status: 500, body: 'upstream broke' }) });
+ svc = await boot({ env: { OV_OAUTH_CLIENT_SECRET: SECRET, OV_OAUTH_CLIENT_ID: 'sources', OV_NETWORK_INTERNAL_URL: 'http://127.0.0.1:9' } });
+ svc.registry.create(sourceDef({ key: 'keyed', endpoints: [`${web.origin}/feed.xml?api_key=${FEED_KEY}`] }), 'test');
+ await svc.ingest.run('keyed', { trigger: 'manual' });
+});
+
+t('every route, as every kind of caller, with real and nonsense ids: the client secret never appears; the feed key only to a source reader', async () => {
+ const now = Math.floor(Date.now() / 1000);
+ const callers = {
+ anonymous: null,
+ garbage: 'not-a-token',
+ 'other audience': serviceToken('news', ['sources.*'], { aud: 'openvibe.elsewhere' }),
+ expired: serviceToken('news', ['sources.*'], { exp: now - 60 }),
+ 'item reader': serviceToken('deals', ['sources.item.read']),
+ 'source reader': serviceToken('news', ['sources.source.read', 'sources.item.read']),
+ };
+ const found = [];
+ const refusedApi = [];
+ for (const route of listRoutes(svc.app)) {
+ for (const method of route.methods) {
+ for (const id of ['keyed', '1', 'nope', "'\"<x>", 'x'.repeat(300)]) {
+ const p = fill(route.path, id);
+ for (const [who, token] of Object.entries(callers)) {
+ const r = await request(svc.base, method.toUpperCase(), `${p}${method === 'get' ? '?q=x&limit=-1' : ''}`, { token, ...(method === 'get' ? {} : { headers: { 'content-type': 'application/json' } }) });
+ const text = r.text + JSON.stringify([...r.headers.entries()]);
+ if (text.includes(SECRET)) found.push(`${who}: ${method} ${p} → ${r.status} carries the client secret`);
+ if (text.includes(FEED_KEY) && who !== 'source reader') found.push(`${who}: ${method} ${p} → ${r.status} carries the feed key`);
+ if (p.startsWith('/api/v1/') && !/^\/api\/v1\/health/.test(p) && ['anonymous', 'garbage', 'other audience', 'expired'].includes(who) && r.status < 400) refusedApi.push(`${who}: ${method} ${p} → ${r.status}`);
+ }
+ }
+ }
+ }
+ assert.deepStrictEqual(found, []);
+ assert.deepStrictEqual(refusedApi, [], 'an /api/v1 route answered without a valid service token');
+});
+
+t('the outbox carries neither', () => {
+ const rows = JSON.stringify(svc.outbox.all ? svc.outbox.all() : svc.db.prepare('SELECT * FROM event_outbox').all());
+ assert.ok(!rows.includes(SECRET), 'client secret in the outbox');
+ assert.ok(!rows.includes(FEED_KEY), 'feed key in the outbox');
+});
+
+t('done', async () => { await svc.stop(); await web.close(); });
+
+t.run();