openvibers/openvibe.network Commits

Network: check the Link release tag over git before spending GitHub REST (#733)

OP OpenVibers committed 4 days ago · f40b33eeea · 3 files, +149 −12

The Node reads OpenVibe.Link's newest tag through ls-refs (git protocol v2) on each fifteen-minute refresh and calls the REST releases/latest endpoint only when that tag is not the cached release. When ls-refs fails the REST call runs every fifteen minutes as before, so nothing gets worse. Without a token this takes the release page from 4 REST calls an hour to 0, plus one per new release.

Changes

ddiff --git a/server/link/release.js b/server/link/release.js
iindex 020dacbd..c04e6fdd 100644
---- a/server/link/release.js
++++ b/server/link/release.js
@@@ -8,7 +8,12 @@
// is kept, and the page says how old it is only once it is more than a day old. A cold start without GitHub
// falls back to the release links that always resolve (the "latest" ones).
//
--// GitHub allows 60 unauthenticated calls an hour per IP. GITHUB_TOKEN, when set, raises that limit. The token
++// The refresh reads the newest tag first through git's own protocol (ls-refs, as `git ls-remote` asks it), which
++// is not GitHub's REST API and spends none of its rate limit. The REST releases/latest endpoint is read only when
++// that tag differs from the cached release's (or nothing was cached). When ls-refs cannot be read the REST call
++// runs every fifteen minutes as it always did.
++//
++// GitHub allows 60 unauthenticated REST calls an hour per IP. GITHUB_TOKEN, when set, raises that limit. The token
// is read from the environment at request time and is never logged.
const fs = require('fs');
@@@ -16,12 +21,18 @@ const path = require('path');
const REPO = 'OpenVibers/OpenVibe.Link';
const API = `https://api.github.com/repos/${REPO}/releases/latest`;
++const GIT_URL = `https://github.com/${REPO}.git/git-upload-pack`;
const TTL_MS = 15 * 60 * 1000; // fifteen minutes
const TIMEOUT_MS = 8000;
const FAIL_COOLDOWN_MS = 60 * 1000; // a cold failure waits a minute before asking GitHub again
const DAY_MS = 24 * 60 * 60 * 1000;
const LATEST = `https://github.com/${REPO}/releases/latest`;
const CACHE_FILE = 'node-release.json';
++// ls-refs for refs/tags/v in git's protocol v2: the command, a delimiter, the ref prefix and a flush. Each pkt-line
++// carries its own length in hex, four bytes included.
++const pktLine = (text) => (text.length + 4).toString(16).padStart(4, '0') + text;
++const LS_REFS_TAGS = pktLine('command=ls-refs\n') + '0001' + pktLine('ref-prefix refs/tags/v\n') + '0000';
++const TAG_REF_RE = /^v(\d+)\.(\d+)\.(\d+)$/;
// What the page says when GitHub cannot be reached and nothing was cached: no version (the page says "the latest
// release" instead of naming one that may be old), and the links that always resolve to the newest release.
@@@ -93,6 +104,51 @@ function saveDisk(entry) {
}
}
++/** The newest release tag in an ls-refs answer, or '' when it names none. Peeled tag lines are ignored. */
++function newestTagFrom(text) {
++ const partsOf = (a, b) => {
++ for (let i = 0; i < 3; i += 1) if (a[i] !== b[i]) return a[i] - b[i];
++ return 0;
++ };
++ let best = '';
++ let bestParts = null;
++ let i = 0;
++ const body = String(text || '');
++ while (i + 4 <= body.length) {
++ const size = parseInt(body.slice(i, i + 4), 16);
++ if (!Number.isFinite(size) || size < 4 || i + size > body.length) break;
++ const line = body.slice(i + 4, i + size).trim();
++ i += size;
++ const space = line.indexOf(' ');
++ if (space < 0) continue;
++ const ref = line.slice(space + 1).split(' ')[0]; // attributes (peeled:…) follow the ref name
++ if (!ref.startsWith('refs/tags/v') || ref.endsWith('^{}')) continue;
++ const m = TAG_REF_RE.exec(ref.slice('refs/tags/'.length));
++ if (!m) continue;
++ const parts = [Number(m[1]), Number(m[2]), Number(m[3])];
++ if (!bestParts || partsOf(parts, bestParts) > 0) {
++ best = `v${m[1]}.${m[2]}.${m[3]}`;
++ bestParts = parts;
++ }
++ }
++ return best;
++}
++
++/** The newest release tag, through git's protocol v2 (no REST call). Rejects when GitHub did not answer with tags. */
++async function newestTag({ fetchImpl = globalThis.fetch } = {}) {
++ const signal = typeof AbortSignal.timeout === 'function' ? AbortSignal.timeout(TIMEOUT_MS) : undefined;
++ const res = await fetchImpl(GIT_URL, {
++ method: 'POST',
++ headers: { 'Content-Type': 'application/x-git-upload-pack-request', 'Git-Protocol': 'version=2', 'User-Agent': 'openvibe-network' },
++ body: LS_REFS_TAGS,
++ signal,
++ });
++ if (!res || !res.ok || typeof res.text !== 'function') throw new Error(`GitHub ls-refs answered ${res && res.status}`);
++ const tag = newestTagFrom(await res.text());
++ if (!tag) throw new Error('GitHub ls-refs answered without a release tag');
++ return tag;
++}
++
/** Ask GitHub once and keep the answer. Resolves to the shaped release; rejects when GitHub could not be read. */
async function fetchLatest({ fetchImpl = globalThis.fetch, now = Date.now } = {}) {
const controller = typeof AbortController === 'function' ? new AbortController() : null;
@@@ -120,10 +176,28 @@ async function fetchLatest({ fetchImpl = globalThis.fetch, now = Date.now } = {}
}
}
--/** The one background refresh that may be running. */
--function refresh(options) {
++/**
++ * The one background refresh that may be running. The tag comes first through git (no REST call); the REST API is
++ * read only when that tag is not the cached release's, or when git did not answer (the old behaviour).
++ */
++function refresh({ fetchImpl = globalThis.fetch, now = Date.now } = {}) {
if (inflight) return inflight;
-- inflight = fetchLatest(options)
++ inflight = (async () => {
++ if (cached) {
++ try {
++ const tag = await newestTag({ fetchImpl });
++ if (tag === cached.data.version) {
++ cached = { at: now(), data: cached.data };
++ failedAt = 0;
++ saveDisk(cached);
++ return cached.data;
++ }
++ } catch {
++ // ls-refs did not answer: ask the REST API as before.
++ }
++ }
++ return fetchLatest({ fetchImpl, now });
++ })()
.catch(() => null)
.finally(() => { inflight = null; });
return inflight;
ddiff --git a/test/node-release-cache.test.js b/test/node-release-cache.test.js
iindex a896413e..0fd656f6 100644
---- a/test/node-release-cache.test.js
++++ b/test/node-release-cache.test.js
@@@ -19,8 +19,13 @@ const FIXTURE = (tag) => ({
{ name: 'SHA256SUMS', browser_download_url: `https://github.com/OpenVibers/OpenVibe.Link/releases/download/${tag}/SHA256SUMS` },
],
});
++const isGit = (url) => String(url).includes('.git/git-upload-pack');
++const pkt = (text) => (text.length + 4).toString(16).padStart(4, '0') + text;
++// An ls-refs answer: version line, a tag ref and its peeled copy (which must be ignored), then the flush.
++const LS = (tags) => pkt('version 2\n') + tags.map((tag) => pkt(`${'a'.repeat(40)} refs/tags/${tag}\n`) + pkt(`${'a'.repeat(40)} refs/tags/${tag}^{}\n`)).join('') + '0000';
const ok = (tag, seen) => async (url, opts) => {
if (seen) seen.push({ url, headers: (opts && opts.headers) || {} });
++ if (isGit(url)) return { ok: true, status: 200, text: async () => LS([tag]) };
return { ok: true, status: 200, json: async () => FIXTURE(tag) };
};
const fresh = () => {
@@@ -74,8 +79,8 @@ const fresh = () => {
assert.strictEqual(stale.version, 'v1.2.3', 'the last good answer is still named');
assert.strictEqual(stale.stale, true, 'and marked stale');
assert.ok(stale.ageMs >= 15 * 60e3, 'its age comes with it');
-- assert.strictEqual(tries, 1, 'the refresh ran in the background');
await restarted.whenIdle();
++ assert.strictEqual(tries, 2, 'the refresh ran in the background: ls-refs, then the REST fallback');
const kept = await restarted.latest({
fetchImpl: async () => { tries += 1; throw new Error('still down'); },
@@@ -83,23 +88,81 @@ const fresh = () => {
dataDir: dir,
});
assert.strictEqual(kept.version, 'v1.2.3', 'the failure did not lose the answer');
-- assert.strictEqual(tries, 1, 'and the cooldown kept the next visitor off GitHub');
++ assert.strictEqual(tries, 2, 'and the cooldown kept the next visitor off GitHub');
// ── Refresh: once GitHub is back, the next request gets the new release from the cache ──
now += 16 * 60e3;
-- let refreshed = 0;
++ let gitRefreshed = 0;
++ let restRefreshed = 0;
const whileExpired = await restarted.latest({
-- fetchImpl: async (u, o) => { refreshed += 1; return ok('v2.0.0')(u, o); },
++ fetchImpl: async (u, o) => {
++ if (isGit(u)) { gitRefreshed += 1; return ok('v2.0.0')(u, o); }
++ restRefreshed += 1;
++ return ok('v2.0.0')(u, o);
++ },
now: () => now,
dataDir: dir,
});
assert.strictEqual(whileExpired.version, 'v1.2.3', 'the expired answer is served without waiting');
-- assert.strictEqual(refreshed, 1, 'and the refresh starts behind the request');
await restarted.whenIdle();
-- const nowFresh = await restarted.latest({ fetchImpl: async () => { refreshed += 1; return ok('v0.0.0')(); }, now: () => now + 60e3, dataDir: dir });
++ assert.strictEqual(gitRefreshed, 1, 'and the refresh starts behind the request with ls-refs');
++ assert.strictEqual(restRefreshed, 1, 'which reads the REST release once because the tag is new');
++ const nowFresh = await restarted.latest({ fetchImpl: async () => { throw new Error('no call expected inside the hold'); }, now: () => now + 60e3, dataDir: dir });
assert.strictEqual(nowFresh.version, 'v2.0.0', 'the refreshed answer is what the next visitor reads');
assert.strictEqual(nowFresh.stale, false);
-- assert.strictEqual(refreshed, 1, 'without another call to GitHub');
++ assert.deepStrictEqual({ gitRefreshed, restRefreshed }, { gitRefreshed: 1, restRefreshed: 1 }, 'without another call to GitHub');
++
++ // ── The tag check: REST only when the release is new; ls-refs down means REST every hold as before ──
++ const tagDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-node-tags-'));
++ try {
++ const tagsModule = fresh();
++ let restCalls = 0;
++ let gitCalls = 0;
++ let tagsNow = ['v1.2.3', 'v1.2.2'];
++ let restTag = 'v1.2.3';
++ const fetchTags = async (url, opts) => {
++ if (isGit(url)) { gitCalls += 1; return { ok: true, status: 200, text: async () => LS(tagsNow) }; }
++ restCalls += 1;
++ return { ok: true, status: 200, json: async () => FIXTURE(restTag) };
++ };
++ const cold = await tagsModule.latest({ fetchImpl: fetchTags, now: () => now, dataDir: tagDir });
++ assert.strictEqual(cold.version, 'v1.2.3');
++ assert.deepStrictEqual({ gitCalls, restCalls }, { gitCalls: 0, restCalls: 1 }, 'nothing cached: REST, no tag check');
++
++ now += 16 * 60e3;
++ await tagsModule.latest({ fetchImpl: fetchTags, now: () => now, dataDir: tagDir });
++ await tagsModule.whenIdle();
++ const held = await tagsModule.latest({ fetchImpl: fetchTags, now: () => now + 60e3, dataDir: tagDir });
++ assert.strictEqual(held.version, 'v1.2.3');
++ assert.strictEqual(held.stale, false, 'the same tag refreshes the cached timestamp');
++ assert.deepStrictEqual({ gitCalls, restCalls }, { gitCalls: 1, restCalls: 1 }, 'same tag: one ls-refs, no REST');
++
++ now += 16 * 60e3;
++ await tagsModule.latest({ fetchImpl: fetchTags, now: () => now, dataDir: tagDir });
++ await tagsModule.whenIdle();
++ assert.deepStrictEqual({ gitCalls, restCalls }, { gitCalls: 2, restCalls: 1 }, 'same tag twice: still one REST call');
++
++ now += 16 * 60e3;
++ tagsNow = ['v10.0.0', 'v9.9.9', 'v1.2.3'];
++ restTag = 'v10.0.0';
++ await tagsModule.latest({ fetchImpl: fetchTags, now: () => now, dataDir: tagDir });
++ await tagsModule.whenIdle();
++ const bumped = await tagsModule.latest({ fetchImpl: fetchTags, now: () => now + 60e3, dataDir: tagDir });
++ assert.strictEqual(bumped.version, 'v10.0.0', 'the highest semver tag is the release');
++ assert.deepStrictEqual({ gitCalls, restCalls }, { gitCalls: 3, restCalls: 2 }, 'a new tag: one more REST call');
++
++ now += 16 * 60e3;
++ const downFetch = async (url, opts) => {
++ if (isGit(url)) { gitCalls += 1; throw new Error('git is down'); }
++ restCalls += 1;
++ return { ok: true, status: 200, json: async () => FIXTURE(restTag) };
++ };
++ await tagsModule.latest({ fetchImpl: downFetch, now: () => now, dataDir: tagDir });
++ await tagsModule.whenIdle();
++ assert.deepStrictEqual({ gitCalls, restCalls }, { gitCalls: 4, restCalls: 3 }, 'ls-refs down: one REST call, as before');
++ } finally {
++ fs.rmSync(tagDir, { recursive: true, force: true });
++ }
// ── The token: GITHUB_TOKEN is sent when set, from the environment only ──
const sent = [];
ddiff --git a/test/security-ssrf.test.js b/test/security-ssrf.test.js
iindex 19760356..bb800f37 100644
---- a/test/security-ssrf.test.js
++++ b/test/security-ssrf.test.js
@@@ -142,7 +142,7 @@ const REVIEWED = {
'server/registry/deploy-drift.js fetch': [2, 'fixed hosts: api.github.com (compare) and github.com (git ls-refs); the repository matches REPO_RE (OpenVibers/ only)'],
'server/registry/ecosystem.js fetch': [2, 'service URLs from openvibe-contracts manifests and operator env overrides'],
'server/registry/library-tags.js fetch': [1, 'fixed host api.github.com'],
-- 'server/link/release.js fetch': [1, 'the Node\'s own release page: a fixed GitHub API path (https://api.github.com/repos/OpenVibers/OpenVibe.Link/releases/latest), 8 s timeout, held 15 minutes in memory and on disk; no request value ever names a host and it only ever reads the public release list'],
++ 'server/link/release.js fetch': [2, 'the Node\'s own release page: fixed hosts only — https://github.com/OpenVibers/OpenVibe.Link.git/git-upload-pack (ls-refs of refs/tags/v, no REST call) and the fixed GitHub API path (https://api.github.com/repos/OpenVibers/OpenVibe.Link/releases/latest), each with an 8 s timeout; the REST answer is held 15 minutes in memory and on disk and read only when a new tag exists; no request value ever names a host and it only ever reads the public release list'],
'server/registry/machines.js fetch': [1, 'the machines hub calls Bot: the internal URL from operator config (OV_BOT_INTERNAL_URL, loopback by default), the paths are fixed, the machine id is a checked dev_ id (percent-encoded) and the person is named in a header, never in the URL'],
'server/presence/index.js fetch': [2, "Live and Chat internal URLs from operator config (loopback defaults): Live's public /api/streams (fixed path, 2 s timeout, held 30 s) and Chat's POST /internal/chat/dm/peers (fixed path, 1.5 s, the subject list in the body); never a request value names a host"],
'server/updates/routes.js fetch': [1, 'OV_BLOG_INTERNAL_URL (operator config)'],