openvibers/openvibe.network Commits

Cache Link's latest release on disk and send GITHUB_TOKEN (#671)

OP OpenVibers committed 5 days ago · 7b4d7936a6 · 5 files, +276 −41

OpenVibe.Network's openvibe.link page and /install read the Node's latest
GitHub release on page loads. The 10-minute in-memory hold did not survive a
restart and sent no token, so the host spent GitHub's 60 unauthenticated calls
an hour and showed "GitHub could not be reached".

Hold the answer 15 minutes in memory and in data/node-release.json, loaded at
startup so a restart serves it at once. When the hold ends, serve the last
answer immediately and refresh behind the request. A failed refresh keeps the
last good copy and waits out a one-minute cooldown; the page says how old it
is only once it is more than a day old. Send GITHUB_TOKEN as a Bearer header
when the environment has one.

Changes

ddiff --git a/server/node/page.js b/server/node/page.js
iindex f4d4a48c..769c72eb 100644
---- a/server/node/page.js
++++ b/server/node/page.js
@@@ -123,9 +123,10 @@ ${kit.snippet({ code: 'sudo apt install ./openvibe-link_<version>_amd64.deb\n# o
/** The release facts: the version, when it came out, the checksums and the release page. */
function releaseBlock(release) {
-- const { versionLabel, dateLabel } = require('./release');
++ const { versionLabel, dateLabel, ageDays } = require('./release');
const version = versionLabel(release);
const date = dateLabel(release);
++ const days = ageDays(release);
const rows = [
kit.row({ title: 'Latest version', sub: release.version || '', meta: date || '', metaIso: date || undefined }),
kit.row({ href: release.checksumsUrl || '#', title: 'SHA256SUMS', sub: 'the checksums of every file in the release, to check the download against' }),
@@@ -134,7 +135,7 @@ function releaseBlock(release) {
return `<section class="np-sec" aria-labelledby="h-release"><h2 id="h-release">The latest release</h2>
<p class="np-fine">${version ? `Version <strong>${esc(version)}</strong>` : 'The download links always point at the latest release.'}${date ? `, published ${esc(date)}` : ''}. The install command above always installs that release.</p>
${kit.list(rows, { flush: true })}
--${release.stale ? '<p class="np-fine np-warn">GitHub could not be reached just now, so this is the last release we read.</p>' : ''}
++${release.stale && days >= 1 ? `<p class="np-fine np-warn">GitHub could not be reached just now, so this is the last release we read, from ${days === 1 ? 'a day' : `${days} days`} ago.</p>` : ''}
</section>`;
}
ddiff --git a/server/node/release.js b/server/node/release.js
iindex 020ec07e..cf664ad7 100644
---- a/server/node/release.js
++++ b/server/node/release.js
@@@ -1,14 +1,27 @@
'use strict';
// The Node's latest release, read from GitHub (OpenVibers/OpenVibe.Node). The page shows the version and the
--// checksums; /install answers with the same release's installer. The answer is cached for ten minutes, so a
--// visitor never waits on GitHub, and when GitHub cannot be reached the last good answer is used; a cold start
--// without GitHub falls back to the release links that always resolve (the "latest" ones).
++// checksums; /install answers with the same release's installer.
++//
++// The answer is held for fifteen minutes in memory and on disk (under DATA_DIR), so a restart serves the last
++// good answer at once instead of waiting on GitHub. When the hold is over the visitor still gets the last
++// answer immediately and a refresh runs in the background; when GitHub cannot be reached the last good answer
++// is kept, and the page says how old it is only once it is more than a day old. A cold start without GitHub
++// falls back to the release links that always resolve (the "latest" ones).
++//
++// GitHub allows 60 unauthenticated calls an hour per IP. GITHUB_TOKEN, when set, raises that limit. The token
++// is read from the environment at request time and is never logged.
++
++const fs = require('fs');
++const path = require('path');
const REPO = 'OpenVibers/OpenVibe.Node';
const API = `https://api.github.com/repos/${REPO}/releases/latest`;
--const TTL_MS = 10 * 60 * 1000; // ten minutes
++const TTL_MS = 15 * 60 * 1000; // fifteen minutes
const TIMEOUT_MS = 8000;
++const FAIL_COOLDOWN_MS = 60 * 1000; // a cold failure waits a minute before asking GitHub again
++const DAY_MS = 24 * 60 * 60 * 1000;
const LATEST = `https://github.com/${REPO}/releases/latest`;
++const CACHE_FILE = 'node-release.json';
// What the page says when GitHub cannot be reached and nothing was cached: no version (the page says "the latest
// release" instead of naming one that may be old), and the links that always resolve to the newest release.
@@@ -20,8 +33,10 @@ const FALLBACK = {
checksumsUrl: `https://github.com/${REPO}/releases/latest/download/SHA256SUMS`,
};
--let cached = null; // { at, data }
--let inflight = null; // one fetch at a time
++let cached = null; // { at, data } — at is when GitHub was read, epoch ms
++let inflight = null; // one fetch at a time
++let diskFile = null; // the file this process reads and writes
++let failedAt = 0; // a cold failure, so the next visitor does not wait 8 s too
/** The release fields the page and /install use. */
function shape(json) {
@@@ -40,38 +55,116 @@ function shape(json) {
};
}
--/** Fetch the latest release. Returns the cached answer while it is fresh, and never throws. */
--async function latest({ fetchImpl = globalThis.fetch, now = Date.now } = {}) {
-- if (cached && now() - cached.at < TTL_MS) return { ...cached.data, cached: true };
-- if (inflight) return inflight;
-- inflight = (async () => {
++/** GITHUB_TOKEN from the environment, or no Authorization at all. Never read from anywhere else, never logged. */
++function authorization() {
++ const token = String(process.env.GITHUB_TOKEN || '').trim();
++ return token ? { authorization: `Bearer ${token}` } : {};
++}
++
++/**
++ * Point the process at a data directory's copy, once per directory, and load what a previous process left.
++ * `null` (or an empty string) turns the disk cache off, which is what tests with an injected clock and fetch use.
++ */
++function useDisk(dataDir) {
++ if (dataDir === null || dataDir === '') return;
++ const dir = dataDir === undefined ? (process.env.DATA_DIR || 'data') : String(dataDir);
++ const file = path.join(path.resolve(dir), CACHE_FILE);
++ if (file === diskFile) return;
++ diskFile = file;
++ try {
++ const saved = JSON.parse(fs.readFileSync(file, 'utf8'));
++ if (!saved || !Number.isFinite(saved.at) || !saved.data || !String(saved.data.version || '').trim()) return;
++ if (!cached || saved.at > cached.at) cached = { at: saved.at, data: saved.data };
++ } catch {
++ // No copy yet, or an unreadable one: the first fetch writes a fresh one.
++ }
++}
++
++/** Keep the answer for the next process. A read-only data dir costs a restart's cache, never the page. */
++function saveDisk(entry) {
++ if (!diskFile) return;
++ try {
++ fs.mkdirSync(path.dirname(diskFile), { recursive: true });
++ const tmp = `${diskFile}.${process.pid}.tmp`;
++ fs.writeFileSync(tmp, JSON.stringify(entry));
++ fs.renameSync(tmp, diskFile);
++ } catch {
++ // The page still works from memory.
++ }
++}
++
++/** Ask GitHub once and keep the answer. Resolves to the shaped release; rejects when GitHub could not be read. */
++async function fetchLatest({ fetchImpl = globalThis.fetch, now = Date.now } = {}) {
++ const controller = typeof AbortController === 'function' ? new AbortController() : null;
++ const timer = controller ? setTimeout(() => controller.abort(), TIMEOUT_MS) : null;
++ try {
++ let res;
try {
-- const controller = typeof AbortController === 'function' ? new AbortController() : null;
-- const timer = controller ? setTimeout(() => controller.abort(), TIMEOUT_MS) : null;
-- let res;
-- try {
-- res = await fetchImpl(API, {
-- signal: controller ? controller.signal : undefined,
-- headers: { accept: 'application/vnd.github+json', 'user-agent': 'openvibe-network' },
-- });
-- } finally {
-- if (timer) clearTimeout(timer);
-- }
-- if (!res || !res.ok) throw new Error(`GitHub releases API answered ${res && res.status}`);
-- const data = shape(await res.json());
-- if (!data.version) throw new Error('GitHub releases API answered without a tag');
-- cached = { at: now(), data };
-- return { ...data, cached: false };
-- } catch {
-- // GitHub is down or slow: the last good answer, or the written-down one. The page still works.
-- return { ...(cached ? cached.data : FALLBACK), cached: true, stale: true };
++ res = await fetchImpl(API, {
++ signal: controller ? controller.signal : undefined,
++ headers: { accept: 'application/vnd.github+json', 'user-agent': 'openvibe-network', ...authorization() },
++ });
} finally {
-- inflight = null;
++ if (timer) clearTimeout(timer);
}
-- })();
++ if (!res || !res.ok) throw new Error(`GitHub releases API answered ${res && res.status}`);
++ const data = shape(await res.json());
++ if (!data.version) throw new Error('GitHub releases API answered without a tag');
++ cached = { at: now(), data };
++ failedAt = 0;
++ saveDisk(cached);
++ return { ...data, cached: false, stale: false, ageMs: 0 };
++ } catch (err) {
++ failedAt = now();
++ throw err;
++ }
++}
++
++/** The one background refresh that may be running. */
++function refresh(options) {
++ if (inflight) return inflight;
++ inflight = fetchLatest(options)
++ .catch(() => null)
++ .finally(() => { inflight = null; });
return inflight;
}
++/**
++ * The latest release. Returns the fresh answer, or the last good answer at once while a refresh runs in the
++ * background; when a refresh fails the last good answer is kept and marked stale with its age. Never throws.
++ */
++async function latest({ fetchImpl = globalThis.fetch, now = Date.now, dataDir } = {}) {
++ useDisk(dataDir);
++ if (cached) {
++ const ageMs = Math.max(0, now() - cached.at);
++ if (ageMs < TTL_MS) return { ...cached.data, cached: true, stale: false, ageMs };
++ // Serving the last answer immediately and refreshing behind the request keeps the visitor off GitHub's
++ // clock. A failed refresh waits out the cooldown, so a long outage does not spend the hour's calls.
++ const cooling = failedAt && now() - failedAt < FAIL_COOLDOWN_MS;
++ if (!inflight && !cooling) refresh({ fetchImpl, now });
++ return { ...cached.data, cached: true, stale: true, ageMs };
++ }
++ if (failedAt && now() - failedAt < FAIL_COOLDOWN_MS) {
++ return { ...FALLBACK, cached: false, stale: true, ageMs: 0 };
++ }
++ try {
++ return await fetchLatest({ fetchImpl, now });
++ } catch {
++ return { ...FALLBACK, cached: false, stale: true, ageMs: 0 };
++ }
++}
++
++/** Resolves when no background refresh is running (tests use it instead of sleeping). */
++function whenIdle() {
++ return inflight || Promise.resolve();
++}
++
++/** How many whole days old the answer is; 0 while it is younger than a day. */
++function ageDays(release) {
++ const ms = Number((release && release.ageMs) || 0);
++ return ms >= DAY_MS ? Math.floor(ms / DAY_MS) : 0;
++}
++
/** The version without the leading v, for a sentence. */
function versionLabel(release) {
return String((release && release.version) || '').replace(/^v/, '');
@@@ -86,4 +179,4 @@ function dateLabel(release) {
return d.toISOString().slice(0, 10);
}
--module.exports = { latest, versionLabel, dateLabel, REPO, API, TTL_MS, FALLBACK, LATEST };
++module.exports = { latest, refresh, whenIdle, ageDays, versionLabel, dateLabel, REPO, API, TTL_MS, TIMEOUT_MS, FAIL_COOLDOWN_MS, DAY_MS, FALLBACK, LATEST };
ddiff --git a/test/link-home.test.js b/test/link-home.test.js
iindex ca51bd9e..bf2bdf5c 100644
---- a/test/link-home.test.js
++++ b/test/link-home.test.js
@@@ -39,23 +39,25 @@ const SHAPED = {
const fetchImpl = async () => { calls += 1; return { ok: true, status: 200, json: async () => FIXTURE }; };
let now = 1000;
const clock = () => now;
-- const first = await release.latest({ fetchImpl, now: clock });
++ const first = await release.latest({ fetchImpl, now: clock, dataDir: null });
assert.strictEqual(first.version, 'v9.9.9', 'the release is read from GitHub');
assert.strictEqual(first.installUrl, 'https://github.com/OpenVibers/OpenVibe.Node/releases/download/v9.9.9/install.sh');
assert.strictEqual(first.checksumsUrl, 'https://github.com/OpenVibers/OpenVibe.Node/releases/download/v9.9.9/SHA256SUMS');
-- const second = await release.latest({ fetchImpl, now: clock });
++ const second = await release.latest({ fetchImpl, now: clock, dataDir: null });
assert.strictEqual(calls, 1, 'the second call is answered from the cache');
assert.strictEqual(second.version, 'v9.9.9');
now += release.TTL_MS + 1;
-- await release.latest({ fetchImpl, now: clock });
-- assert.strictEqual(calls, 2, 'after ten minutes GitHub is asked again');
++ await release.latest({ fetchImpl, now: clock, dataDir: null });
++ assert.strictEqual(calls, 2, 'after fifteen minutes GitHub is asked again in the background');
++ await release.whenIdle();
now += release.TTL_MS + 1;
-- const broken = await release.latest({ fetchImpl: async () => { throw new Error('no GitHub'); }, now: clock });
++ const broken = await release.latest({ fetchImpl: async () => { throw new Error('no GitHub'); }, now: clock, dataDir: null });
assert.ok(broken.version, 'a release is still named');
assert.strictEqual(broken.stale, true, 'and it is marked as the last one read');
assert.ok(broken.installUrl.startsWith('https://github.com/OpenVibers/OpenVibe.Node/releases/download/'), 'the installer link still works');
++ await release.whenIdle();
// The vhost must send /machines and /pair to the app (which signposts them), not the catch-all to Network.
const vhost = fs.readFileSync(path.join(__dirname, '..', 'deploy', 'nginx', 'parked-domains.conf'), 'utf8');
@@@ -66,6 +68,10 @@ const SHAPED = {
// ── The page: OpenVibe Link, and every fact the Node page had ──
const html = page.render({ release: first, releaseId: 'rel_test' });
++ const oldCopy = page.render({ release: { ...first, stale: true, ageMs: 3 * release.DAY_MS }, releaseId: 'rel_test' });
++ assert.ok(oldCopy.includes('from 3 days ago'), 'a stale copy older than a day says how old it is');
++ const youngCopy = page.render({ release: { ...first, stale: true, ageMs: release.TTL_MS }, releaseId: 'rel_test' });
++ assert.ok(!youngCopy.includes('could not be reached'), 'a stale copy younger than a day says nothing about the outage');
assert.ok(html.includes('<title>OpenVibe Link: your machines, connected</title>'), 'the title is Link\'s');
assert.ok(html.includes('<h1>Your machines, connected</h1>'), 'the h1 is Link\'s');
assert.ok(html.includes('<p class="np-kicker">OpenVibe Link</p>'), 'the kicker names Link');
ddiff --git a/test/node-release-cache.test.js b/test/node-release-cache.test.js
nnew file mode 100644
iindex 00000000..f83eeac8
---- /dev/null
++++ b/test/node-release-cache.test.js
@@@ -0,0 +1,135 @@
++'use strict';
++// server/node/release.js: the Node's latest release held for fifteen minutes, in memory and on disk, refreshed in
++// the background, kept when GitHub cannot be read, with GITHUB_TOKEN sent when the environment has one.
++// node test/node-release-cache.test.js
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++
++const MODULE = require.resolve('../server/node/release');
++const DAY = 24 * 60 * 60 * 1000;
++
++const FIXTURE = (tag) => ({
++ tag_name: tag,
++ published_at: '2026-11-06T12:00:00Z',
++ html_url: `https://github.com/OpenVibers/OpenVibe.Node/releases/tag/${tag}`,
++ assets: [
++ { name: 'install.sh', browser_download_url: `https://github.com/OpenVibers/OpenVibe.Node/releases/download/${tag}/install.sh` },
++ { name: 'SHA256SUMS', browser_download_url: `https://github.com/OpenVibers/OpenVibe.Node/releases/download/${tag}/SHA256SUMS` },
++ ],
++});
++const ok = (tag, seen) => async (url, opts) => {
++ if (seen) seen.push({ url, headers: (opts && opts.headers) || {} });
++ return { ok: true, status: 200, json: async () => FIXTURE(tag) };
++};
++const fresh = () => {
++ delete require.cache[MODULE];
++ return require(MODULE);
++};
++
++(async () => {
++ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-node-release-'));
++ let now = Date.now();
++
++ try {
++ // ── Fresh: the first answer is read once and held; the disk copy survives a restart ──
++ const first = fresh();
++ let calls = 0;
++ const one = await first.latest({ fetchImpl: async (u, o) => { calls += 1; return ok('v1.2.3')(u, o); }, now: () => now, dataDir: dir });
++ assert.strictEqual(one.version, 'v1.2.3');
++ assert.strictEqual(one.cached, false);
++ assert.strictEqual(one.stale, false);
++ assert.strictEqual(calls, 1, 'the first answer is read from GitHub');
++
++ const again = await first.latest({ fetchImpl: async () => { calls += 1; return ok('v9.9.9')(); }, now: () => now + 60e3, dataDir: dir });
++ assert.strictEqual(calls, 1, 'a second visitor inside the fifteen minutes does not reach GitHub');
++ assert.strictEqual(again.version, 'v1.2.3');
++ assert.strictEqual(again.cached, true);
++
++ const saved = JSON.parse(fs.readFileSync(path.join(dir, 'node-release.json'), 'utf8'));
++ assert.strictEqual(saved.data.version, 'v1.2.3', 'the answer is on disk');
++ assert.ok(Number.isFinite(saved.at), 'with the time it was read');
++
++ // A new process (module reload) answers from that file at once, and only asks GitHub in the background.
++ const restarted = fresh();
++ let afterRestart = 0;
++ const fromDisk = await restarted.latest({
++ fetchImpl: async (u, o) => { afterRestart += 1; return ok('v9.9.9')(u, o); },
++ now: () => now + 120e3,
++ dataDir: dir,
++ });
++ assert.strictEqual(fromDisk.version, 'v1.2.3', 'a restart serves the last good answer at once');
++ assert.strictEqual(fromDisk.cached, true);
++ assert.strictEqual(afterRestart, 0, 'and does not wait on GitHub for it');
++
++ // ── Stale while GitHub is down: the last answer is kept, its age kept, nothing thrown ──
++ now += 16 * 60e3;
++ let tries = 0;
++ const stale = await restarted.latest({
++ fetchImpl: async () => { tries += 1; throw new Error('GitHub is down'); },
++ now: () => now,
++ dataDir: dir,
++ });
++ assert.strictEqual(stale.version, 'v1.2.3', 'the last good answer is still named');
++ assert.strictEqual(stale.stale, true, 'and marked stale');
++ assert.ok(stale.ageMs >= 15 * 60e3, 'its age comes with it');
++ assert.strictEqual(tries, 1, 'the refresh ran in the background');
++ await restarted.whenIdle();
++
++ const kept = await restarted.latest({
++ fetchImpl: async () => { tries += 1; throw new Error('still down'); },
++ now: () => now + 1000,
++ dataDir: dir,
++ });
++ assert.strictEqual(kept.version, 'v1.2.3', 'the failure did not lose the answer');
++ assert.strictEqual(tries, 1, 'and the cooldown kept the next visitor off GitHub');
++
++ // ── Refresh: once GitHub is back, the next request gets the new release from the cache ──
++ now += 16 * 60e3;
++ let refreshed = 0;
++ const whileExpired = await restarted.latest({
++ fetchImpl: async (u, o) => { refreshed += 1; return ok('v2.0.0')(u, o); },
++ now: () => now,
++ dataDir: dir,
++ });
++ assert.strictEqual(whileExpired.version, 'v1.2.3', 'the expired answer is served without waiting');
++ assert.strictEqual(refreshed, 1, 'and the refresh starts behind the request');
++ await restarted.whenIdle();
++ const nowFresh = await restarted.latest({ fetchImpl: async () => { refreshed += 1; return ok('v0.0.0')(); }, now: () => now + 60e3, dataDir: dir });
++ assert.strictEqual(nowFresh.version, 'v2.0.0', 'the refreshed answer is what the next visitor reads');
++ assert.strictEqual(nowFresh.stale, false);
++ assert.strictEqual(refreshed, 1, 'without another call to GitHub');
++
++ // ── The token: GITHUB_TOKEN is sent when set, from the environment only ──
++ const sent = [];
++ const tokenDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-node-token-'));
++ try {
++ const tokenModule = fresh();
++ process.env.GITHUB_TOKEN = 'ghp_test_only';
++ await tokenModule.latest({ fetchImpl: ok('v3.0.0', sent), now: () => now, dataDir: tokenDir });
++ assert.strictEqual(sent.length, 1);
++ assert.strictEqual(sent[0].headers.authorization, 'Bearer ghp_test_only', 'the token rides along as a Bearer header');
++ delete process.env.GITHUB_TOKEN;
++
++ const bare = [];
++ const bareModule = fresh();
++ await bareModule.latest({ fetchImpl: ok('v3.0.0', bare), now: () => now, dataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'ov-node-bare-')) });
++ assert.ok(!('authorization' in bare[0].headers), 'no token, no Authorization header');
++ assert.ok(!JSON.stringify(bare[0].headers).includes('ghp_'), 'and nothing leaks into the request');
++ } finally {
++ delete process.env.GITHUB_TOKEN;
++ fs.rmSync(tokenDir, { recursive: true, force: true });
++ }
++
++ // ── The age is spoken only past a day ──
++ assert.strictEqual(restarted.ageDays({ ageMs: 60e3 }), 0);
++ assert.strictEqual(restarted.ageDays({ ageMs: DAY + 1 }), 1);
++ assert.strictEqual(restarted.ageDays({ ageMs: 3 * DAY }), 3);
++ assert.strictEqual(restarted.ageDays({}), 0);
++
++ console.log('node-release-cache: fresh, restart from disk, stale while down, background refresh and token ok');
++ } finally {
++ fs.rmSync(dir, { recursive: true, force: true });
++ }
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/test/security-ssrf.test.js b/test/security-ssrf.test.js
iindex 8cd50118..5e45505a 100644
---- a/test/security-ssrf.test.js
++++ b/test/security-ssrf.test.js
@@@ -138,7 +138,7 @@ const REVIEWED = {
'server/registry/deploy-drift.js fetch': [1, 'fixed host api.github.com'],
'server/registry/ecosystem.js fetch': [2, 'service URLs from openvibe-contracts manifests and operator env overrides'],
'server/registry/library-tags.js fetch': [1, 'fixed host api.github.com'],
-- 'server/node/release.js fetch': [1, 'the Node\'s own release page: a fixed GitHub API path (https://api.github.com/repos/OpenVibers/OpenVibe.Node/releases/latest), 8 s timeout, held 10 minutes; no request value ever names a host and it only ever reads the public release list'],
++ 'server/node/release.js fetch': [1, 'the Node\'s own release page: a fixed GitHub API path (https://api.github.com/repos/OpenVibers/OpenVibe.Node/releases/latest), 8 s timeout, held 15 minutes in memory and on disk; no request value ever names a host and it only ever reads the public release list'],
'server/registry/machines.js fetch': [1, 'the machines hub calls Bot: the internal URL from operator config (OV_BOT_INTERNAL_URL, loopback by default), the paths are fixed, the machine id is a checked dev_ id (percent-encoded) and the person is named in a header, never in the URL'],
'server/presence/index.js fetch': [2, "Live and Chat internal URLs from operator config (loopback defaults): Live's public /api/streams (fixed path, 2 s timeout, held 30 s) and Chat's POST /internal/chat/dm/peers (fixed path, 1.5 s, the subject list in the body); never a request value names a host"],
'server/updates/routes.js fetch': [1, 'OV_BLOG_INTERNAL_URL (operator config)'],