openvibers/openvibe.live Commits

Channel weather: an expired forecast is answered at once and refreshed behind it (p95 was 2.4 s), one shared refresh, 5 s timeouts, error answers never cached, unknown zips remembered (#188)

OP OpenVibers committed 27 days ago · 2ac32f765b · 4 files, +177 −61

* Channel weather: an expired forecast is answered at once and refreshed behind it (p95 was 2.4 s), one shared refresh, 5 s timeouts, error answers never cached, unknown zips remembered

* security-ssrf: server/streaming/weather.js on the reviewed outbound list (fixed hosts)

Changes

ddiff --git a/server/streaming/routes.js b/server/streaming/routes.js
iindex 5e4eadfc..e35b2042 100644
---- a/server/streaming/routes.js
++++ b/server/streaming/routes.js
@@@ -978,66 +978,8 @@ router.post('/channel/offline-screen', requireAuth, offlineUpload.single('file')
}
});
--// ── Weather for Channel (privacy-preserving) ─────────────────
--const weatherCache = new Map(); // key: zip, value: { data, ts }
--const WEATHER_CACHE_TTL = 15 * 60 * 1000; // 15 min
--const GEOCODE_CACHE = new Map();
--
--async function geocodeZip(zip) {
-- if (GEOCODE_CACHE.has(zip)) return GEOCODE_CACHE.get(zip);
-- try {
-- const url = `https://geocoding-api.open-meteo.com/v1/search?name=${encodeURIComponent(zip)}&count=1&language=en&format=json`;
-- const resp = await fetch(url);
-- const json = await resp.json();
-- if (json.results && json.results.length > 0) {
-- const r = json.results[0];
-- const result = { lat: r.latitude, lon: r.longitude, name: r.name, region: r.admin1 || '', country: r.country_code || '' };
-- GEOCODE_CACHE.set(zip, result);
-- return result;
-- }
-- } catch (e) { console.warn('[Weather] Geocode error:', e.message); }
-- // Fallback: try US zip via zip-coordinates API
-- try {
-- const url = `https://api.zippopotam.us/us/${encodeURIComponent(zip)}`;
-- const resp = await fetch(url);
-- if (resp.ok) {
-- const json = await resp.json();
-- const place = json.places?.[0];
-- if (place) {
-- const result = { lat: parseFloat(place.latitude), lon: parseFloat(place.longitude), name: place['place name'], region: place['state abbreviation'] || '', country: 'US' };
-- GEOCODE_CACHE.set(zip, result);
-- return result;
-- }
-- }
-- } catch (e) { /* silent fallback */ }
-- return null;
--}
--
--async function fetchWeather(zip) {
-- const now = Date.now();
-- const cached = weatherCache.get(zip);
-- if (cached && (now - cached.ts) < WEATHER_CACHE_TTL) return cached.data;
--
-- const geo = await geocodeZip(zip);
-- if (!geo) return null;
--
-- try {
-- const url = `https://api.open-meteo.com/v1/forecast?latitude=${geo.lat}&longitude=${geo.lon}`
-- + `&hourly=temperature_2m,apparent_temperature,relative_humidity_2m,precipitation_probability,precipitation,weather_code,wind_speed_10m,wind_gusts_10m,wind_direction_10m,cloud_cover,visibility,uv_index`
-- + `&daily=temperature_2m_max,temperature_2m_min,weather_code,precipitation_sum,precipitation_probability_max,wind_speed_10m_max,sunrise,sunset`
-- + `&current=temperature_2m,apparent_temperature,relative_humidity_2m,weather_code,wind_speed_10m,wind_direction_10m,is_day`
-- + `&temperature_unit=fahrenheit&wind_speed_unit=mph&precipitation_unit=inch`
-- + `&timezone=auto&forecast_days=7`;
-- const resp = await fetch(url);
-- const data = await resp.json();
-- const result = { ...data, location: { name: geo.name, region: geo.region, country: geo.country } };
-- weatherCache.set(zip, { data: result, ts: now });
-- return result;
-- } catch (e) {
-- console.warn('[Weather] Fetch error:', e.message);
-- return null;
-- }
--}
++// ── Weather for Channel (privacy-preserving): server/streaming/weather.js ──
++const { fetchWeather } = require('./weather');
// ── Game summary (roadmap WS-M task 2, PF7) ──────────────────
// The channel owner's public games.progress.summary (level, achievements, playtime) from Network; 204 when none.
ddiff --git a/server/streaming/weather.js b/server/streaming/weather.js
nnew file mode 100644
iindex 00000000..296e4418
---- /dev/null
++++ b/server/streaming/weather.js
@@@ -0,0 +1,94 @@
++'use strict';
++/**
++ * The channel weather (GET /api/streams/channel/:username/weather): a streamer's zip code → a 7-day Open-Meteo
++ * forecast. The zip never leaves the server.
++ *
++ * fetchWeather(zip) the forecast with { location }, or null when there has never been one for this zip
++ *
++ * A visitor never waits for a refresh once a zip has a forecast: an entry older than 15 minutes is answered at once
++ * and refreshed behind the answer (the route's p95 was 2.4 s on 2026-10-17: each expiry made one visitor wait for
++ * Open-Meteo), and concurrent requests share one refresh. A failed refresh keeps the last forecast and is tried
++ * again after a minute; an error answer from Open-Meteo is never cached as a forecast. Every upstream call gives up
++ * after 5 seconds. A zip no geocoder knows is remembered for an hour, so its channel does not cost two lookups per
++ * viewer.
++ */
++
++const WEATHER_TTL_MS = 15 * 60 * 1000;
++const RETRY_AFTER_FAILURE_MS = 60 * 1000;
++const GEOCODE_MISS_MS = 60 * 60 * 1000;
++const TIMEOUT_MS = 5000;
++
++const weatherCache = new Map(); // zip → { data, ts }
++const inflight = new Map(); // zip → the refresh in progress
++const geocodeCache = new Map(); // zip → { lat, lon, name, region, country }
++const geocodeMiss = new Map(); // zip → until (ms)
++
++const get = (url) => fetch(url, { signal: AbortSignal.timeout(TIMEOUT_MS) });
++
++async function geocodeZip(zip) {
++ if (geocodeCache.has(zip)) return geocodeCache.get(zip);
++ if ((geocodeMiss.get(zip) || 0) > Date.now()) return null;
++ try {
++ const resp = await get(`https://geocoding-api.open-meteo.com/v1/search?name=${encodeURIComponent(zip)}&count=1&language=en&format=json`);
++ const json = resp.ok ? await resp.json() : null;
++ const r = json && json.results && json.results[0];
++ if (r) {
++ const result = { lat: r.latitude, lon: r.longitude, name: r.name, region: r.admin1 || '', country: r.country_code || '' };
++ geocodeCache.set(zip, result);
++ return result;
++ }
++ } catch (e) { console.warn('[Weather] Geocode error:', e.message); }
++ // Fallback: a US zip through zippopotam.us.
++ try {
++ const resp = await get(`https://api.zippopotam.us/us/${encodeURIComponent(zip)}`);
++ if (resp.ok) {
++ const place = (await resp.json()).places?.[0];
++ if (place) {
++ const result = { lat: parseFloat(place.latitude), lon: parseFloat(place.longitude), name: place['place name'], region: place['state abbreviation'] || '', country: 'US' };
++ geocodeCache.set(zip, result);
++ return result;
++ }
++ }
++ } catch { /* silent fallback */ }
++ geocodeMiss.set(zip, Date.now() + GEOCODE_MISS_MS);
++ return null;
++}
++
++function refresh(zip) {
++ if (inflight.has(zip)) return inflight.get(zip);
++ const run = (async () => {
++ const geo = await geocodeZip(zip);
++ if (!geo) return weatherCache.get(zip)?.data || null;
++ try {
++ const resp = await get(`https://api.open-meteo.com/v1/forecast?latitude=${geo.lat}&longitude=${geo.lon}`
++ + '&hourly=temperature_2m,apparent_temperature,relative_humidity_2m,precipitation_probability,precipitation,weather_code,wind_speed_10m,wind_gusts_10m,wind_direction_10m,cloud_cover,visibility,uv_index'
++ + '&daily=temperature_2m_max,temperature_2m_min,weather_code,precipitation_sum,precipitation_probability_max,wind_speed_10m_max,sunrise,sunset'
++ + '&current=temperature_2m,apparent_temperature,relative_humidity_2m,weather_code,wind_speed_10m,wind_direction_10m,is_day'
++ + '&temperature_unit=fahrenheit&wind_speed_unit=mph&precipitation_unit=inch'
++ + '&timezone=auto&forecast_days=7');
++ if (!resp.ok) throw new Error(`Open-Meteo answered ${resp.status}`);
++ const result = { ...(await resp.json()), location: { name: geo.name, region: geo.region, country: geo.country } };
++ weatherCache.set(zip, { data: result, ts: Date.now() });
++ return result;
++ } catch (e) {
++ console.warn('[Weather] Fetch error:', e.message);
++ const stale = weatherCache.get(zip);
++ if (stale) stale.ts = Date.now() - WEATHER_TTL_MS + RETRY_AFTER_FAILURE_MS; // the last forecast, tried again in a minute
++ return stale ? stale.data : null;
++ }
++ })().finally(() => inflight.delete(zip));
++ inflight.set(zip, run);
++ return run;
++}
++
++async function fetchWeather(zip) {
++ const cached = weatherCache.get(zip);
++ if (cached && Date.now() - cached.ts < WEATHER_TTL_MS) return cached.data;
++ const next = refresh(zip);
++ if (cached) return cached.data; // the refresh finishes behind this answer
++ return await next;
++}
++
++function _reset() { weatherCache.clear(); inflight.clear(); geocodeCache.clear(); geocodeMiss.clear(); }
++
++module.exports = { fetchWeather, geocodeZip, WEATHER_TTL_MS, _reset, _cache: weatherCache };
ddiff --git a/test/security-ssrf.test.js b/test/security-ssrf.test.js
iindex 769d7d95..f8635b61 100644
---- a/test/security-ssrf.test.js
++++ b/test/security-ssrf.test.js
@@@ -379,7 +379,8 @@ async function check(name, fn) {
'server/kiosk/routes.js': 'OpenVibe.Tools (configured) and DuckDuckGo favicons; the user URL via egress.fetchText',
'server/media/media-queue.js': 'YouTube oEmbed (fixed host); user links checked by egress, yt-dlp through the egress proxy',
'server/streaming/restream-manager.js': 'Twitch / Kick / YouTube APIs (fixed hosts)',
-- 'server/streaming/routes.js': 'weather APIs (fixed hosts) and the local RTMP server',
++ 'server/streaming/routes.js': 'the local RTMP server',
++ 'server/streaming/weather.js': 'Open-Meteo forecast/geocoding and zippopotam.us (fixed hosts; the zip is URL-encoded into the query)',
};
const root = path.join(__dirname, '..');
const found = [];
ddiff --git a/test/weather-cache.test.js b/test/weather-cache.test.js
nnew file mode 100644
iindex 00000000..b8aebac3
---- /dev/null
++++ b/test/weather-cache.test.js
@@@ -0,0 +1,79 @@
++'use strict';
++/**
++ * The channel weather cache (server/streaming/weather.js): a visitor never waits for a refresh once a zip has a
++ * forecast (the route's p95 was 2.4 s on 2026-10-17), concurrent requests share one refresh, an error answer is
++ * never cached, a failed refresh keeps the last forecast, and an unknown zip is remembered.
++ *
++ * node test/weather-cache.test.js
++ */
++const assert = require('assert');
++
++console.warn = () => {};
++
++(async () => {
++ const calls = [];
++ let forecastStatus = 200;
++ let forecastTemp = 70;
++ let gate = null; // a promise the next forecast answer waits for
++ global.fetch = async (url, opts) => {
++ calls.push(url);
++ assert.ok(opts && opts.signal, 'every upstream call has a timeout');
++ if (url.includes('geocoding-api')) {
++ if (url.includes('00000')) return { ok: true, json: async () => ({}) };
++ return { ok: true, json: async () => ({ results: [{ latitude: 34.1, longitude: -118.4, name: 'Beverly Hills', admin1: 'California', country_code: 'US' }] }) };
++ }
++ if (url.includes('zippopotam')) return { ok: false, json: async () => ({}) };
++ if (gate) await gate;
++ const status = forecastStatus;
++ return { ok: status === 200, status, json: async () => (status === 200 ? { current: { temperature_2m: forecastTemp } } : { error: true, reason: 'busy' }) };
++ };
++ const weather = require('../server/streaming/weather');
++ const forecasts = () => calls.filter((u) => u.includes('api.open-meteo.com/v1/forecast')).length;
++
++ // First request waits for the forecast; the next ones are served from the cache.
++ const first = await weather.fetchWeather('90210');
++ assert.strictEqual(first.current.temperature_2m, 70);
++ assert.deepStrictEqual(first.location, { name: 'Beverly Hills', region: 'California', country: 'US' });
++ await weather.fetchWeather('90210');
++ assert.strictEqual(forecasts(), 1, 'cached within 15 minutes');
++
++ // Expired: answered at once with the last forecast while one shared refresh runs behind it.
++ weather._cache.get('90210').ts -= weather.WEATHER_TTL_MS + 1;
++ let release;
++ gate = new Promise((r) => { release = r; });
++ forecastTemp = 72;
++ const [a, b] = await Promise.all([weather.fetchWeather('90210'), weather.fetchWeather('90210')]);
++ assert.strictEqual(a.current.temperature_2m, 70, 'the stale forecast, without waiting');
++ assert.strictEqual(b.current.temperature_2m, 70);
++ assert.strictEqual(forecasts(), 2, 'one refresh for both');
++ release(); gate = null;
++ await new Promise((r) => setImmediate(r));
++ await new Promise((r) => setImmediate(r));
++ assert.strictEqual((await weather.fetchWeather('90210')).current.temperature_2m, 72, 'the refresh landed');
++
++ // A failed refresh (Open-Meteo answering 429) keeps the last forecast and is not cached as one.
++ weather._cache.get('90210').ts -= weather.WEATHER_TTL_MS + 1;
++ forecastStatus = 429;
++ assert.strictEqual((await weather.fetchWeather('90210')).current.temperature_2m, 72);
++ await new Promise((r) => setImmediate(r));
++ await new Promise((r) => setImmediate(r));
++ assert.strictEqual((await weather.fetchWeather('90210')).current.temperature_2m, 72, 'still the last good forecast');
++ const n = forecasts();
++ await weather.fetchWeather('90210');
++ assert.strictEqual(forecasts(), n, 'a failure is tried again after a minute, not on every request');
++
++ // A zip with no forecast yet and an error answer: null, nothing cached.
++ weather._reset();
++ calls.length = 0;
++ assert.strictEqual(await weather.fetchWeather('10001'), null);
++ assert.strictEqual(weather._cache.has('10001'), false, 'an error answer is not a forecast');
++
++ // A zip no geocoder knows costs its lookups once an hour, not on every viewer's request.
++ calls.length = 0;
++ assert.strictEqual(await weather.fetchWeather('00000'), null);
++ const lookups = calls.length;
++ assert.strictEqual(await weather.fetchWeather('00000'), null);
++ assert.strictEqual(calls.length, lookups, 'remembered as unknown');
++
++ console.log('weather cache: all checks passed');
++})().catch((e) => { console.error(e); process.exit(1); });