openvibers/openvibe.food Commits

Food near you tries Overpass servers in order, each with a share of the budget, and falls back to the last cached answer: one overloaded server held every search for 25 s (#80)

OP OpenVibers committed 20 days ago · 1d8abf1ba9 · 7 files, +127 −18

Changes

ddiff --git a/server/config.js b/server/config.js
iindex 8ea4000..124431b 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -50,6 +50,12 @@ function load(env = process.env) {
osm: {
nominatimUrl: trim(env.FOOD_NOMINATIM_URL || 'https://nominatim.openstreetmap.org'),
overpassUrl: trim(env.FOOD_OVERPASS_URL || 'https://overpass-api.de/api/interpreter'),
++ // Public Overpass servers tried in order, each with overpassTryMs of the timeoutMs budget: one is often
++ // overloaded (2026-10-24: overpass-api.de took 26 s and returned nothing, so every near search took 25 s).
++ overpassUrls: [...new Set([trim(env.FOOD_OVERPASS_URL || 'https://overpass-api.de/api/interpreter'),
++ ...String(env.FOOD_OVERPASS_FALLBACK_URLS ?? 'https://overpass.kumi.systems/api/interpreter,https://overpass.private.coffee/api/interpreter')
++ .split(',').map((u) => trim(u)).filter(Boolean)])],
++ overpassTryMs: Math.max(1000, int(env.FOOD_OVERPASS_TRY_MS, 8_000)),
// Identify to OSM with a contact URL, as both policies require; never a browser User-Agent.
userAgent: env.FOOD_OSM_USER_AGENT || `OpenVibeFood/0.1 (+${baseUrl})`,
minIntervalMs: Math.max(0, int(env.FOOD_OSM_MIN_INTERVAL_MS, 1000)),
ddiff --git a/server/food/store.js b/server/food/store.js
iindex c6b2bff..ba73ed9 100644
---- a/server/food/store.js
++++ b/server/food/store.js
@@@ -13,10 +13,12 @@ const parse = (v) => (v == null ? null : typeof v === 'string' ? JSON.parse(v) :
// ── food_geo_cache ───────────────────────────────────────────
/** A cached upstream answer that has not expired, or null. */
--async function cacheGet(s, key) {
++// An expired answer is kept this long after its TTL, only to stand in when OpenStreetMap is not answering (stale).
++const STALE_KEEP_MS = 30 * 24 * 60 * 60 * 1000;
++async function cacheGet(s, key, { stale = false } = {}) {
const row = await s.db.maybe('SELECT body, fetched_at, expires_at FROM food_geo_cache WHERE key = $1', [key]);
-- if (!row || row.expires_at <= s.iso()) return null;
-- return { body: parse(row.body), fetched_at: row.fetched_at, expires_at: row.expires_at };
++ if (!row || (!stale && row.expires_at <= s.iso())) return null;
++ return { body: parse(row.body), fetched_at: row.fetched_at, expires_at: row.expires_at, stale: row.expires_at <= s.iso() };
}
async function cachePut(s, key, body, ttlMs) {
const fetchedAt = s.iso();
@@@ -28,10 +30,11 @@ async function cachePut(s, key, body, ttlMs) {
return { fetched_at: fetchedAt, expires_at: expiresAt };
}
/** Drop expired rows (called at boot; the table stays small because every answer is bounded). */
--const cachePrune = (s) => s.db.exec('DELETE FROM food_geo_cache WHERE expires_at <= $1', [s.iso()]);
++const cachePrune = (s) => s.db.exec('DELETE FROM food_geo_cache WHERE expires_at <= $1', [new Date(s.now() - STALE_KEEP_MS).toISOString()]);
/**
-- * Prune the geo cache now, then hourly: an expired row is ignored anyway, this only keeps the table small. The
++ * Prune the geo cache now, then hourly: a row 30 days past its TTL goes (until then it only stands in when OSM is down;
++ * an expired row is otherwise ignored), which keeps the table small. The
* interval is unref'd so it never holds the process open; the returned timer is what the entry point clears on
* shutdown (server/index.js). This repository has no restore-drill mode; if one is added it must skip this timer,
* because a drill reads a restored database and must not write to it.
ddiff --git a/server/food/upstream.js b/server/food/upstream.js
iindex 7d8106b..fe9ad6c 100644
---- a/server/food/upstream.js
++++ b/server/food/upstream.js
@@@ -53,13 +53,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
};
/** Fetch a URL built from `base` (never from caller input), with the OSM User-Agent and a hard timeout. */
-- async function fetchJson(url, init = {}) {
++ async function fetchJson(url, init = {}, { timeoutMs = osm.timeoutMs } = {}) {
let res;
try {
res = await fetchImpl(url.toString(), {
...init,
headers: { 'User-Agent': osm.userAgent, Accept: 'application/json', ...(init.headers || {}) },
-- signal: AbortSignal.timeout(osm.timeoutMs),
++ signal: AbortSignal.timeout(timeoutMs),
});
} catch (err) {
throw new UpstreamError('food.place.upstream', `the place service could not be reached (${err && err.name === 'TimeoutError' ? 'timed out' : 'network error'})`);
@@@ -71,7 +71,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
async function cached(key, fn) {
const hit = await geo.cacheGet(s, key);
if (hit) return { body: hit.body, cached: true, fetched_at: hit.fetched_at };
-- const body = await fn();
++ let body;
++ try { body = await fn(); } catch (err) {
++ // OSM not answering: the last answer for the same question, however old (kept 30 days past its TTL).
++ const old = await geo.cacheGet(s, key, { stale: true });
++ if (old) return { body: old.body, cached: true, stale: true, fetched_at: old.fetched_at };
++ throw err;
++ }
const written = await geo.cachePut(s, key, body, osm.cacheTtlMs);
return { body, cached: false, fetched_at: written.fetched_at };
}
@@@ -86,14 +92,34 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
return cached(`nominatim:${keyOf(query)}:${limit}`, () => pacerFor(osm.nominatimUrl).run(() => fetchJson(url)));
}
-- /** One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. */
++ /**
++ * One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. The servers
++ * in osm.overpassUrls are tried in order, each for osm.overpassTryMs (the last one for what is left of timeoutMs),
++ * so one overloaded server no longer holds the page for the whole budget.
++ */
function overpass(query) {
const body = `data=${encodeURIComponent(query)}`;
-- return cached(`overpass:${crypto.createHash('sha256').update(query).digest('hex')}`, () => pacerFor(osm.overpassUrl).run(() => fetchJson(new URL(osm.overpassUrl), {
-- method: 'POST',
-- headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
-- body,
-- })));
++ const urls = osm.overpassUrls && osm.overpassUrls.length ? osm.overpassUrls : [osm.overpassUrl];
++ return cached(`overpass:${crypto.createHash('sha256').update(query).digest('hex')}`, async () => {
++ const started = now();
++ let tried = 0;
++ for (const [i, base] of urls.entries()) {
++ const left = osm.timeoutMs - (now() - started);
++ if (left < 1000) break;
++ const timeoutMs = i === urls.length - 1 ? left : Math.min(left, osm.overpassTryMs);
++ tried++;
++ try {
++ return await pacerFor(base).run(() => fetchJson(new URL(base), {
++ method: 'POST',
++ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
++ body,
++ }, { timeoutMs }));
++ } catch (err) {
++ log.warn && log.warn(`[OpenVibe.Food] Overpass ${new URL(base).host}: ${err.message}`);
++ }
++ }
++ throw new UpstreamError('food.place.upstream', `OpenStreetMap's place servers are busy right now (${tried} tried). Try again in a few minutes.`);
++ });
}
return { geocode, overpass, fetchJson, UpstreamError };
ddiff --git a/test/helpers/boot.js b/test/helpers/boot.js
iindex 1ca2519..c4d3b7f 100644
---- a/test/helpers/boot.js
++++ b/test/helpers/boot.js
@@@ -8,6 +8,8 @@ process.env.FOOD_API_RATE_LIMIT_PER_MIN = process.env.FOOD_API_RATE_LIMIT_PER_MI
const OSM_OFFLINE = {
FOOD_NOMINATIM_URL: 'http://127.0.0.1:9/nominatim',
FOOD_OVERPASS_URL: 'http://127.0.0.1:9/overpass',
++ // No public fallback servers either: the suite never reaches the real Overpass.
++ FOOD_OVERPASS_FALLBACK_URLS: '',
// The one-request-a-second pace is asserted by places.test.js with the policy's real second; the rest of the
// suite must not pay it.
FOOD_OSM_MIN_INTERVAL_MS: '1',
ddiff --git a/test/helpers/osm.js b/test/helpers/osm.js
iindex 4d8364b..1e7cb59 100644
---- a/test/helpers/osm.js
++++ b/test/helpers/osm.js
@@@ -40,6 +40,8 @@ async function startOsm(opts = {}) {
geocode: opts.geocode || [SPRINGFIELD],
overpass: opts.overpass || DEFAULT_OVER,
fail: false,
++ overpassFail: false, // Overpass alone answers 503 (Nominatim still works)
++ overpassDelayMs: 0, // Overpass waits this long before answering (a slow public server)
respond: null, // (kind, query) → body, to override per call
};
@@@ -58,6 +60,8 @@ async function startOsm(opts = {}) {
return json(200, state.geocode.filter((g) => words.some((w) => String(g.display_name).toLowerCase().includes(w))));
}
if (url.pathname === '/api/interpreter' && req.method === 'POST') {
++ if (state.overpassFail) return json(503, { error: 'the stand-in Overpass is busy' });
++ if (state.overpassDelayMs) { const ms = state.overpassDelayMs; state.overpassDelayMs = 0; setTimeout(() => { if (!res.writableEnded) json(200, { elements: [] }); }, ms); return; }
const data = decodeURIComponent(new URLSearchParams(raw).get('data') || '');
const kind = /social_facility|food_sharing|give_box/.test(data) ? 'foodbank' : 'grocery';
if (state.respond) return json(200, state.respond(kind, data));
@@@ -70,8 +74,9 @@ async function startOsm(opts = {}) {
const url = `http://127.0.0.1:${server.address().port}`;
return {
url, state, requests,
-- env: { FOOD_NOMINATIM_URL: url, FOOD_OVERPASS_URL: `${url}/api/interpreter` },
-- reset() { requests.length = 0; state.fail = false; state.respond = null; state.geocode = [SPRINGFIELD]; state.overpass = DEFAULT_OVER; },
++ // No fallback servers: a test never reaches the real Overpass (places.test.js sets its own stand-in list).
++ env: { FOOD_NOMINATIM_URL: url, FOOD_OVERPASS_URL: `${url}/api/interpreter`, FOOD_OVERPASS_FALLBACK_URLS: '' },
++ reset() { requests.length = 0; state.fail = false; state.overpassFail = false; state.overpassDelayMs = 0; state.respond = null; state.geocode = [SPRINGFIELD]; state.overpass = DEFAULT_OVER; },
/** The gaps, in ms, between consecutive upstream calls — the pace a test checks. */
gaps() { return requests.slice(1).map((r, i) => r.at - requests[i].at); },
close: () => new Promise((r) => server.close(r)),
ddiff --git a/test/places-fallback.test.js b/test/places-fallback.test.js
nnew file mode 100644
iindex 0000000..3f713b6
---- /dev/null
++++ b/test/places-fallback.test.js
@@@ -0,0 +1,66 @@
++'use strict';
++/**
++ * Overpass servers in order (config osm.overpassUrls): when the first one fails or is too slow, the next answers within
++ * the same search; when none answers, the last cached answer for the same area stands in (stale); with none, the
++ * answer says in plain words that OpenStreetMap is busy. On 2026-10-24 overpass-api.de took 26 s and returned nothing,
++ * so every "food near" search waited the whole 25 s budget and said "the place service answered 502". Two stand-in
++ * OSMs here; no test calls the real OpenStreetMap.
++ */
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++const { startOsm } = require('./helpers/osm');
++
++(async () => {
++ const first = await startOsm();
++ const second = await startOsm();
++ const t = await boot({ env: {
++ ...first.env, FOOD_OSM_MIN_INTERVAL_MS: '1', FOOD_OVERPASS_TRY_MS: '1000',
++ FOOD_OVERPASS_FALLBACK_URLS: `${second.url}/api/interpreter`,
++ } });
++ const search = (q = 'Springfield, Illinois') => t.get(`/api/v1/places?q=${encodeURIComponent(q)}&kind=foodbank`);
++ const overpassCalls = (osm) => osm.requests.filter((r) => r.path === '/api/interpreter').length;
++ const clearOverpassCache = () => t.ctx.s.db.query("DELETE FROM food_geo_cache WHERE key LIKE 'overpass:%'");
++
++ try {
++ await check('the first Overpass server answers 503: the next one gives the food banks in the same search', async () => {
++ first.state.overpassFail = true;
++ const r = await search();
++ assert.strictEqual(r.status, 200, r.text);
++ assert.strictEqual(r.json().results.length, 4);
++ assert.strictEqual(overpassCalls(first), 1);
++ assert.strictEqual(overpassCalls(second), 1, 'the fallback server was asked once');
++ first.state.overpassFail = false;
++ });
++
++ await check('the first server is too slow: after its share of the budget the next one answers', async () => {
++ await clearOverpassCache();
++ first.state.overpassDelayMs = 5000;
++ const started = Date.now();
++ const r = await search();
++ assert.strictEqual(r.status, 200, r.text);
++ assert.strictEqual(r.json().results.length, 4);
++ assert.ok(Date.now() - started < 4000, `answered in ${Date.now() - started} ms, not after the slow server`);
++ });
++
++ await check('no server answers: the cached answer for the same area stands in, even past its TTL', async () => {
++ await t.ctx.s.db.query("UPDATE food_geo_cache SET expires_at = '2020-01-01T00:00:00.000Z' WHERE key LIKE 'overpass:%'");
++ first.state.overpassFail = true;
++ second.state.overpassFail = true;
++ const r = await search();
++ assert.strictEqual(r.status, 200, r.text);
++ assert.strictEqual(r.json().results.length, 4, 'the stale answer');
++ });
++
++ await check('no server answers and nothing is cached: OpenStreetMap is busy, said plainly', async () => {
++ await clearOverpassCache();
++ const r = await search();
++ assert.strictEqual(r.status, 502, r.text);
++ assert.match(r.text, /OpenStreetMap's place servers are busy right now \(2 tried\)/);
++ });
++ } finally {
++ await t.close();
++ await first.close();
++ await second.close();
++ }
++ done();
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/test/places.test.js b/test/places.test.js
iindex 66dd6e0..18e94c3 100644
---- a/test/places.test.js
++++ b/test/places.test.js
@@@ -195,7 +195,8 @@ const OVERPASS = '/api/interpreter';
assert.strictEqual(r.json().code, 'food.place.upstream');
const page = await t.get('/near?lat=41&lon=-91');
assert.strictEqual(page.status, 200, 'the page still renders');
-- assert.match(page.text, /the place service answered 503/, 'the page says what went wrong');
++ // Geocoding is cached by now, so it is Overpass that fails: said plainly (places-fallback.test.js).
++ assert.match(page.text, /the place service answered 503|OpenStreetMap&#39;s place servers are busy|OpenStreetMap's place servers are busy/, 'the page says what went wrong');
osm.state.fail = false;
});
} finally {
@@@ -222,7 +223,7 @@ const OVERPASS = '/api/interpreter';
const r = await offline.get('/api/v1/places?lat=51.5&lon=-0.1&kind=foodbank');
assert.strictEqual(r.status, 502);
assert.strictEqual(r.json().code, 'food.place.upstream');
-- assert.match(r.json().detail, /could not be reached/);
++ assert.match(r.json().detail, /could not be reached|place servers are busy right now \(1 tried\)/);
} finally { await offline.close(); }
});