openvibers/openvibe.food Commits
Food near you tries Overpass servers in order, each with a share of the budget, and falls back to the last cached answer: one overloaded server held every search for 25 s (#80)
Changes
@@@ -50,6 +50,12 @@ function load(env = process.env) {osm: {nominatimUrl: trim(env.FOOD_NOMINATIM_URL || 'https://nominatim.openstreetmap.org'),overpassUrl: trim(env.FOOD_OVERPASS_URL || 'https://overpass-api.de/api/interpreter'),++ // Public Overpass servers tried in order, each with overpassTryMs of the timeoutMs budget: one is often++ // overloaded (2026-10-24: overpass-api.de took 26 s and returned nothing, so every near search took 25 s).++ overpassUrls: [...new Set([trim(env.FOOD_OVERPASS_URL || 'https://overpass-api.de/api/interpreter'),++ ...String(env.FOOD_OVERPASS_FALLBACK_URLS ?? 'https://overpass.kumi.systems/api/interpreter,https://overpass.private.coffee/api/interpreter')++ .split(',').map((u) => trim(u)).filter(Boolean)])],++ overpassTryMs: Math.max(1000, int(env.FOOD_OVERPASS_TRY_MS, 8_000)),// Identify to OSM with a contact URL, as both policies require; never a browser User-Agent.userAgent: env.FOOD_OSM_USER_AGENT || `OpenVibeFood/0.1 (+${baseUrl})`,minIntervalMs: Math.max(0, int(env.FOOD_OSM_MIN_INTERVAL_MS, 1000)),@@@ -13,10 +13,12 @@ const parse = (v) => (v == null ? null : typeof v === 'string' ? JSON.parse(v) :// ── food_geo_cache ───────────────────────────────────────────/** A cached upstream answer that has not expired, or null. */--async function cacheGet(s, key) {++// An expired answer is kept this long after its TTL, only to stand in when OpenStreetMap is not answering (stale).++const STALE_KEEP_MS = 30 * 24 * 60 * 60 * 1000;++async function cacheGet(s, key, { stale = false } = {}) {const row = await s.db.maybe('SELECT body, fetched_at, expires_at FROM food_geo_cache WHERE key = $1', [key]);-- if (!row || row.expires_at <= s.iso()) return null;-- return { body: parse(row.body), fetched_at: row.fetched_at, expires_at: row.expires_at };++ if (!row || (!stale && row.expires_at <= s.iso())) return null;++ return { body: parse(row.body), fetched_at: row.fetched_at, expires_at: row.expires_at, stale: row.expires_at <= s.iso() };}async function cachePut(s, key, body, ttlMs) {const fetchedAt = s.iso();@@@ -28,10 +30,11 @@ async function cachePut(s, key, body, ttlMs) {return { fetched_at: fetchedAt, expires_at: expiresAt };}/** Drop expired rows (called at boot; the table stays small because every answer is bounded). */--const cachePrune = (s) => s.db.exec('DELETE FROM food_geo_cache WHERE expires_at <= $1', [s.iso()]);++const cachePrune = (s) => s.db.exec('DELETE FROM food_geo_cache WHERE expires_at <= $1', [new Date(s.now() - STALE_KEEP_MS).toISOString()]);/**-- * Prune the geo cache now, then hourly: an expired row is ignored anyway, this only keeps the table small. The++ * Prune the geo cache now, then hourly: a row 30 days past its TTL goes (until then it only stands in when OSM is down;++ * an expired row is otherwise ignored), which keeps the table small. The* interval is unref'd so it never holds the process open; the returned timer is what the entry point clears on* shutdown (server/index.js). This repository has no restore-drill mode; if one is added it must skip this timer,* because a drill reads a restored database and must not write to it.@@@ -53,13 +53,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D};/** Fetch a URL built from `base` (never from caller input), with the OSM User-Agent and a hard timeout. */-- async function fetchJson(url, init = {}) {++ async function fetchJson(url, init = {}, { timeoutMs = osm.timeoutMs } = {}) {let res;try {res = await fetchImpl(url.toString(), {...init,headers: { 'User-Agent': osm.userAgent, Accept: 'application/json', ...(init.headers || {}) },-- signal: AbortSignal.timeout(osm.timeoutMs),++ signal: AbortSignal.timeout(timeoutMs),});} catch (err) {throw new UpstreamError('food.place.upstream', `the place service could not be reached (${err && err.name === 'TimeoutError' ? 'timed out' : 'network error'})`);@@@ -71,7 +71,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => Dasync function cached(key, fn) {const hit = await geo.cacheGet(s, key);if (hit) return { body: hit.body, cached: true, fetched_at: hit.fetched_at };-- const body = await fn();++ let body;++ try { body = await fn(); } catch (err) {++ // OSM not answering: the last answer for the same question, however old (kept 30 days past its TTL).++ const old = await geo.cacheGet(s, key, { stale: true });++ if (old) return { body: old.body, cached: true, stale: true, fetched_at: old.fetched_at };++ throw err;++ }const written = await geo.cachePut(s, key, body, osm.cacheTtlMs);return { body, cached: false, fetched_at: written.fetched_at };}@@@ -86,14 +92,34 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => Dreturn cached(`nominatim:${keyOf(query)}:${limit}`, () => pacerFor(osm.nominatimUrl).run(() => fetchJson(url)));}-- /** One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. */++ /**++ * One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. The servers++ * in osm.overpassUrls are tried in order, each for osm.overpassTryMs (the last one for what is left of timeoutMs),++ * so one overloaded server no longer holds the page for the whole budget.++ */function overpass(query) {const body = `data=${encodeURIComponent(query)}`;-- return cached(`overpass:${crypto.createHash('sha256').update(query).digest('hex')}`, () => pacerFor(osm.overpassUrl).run(() => fetchJson(new URL(osm.overpassUrl), {-- method: 'POST',-- headers: { 'Content-Type': 'application/x-www-form-urlencoded' },-- body,-- })));++ const urls = osm.overpassUrls && osm.overpassUrls.length ? osm.overpassUrls : [osm.overpassUrl];++ return cached(`overpass:${crypto.createHash('sha256').update(query).digest('hex')}`, async () => {++ const started = now();++ let tried = 0;++ for (const [i, base] of urls.entries()) {++ const left = osm.timeoutMs - (now() - started);++ if (left < 1000) break;++ const timeoutMs = i === urls.length - 1 ? left : Math.min(left, osm.overpassTryMs);++ tried++;++ try {++ return await pacerFor(base).run(() => fetchJson(new URL(base), {++ method: 'POST',++ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },++ body,++ }, { timeoutMs }));++ } catch (err) {++ log.warn && log.warn(`[OpenVibe.Food] Overpass ${new URL(base).host}: ${err.message}`);++ }++ }++ throw new UpstreamError('food.place.upstream', `OpenStreetMap's place servers are busy right now (${tried} tried). Try again in a few minutes.`);++ });}return { geocode, overpass, fetchJson, UpstreamError };@@@ -8,6 +8,8 @@ process.env.FOOD_API_RATE_LIMIT_PER_MIN = process.env.FOOD_API_RATE_LIMIT_PER_MIconst OSM_OFFLINE = {FOOD_NOMINATIM_URL: 'http://127.0.0.1:9/nominatim',FOOD_OVERPASS_URL: 'http://127.0.0.1:9/overpass',++ // No public fallback servers either: the suite never reaches the real Overpass.++ FOOD_OVERPASS_FALLBACK_URLS: '',// The one-request-a-second pace is asserted by places.test.js with the policy's real second; the rest of the// suite must not pay it.FOOD_OSM_MIN_INTERVAL_MS: '1',@@@ -40,6 +40,8 @@ async function startOsm(opts = {}) {geocode: opts.geocode || [SPRINGFIELD],overpass: opts.overpass || DEFAULT_OVER,fail: false,++ overpassFail: false, // Overpass alone answers 503 (Nominatim still works)++ overpassDelayMs: 0, // Overpass waits this long before answering (a slow public server)respond: null, // (kind, query) → body, to override per call};@@@ -58,6 +60,8 @@ async function startOsm(opts = {}) {return json(200, state.geocode.filter((g) => words.some((w) => String(g.display_name).toLowerCase().includes(w))));}if (url.pathname === '/api/interpreter' && req.method === 'POST') {++ if (state.overpassFail) return json(503, { error: 'the stand-in Overpass is busy' });++ if (state.overpassDelayMs) { const ms = state.overpassDelayMs; state.overpassDelayMs = 0; setTimeout(() => { if (!res.writableEnded) json(200, { elements: [] }); }, ms); return; }const data = decodeURIComponent(new URLSearchParams(raw).get('data') || '');const kind = /social_facility|food_sharing|give_box/.test(data) ? 'foodbank' : 'grocery';if (state.respond) return json(200, state.respond(kind, data));@@@ -70,8 +74,9 @@ async function startOsm(opts = {}) {const url = `http://127.0.0.1:${server.address().port}`;return {url, state, requests,-- env: { FOOD_NOMINATIM_URL: url, FOOD_OVERPASS_URL: `${url}/api/interpreter` },-- reset() { requests.length = 0; state.fail = false; state.respond = null; state.geocode = [SPRINGFIELD]; state.overpass = DEFAULT_OVER; },++ // No fallback servers: a test never reaches the real Overpass (places.test.js sets its own stand-in list).++ env: { FOOD_NOMINATIM_URL: url, FOOD_OVERPASS_URL: `${url}/api/interpreter`, FOOD_OVERPASS_FALLBACK_URLS: '' },++ reset() { requests.length = 0; state.fail = false; state.overpassFail = false; state.overpassDelayMs = 0; state.respond = null; state.geocode = [SPRINGFIELD]; state.overpass = DEFAULT_OVER; },/** The gaps, in ms, between consecutive upstream calls — the pace a test checks. */gaps() { return requests.slice(1).map((r, i) => r.at - requests[i].at); },close: () => new Promise((r) => server.close(r)),nnew file mode 100644@@@ -0,0 +1,66 @@++'use strict';++/**++ * Overpass servers in order (config osm.overpassUrls): when the first one fails or is too slow, the next answers within++ * the same search; when none answers, the last cached answer for the same area stands in (stale); with none, the++ * answer says in plain words that OpenStreetMap is busy. On 2026-10-24 overpass-api.de took 26 s and returned nothing,++ * so every "food near" search waited the whole 25 s budget and said "the place service answered 502". Two stand-in++ * OSMs here; no test calls the real OpenStreetMap.++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const { startOsm } = require('./helpers/osm');++++(async () => {++ const first = await startOsm();++ const second = await startOsm();++ const t = await boot({ env: {++ ...first.env, FOOD_OSM_MIN_INTERVAL_MS: '1', FOOD_OVERPASS_TRY_MS: '1000',++ FOOD_OVERPASS_FALLBACK_URLS: `${second.url}/api/interpreter`,++ } });++ const search = (q = 'Springfield, Illinois') => t.get(`/api/v1/places?q=${encodeURIComponent(q)}&kind=foodbank`);++ const overpassCalls = (osm) => osm.requests.filter((r) => r.path === '/api/interpreter').length;++ const clearOverpassCache = () => t.ctx.s.db.query("DELETE FROM food_geo_cache WHERE key LIKE 'overpass:%'");++++ try {++ await check('the first Overpass server answers 503: the next one gives the food banks in the same search', async () => {++ first.state.overpassFail = true;++ const r = await search();++ assert.strictEqual(r.status, 200, r.text);++ assert.strictEqual(r.json().results.length, 4);++ assert.strictEqual(overpassCalls(first), 1);++ assert.strictEqual(overpassCalls(second), 1, 'the fallback server was asked once');++ first.state.overpassFail = false;++ });++++ await check('the first server is too slow: after its share of the budget the next one answers', async () => {++ await clearOverpassCache();++ first.state.overpassDelayMs = 5000;++ const started = Date.now();++ const r = await search();++ assert.strictEqual(r.status, 200, r.text);++ assert.strictEqual(r.json().results.length, 4);++ assert.ok(Date.now() - started < 4000, `answered in ${Date.now() - started} ms, not after the slow server`);++ });++++ await check('no server answers: the cached answer for the same area stands in, even past its TTL', async () => {++ await t.ctx.s.db.query("UPDATE food_geo_cache SET expires_at = '2020-01-01T00:00:00.000Z' WHERE key LIKE 'overpass:%'");++ first.state.overpassFail = true;++ second.state.overpassFail = true;++ const r = await search();++ assert.strictEqual(r.status, 200, r.text);++ assert.strictEqual(r.json().results.length, 4, 'the stale answer');++ });++++ await check('no server answers and nothing is cached: OpenStreetMap is busy, said plainly', async () => {++ await clearOverpassCache();++ const r = await search();++ assert.strictEqual(r.status, 502, r.text);++ assert.match(r.text, /OpenStreetMap's place servers are busy right now \(2 tried\)/);++ });++ } finally {++ await t.close();++ await first.close();++ await second.close();++ }++ done();++})().catch((e) => { console.error(e); process.exit(1); });@@@ -195,7 +195,8 @@ const OVERPASS = '/api/interpreter';assert.strictEqual(r.json().code, 'food.place.upstream');const page = await t.get('/near?lat=41&lon=-91');assert.strictEqual(page.status, 200, 'the page still renders');-- assert.match(page.text, /the place service answered 503/, 'the page says what went wrong');++ // Geocoding is cached by now, so it is Overpass that fails: said plainly (places-fallback.test.js).++ assert.match(page.text, /the place service answered 503|OpenStreetMap's place servers are busy|OpenStreetMap's place servers are busy/, 'the page says what went wrong');osm.state.fail = false;});} finally {@@@ -222,7 +223,7 @@ const OVERPASS = '/api/interpreter';const r = await offline.get('/api/v1/places?lat=51.5&lon=-0.1&kind=foodbank');assert.strictEqual(r.status, 502);assert.strictEqual(r.json().code, 'food.place.upstream');-- assert.match(r.json().detail, /could not be reached/);++ assert.match(r.json().detail, /could not be reached|place servers are busy right now \(1 tried\)/);} finally { await offline.close(); }});
@@ -50,6 +50,12 @@ function load(env = process.env) {
@@ -50,6 +50,12 @@ function load(env = process.env) {
osm: {
osm: {
nominatimUrl: trim(env.FOOD_NOMINATIM_URL || 'https://nominatim.openstreetmap.org'),
nominatimUrl: trim(env.FOOD_NOMINATIM_URL || 'https://nominatim.openstreetmap.org'),
overpassUrl: trim(env.FOOD_OVERPASS_URL || 'https://overpass-api.de/api/interpreter'),
overpassUrl: trim(env.FOOD_OVERPASS_URL || 'https://overpass-api.de/api/interpreter'),
+ // Public Overpass servers tried in order, each with overpassTryMs of the timeoutMs budget: one is often
+ // overloaded (2026-10-24: overpass-api.de took 26 s and returned nothing, so every near search took 25 s).
+ overpassUrls: [...new Set([trim(env.FOOD_OVERPASS_URL || 'https://overpass-api.de/api/interpreter'),
+ ...String(env.FOOD_OVERPASS_FALLBACK_URLS ?? 'https://overpass.kumi.systems/api/interpreter,https://overpass.private.coffee/api/interpreter')
+ .split(',').map((u) => trim(u)).filter(Boolean)])],
+ overpassTryMs: Math.max(1000, int(env.FOOD_OVERPASS_TRY_MS, 8_000)),
// Identify to OSM with a contact URL, as both policies require; never a browser User-Agent.
// Identify to OSM with a contact URL, as both policies require; never a browser User-Agent.
userAgent: env.FOOD_OSM_USER_AGENT || `OpenVibeFood/0.1 (+${baseUrl})`,
userAgent: env.FOOD_OSM_USER_AGENT || `OpenVibeFood/0.1 (+${baseUrl})`,
minIntervalMs: Math.max(0, int(env.FOOD_OSM_MIN_INTERVAL_MS, 1000)),
minIntervalMs: Math.max(0, int(env.FOOD_OSM_MIN_INTERVAL_MS, 1000)),
@@ -13,10 +13,12 @@ const parse = (v) => (v == null ? null : typeof v === 'string' ? JSON.parse(v) :
@@ -13,10 +13,12 @@ const parse = (v) => (v == null ? null : typeof v === 'string' ? JSON.parse(v) :
// ── food_geo_cache ───────────────────────────────────────────
// ── food_geo_cache ───────────────────────────────────────────
/** A cached upstream answer that has not expired, or null. */
/** A cached upstream answer that has not expired, or null. */
-async function cacheGet(s, key) {
+// An expired answer is kept this long after its TTL, only to stand in when OpenStreetMap is not answering (stale).
+const STALE_KEEP_MS = 30 * 24 * 60 * 60 * 1000;
+async function cacheGet(s, key, { stale = false } = {}) {
const row = await s.db.maybe('SELECT body, fetched_at, expires_at FROM food_geo_cache WHERE key = $1', [key]);
const row = await s.db.maybe('SELECT body, fetched_at, expires_at FROM food_geo_cache WHERE key = $1', [key]);
- if (!row || row.expires_at <= s.iso()) return null;
- return { body: parse(row.body), fetched_at: row.fetched_at, expires_at: row.expires_at };
+ if (!row || (!stale && row.expires_at <= s.iso())) return null;
+ return { body: parse(row.body), fetched_at: row.fetched_at, expires_at: row.expires_at, stale: row.expires_at <= s.iso() };
}
}
async function cachePut(s, key, body, ttlMs) {
async function cachePut(s, key, body, ttlMs) {
const fetchedAt = s.iso();
const fetchedAt = s.iso();
@@ -28,10 +30,11 @@ async function cachePut(s, key, body, ttlMs) {
@@ -28,10 +30,11 @@ async function cachePut(s, key, body, ttlMs) {
return { fetched_at: fetchedAt, expires_at: expiresAt };
return { fetched_at: fetchedAt, expires_at: expiresAt };
}
}
/** Drop expired rows (called at boot; the table stays small because every answer is bounded). */
/** Drop expired rows (called at boot; the table stays small because every answer is bounded). */
-const cachePrune = (s) => s.db.exec('DELETE FROM food_geo_cache WHERE expires_at <= $1', [s.iso()]);
+const cachePrune = (s) => s.db.exec('DELETE FROM food_geo_cache WHERE expires_at <= $1', [new Date(s.now() - STALE_KEEP_MS).toISOString()]);
/**
/**
- * Prune the geo cache now, then hourly: an expired row is ignored anyway, this only keeps the table small. The
+ * Prune the geo cache now, then hourly: a row 30 days past its TTL goes (until then it only stands in when OSM is down;
+ * an expired row is otherwise ignored), which keeps the table small. The
* interval is unref'd so it never holds the process open; the returned timer is what the entry point clears on
* interval is unref'd so it never holds the process open; the returned timer is what the entry point clears on
* shutdown (server/index.js). This repository has no restore-drill mode; if one is added it must skip this timer,
* shutdown (server/index.js). This repository has no restore-drill mode; if one is added it must skip this timer,
* because a drill reads a restored database and must not write to it.
* because a drill reads a restored database and must not write to it.
@@ -53,13 +53,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
@@ -53,13 +53,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
};
};
/** Fetch a URL built from `base` (never from caller input), with the OSM User-Agent and a hard timeout. */
/** Fetch a URL built from `base` (never from caller input), with the OSM User-Agent and a hard timeout. */
- async function fetchJson(url, init = {}) {
+ async function fetchJson(url, init = {}, { timeoutMs = osm.timeoutMs } = {}) {
let res;
let res;
try {
try {
res = await fetchImpl(url.toString(), {
res = await fetchImpl(url.toString(), {
...init,
...init,
headers: { 'User-Agent': osm.userAgent, Accept: 'application/json', ...(init.headers || {}) },
headers: { 'User-Agent': osm.userAgent, Accept: 'application/json', ...(init.headers || {}) },
- signal: AbortSignal.timeout(osm.timeoutMs),
+ signal: AbortSignal.timeout(timeoutMs),
});
});
} catch (err) {
} catch (err) {
throw new UpstreamError('food.place.upstream', `the place service could not be reached (${err && err.name === 'TimeoutError' ? 'timed out' : 'network error'})`);
throw new UpstreamError('food.place.upstream', `the place service could not be reached (${err && err.name === 'TimeoutError' ? 'timed out' : 'network error'})`);
@@ -71,7 +71,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
@@ -71,7 +71,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
async function cached(key, fn) {
async function cached(key, fn) {
const hit = await geo.cacheGet(s, key);
const hit = await geo.cacheGet(s, key);
if (hit) return { body: hit.body, cached: true, fetched_at: hit.fetched_at };
if (hit) return { body: hit.body, cached: true, fetched_at: hit.fetched_at };
- const body = await fn();
+ let body;
+ try { body = await fn(); } catch (err) {
+ // OSM not answering: the last answer for the same question, however old (kept 30 days past its TTL).
+ const old = await geo.cacheGet(s, key, { stale: true });
+ if (old) return { body: old.body, cached: true, stale: true, fetched_at: old.fetched_at };
+ throw err;
+ }
const written = await geo.cachePut(s, key, body, osm.cacheTtlMs);
const written = await geo.cachePut(s, key, body, osm.cacheTtlMs);
return { body, cached: false, fetched_at: written.fetched_at };
return { body, cached: false, fetched_at: written.fetched_at };
}
}
@@ -86,14 +92,34 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
@@ -86,14 +92,34 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
return cached(`nominatim:${keyOf(query)}:${limit}`, () => pacerFor(osm.nominatimUrl).run(() => fetchJson(url)));
return cached(`nominatim:${keyOf(query)}:${limit}`, () => pacerFor(osm.nominatimUrl).run(() => fetchJson(url)));
}
}
- /** One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. */
+ /**
+ * One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. The servers
+ * in osm.overpassUrls are tried in order, each for osm.overpassTryMs (the last one for what is left of timeoutMs),
+ * so one overloaded server no longer holds the page for the whole budget.
+ */
function overpass(query) {
function overpass(query) {
const body = `data=${encodeURIComponent(query)}`;
const body = `data=${encodeURIComponent(query)}`;
- return cached(`overpass:${crypto.createHash('sha256').update(query).digest('hex')}`, () => pacerFor(osm.overpassUrl).run(() => fetchJson(new URL(osm.overpassUrl), {
- method: 'POST',
- headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
- body,
- })));
+ const urls = osm.overpassUrls && osm.overpassUrls.length ? osm.overpassUrls : [osm.overpassUrl];
+ return cached(`overpass:${crypto.createHash('sha256').update(query).digest('hex')}`, async () => {
+ const started = now();
+ let tried = 0;
+ for (const [i, base] of urls.entries()) {
+ const left = osm.timeoutMs - (now() - started);
+ if (left < 1000) break;
+ const timeoutMs = i === urls.length - 1 ? left : Math.min(left, osm.overpassTryMs);
+ tried++;
+ try {
+ return await pacerFor(base).run(() => fetchJson(new URL(base), {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+ body,
+ }, { timeoutMs }));
+ } catch (err) {
+ log.warn && log.warn(`[OpenVibe.Food] Overpass ${new URL(base).host}: ${err.message}`);
+ }
+ }
+ throw new UpstreamError('food.place.upstream', `OpenStreetMap's place servers are busy right now (${tried} tried). Try again in a few minutes.`);
+ });
}
}
return { geocode, overpass, fetchJson, UpstreamError };
return { geocode, overpass, fetchJson, UpstreamError };
@@ -8,6 +8,8 @@ process.env.FOOD_API_RATE_LIMIT_PER_MIN = process.env.FOOD_API_RATE_LIMIT_PER_MI
@@ -8,6 +8,8 @@ process.env.FOOD_API_RATE_LIMIT_PER_MIN = process.env.FOOD_API_RATE_LIMIT_PER_MI
const OSM_OFFLINE = {
const OSM_OFFLINE = {
FOOD_NOMINATIM_URL: 'http://127.0.0.1:9/nominatim',
FOOD_NOMINATIM_URL: 'http://127.0.0.1:9/nominatim',
FOOD_OVERPASS_URL: 'http://127.0.0.1:9/overpass',
FOOD_OVERPASS_URL: 'http://127.0.0.1:9/overpass',
+ // No public fallback servers either: the suite never reaches the real Overpass.
+ FOOD_OVERPASS_FALLBACK_URLS: '',
// The one-request-a-second pace is asserted by places.test.js with the policy's real second; the rest of the
// The one-request-a-second pace is asserted by places.test.js with the policy's real second; the rest of the
// suite must not pay it.
// suite must not pay it.
FOOD_OSM_MIN_INTERVAL_MS: '1',
FOOD_OSM_MIN_INTERVAL_MS: '1',
@@ -40,6 +40,8 @@ async function startOsm(opts = {}) {
@@ -40,6 +40,8 @@ async function startOsm(opts = {}) {
geocode: opts.geocode || [SPRINGFIELD],
geocode: opts.geocode || [SPRINGFIELD],
overpass: opts.overpass || DEFAULT_OVER,
overpass: opts.overpass || DEFAULT_OVER,
fail: false,
fail: false,
+ overpassFail: false, // Overpass alone answers 503 (Nominatim still works)
+ overpassDelayMs: 0, // Overpass waits this long before answering (a slow public server)
respond: null, // (kind, query) → body, to override per call
respond: null, // (kind, query) → body, to override per call
};
};
@@ -58,6 +60,8 @@ async function startOsm(opts = {}) {
@@ -58,6 +60,8 @@ async function startOsm(opts = {}) {
return json(200, state.geocode.filter((g) => words.some((w) => String(g.display_name).toLowerCase().includes(w))));
return json(200, state.geocode.filter((g) => words.some((w) => String(g.display_name).toLowerCase().includes(w))));
}
}
if (url.pathname === '/api/interpreter' && req.method === 'POST') {
if (url.pathname === '/api/interpreter' && req.method === 'POST') {
+ if (state.overpassFail) return json(503, { error: 'the stand-in Overpass is busy' });
+ if (state.overpassDelayMs) { const ms = state.overpassDelayMs; state.overpassDelayMs = 0; setTimeout(() => { if (!res.writableEnded) json(200, { elements: [] }); }, ms); return; }
const data = decodeURIComponent(new URLSearchParams(raw).get('data') || '');
const data = decodeURIComponent(new URLSearchParams(raw).get('data') || '');
const kind = /social_facility|food_sharing|give_box/.test(data) ? 'foodbank' : 'grocery';
const kind = /social_facility|food_sharing|give_box/.test(data) ? 'foodbank' : 'grocery';
if (state.respond) return json(200, state.respond(kind, data));
if (state.respond) return json(200, state.respond(kind, data));
@@ -70,8 +74,9 @@ async function startOsm(opts = {}) {
@@ -70,8 +74,9 @@ async function startOsm(opts = {}) {
const url = `http://127.0.0.1:${server.address().port}`;
const url = `http://127.0.0.1:${server.address().port}`;
return {
return {
url, state, requests,
url, state, requests,
- env: { FOOD_NOMINATIM_URL: url, FOOD_OVERPASS_URL: `${url}/api/interpreter` },
- reset() { requests.length = 0; state.fail = false; state.respond = null; state.geocode = [SPRINGFIELD]; state.overpass = DEFAULT_OVER; },
+ // No fallback servers: a test never reaches the real Overpass (places.test.js sets its own stand-in list).
+ env: { FOOD_NOMINATIM_URL: url, FOOD_OVERPASS_URL: `${url}/api/interpreter`, FOOD_OVERPASS_FALLBACK_URLS: '' },
+ reset() { requests.length = 0; state.fail = false; state.overpassFail = false; state.overpassDelayMs = 0; state.respond = null; state.geocode = [SPRINGFIELD]; state.overpass = DEFAULT_OVER; },
/** The gaps, in ms, between consecutive upstream calls — the pace a test checks. */
/** The gaps, in ms, between consecutive upstream calls — the pace a test checks. */
gaps() { return requests.slice(1).map((r, i) => r.at - requests[i].at); },
gaps() { return requests.slice(1).map((r, i) => r.at - requests[i].at); },
close: () => new Promise((r) => server.close(r)),
close: () => new Promise((r) => server.close(r)),
new file mode 100644
new file mode 100644
@@ -0,0 +1,66 @@
@@ -0,0 +1,66 @@
+'use strict';
+/**
+ * Overpass servers in order (config osm.overpassUrls): when the first one fails or is too slow, the next answers within
+ * the same search; when none answers, the last cached answer for the same area stands in (stale); with none, the
+ * answer says in plain words that OpenStreetMap is busy. On 2026-10-24 overpass-api.de took 26 s and returned nothing,
+ * so every "food near" search waited the whole 25 s budget and said "the place service answered 502". Two stand-in
+ * OSMs here; no test calls the real OpenStreetMap.
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const { startOsm } = require('./helpers/osm');
+
+(async () => {
+ const first = await startOsm();
+ const second = await startOsm();
+ const t = await boot({ env: {
+ ...first.env, FOOD_OSM_MIN_INTERVAL_MS: '1', FOOD_OVERPASS_TRY_MS: '1000',
+ FOOD_OVERPASS_FALLBACK_URLS: `${second.url}/api/interpreter`,
+ } });
+ const search = (q = 'Springfield, Illinois') => t.get(`/api/v1/places?q=${encodeURIComponent(q)}&kind=foodbank`);
+ const overpassCalls = (osm) => osm.requests.filter((r) => r.path === '/api/interpreter').length;
+ const clearOverpassCache = () => t.ctx.s.db.query("DELETE FROM food_geo_cache WHERE key LIKE 'overpass:%'");
+
+ try {
+ await check('the first Overpass server answers 503: the next one gives the food banks in the same search', async () => {
+ first.state.overpassFail = true;
+ const r = await search();
+ assert.strictEqual(r.status, 200, r.text);
+ assert.strictEqual(r.json().results.length, 4);
+ assert.strictEqual(overpassCalls(first), 1);
+ assert.strictEqual(overpassCalls(second), 1, 'the fallback server was asked once');
+ first.state.overpassFail = false;
+ });
+
+ await check('the first server is too slow: after its share of the budget the next one answers', async () => {
+ await clearOverpassCache();
+ first.state.overpassDelayMs = 5000;
+ const started = Date.now();
+ const r = await search();
+ assert.strictEqual(r.status, 200, r.text);
+ assert.strictEqual(r.json().results.length, 4);
+ assert.ok(Date.now() - started < 4000, `answered in ${Date.now() - started} ms, not after the slow server`);
+ });
+
+ await check('no server answers: the cached answer for the same area stands in, even past its TTL', async () => {
+ await t.ctx.s.db.query("UPDATE food_geo_cache SET expires_at = '2020-01-01T00:00:00.000Z' WHERE key LIKE 'overpass:%'");
+ first.state.overpassFail = true;
+ second.state.overpassFail = true;
+ const r = await search();
+ assert.strictEqual(r.status, 200, r.text);
+ assert.strictEqual(r.json().results.length, 4, 'the stale answer');
+ });
+
+ await check('no server answers and nothing is cached: OpenStreetMap is busy, said plainly', async () => {
+ await clearOverpassCache();
+ const r = await search();
+ assert.strictEqual(r.status, 502, r.text);
+ assert.match(r.text, /OpenStreetMap's place servers are busy right now \(2 tried\)/);
+ });
+ } finally {
+ await t.close();
+ await first.close();
+ await second.close();
+ }
+ done();
+})().catch((e) => { console.error(e); process.exit(1); });
@@ -195,7 +195,8 @@ const OVERPASS = '/api/interpreter';
@@ -195,7 +195,8 @@ const OVERPASS = '/api/interpreter';
assert.strictEqual(r.json().code, 'food.place.upstream');
assert.strictEqual(r.json().code, 'food.place.upstream');
const page = await t.get('/near?lat=41&lon=-91');
const page = await t.get('/near?lat=41&lon=-91');
assert.strictEqual(page.status, 200, 'the page still renders');
assert.strictEqual(page.status, 200, 'the page still renders');
- assert.match(page.text, /the place service answered 503/, 'the page says what went wrong');
+ // Geocoding is cached by now, so it is Overpass that fails: said plainly (places-fallback.test.js).
+ assert.match(page.text, /the place service answered 503|OpenStreetMap's place servers are busy|OpenStreetMap's place servers are busy/, 'the page says what went wrong');
osm.state.fail = false;
osm.state.fail = false;
});
});
} finally {
} finally {
@@ -222,7 +223,7 @@ const OVERPASS = '/api/interpreter';
@@ -222,7 +223,7 @@ const OVERPASS = '/api/interpreter';
const r = await offline.get('/api/v1/places?lat=51.5&lon=-0.1&kind=foodbank');
const r = await offline.get('/api/v1/places?lat=51.5&lon=-0.1&kind=foodbank');
assert.strictEqual(r.status, 502);
assert.strictEqual(r.status, 502);
assert.strictEqual(r.json().code, 'food.place.upstream');
assert.strictEqual(r.json().code, 'food.place.upstream');
- assert.match(r.json().detail, /could not be reached/);
+ assert.match(r.json().detail, /could not be reached|place servers are busy right now \(1 tried\)/);
} finally { await offline.close(); }
} finally { await offline.close(); }
});
});