openvibers/openvibe.food Commits

Food near you: the home's cities are warmed in the background, a query that just failed answers at once, and VK Maps replaces kumi.systems (private.coffee's old name) among the Overpass fallbacks (#88)

OP OpenVibers committed 19 days ago · ce5c15fd06 · 10 files, +172 −23

Changes

ddiff --git a/.env.example b/.env.example
iindex 8d82d64..656e835 100644
---- a/.env.example
++++ b/.env.example
@@@ -43,6 +43,16 @@ FOOD_EVENTS_URL=
# The pace between two requests to the same host, in ms (the policies ask for at least 1000).
#FOOD_OSM_MIN_INTERVAL_MS=1000
#FOOD_OSM_TIMEOUT_MS=25000
++# Overpass servers tried after FOOD_OVERPASS_URL, comma-separated (default: private.coffee, then VK Maps: the OSM
++# wiki's public instances without request limits), each for FOOD_OVERPASS_TRY_MS of the budget.
++#FOOD_OVERPASS_FALLBACK_URLS=https://overpass.private.coffee/api/interpreter,https://maps.mail.ru/osm/tools/overpass/api/interpreter
++#FOOD_OVERPASS_TRY_MS=8000
++# After an area query failed, the same query answers at once (its last result, or "busy") for this long, in ms.
++#FOOD_OSM_RETRY_AFTER_MS=180000
++# The home's "food near" cities refreshed in the background (server/food/near-warm.js): how often, in ms (0 = off),
++# and each area query's budget there.
++#FOOD_NEAR_WARM_MS=21600000
++#FOOD_NEAR_WARM_BUDGET_MS=150000
# How long a geocode or an area answer is kept, in ms (default 7 days).
#FOOD_GEO_CACHE_TTL_MS=604800000
ddiff --git a/server/app.js b/server/app.js
iindex 0c8a303..ccadb5e 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -62,6 +62,8 @@ async function createApp(opts = {}) {
// it from here, so every test points it at a stand-in through FOOD_NOMINATIM_URL / FOOD_OVERPASS_URL.
// Its own wall clock, not the store's: the pace between two calls must not depend on an injected test clock.
ctx.upstream = createUpstream({ config, s, fetchImpl, log });
++ // The home's "food near" cities, warmed in the background so their links read the cache (server/index.js starts it).
++ ctx.nearWarm = require('./food/near-warm').createNearWarm({ config, upstream: ctx.upstream, log });
ctx.barcodes = opts.barcodes || require('./food/barcode').createBarcodes({ config, fetchImpl });
// Drop expired geocode/area answers now and hourly; an expired row is ignored anyway, this only keeps the table
// small. The interval is unref'd and kept on ctx so the entry point clears it on shutdown (server/index.js).
ddiff --git a/server/config.js b/server/config.js
iindex 124431b..fe3a016 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -52,14 +52,24 @@ function load(env = process.env) {
overpassUrl: trim(env.FOOD_OVERPASS_URL || 'https://overpass-api.de/api/interpreter'),
// Public Overpass servers tried in order, each with overpassTryMs of the timeoutMs budget: one is often
// overloaded (2026-10-24: overpass-api.de took 26 s and returned nothing, so every near search took 25 s).
++ // The fallbacks are the OSM wiki's public instances without request limits: private.coffee (formerly
++ // overpass.kumi.systems, the same servers) and VK Maps (2026-10-25: the only one answering while the others
++ // timed out).
overpassUrls: [...new Set([trim(env.FOOD_OVERPASS_URL || 'https://overpass-api.de/api/interpreter'),
-- ...String(env.FOOD_OVERPASS_FALLBACK_URLS ?? 'https://overpass.kumi.systems/api/interpreter,https://overpass.private.coffee/api/interpreter')
++ ...String(env.FOOD_OVERPASS_FALLBACK_URLS ?? 'https://overpass.private.coffee/api/interpreter,https://maps.mail.ru/osm/tools/overpass/api/interpreter')
.split(',').map((u) => trim(u)).filter(Boolean)])],
overpassTryMs: Math.max(1000, int(env.FOOD_OVERPASS_TRY_MS, 8_000)),
// Identify to OSM with a contact URL, as both policies require; never a browser User-Agent.
userAgent: env.FOOD_OSM_USER_AGENT || `OpenVibeFood/0.1 (+${baseUrl})`,
minIntervalMs: Math.max(0, int(env.FOOD_OSM_MIN_INTERVAL_MS, 1000)),
timeoutMs: Math.max(1000, int(env.FOOD_OSM_TIMEOUT_MS, 25_000)),
++ // After an area query failed, the same query refuses at once for this long (or answers its last result):
++ // the next visitor does not wait out the whole budget on servers that just failed.
++ retryAfterMs: Math.max(0, int(env.FOOD_OSM_RETRY_AFTER_MS, 180_000)),
++ // The home's "food near" cities warmed in the background (server/food/near-warm.js): how often (0 = off),
++ // and the budget of each area query there, off any visitor's path.
++ nearWarmMs: Math.max(0, int(env.FOOD_NEAR_WARM_MS, 6 * 60 * 60 * 1000)),
++ nearWarmBudgetMs: Math.max(1000, int(env.FOOD_NEAR_WARM_BUDGET_MS, 150_000)),
cacheTtlMs: Math.max(60_000, int(env.FOOD_GEO_CACHE_TTL_MS, 7 * 24 * 60 * 60 * 1000)),
defaultRadiusKm: 8,
maxRadiusKm: 50,
ddiff --git a/server/food/near-warm.js b/server/food/near-warm.js
nnew file mode 100644
iindex 0000000..9970b44
---- /dev/null
++++ b/server/food/near-warm.js
@@@ -0,0 +1,69 @@
++'use strict';
++
++/**
++ * The home's "food near" cities, kept warm. The home links each of NEAR_CITIES as /near?q=<city>; a visitor who follows
++ * one should read the cache, not wait on OpenStreetMap's public Overpass servers, which are often overloaded (2026-10-25:
++ * every one of these searches failed after 25 s, so each link showed "busy").
++ *
++ * Every osm.nearWarmMs (6 hours; 0 turns it off), each city is geocoded (cached like any search) and its food bank and
++ * grocery area queries are asked again in the background, one at a time and paced like every other call
++ * (server/food/upstream.js), when their cached answer is missing or expires within a day. There the budget is
++ * osm.nearWarmBudgetMs per query, off any visitor's path. A query that still fails keeps its last answer, which the
++ * page shows as it does whenever OSM is down.
++ *
++ * createNearWarm({ config, upstream, log, now }) → { start(), stop(), runOnce() → { warmed, fresh, failed } }
++ */
++const { KINDS, NEAR_CITIES, queryFor, radiusMetres } = require('./places');
++
++const REFRESH_BEFORE_MS = 24 * 60 * 60 * 1000;
++const FIRST_RUN_MS = 2 * 60_000;
++
++function createNearWarm({ config, upstream, log = console, now = () => Date.now(), cities = NEAR_CITIES }) {
++ const osm = config.osm;
++ let timer = null;
++ let first = null;
++ let running = false;
++
++ async function runOnce() {
++ const out = { warmed: 0, fresh: 0, failed: 0 };
++ if (running) return out;
++ running = true;
++ try {
++ for (const city of cities) {
++ let hit;
++ try {
++ const g = await upstream.geocode(city, { limit: 1 });
++ hit = Array.isArray(g.body) ? g.body[0] : null;
++ } catch { hit = null; }
++ const lat = hit && Number(hit.lat);
++ const lon = hit && Number(hit.lon);
++ if (!Number.isFinite(lat) || !Number.isFinite(lon)) { out.failed += KINDS.length; continue; }
++ for (const kind of KINDS) {
++ const query = queryFor(kind, lat, lon, radiusMetres(null, config));
++ const until = await upstream.cachedUntil(query);
++ if (until && Date.parse(until) - now() > REFRESH_BEFORE_MS) { out.fresh++; continue; }
++ try {
++ const r = await upstream.overpass(query, { force: true, budgetMs: osm.nearWarmBudgetMs, tryMs: Math.ceil(osm.nearWarmBudgetMs / 3) });
++ if (r.stale) out.failed++; else out.warmed++;
++ } catch { out.failed++; }
++ }
++ }
++ } finally { running = false; }
++ if (out.warmed || out.failed) log.log && log.log(`[OpenVibe.Food] near cities: ${out.warmed} warmed, ${out.fresh} fresh, ${out.failed} not reached`);
++ return out;
++ }
++
++ return {
++ runOnce,
++ start() {
++ if (!osm.nearWarmMs || timer) return;
++ timer = setInterval(() => { runOnce().catch(() => {}); }, osm.nearWarmMs);
++ timer.unref();
++ first = setTimeout(() => { runOnce().catch(() => {}); }, FIRST_RUN_MS);
++ first.unref();
++ },
++ stop() { clearInterval(timer); clearTimeout(first); timer = null; first = null; },
++ };
++}
++
++module.exports = { createNearWarm, REFRESH_BEFORE_MS };
ddiff --git a/server/food/places.js b/server/food/places.js
iindex 53b55e4..c93f9fa 100644
---- a/server/food/places.js
++++ b/server/food/places.js
@@@ -141,6 +141,15 @@ function elementsToResults(elements, kind, origin, limit) {
const clamp = (n, lo, hi) => Math.min(Math.max(n, lo), hi);
++/** The cities the home links to (/near?q=<city>), kept warm in the background (server/food/near-warm.js). */
++const NEAR_CITIES = ['New York, NY', 'Los Angeles, CA', 'Chicago, IL', 'Houston, TX', 'Phoenix, AZ', 'Philadelphia, PA', 'San Antonio, TX', 'Seattle, WA'];
++
++/** The area search's radius in metres for a requested radius in km (the default when none), within the configured bounds. */
++function radiusMetres(radiusKm, config) {
++ const maxRadius = (config && config.osm.maxRadiusKm) || 50;
++ return Math.round(clamp(Number(radiusKm) || (config && config.osm.defaultRadiusKm) || 8, 0.5, maxRadius) * 1000);
++}
++
/**
* The place search behind both the API and the pages.
*
@@@ -167,10 +176,9 @@ async function searchPlaces(upstream, { kind = 'foodbank', q = null, lat = null,
throw new UpstreamError('food.place.invalid', 'q: a place to search for (or lat and lon)', { status: 422 });
}
-- const maxRadius = (config && config.osm.maxRadiusKm) || 50;
-- const rad = clamp(Number(radiusKm) || (config && config.osm.defaultRadiusKm) || 8, 0.5, maxRadius);
++ const radiusM = radiusMetres(radiusKm, config);
++ const rad = radiusM / 1000;
const cap = clamp(Number(limit) || (config && config.osm.maxResults) || 60, 1, (config && config.osm.maxResults) || 60);
-- const radiusM = Math.round(rad * 1000);
const perKind = {};
let fetchedAt = null;
@@@ -199,4 +207,4 @@ async function searchPlaces(upstream, { kind = 'foodbank', q = null, lat = null,
};
}
--module.exports = { searchPlaces, haversineKm, formatAddress, isBudgetStore, safeUrl, queryFor, elementsToResults, ATTRIBUTION, SOURCE, FOOD_BANK_TYPES, KINDS };
++module.exports = { searchPlaces, haversineKm, formatAddress, isBudgetStore, safeUrl, queryFor, radiusMetres, elementsToResults, ATTRIBUTION, SOURCE, FOOD_BANK_TYPES, KINDS, NEAR_CITIES };
ddiff --git a/server/food/upstream.js b/server/food/upstream.js
iindex fe9ad6c..4c1cd6f 100644
---- a/server/food/upstream.js
++++ b/server/food/upstream.js
@@@ -45,6 +45,7 @@ const keyOf = (q) => String(q).trim().toLowerCase().replace(/\s+/g, ' ');
function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console } = {}) {
const osm = config.osm;
++ const failed = new Map(); // area query key → when it last failed (this process)
const pacers = new Map();
const pacerFor = (rawUrl) => {
const host = new URL(rawUrl).host;
@@@ -68,11 +69,21 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
try { return await res.json(); } catch { throw new UpstreamError('food.place.upstream', 'the place service answered something that is not JSON'); }
}
-- async function cached(key, fn) {
-- const hit = await geo.cacheGet(s, key);
++ async function cached(key, fn, { failedWithinMs = 0, force = false } = {}) {
++ const hit = force ? null : await geo.cacheGet(s, key);
if (hit) return { body: hit.body, cached: true, fetched_at: hit.fetched_at };
++ // The same question failed a moment ago: its last answer if there is one, else the refusal at once.
++ if (failedWithinMs && !force && now() - (failed.get(key) || -Infinity) < failedWithinMs) {
++ const old = await geo.cacheGet(s, key, { stale: true });
++ if (old) return { body: old.body, cached: true, stale: true, fetched_at: old.fetched_at };
++ throw new UpstreamError('food.place.upstream', 'OpenStreetMap\'s place servers were busy a moment ago. Try again in a few minutes.');
++ }
let body;
-- try { body = await fn(); } catch (err) {
++ try { body = await fn(); failed.delete(key); } catch (err) {
++ if (failedWithinMs) {
++ if (failed.size > 1000) failed.clear();
++ failed.set(key, now());
++ }
// OSM not answering: the last answer for the same question, however old (kept 30 days past its TTL).
const old = await geo.cacheGet(s, key, { stale: true });
if (old) return { body: old.body, cached: true, stale: true, fetched_at: old.fetched_at };
@@@ -92,21 +103,24 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
return cached(`nominatim:${keyOf(query)}:${limit}`, () => pacerFor(osm.nominatimUrl).run(() => fetchJson(url)));
}
++ const overpassKey = (query) => `overpass:${crypto.createHash('sha256').update(query).digest('hex')}`;
++
/**
* One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. The servers
-- * in osm.overpassUrls are tried in order, each for osm.overpassTryMs (the last one for what is left of timeoutMs),
-- * so one overloaded server no longer holds the page for the whole budget.
++ * in osm.overpassUrls are tried in order, each for `tryMs` (the last one for what is left of `budgetMs`), so one
++ * overloaded server no longer holds the page for the whole budget. A query that just failed refuses at once for
++ * osm.retryAfterMs. `force` (the background warmer) asks again even when a cached answer is still fresh.
*/
-- function overpass(query) {
++ function overpass(query, { budgetMs = osm.timeoutMs, tryMs = osm.overpassTryMs, force = false } = {}) {
const body = `data=${encodeURIComponent(query)}`;
const urls = osm.overpassUrls && osm.overpassUrls.length ? osm.overpassUrls : [osm.overpassUrl];
-- return cached(`overpass:${crypto.createHash('sha256').update(query).digest('hex')}`, async () => {
++ return cached(overpassKey(query), async () => {
const started = now();
let tried = 0;
for (const [i, base] of urls.entries()) {
-- const left = osm.timeoutMs - (now() - started);
++ const left = budgetMs - (now() - started);
if (left < 1000) break;
-- const timeoutMs = i === urls.length - 1 ? left : Math.min(left, osm.overpassTryMs);
++ const timeoutMs = i === urls.length - 1 ? left : Math.min(left, tryMs);
tried++;
try {
return await pacerFor(base).run(() => fetchJson(new URL(base), {
@@@ -119,10 +133,16 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
}
}
throw new UpstreamError('food.place.upstream', `OpenStreetMap's place servers are busy right now (${tried} tried). Try again in a few minutes.`);
-- });
++ }, { failedWithinMs: osm.retryAfterMs, force });
++ }
++
++ /** When an area query's cached answer expires (an ISO time), or null when there is none or it already has. */
++ async function cachedUntil(query) {
++ const hit = await geo.cacheGet(s, overpassKey(query));
++ return hit ? hit.expires_at : null;
}
-- return { geocode, overpass, fetchJson, UpstreamError };
++ return { geocode, overpass, cachedUntil, fetchJson, UpstreamError };
}
module.exports = { createUpstream, createPacer, UpstreamError };
ddiff --git a/server/http/pages.js b/server/http/pages.js
iindex 5bb370d..b671ce6 100644
---- a/server/http/pages.js
++++ b/server/http/pages.js
@@@ -30,7 +30,7 @@ const { send } = require('../render/layout');
const foods = require('../food/foods');
const { NOTE: OFFICIAL_NOTE } = require('../food/official-prices');
const { planMeals } = require('../food/planner');
--const { searchPlaces, safeUrl, ATTRIBUTION } = require('../food/places');
++const { searchPlaces, safeUrl, ATTRIBUTION, NEAR_CITIES } = require('../food/places');
const { matchPantry } = require('../food/pantry-match');
const { suggestRecipes } = require('../food/recipes');
const store = require('../food/store');
@@@ -111,7 +111,6 @@ ${watchable ? html`<p class="muted small">Watch for changes opens OpenVibe.Watch
const byProtein = foods.FOODS.filter((f) => proteinPerDollar(f)).sort((a, b) => proteinPerDollar(b) - proteinPerDollar(a)).slice(0, 8);
const shortName = (f) => f.name.replace(/\s*\([^)]*\)\s*$/, '');
const foodCard = (f, main, label) => html`<li><a href="/foods/${f.slug}"><b>${shortName(f)}</b><span class="hc-main"><strong>${main}</strong> ${label}</span><span class="hc-sub">${money(f.price)} at ${f.cheapest_store_name}, typical</span></a></li>`;
-- const NEAR_CITIES = ['New York, NY', 'Los Angeles, CA', 'Chicago, IL', 'Houston, TX', 'Phoenix, AZ', 'Philadelphia, PA', 'San Antonio, TX', 'Seattle, WA'];
r.get('/', async (req, res) => {
let lib = { total: 0, cheap: [] };
if (ctx.library) {
ddiff --git a/server/index.js b/server/index.js
iindex d876534..11cfd2e 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -48,7 +48,9 @@ async function start() {
// server/food/official-prices.js).
ctx.recalls.start();
ctx.officialPrices.start();
-- const extra = [() => { if (subscriptions) subscriptions.stop(); }, () => ctx.library.stop(), () => ctx.recipeSearch.stop(), () => ctx.recalls.stop(), () => ctx.officialPrices.stop()];
++ // The home's "food near" cities: their area searches refreshed in the background (server/food/near-warm.js).
++ ctx.nearWarm.start();
++ const extra = [() => { if (subscriptions) subscriptions.stop(); }, () => ctx.library.stop(), () => ctx.recipeSearch.stop(), () => ctx.recalls.stop(), () => ctx.officialPrices.stop(), () => ctx.nearWarm.stop()];
createLifecycle({ server, ctx, extra });
return { server, ctx };
}
ddiff --git a/test/places-fallback.test.js b/test/places-fallback.test.js
iindex 3f713b6..642dc37 100644
---- a/test/places-fallback.test.js
++++ b/test/places-fallback.test.js
@@@ -3,8 +3,10 @@
* Overpass servers in order (config osm.overpassUrls): when the first one fails or is too slow, the next answers within
* the same search; when none answers, the last cached answer for the same area stands in (stale); with none, the
* answer says in plain words that OpenStreetMap is busy. On 2026-10-24 overpass-api.de took 26 s and returned nothing,
-- * so every "food near" search waited the whole 25 s budget and said "the place service answered 502". Two stand-in
-- * OSMs here; no test calls the real OpenStreetMap.
++ * so every "food near" search waited the whole 25 s budget and said "the place service answered 502". A query that just
++ * failed refuses at once for a while instead of waiting the budget again; the home's cities are warmed in the background
++ * (server/food/near-warm.js), past that memory, with a budget of their own. Two stand-in OSMs here; no test calls the
++ * real OpenStreetMap.
*/
const assert = require('assert');
const { boot, check, done } = require('./helpers/boot');
@@@ -18,6 +20,8 @@ const { startOsm } = require('./helpers/osm');
FOOD_OVERPASS_FALLBACK_URLS: `${second.url}/api/interpreter`,
} });
const search = (q = 'Springfield, Illinois') => t.get(`/api/v1/places?q=${encodeURIComponent(q)}&kind=foodbank`);
++ // A point search: its own area query, so the failure remembered for Springfield's (just above) does not answer it.
++ const point = () => t.get('/api/v1/places?lat=39.79&lon=-89.65&kind=foodbank');
const overpassCalls = (osm) => osm.requests.filter((r) => r.path === '/api/interpreter').length;
const clearOverpassCache = () => t.ctx.s.db.query("DELETE FROM food_geo_cache WHERE key LIKE 'overpass:%'");
@@@ -53,10 +57,34 @@ const { startOsm } = require('./helpers/osm');
await check('no server answers and nothing is cached: OpenStreetMap is busy, said plainly', async () => {
await clearOverpassCache();
-- const r = await search();
++ const r = await point();
assert.strictEqual(r.status, 502, r.text);
assert.match(r.text, /OpenStreetMap's place servers are busy right now \(2 tried\)/);
});
++
++ await check('asked again at once after that: the same answer without waiting on the servers again', async () => {
++ const before = overpassCalls(first) + overpassCalls(second);
++ const started = Date.now();
++ const r = await point();
++ assert.strictEqual(r.status, 502, r.text);
++ assert.match(r.text, /were busy a moment ago/);
++ assert.strictEqual(overpassCalls(first) + overpassCalls(second), before, 'no server was asked');
++ assert.ok(Date.now() - started < 1000, `answered in ${Date.now() - started} ms`);
++ });
++
++ await check('the home\'s cities warm in the background, past that memory; then a visitor reads the cache, and fresh ones are left alone', async () => {
++ first.state.overpassFail = false;
++ second.state.overpassFail = false;
++ const { createNearWarm } = require('../server/food/near-warm');
++ const warm = createNearWarm({ config: t.ctx.config, upstream: t.ctx.upstream, cities: ['Springfield, Illinois'], log: {} });
++ assert.deepStrictEqual(await warm.runOnce(), { warmed: 2, fresh: 0, failed: 0 }, 'food banks and grocery stores, asked again though the food bank query failed a moment ago');
++ const before = overpassCalls(first) + overpassCalls(second);
++ const r = await search();
++ assert.strictEqual(r.status, 200, r.text);
++ assert.strictEqual(r.json().results.length, 4);
++ assert.strictEqual(overpassCalls(first) + overpassCalls(second), before, 'the visitor read the cache');
++ assert.deepStrictEqual(await warm.runOnce(), { warmed: 0, fresh: 2, failed: 0 }, 'not asked again while fresh');
++ });
} finally {
await t.close();
await first.close();
ddiff --git a/test/places.test.js b/test/places.test.js
iindex 18e94c3..45de540 100644
---- a/test/places.test.js
++++ b/test/places.test.js
@@@ -196,7 +196,8 @@ const OVERPASS = '/api/interpreter';
const page = await t.get('/near?lat=41&lon=-91');
assert.strictEqual(page.status, 200, 'the page still renders');
// Geocoding is cached by now, so it is Overpass that fails: said plainly (places-fallback.test.js).
-- assert.match(page.text, /the place service answered 503|OpenStreetMap&#39;s place servers are busy|OpenStreetMap's place servers are busy/, 'the page says what went wrong');
++ // The API call just above failed the same area query, so the page gets the remembered refusal at once.
++ assert.match(page.text, /the place service answered 503|OpenStreetMap(&#39;|')s place servers (are|were) busy/, 'the page says what went wrong');
osm.state.fail = false;
});
} finally {