openvibers/openvibe.food Commits
Overpass: the server that answered last is asked first (a search spent 16 s on two dead servers before reaching the one that answered) (#89)
Changes
@@@ -46,6 +46,7 @@ const keyOf = (q) => String(q).trim().toLowerCase().replace(/\s+/g, ' ');function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console } = {}) {const osm = config.osm;const failed = new Map(); // area query key → when it last failed (this process)++ let lastGood = null; // the Overpass server that answered last: asked first next timeconst pacers = new Map();const pacerFor = (rawUrl) => {const host = new URL(rawUrl).host;@@@ -108,12 +109,14 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D/*** One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. The servers* in osm.overpassUrls are tried in order, each for `tryMs` (the last one for what is left of `budgetMs`), so one-- * overloaded server no longer holds the page for the whole budget. A query that just failed refuses at once for++ * overloaded server no longer holds the page for the whole budget; the one that answered last goes first (on++ * 2026-10-25 only the third answered, and a search spent 16 s on the first two before reaching it). A query that just failed refuses at once for* osm.retryAfterMs. `force` (the background warmer) asks again even when a cached answer is still fresh.*/function overpass(query, { budgetMs = osm.timeoutMs, tryMs = osm.overpassTryMs, force = false } = {}) {const body = `data=${encodeURIComponent(query)}`;-- const urls = osm.overpassUrls && osm.overpassUrls.length ? osm.overpassUrls : [osm.overpassUrl];++ const all = osm.overpassUrls && osm.overpassUrls.length ? osm.overpassUrls : [osm.overpassUrl];++ const urls = lastGood && all.includes(lastGood) ? [lastGood, ...all.filter((u) => u !== lastGood)] : all;return cached(overpassKey(query), async () => {const started = now();let tried = 0;@@@ -123,11 +126,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => Dconst timeoutMs = i === urls.length - 1 ? left : Math.min(left, tryMs);tried++;try {-- return await pacerFor(base).run(() => fetchJson(new URL(base), {++ const answer = await pacerFor(base).run(() => fetchJson(new URL(base), {method: 'POST',headers: { 'Content-Type': 'application/x-www-form-urlencoded' },body,}, { timeoutMs }));++ lastGood = base;++ return answer;} catch (err) {log.warn && log.warn(`[OpenVibe.Food] Overpass ${new URL(base).host}: ${err.message}`);}@@@ -36,6 +36,19 @@ const { startOsm } = require('./helpers/osm');first.state.overpassFail = false;});++ await check('the server that answered last is asked first: the next search goes straight to it', async () => {++ await clearOverpassCache();++ first.state.overpassFail = false;++ const before = [overpassCalls(first), overpassCalls(second)];++ const r = await search();++ assert.strictEqual(r.status, 200, r.text);++ assert.deepStrictEqual([overpassCalls(first) - before[0], overpassCalls(second) - before[1]], [0, 1], 'the fallback that answered went first');++ second.state.overpassFail = true;++ await clearOverpassCache();++ assert.strictEqual((await search()).status, 200, 'it fails: the others are still tried in order');++ second.state.overpassFail = false;++ });++await check('the first server is too slow: after its share of the budget the next one answers', async () => {await clearOverpassCache();first.state.overpassDelayMs = 5000;
@@ -46,6 +46,7 @@ const keyOf = (q) => String(q).trim().toLowerCase().replace(/\s+/g, ' ');
@@ -46,6 +46,7 @@ const keyOf = (q) => String(q).trim().toLowerCase().replace(/\s+/g, ' ');
function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console } = {}) {
function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console } = {}) {
const osm = config.osm;
const osm = config.osm;
const failed = new Map(); // area query key → when it last failed (this process)
const failed = new Map(); // area query key → when it last failed (this process)
+ let lastGood = null; // the Overpass server that answered last: asked first next time
const pacers = new Map();
const pacers = new Map();
const pacerFor = (rawUrl) => {
const pacerFor = (rawUrl) => {
const host = new URL(rawUrl).host;
const host = new URL(rawUrl).host;
@@ -108,12 +109,14 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
@@ -108,12 +109,14 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
/**
/**
* One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. The servers
* One bounded Overpass query → its elements. Cached by the query text, so the same area is asked once. The servers
* in osm.overpassUrls are tried in order, each for `tryMs` (the last one for what is left of `budgetMs`), so one
* in osm.overpassUrls are tried in order, each for `tryMs` (the last one for what is left of `budgetMs`), so one
- * overloaded server no longer holds the page for the whole budget. A query that just failed refuses at once for
+ * overloaded server no longer holds the page for the whole budget; the one that answered last goes first (on
+ * 2026-10-25 only the third answered, and a search spent 16 s on the first two before reaching it). A query that just failed refuses at once for
* osm.retryAfterMs. `force` (the background warmer) asks again even when a cached answer is still fresh.
* osm.retryAfterMs. `force` (the background warmer) asks again even when a cached answer is still fresh.
*/
*/
function overpass(query, { budgetMs = osm.timeoutMs, tryMs = osm.overpassTryMs, force = false } = {}) {
function overpass(query, { budgetMs = osm.timeoutMs, tryMs = osm.overpassTryMs, force = false } = {}) {
const body = `data=${encodeURIComponent(query)}`;
const body = `data=${encodeURIComponent(query)}`;
- const urls = osm.overpassUrls && osm.overpassUrls.length ? osm.overpassUrls : [osm.overpassUrl];
+ const all = osm.overpassUrls && osm.overpassUrls.length ? osm.overpassUrls : [osm.overpassUrl];
+ const urls = lastGood && all.includes(lastGood) ? [lastGood, ...all.filter((u) => u !== lastGood)] : all;
return cached(overpassKey(query), async () => {
return cached(overpassKey(query), async () => {
const started = now();
const started = now();
let tried = 0;
let tried = 0;
@@ -123,11 +126,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
@@ -123,11 +126,13 @@ function createUpstream({ config, s, fetchImpl = globalThis.fetch, now = () => D
const timeoutMs = i === urls.length - 1 ? left : Math.min(left, tryMs);
const timeoutMs = i === urls.length - 1 ? left : Math.min(left, tryMs);
tried++;
tried++;
try {
try {
- return await pacerFor(base).run(() => fetchJson(new URL(base), {
+ const answer = await pacerFor(base).run(() => fetchJson(new URL(base), {
method: 'POST',
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body,
body,
}, { timeoutMs }));
}, { timeoutMs }));
+ lastGood = base;
+ return answer;
} catch (err) {
} catch (err) {
log.warn && log.warn(`[OpenVibe.Food] Overpass ${new URL(base).host}: ${err.message}`);
log.warn && log.warn(`[OpenVibe.Food] Overpass ${new URL(base).host}: ${err.message}`);
}
}
@@ -36,6 +36,19 @@ const { startOsm } = require('./helpers/osm');
@@ -36,6 +36,19 @@ const { startOsm } = require('./helpers/osm');
first.state.overpassFail = false;
first.state.overpassFail = false;
});
});
+ await check('the server that answered last is asked first: the next search goes straight to it', async () => {
+ await clearOverpassCache();
+ first.state.overpassFail = false;
+ const before = [overpassCalls(first), overpassCalls(second)];
+ const r = await search();
+ assert.strictEqual(r.status, 200, r.text);
+ assert.deepStrictEqual([overpassCalls(first) - before[0], overpassCalls(second) - before[1]], [0, 1], 'the fallback that answered went first');
+ second.state.overpassFail = true;
+ await clearOverpassCache();
+ assert.strictEqual((await search()).status, 200, 'it fails: the others are still tried in order');
+ second.state.overpassFail = false;
+ });
+
await check('the first server is too slow: after its share of the budget the next one answers', async () => {
await check('the first server is too slow: after its share of the budget the next one answers', async () => {
await clearOverpassCache();
await clearOverpassCache();
first.state.overpassDelayMs = 5000;
first.state.overpassDelayMs = 5000;