openvibers/openvibe.tools Commits
T8 job 3: maps egress adapter (#5)
Changes
@@@ -26,10 +26,8 @@ const https = require('https');const path = require('path');// The public-address rule is openvibe-shared/egress (the one Live and Events use too). apps/_shared has--// no node_modules of its own: resolve it from either satellite that uses this adapter.--const shared = require(require.resolve('openvibe-shared/egress', {-- paths: [path.join(__dirname, '..', 'gateway'), path.join(__dirname, '..', 'maps')],--}));++// no node_modules of its own: resolve it from the gateway and maps, the apps that load this file.++const shared = require(require.resolve('openvibe-shared/egress', { paths: [path.join(__dirname, '..', 'gateway'), path.join(__dirname, '..', 'maps')] }));const { isPublicAddress, embeddedV4, normalizeHost, isInternalName } = shared;class TargetRefused extends Error {@@@ -158,12 +156,14 @@ function createEgress({/*** One HTTP(S) request to a user-chosen URL; redirects are NOT followed.* → Promise<{ url, status, statusText, headers, body, truncated }>. body is a Buffer ('' for HEAD-- * or when maxBytes is 0).++ * or when maxBytes is 0). `body` is the request body (string or Buffer, default none).*/-- async function request(rawUrl, { method = 'GET', headers = {}, timeoutMs = 12000, maxBytes = 0 } = {}) {++ async function request(rawUrl, { method = 'GET', headers = {}, timeoutMs = 12000, maxBytes = 0, body } = {}) {const url = parseUrl(rawUrl);const target = await resolve(url.hostname);const secure = url.protocol === 'https:';++ const outHeaders = body == null || Object.keys(headers).some(h => h.toLowerCase() === 'content-length')++ ? headers : { ...headers, 'Content-Length': Buffer.byteLength(body) };return new Promise((ok, fail) => {let settled = false;let deadline = null;@@@ -173,7 +173,7 @@ function createEgress({port: url.port || (secure ? 443 : 80),path: `${url.pathname}${url.search}`,method,-- headers,++ headers: outHeaders,agent: false,// Only the checked addresses: no second DNS answer is ever used.lookup: target.literal ? undefined : pinnedLookup(target.addresses),@@@ -204,7 +204,7 @@ function createEgress({deadline = setTimeout(timeout, timeoutMs);req.on('timeout', timeout);req.on('error', (err) => done(fail, err));-- req.end();++ req.end(body == null ? undefined : body);});}@@@ -103,6 +103,30 @@ const { createEgress, isPublicAddress, embeddedV4, TargetRefused } = require('..const head = await g3.request(`http://127.0.0.1:${port}/`, { method: 'HEAD' });assert.equal(head.status, 200); assert.equal(head.body.length, 0);++ // A request body is written verbatim, with its Content-Length.++ let sent = null;++ const { EventEmitter: EE } = require('events');++ const poster = createEgress({++ lookup: (h, o, cb) => cb(null, [{ address: '127.0.0.1', family: 4 }]),++ isAllowed: (a) => a === '127.0.0.1' || isPublicAddress(a),++ httpRequest: (opts, cb) => {++ const req = new EE();++ req.end = (b) => { sent = { headers: opts.headers, body: b }; };++ req.destroy = () => {};++ process.nextTick(() => {++ const res = new EE();++ res.statusCode = 200; res.statusMessage = 'OK'; res.headers = {}; res.destroy = () => {};++ cb(res);++ process.nextTick(() => res.emit('end'));++ });++ return req;++ },++ });++ const posted = await poster.request('http://site.example/post', { method: 'POST', body: 'data=x', maxBytes: 100 });++ assert.equal(posted.status, 200);++ assert.equal(sent.body, 'data=x');++ assert.equal(sent.headers['Content-Length'], 6);++// request() never follows; follow() re-checks each hop and refuses before dialling.const one = await g3.request(`http://127.0.0.1:${port}/to-private`);assert.equal(one.status, 302);@@@ -51,7 +51,10 @@ function mergeForm(files, job) {const merging = fetch(`${app.base}/api/v1/tools/mergepdf/run?wait_ms=60000`, { method: 'POST', body: mergeForm([big, big]), headers: { cookie } }).then(async r => ({ status: r.status, body: await r.json(), at: Date.now() }));const latencies = [];let done = null;-- merging.then(r => { done = r; });++ let sawBusy = false;++ // Settle `done` either way: an unhandled rejection here would end the process before the++ // real assertion failure is printed.++ merging.then(r => { done = r; }, err => { done = { error: err }; });await sleep(150); // the upload is in and pdf-lib is workingwhile (!done) {const s = Date.now();@@@ -59,9 +62,13 @@ function mergeForm(files, job) {assert.strictEqual(h.status, 200);const body = await h.json();latencies.push(Date.now() - s);-- if (!done && latencies.length === 3) assert.strictEqual(body.workers.busy, 1, 'the merge is in a worker thread');++ if (!done && body.workers.busy === 1) sawBusy = true;await sleep(40);}++ if (done.error) throw done.error;++ // On a loaded machine the upload can outlast the first few health calls, so look for the busy++ // worker at any point during the merge rather than at a fixed call.++ assert.ok(sawBusy, 'the merge is in a worker thread');const took = done.at - t0;assert.strictEqual(done.status, 200, JSON.stringify(done.body));assert.strictEqual(done.body.state, 'succeeded', JSON.stringify(done.body.error));@@@ -111,12 +111,12 @@ function rawGet(port, p, headers = {}) {if (e.isDirectory()) walk(f);else if (e.name.endsWith('.js')) {const src = fs.readFileSync(f, 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1');-- if (/(^|[^.\w])(fetch|fetchImpl)\(|\bhttps?\.(get|request)\(|\bmod\.(get|request)\(|new WebSocket\(|require\(['"](axios|got|node-fetch|undici)['"]\)/m.test(src)) found.push(path.relative(APPS, f));++ if (/(^|[^.\w])(fetch|fetchImpl)\(|\bhttps?\.(get|post|request)\(|\bmod\.(get|request)\(|new WebSocket\(|require\(['"](axios|got|node-fetch|undici)['"]\)/m.test(src)) found.push(path.relative(APPS, f));}}};for (const app of fs.readdirSync(APPS)) {-- if (app === '_shared') { for (const e of fs.readdirSync(path.join(APPS, '_shared'), { withFileTypes: true })) { if (e.isFile() && e.name.endsWith('.js')) { const f = path.join(APPS, '_shared', e.name); const src = fs.readFileSync(f, 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1'); if (/(^|[^.\w])(fetch|fetchImpl)\(|\bhttps?\.(get|request)\(|\bmod\.(get|request)\(|new WebSocket\(|require\(['"](axios|got|node-fetch|undici)['"]\)/m.test(src)) found.push(path.relative(APPS, f)); } else if (e.isDirectory() && !['node_modules', 'test'].includes(e.name)) walk(path.join(APPS, '_shared', e.name)); } continue; }++ if (app === '_shared') { for (const e of fs.readdirSync(path.join(APPS, '_shared'), { withFileTypes: true })) { if (e.isFile() && e.name.endsWith('.js')) { const f = path.join(APPS, '_shared', e.name); const src = fs.readFileSync(f, 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1'); if (/(^|[^.\w])(fetch|fetchImpl)\(|\bhttps?\.(get|post|request)\(|\bmod\.(get|request)\(|new WebSocket\(|require\(['"](axios|got|node-fetch|undici)['"]\)/m.test(src)) found.push(path.relative(APPS, f)); } else if (e.isDirectory() && !['node_modules', 'test'].includes(e.name)) walk(path.join(APPS, '_shared', e.name)); } continue; }if (fs.existsSync(path.join(APPS, app, 'server'))) walk(path.join(APPS, app, 'server'));}assert.ok(found.length >= 20, `the scan finds the known sites (${found.length})`);nnew file mode 100644@@@ -0,0 +1,151 @@++'use strict';++/**++ * The one door out of apps/maps. Every outbound HTTP request from this server goes through `get` /++ * `post` here, and this file reaches the network only through the shared SSRF guard++ * (apps/_shared/egress.js): the hostname is resolved, every answer must be a public address, and the++ * connection is pinned to the checked address. On top of that guard this adapter enforces:++ *++ * - HTTPS only, to the allowlisted source hosts below and nothing else. The check runs before++ * every request, redirect hops included (a redirect to a new host is re-checked), so a source++ * answering `Location: http://…` or a non-allowlisted name is refused before any connection.++ * - a per-host response-size cap (the number after each host). A body over the cap is refused as++ * tools.maps.response_too_large, whether the server sent it raw (truncated at the socket) or++ * gzip/deflate/brotli (the decompressor is capped at the same number).++ * - at most 3 redirects, GET only; a redirect in answer to a POST is refused.++ * - host and pathname only in errors: source API keys ride in query strings (NPS, OpenChargeMap)++ * and headers (RIDB), so a message that echoed the URL would leak a key into logs.++ *++ * Allowlist (host -> max response bytes):++ * nominatim.openstreetmap.org 256 KB — small JSON, a handful of geocode hits++ * overpass-api.de 32 MB — Overpass QL can answer a whole OSM extract++ * ridb.recreation.gov 2 MB — facility/recarea JSON lists++ * developer.nps.gov 8 MB — campground, visitor-centre and parking JSON++ * api.openchargemap.io 4 MB — charge-point POI pages++ * api.weather.gov 2 MB — points and active-alerts JSON++ * api.open-meteo.com 1 MB — forecast and elevation JSON++ * apps.fs.usda.gov 16 MB — ArcGIS USFS recreation features (GeoJSON)++ * services.arcgis.com 16 MB — ArcGIS National Bridge Inventory features++ * www.refugerestrooms.org 1 MB — restroom JSON, small++ * www.ioverlander.com 4 MB — scraped place pages (HTML)++ * freecampsites.net 8 MB — scraped WordPress pages (HTML)++ * www.campendium.com 2 MB — campground API JSON++ * edits.nationalmap.gov 2 MB — gazetteer search JSON++ * www.walmart.com 4 MB — scraped store-search page (HTML)++ */++const zlib = require('zlib');++const { createEgress } = require('../../_shared/egress');++++const KB = 1024;++++const ALLOW = Object.freeze({++ 'nominatim.openstreetmap.org': 256 * KB,++ 'overpass-api.de': 32 * KB * KB,++ 'ridb.recreation.gov': 2 * KB * KB,++ 'developer.nps.gov': 8 * KB * KB,++ 'api.openchargemap.io': 4 * KB * KB,++ 'api.weather.gov': 2 * KB * KB,++ 'api.open-meteo.com': 1 * KB * KB,++ 'apps.fs.usda.gov': 16 * KB * KB,++ 'services.arcgis.com': 16 * KB * KB,++ 'www.refugerestrooms.org': 1 * KB * KB,++ 'www.ioverlander.com': 4 * KB * KB,++ 'freecampsites.net': 8 * KB * KB,++ 'www.campendium.com': 2 * KB * KB,++ 'edits.nationalmap.gov': 2 * KB * KB,++ 'www.walmart.com': 4 * KB * KB,++});++++let egress = createEgress();++++/** Replace the guard; tests inject createEgress({ lookup, httpsRequest }) mocks. */++function setEgress(e) { egress = e; }++++class MapsEgressError extends Error {++ constructor(message, code) {++ super(message);++ this.name = 'MapsEgressError';++ this.code = code;++ this.status = 502;++ }++}++++const REDIRECTS = [301, 302, 303, 307, 308];++const tooBig = (host) => `${host} answered more than ${ALLOW[host]} bytes`;++++async function send(method, rawUrl, { params, headers = {}, timeout = 10000, body } = {}) {++ let url = new URL(rawUrl);++ if (params) {++ for (const [k, v] of Object.entries(params)) {++ if (v !== undefined && v !== null) url.searchParams.append(k, String(v));++ }++ }++++ // The caller's headers (an API key rides in RIDB's apikey) belong to the origin the caller asked++ // for; a redirect to another allowlisted host must not carry them across.++ const origin = url.origin;++++ for (let redirects = 0; ;) {++ const host = url.hostname;++ if (url.protocol !== 'https:' || !Object.prototype.hasOwnProperty.call(ALLOW, host)) {++ throw new MapsEgressError(`${host} is not an allowed maps source`, 'tools.maps.host_not_allowed');++ }++++ let r;++ try {++ r = await egress.request(url.href, {++ method,++ headers: { 'Accept-Encoding': 'identity', ...(url.origin === origin ? headers : {}) },++ timeoutMs: timeout,++ maxBytes: ALLOW[host],++ body,++ });++ } catch (err) {++ if (err && (err.name === 'TimeoutError' || err.message === 'Timeout')) {++ const e = new Error(`timeout of ${timeout}ms exceeded`);++ e.code = 'ECONNABORTED';++ throw e;++ }++ throw err;++ }++++ const location = r.headers.location;++ if (REDIRECTS.includes(r.status) && location) {++ if (method !== 'GET') throw new MapsEgressError(`${host}${url.pathname} redirected a POST`, 'tools.maps.redirect_refused');++ if (redirects >= 3) throw new MapsEgressError(`too many redirects from ${host}`, 'tools.maps.too_many_redirects');++ redirects++;++ url = new URL(location, url);++ continue;++ }++++ if (r.truncated === true) throw new MapsEgressError(tooBig(host), 'tools.maps.response_too_large');++++ let buf = r.body;++ const encoding = String(r.headers['content-encoding'] || '').toLowerCase();++ if (encoding === 'gzip' || encoding === 'deflate' || encoding === 'br') {++ try {++ const opts = { maxOutputLength: ALLOW[host] };++ buf = encoding === 'gzip' ? zlib.gunzipSync(buf, opts)++ : encoding === 'deflate' ? zlib.inflateSync(buf, opts)++ : zlib.brotliDecompressSync(buf, opts);++ } catch (err) {++ if (err instanceof RangeError) throw new MapsEgressError(tooBig(host), 'tools.maps.response_too_large');++ throw err;++ }++ }++++ let data = buf.toString('utf8');++ try { data = JSON.parse(data); } catch { /* not JSON: keep the string, cheerio callers want HTML */ }++++ if (r.status < 200 || r.status > 299) {++ const err = new Error(`Request failed with status code ${r.status}`);++ err.response = { status: r.status, headers: r.headers, data };++ throw err;++ }++ return { status: r.status, headers: r.headers, data };++ }++}++++const get = (url, opts) => send('GET', url, opts);++const post = (url, body, opts) => send('POST', url, { ...opts, body });++++module.exports = { get, post, ALLOW, setEgress, MapsEgressError };@@@ -16,6 +16,7 @@ const NodeCache = require('node-cache');const cookieParser = require('cookie-parser');const config = require('./config');++const http = require('./egress');// ── Analytics ────────────────────────────────────────────────const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recorded@@@ -164,8 +165,7 @@ app.get('/api/geocode', async (req, res) => {if (cached) return res.json(cached);try {-- const axios = require('axios');-- const resp = await nominatim.run(() => axios.get('https://nominatim.openstreetmap.org/search', {++ const resp = await nominatim.run(() => http.get('https://nominatim.openstreetmap.org/search', {params: { q, format: 'json', limit: 5, countrycodes: 'us,ca,mx' },headers: { 'User-Agent': 'Maps.OpenVibe/1.0 (maps.openvibe.tools)' },timeout: 8000,@@@ -11,7 +11,7 @@** Combined → deduplicated, enriched with distance/walking time, scored for stealth-camper utility.*/--const axios = require('axios');++const http = require('../egress');const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');// ─── Constants ─────────────────────────────────────────────────────@@@ -105,7 +105,7 @@ async function fetchRefugeRestrooms(lat, lon, perPage = 30) {for (let attempt = 0; attempt < maxAttempts; attempt++) {try {const timeout = attempt === 0 ? 15000 : 25000;-- const resp = await axios.get(REFUGE_API, {++ const resp = await http.get(REFUGE_API, {params: { lat, lng: lon, per_page: perPage },timeout,headers: { 'Accept': 'application/json' },@@@ -16,7 +16,7 @@* - Finer-grained local bridge data with name, surface, layer info* - Can determine what a bridge crosses by checking nearby waterways*/--const axios = require('axios');++const http = require('../egress');const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');// ─── Constants ─────────────────────────────────────────────────────@@@ -177,7 +177,7 @@ async function fetchNbiBridges(lat, lon, radiusMiles = 5) {f: 'json',};-- const resp = await axios.get(NBI_API, { params, timeout: 15000 });++ const resp = await http.get(NBI_API, { params, timeout: 15000 });if (!resp.data || !resp.data.features) return [];return resp.data.features.map(f => {@@@ -5,7 +5,7 @@** We fetch their API-like endpoints that serve map data.*/--const axios = require('axios');++const http = require('../egress');const cheerio = require('cheerio');const { haversine } = require('./utils');@@@ -23,7 +23,7 @@ async function search(lat, lon, radiusMiles) {try {// Try the WordPress REST API that many WP sites expose-- const { data } = await axios.get(`${BASE_URL}/wp-json/wp/v2/posts`, {++ const { data } = await http.get(`${BASE_URL}/wp-json/wp/v2/posts`, {params: {per_page: 50,search: 'camping',@@@ -68,7 +68,7 @@ async function search(lat, lon, radiusMiles) {// Also try scraping their map page for WA state campsite markerstry {-- const { data: html } = await axios.get(`${BASE_URL}/?s=washington+camping`, {++ const { data: html } = await http.get(`${BASE_URL}/?s=washington+camping`, {headers: {'User-Agent': 'WA-StealthCampLocator/1.0 (research-project)',},@@@ -12,7 +12,7 @@* - Fred Meyer / Kroger (API + built-in pricing)*/--const axios = require('axios');++const http = require('../egress');const cheerio = require('cheerio');const { haversine } = require('./utils');@@@ -821,7 +821,7 @@ async function findNearbyStores(lat, lon, radiusMeters = 8000) {const query = `[out:json][timeout:15];(\n${filters}\n);out body;`;try {-- const resp = await axios.post('https://overpass-api.de/api/interpreter', `data=${encodeURIComponent(query)}`, {++ const resp = await http.post('https://overpass-api.de/api/interpreter', `data=${encodeURIComponent(query)}`, {headers: { 'Content-Type': 'application/x-www-form-urlencoded' },timeout: 15000,});@@@ -894,7 +894,7 @@ out skel qt;`;try {-- const resp = await axios.post('https://overpass-api.de/api/interpreter',++ const resp = await http.post('https://overpass-api.de/api/interpreter',`data=${encodeURIComponent(query)}`,{ headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, timeout: 30000 });@@@ -985,7 +985,7 @@ out skel qt;async function scrapeWalmartPrices(searchTerm) {try {const url = `https://www.walmart.com/search?q=${encodeURIComponent(searchTerm)}`;-- const resp = await axios.get(url, {++ const resp = await http.get(url, {headers: {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',@@@ -3,7 +3,7 @@* Scrapes camping/overlanding spots from iOverlander.com for Washington State.* iOverlander has a web interface with location data we can extract.*/--const axios = require('axios');++const http = require('../egress');const cheerio = require('cheerio');const { haversine } = require('./utils');@@@ -45,7 +45,7 @@ async function search(lat, lon, radiusMiles = 25) {// Try iOverlander's search pageconst url = `https://www.ioverlander.com/places?lat=${lat}&lng=${lon}&zoom=10`;-- const resp = await axios.get(url, {++ const resp = await http.get(url, {headers: { 'User-Agent': UA },timeout: 12000,});@@@ -9,7 +9,7 @@* Free API key required: https://www.nps.gov/subjects/developer/get-started.htm*/--const axios = require('axios');++const http = require('../egress');const { haversine } = require('./utils');const BASE_URL = 'https://developer.nps.gov/api/v1';@@@ -73,7 +73,7 @@ async function searchCampgrounds(lat, lon, radiusMiles, apiKey) {stateCode,};-- const resp = await axios.get(`${BASE_URL}/campgrounds`, {++ const resp = await http.get(`${BASE_URL}/campgrounds`, {params,timeout: 15000,headers: { 'User-Agent': 'OpenVibeApp/2.0' },@@@ -145,7 +145,7 @@ async function searchVisitorCenters(lat, lon, radiusMiles, apiKey) {if (!apiKey) return [];try {-- const resp = await axios.get(`${BASE_URL}/visitorcenters`, {++ const resp = await http.get(`${BASE_URL}/visitorcenters`, {params: { api_key: apiKey, limit: 100, stateCode: getNearbyCodes(lat, lon, radiusMiles) || 'WA' },timeout: 10000,headers: { 'User-Agent': 'OpenVibeApp/2.0' },@@@ -197,7 +197,7 @@ async function searchParkingLots(lat, lon, radiusMiles, apiKey) {if (!apiKey) return [];try {-- const resp = await axios.get(`${BASE_URL}/parkinglots`, {++ const resp = await http.get(`${BASE_URL}/parkinglots`, {params: { api_key: apiKey, limit: 100, stateCode: getNearbyCodes(lat, lon, radiusMiles) || 'WA' },timeout: 10000,headers: { 'User-Agent': 'OpenVibeApp/2.0' },@@@ -8,11 +8,10 @@* Free & open — works without key but key gives higher rate limits*/--const { createEgress } = require('../../../_shared/egress');++const http = require('../egress');const { haversine } = require('./utils');const BASE_URL = 'https://api.openchargemap.io/v3/poi';--const egress = createEgress();/*** Search for EV charging stations near a location@@@ -33,19 +32,15 @@ async function search(lat, lon, radiusMiles, apiKey) {if (apiKey) params.key = apiKey;-- const url = new URL(BASE_URL);-- for (const [name, value] of Object.entries(params)) url.searchParams.set(name, value);-- const resp = await egress.follow(url.href, {-- timeoutMs: 12000,-- maxBytes: 2 * 1024 * 1024,-- headers: { 'User-Agent': 'OpenVibeApp/2.0', Accept: 'application/json' },++ const resp = await http.get(BASE_URL, {++ params,++ timeout: 12000,++ headers: { 'User-Agent': 'OpenVibeApp/2.0' },});-- if (resp.status < 200 || resp.status >= 300) throw new Error(`HTTP ${resp.status}`);-- if (resp.truncated) throw new Error('Response too large');-- const data = JSON.parse(resp.body.toString('utf8'));-- if (!Array.isArray(data)) return [];-- return data++ if (!Array.isArray(resp.data)) return [];++++ return resp.data.filter(s => s.AddressInfo?.Latitude && s.AddressInfo?.Longitude).map(s => {const addr = s.AddressInfo;@@@ -11,7 +11,6 @@** Overpass API: https://overpass-api.de/api/interpreter*/--const axios = require('axios');const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');/**@@@ -11,7 +11,6 @@** All data sourced from OpenStreetMap via Overpass API.*/--const axios = require('axios');const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');// ═══════════════════════════════════════════════════════════════════@@@ -6,7 +6,7 @@* NOTE: RIDB requires an API key; the caller passes it in (server/config.js, RIDB_API_KEY).* Register at https://ridb.recreation.gov/ for one. Without it RIDB is skipped.*/--const axios = require('axios');++const http = require('../egress');const { haversine } = require('./utils');// The RIDB key is passed in by the caller (server/config.js); never a key in the code.@@@ -33,7 +33,7 @@ async function search(lat, lon, radiusMiles, apiKey) {try {// Search facilities by lat/lon with radius-- const { data } = await axios.get(`${BASE_URL}/facilities`, {++ const { data } = await http.get(`${BASE_URL}/facilities`, {params: {latitude: lat,longitude: lon,@@@ -89,7 +89,7 @@ async function search(lat, lon, radiusMiles, apiKey) {// Also try RecAreas (larger tracts like national forests)if (_ridbDisabled) return results;try {-- const { data } = await axios.get(`${BASE_URL}/recareas`, {++ const { data } = await http.get(`${BASE_URL}/recareas`, {params: {latitude: lat,longitude: lon,@@@ -144,7 +144,7 @@ async function search(lat, lon, radiusMiles, apiKey) {* Get detailed facility info from RIDB*/async function getDetail(facilityId, apiKey) {-- const { data } = await axios.get(`${BASE_URL}/facilities/${facilityId}`, {++ const { data } = await http.get(`${BASE_URL}/facilities/${facilityId}`, {headers: { apikey: apiKey, Accept: 'application/json' },timeout: 10000,});@@@ -12,7 +12,7 @@* - USGS Waterfall database*/--const axios = require('axios');++const http = require('../egress');const cheerio = require('cheerio');const { haversine, overpassQuery } = require('./utils');@@@ -27,7 +27,7 @@ const HEADERS = {*/async function scrapeCampendium(lat, lon, radiusMiles) {try {-- const resp = await axios.get('https://www.campendium.com/api/v2/campgrounds', {++ const resp = await http.get('https://www.campendium.com/api/v2/campgrounds', {params: {lat,lng: lon,@@@ -91,7 +91,7 @@ async function scrapeCampendium(lat, lon, radiusMiles) {async function scrapeUSGSFeatures(lat, lon, radiusMiles) {try {// GNIS Feature Search API-- const resp = await axios.get('https://edits.nationalmap.gov/apps/gaz-domestic/api/search', {++ const resp = await http.get('https://edits.nationalmap.gov/apps/gaz-domestic/api/search', {params: {lat,lon,@@@ -5,7 +5,7 @@* - OpenStreetMap land use data via Overpass* - Slope/terrain classification*/--const axios = require('axios');++const http = require('../egress');/*** Get elevation data for a point.@@@ -13,7 +13,7 @@ const axios = require('axios');async function getElevation(lat, lon) {try {const url = `https://api.open-meteo.com/v1/elevation?latitude=${lat}&longitude=${lon}`;-- const resp = await axios.get(url, { timeout: 5000 });++ const resp = await http.get(url, { timeout: 5000 });return resp.data?.elevation?.[0] ?? null;} catch (e) {return null;@@@ -29,7 +29,7 @@ async function getBulkElevation(coords) {const lons = coords.map(c => c.lon).join(',');try {const url = `https://api.open-meteo.com/v1/elevation?latitude=${lats}&longitude=${lons}`;-- const resp = await axios.get(url, { timeout: 8000 });++ const resp = await http.get(url, { timeout: 8000 });return resp.data?.elevation || [];} catch (e) {return coords.map(() => null);@@@ -62,7 +62,7 @@ async function analyzeLandUse(lat, lon, radiusMeters = 500) {);out tags;`;-- const resp = await axios.post('https://overpass-api.de/api/interpreter', `data=${encodeURIComponent(query)}`, {++ const resp = await http.post('https://overpass-api.de/api/interpreter', `data=${encodeURIComponent(query)}`, {headers: { 'Content-Type': 'application/x-www-form-urlencoded' },timeout: 10000,});@@@ -12,7 +12,7 @@* OPERATIONAL_HOURS, FORESTNAME, RECAREAURL, OPENSTATUS,* RESERVATION_INFO, RESTRICTIONS, ACCESSIBILITY*/--const axios = require('axios');++const http = require('../egress');const { haversine } = require('./utils');const USFS_API = 'https://apps.fs.usda.gov/arcx/rest/services/EDW/EDW_RecreationOpportunities_01/MapServer/0/query';@@@ -109,7 +109,7 @@ async function search(lat, lon, radiusMiles = 25) {resultRecordCount: '200',});-- const { data } = await axios.get(`${USFS_API}?${params.toString()}`, {++ const { data } = await http.get(`${USFS_API}?${params.toString()}`, {timeout: 30000,headers: {'User-Agent': 'OpenVibeApp/2.0 (stealth camping tool)',@@@ -3,7 +3,7 @@* Common functions used across multiple modules.*/--const axios = require('axios');++const http = require('../egress');const OVERPASS_URL = 'https://overpass-api.de/api/interpreter';@@@ -29,7 +29,7 @@ async function _processOverpassQueue() {async function _runOverpassJob({ query, timeout, resolve, reject, attempt = 0 }) {try {-- const resp = await axios.post(OVERPASS_URL, `data=${encodeURIComponent(query)}`, {++ const resp = await http.post(OVERPASS_URL, `data=${encodeURIComponent(query)}`, {headers: { 'Content-Type': 'application/x-www-form-urlencoded' },timeout,});@@@ -5,7 +5,7 @@** Returns unified: { current, daily[], hourly[], alerts[], sun, moon }*/--const axios = require('axios');++const http = require('../egress');const UA = 'GhostCamp-WA/2.0 (stealth-camping-locator)';@@@ -82,7 +82,7 @@ async function getWeather(lat, lon) {'forecast_hours=48',].join('&');-- const resp = await axios.get(`https://api.open-meteo.com/v1/forecast?${params}`, { timeout: 10000 });++ const resp = await http.get(`https://api.open-meteo.com/v1/forecast?${params}`, { timeout: 10000 });const d = resp.data;// Current conditions@@@ -176,14 +176,14 @@ async function getWeather(lat, lon) {// ─── NWS Alerts only ───try {-- const ptResp = await axios.get(`https://api.weather.gov/points/${lat},${lon}`, {++ const ptResp = await http.get(`https://api.weather.gov/points/${lat},${lon}`, {headers: { 'User-Agent': UA, Accept: 'application/geo+json' },timeout: 5000,});const zone = ptResp.data?.properties?.forecastZone;if (zone) {const zoneId = zone.split('/').pop();-- const alertResp = await axios.get(`https://api.weather.gov/alerts/active?zone=${zoneId}`, {++ const alertResp = await http.get(`https://api.weather.gov/alerts/active?zone=${zoneId}`, {headers: { 'User-Agent': UA },timeout: 5000,});nnew file mode 100644@@@ -0,0 +1,194 @@++'use strict';++// The maps egress adapter (apps/maps/server/egress.js): one allowlisted HTTPS door for every++// outbound request. Every transport is a mock — no DNS, no socket — so any real network use would++// fail the test. Covers the allowlist and redirect hops, the pinned-DNS rule from the shared guard,++// size caps, key handling, and the grep guard that no other maps file reaches the network itself.++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const { EventEmitter } = require('events');++const { createEgress, TargetRefused } = require('../../_shared/egress');++++const maps = require('../server/egress');++const { setEgress, MapsEgressError } = maps;++++const PUBLIC = '93.184.216.34';++const RIDB_KEY = 'ridb-sentinel-key';++const NPS_KEY = 'nps-sentinel-key';++const OCM_KEY = 'ocm-sentinel-key';++++// A guard whose httpsRequest records the request and answers with `respond(rec)` (or a fixed answer).++function makeMock(respond) {++ const seen = [];++ const lookups = [];++ const guard = createEgress({++ lookup: (host, opts, cb) => { lookups.push(host); cb(null, [{ address: PUBLIC, family: 4 }]); },++ httpsRequest: (opts, cb) => {++ const rec = { method: opts.method, hostname: opts.hostname, path: opts.path, headers: opts.headers, body: null };++ seen.push(rec);++ const req = new EventEmitter();++ req.destroy = () => {};++ req.end = (b) => {++ rec.body = b;++ process.nextTick(() => {++ const answer = (typeof respond === 'function' ? respond(rec) : respond) || {};++ const res = new EventEmitter();++ res.statusCode = answer.status || 200;++ res.statusMessage = 'OK';++ res.headers = answer.headers || {};++ res.destroy = () => {};++ cb(res);++ process.nextTick(() => {++ res.emit('data', Buffer.from(answer.body != null ? answer.body : '{}'));++ res.emit('end');++ });++ });++ };++ return req;++ },++ });++ return { guard, seen, lookups };++}++++(async () => {++ let failures = 0;++ const check = async (name, fn) => {++ try { await fn(); console.log(` ✓ ${name}`); }++ catch (e) { failures++; console.log(` ✗ ${name}\n ${String(e.stack || e.message).split('\n').slice(0, 8).join('\n ')}`); }++ };++++ const ridb = require('../server/sources/ridb');++ const nps = require('../server/sources/nps');++ const ocm = require('../server/sources/openchargemap');++ const terrain = require('../server/sources/terrain');++ const { overpassQuery } = require('../server/sources/utils');++++ await check('injected RIDB, NPS and OpenChargeMap keys reach their requests and never a log', async () => {++ const mock = makeMock({ status: 200, body: '{"data":[],"RECDATA":[]}' });++ setEgress(mock.guard);++ const logs = [];++ const real = { log: console.log, warn: console.warn, error: console.error };++ console.log = console.warn = console.error = (...a) => logs.push(a.map(String).join(' '));++ let results;++ try {++ const r1 = await ridb.search(47.6062, -122.3321, 25, RIDB_KEY);++ const r2 = await nps.search(47.6062, -122.3321, 25, NPS_KEY);++ const r3 = await ocm.search(47.6062, -122.3321, 25, OCM_KEY);++ results = JSON.stringify([r1, r2, r3]);++ } finally { Object.assign(console, real); }++++ const ridbReqs = mock.seen.filter(r => r.hostname === 'ridb.recreation.gov');++ assert.ok(ridbReqs.length > 0, 'RIDB sent a request');++ for (const r of ridbReqs) assert.strictEqual(r.headers.apikey, RIDB_KEY, 'RIDB key is the apikey header');++ assert.ok(mock.seen.some(r => r.hostname === 'developer.nps.gov' && r.path.includes(`api_key=${NPS_KEY}`)), 'NPS key is the api_key query param');++ assert.ok(mock.seen.some(r => r.hostname === 'api.openchargemap.io' && r.path.includes(`key=${OCM_KEY}`)), 'OpenChargeMap key is the key query param');++ const all = results + '\n' + logs.join('\n');++ for (const key of [RIDB_KEY, NPS_KEY, OCM_KEY]) assert.ok(!all.includes(key), `${key} leaked into results or logs`);++ });++++ await check('with an empty key RIDB and NPS send nothing and still resolve', async () => {++ const mock = makeMock({ status: 200, body: '{"data":[],"RECDATA":[]}' });++ setEgress(mock.guard);++ const r1 = await ridb.search(47.6062, -122.3321, 25, '');++ const r2 = await nps.search(47.6062, -122.3321, 25, '');++ assert.deepStrictEqual(mock.seen.length, 0, 'no request was sent');++ for (const r of [r1, r2]) assert.deepStrictEqual(r, [], 'search resolves empty');++ });++++ await check('a redirect to a non-allowlisted host is refused before any DNS lookup', async () => {++ const mock = makeMock(() => ({ status: 302, headers: { location: 'https://evil.example/' } }));++ setEgress(mock.guard);++ await assert.rejects(maps.get('https://api.weather.gov/evil'),++ (e) => e instanceof MapsEgressError && e.code === 'tools.maps.host_not_allowed');++ assert.ok(!mock.lookups.includes('evil.example'), 'the refused host was never resolved');++ });++++ await check('a redirect that drops to http is refused', async () => {++ const mock = makeMock(() => ({ status: 302, headers: { location: 'http://api.weather.gov/plain' } }));++ setEgress(mock.guard);++ await assert.rejects(maps.get('https://api.weather.gov/start'),++ (e) => e instanceof MapsEgressError && e.code === 'tools.maps.host_not_allowed');++ });++++ await check('a fourth redirect between allowlisted hosts is refused', async () => {++ const mock = makeMock(() => ({ status: 302, headers: { location: 'https://api.weather.gov/next' } }));++ setEgress(mock.guard);++ await assert.rejects(maps.get('https://api.weather.gov/start'),++ (e) => e instanceof MapsEgressError && e.code === 'tools.maps.too_many_redirects');++ });++++ await check('a redirect to another allowlisted host drops the caller headers', async () => {++ const mock = makeMock((rec) => rec.hostname === 'ridb.recreation.gov'++ ? { status: 302, headers: { location: 'https://api.weather.gov/next' } }++ : { status: 200, body: '{}' });++ setEgress(mock.guard);++ await maps.get('https://ridb.recreation.gov/api/v1/facilities', { headers: { apikey: 'k' } });++ assert.strictEqual(mock.seen.length, 2, 'both hops were made');++ assert.strictEqual(mock.seen[0].headers.apikey, 'k', 'the key goes to the host the caller asked for');++ assert.strictEqual(mock.seen[1].hostname, 'api.weather.gov');++ assert.strictEqual(mock.seen[1].headers.apikey, undefined, 'the key is not resent cross-origin');++ assert.strictEqual(mock.seen[1].headers['Accept-Encoding'], 'identity');++ });++++ await check('a same-origin redirect keeps the caller headers', async () => {++ const mock = makeMock((rec) => rec.path.endsWith('/start')++ ? { status: 302, headers: { location: 'https://ridb.recreation.gov/api/v1/facilities' } }++ : { status: 200, body: '{}' });++ setEgress(mock.guard);++ await maps.get('https://ridb.recreation.gov/api/v1/start', { headers: { apikey: 'k' } });++ assert.strictEqual(mock.seen.length, 2, 'both hops were made');++ assert.strictEqual(mock.seen[1].headers.apikey, 'k', 'a same-origin redirect keeps the key');++ });++++ await check('a host resolving to a private address is refused by the pinned guard', async () => {++ setEgress(createEgress({ lookup: (h, o, cb) => cb(null, [{ address: '10.0.0.5', family: 4 }]) }));++ await assert.rejects(maps.get('https://api.weather.gov/points/47,-122'), (e) => e instanceof TargetRefused);++ });++++ await check('an oversized answer is refused and the terrain source falls back to null', async () => {++ const mock = makeMock({ status: 200, body: 'x'.repeat(1.5 * 1024 * 1024) });++ setEgress(mock.guard);++ await assert.rejects(maps.get('https://api.open-meteo.com/v1/elevation?latitude=47&longitude=-122'),++ (e) => e instanceof MapsEgressError && e.code === 'tools.maps.response_too_large');++ assert.strictEqual(await terrain.getElevation(47, -122), null);++ });++++ await check('overpassQuery POSTs the encoded QL as a form body', async () => {++ const mock = makeMock({ status: 200, body: '{"elements":[]}' });++ setEgress(mock.guard);++ await overpassQuery('node(1);');++ const rec = mock.seen.find(r => r.hostname === 'overpass-api.de');++ assert.ok(rec, 'Overpass was asked');++ assert.strictEqual(rec.method, 'POST');++ assert.strictEqual(rec.body, `data=${encodeURIComponent('node(1);')}`);++ assert.strictEqual(rec.headers['Content-Type'], 'application/x-www-form-urlencoded');++ });++++ await check('a non-2xx answer surfaces err.response.status', async () => {++ const mock = makeMock({ status: 429, body: '{"error":"slow down"}' });++ setEgress(mock.guard);++ await assert.rejects(maps.get('https://api.weather.gov/alerts/active'),++ (e) => e.response && e.response.status === 429);++ });++++ await check('no maps server file outside egress.js reaches the network itself', async () => {++ const serverDir = path.join(__dirname, '..', 'server');++ const moduleRequire = /require\(\s*['"](axios|node-fetch|got|undici|node:https?|https?)['"]\s*\)/;++ const bad = [];++ (function walk(dir) {++ for (const e of fs.readdirSync(dir, { withFileTypes: true })) {++ const p = path.join(dir, e.name);++ if (e.isDirectory()) walk(p);++ else if (e.name.endsWith('.js') && e.name !== 'egress.js') {++ const src = fs.readFileSync(p, 'utf8');++ if (moduleRequire.test(src) || /\bfetch\s*\(/.test(src) || /\baxios\b/.test(src)) bad.push(path.relative(serverDir, p));++ }++ }++ })(serverDir);++ assert.deepStrictEqual(bad, [], `files still reaching the network: ${bad.join(', ')}`);++ });++++ if (failures) { console.log(`\n${failures} failure(s)`); process.exit(1); }++ console.log('\nmaps egress adapter: all checks passed');++ process.exit(0);++})().catch((e) => { console.error(e); process.exit(1); });@@@ -1,63 +1,88 @@'use strict';++// OpenChargeMap leaves the process only through the maps egress adapter++// (apps/maps/server/egress.js): the request carries the configured key, a key-less source still++// works (free tier), and a failed answer resolves empty instead of throwing. Every transport is a++// mock — no DNS, no socket — so any real network use fails the test.const assert = require('assert');--const Module = require('module');++const { EventEmitter } = require('events');++const { createEgress } = require('../../_shared/egress');++const maps = require('../server/egress');++const { setEgress } = maps;++const openchargemap = require('../server/sources/openchargemap');++++const PUBLIC = '93.184.216.34';const requests = [];--let response = { status: 200, truncated: false, body: Buffer.from('[]') };--const egress = {-- follow: async (url, options) => {-- requests.push({ url, options });-- return response;-- },--};----// Keep the upstream offline. utils loads axios for other sources, but this source must use egress.--const originalLoad = Module._load;--Module._load = function (request, parent, isMain) {-- if (request === '../../../_shared/egress') return { createEgress: () => egress };-- if (request === 'axios') return {};-- return originalLoad.call(this, request, parent, isMain);--};--let openchargemap;--try {-- openchargemap = require('../server/sources/openchargemap');--} finally {-- Module._load = originalLoad;++let answer = { status: 200, body: '[]' };++++function install() {++ const guard = createEgress({++ lookup: (host, opts, cb) => cb(null, [{ address: PUBLIC, family: 4 }]),++ httpsRequest: (opts, cb) => {++ const rec = { method: opts.method, hostname: opts.hostname, path: opts.path, headers: opts.headers };++ requests.push(rec);++ const req = new EventEmitter();++ req.destroy = () => {};++ req.end = () => {++ process.nextTick(() => {++ const res = new EventEmitter();++ res.statusCode = answer.status;++ res.statusMessage = 'OK';++ res.headers = {};++ res.destroy = () => {};++ cb(res);++ process.nextTick(() => {++ res.emit('data', Buffer.from(answer.body));++ res.emit('end');++ });++ });++ };++ return req;++ },++ });++ setEgress(guard);}(async () => {++ install();const station = {ID: 42,AddressInfo: { Title: 'Test charger', Latitude: 47.61, Longitude: -122.33 },UsageCost: 'Free',};-- response = { status: 200, truncated: false, body: Buffer.from(JSON.stringify([station])) };const key = 'sentinel-ocm-key-not-a-secret';++++ answer = { status: 200, body: JSON.stringify([station]) };const configured = await openchargemap.search(47.61, -122.33, 25, key);-- assert.equal(requests.length, 1);-- const url = new URL(requests[0].url);-- assert.equal(url.origin, 'https://api.openchargemap.io');++ assert.equal(requests.length, 1, 'one request');++ const req = requests[0];++ assert.equal(req.method, 'GET');++ assert.equal(req.hostname, 'api.openchargemap.io');++ const url = new URL(req.path, 'https://api.openchargemap.io');assert.equal(url.pathname, '/v3/poi');assert.equal(url.searchParams.get('key'), key);assert.equal(url.searchParams.get('latitude'), '47.61');assert.equal(url.searchParams.get('longitude'), '-122.33');assert.equal(url.searchParams.get('distance'), '25');assert.equal(url.searchParams.get('maxresults'), '50');-- assert.equal(requests[0].options.timeoutMs, 12000);-- assert.ok(requests[0].options.maxBytes > 0);-- assert.equal(requests[0].options.headers['User-Agent'], 'OpenVibeApp/2.0');++ assert.equal(req.headers['User-Agent'], 'OpenVibeApp/2.0');++ assert.ok(req.headers['Accept-Encoding'], 'the adapter asks for identity encoding');assert.equal(configured[0].id, 'ocm-42');assert.ok(!JSON.stringify(configured).includes(key));++ // Free tier: no key configured still sends (without a key) and still maps.requests.length = 0;const unconfigured = await openchargemap.search(47.61, -122.33, 25, '');assert.equal(requests.length, 1);-- assert.equal(new URL(requests[0].url).searchParams.has('key'), false);++ assert.equal(new URL(requests[0].path, 'https://api.openchargemap.io').searchParams.has('key'), false);assert.equal(unconfigured[0].id, 'ocm-42');-- response = { status: 503, truncated: false, body: Buffer.from('[]') };-- assert.deepEqual(await openchargemap.search(47.61, -122.33, 25, key), []);-- response = { status: 200, truncated: true, body: Buffer.from('[]') };-- assert.deepEqual(await openchargemap.search(47.61, -122.33, 25, key), []);-- console.log('OpenChargeMap egress: configured, unconfigured, and failed responses passed');++ // A failed upstream resolves empty: the adapter throws, the source catches.++ for (const status of [401, 503]) {++ requests.length = 0;++ answer = { status, body: '[]' };++ assert.deepEqual(await openchargemap.search(47.61, -122.33, 25, key), []);++ }++++ console.log('OpenChargeMap egress: configured, key-less and failed responses passed');})().catch(err => { console.error(err); process.exitCode = 1; });@@@ -1,7 +1,7 @@'use strict';// Provider keys live in the satellite's config (apps/maps/server/config.js) and are handed to each// source — never read from the environment inside a source module, never a key in the code.--// Here the transport the sources use (axios) is stubbed, so nothing touches the network:++// The transport is mocked (setEgress), so nothing touches the network:// 1. the RIDB key the caller passes reaches the RIDB request,// 2. that key never appears in the search results a caller gets back,// 3. with no RIDB key, RIDB sends nothing and search still answers (an empty list),@@@ -9,16 +9,32 @@const assert = require('assert');const fs = require('fs');const path = require('path');++const { EventEmitter } = require('events');++const { createEgress } = require('../../_shared/egress');++const { setEgress } = require('../server/egress');--// Stub axios before the source module is loaded: both hold the same cached module object.--const axios = require('axios');++const RIDB_KEY = 'sentinel-ridb-key-not-a-secret';++++// A guard whose transport records every request and answers 200 with a small JSON body.const requests = [];--axios.get = async (url, opts = {}) => { requests.push({ url, opts }); return { data: {} }; };++setEgress(createEgress({++ lookup: (host, opts, cb) => cb(null, [{ address: '93.184.216.34', family: 4 }]),++ httpsRequest: (opts, cb) => {++ requests.push({ hostname: opts.hostname, path: opts.path, opts: { headers: opts.headers } });++ const req = new EventEmitter();++ req.destroy = () => {};++ req.end = () => process.nextTick(() => {++ const res = new EventEmitter();++ res.statusCode = 200; res.statusMessage = 'OK'; res.headers = {}; res.destroy = () => {};++ cb(res);++ process.nextTick(() => { res.emit('data', Buffer.from('{"RECDATA":[]}')); res.emit('end'); });++ });++ return req;++ },++}));const ridb = require('../server/sources/ridb');--const RIDB_KEY = 'sentinel-ridb-key-not-a-secret';--(async () => {let failures = 0;const check = async (name, fn) => {@@@ -31,8 +47,8 @@ const RIDB_KEY = 'sentinel-ridb-key-not-a-secret';await ridb.search(47.6062, -122.3321, 25, RIDB_KEY);assert.ok(requests.length > 0, 'RIDB sent a request');for (const r of requests) {-- assert.ok(/recreation\.gov/.test(r.url), `request goes to RIDB: ${r.url}`);-- assert.strictEqual(r.opts.headers && r.opts.headers.apikey, RIDB_KEY, `apikey header on ${r.url}`);++ assert.ok(/recreation\.gov/.test(`https://${r.hostname}${r.path}`), `request goes to RIDB: ${r.hostname}${r.path}`);++ assert.strictEqual(r.opts.headers && r.opts.headers.apikey, RIDB_KEY, `apikey header on ${r.hostname}${r.path}`);}});
@@ -26,10 +26,8 @@ const https = require('https');
@@ -26,10 +26,8 @@ const https = require('https');
const path = require('path');
const path = require('path');
// The public-address rule is openvibe-shared/egress (the one Live and Events use too). apps/_shared has
// The public-address rule is openvibe-shared/egress (the one Live and Events use too). apps/_shared has
-// no node_modules of its own: resolve it from either satellite that uses this adapter.
-const shared = require(require.resolve('openvibe-shared/egress', {
- paths: [path.join(__dirname, '..', 'gateway'), path.join(__dirname, '..', 'maps')],
-}));
+// no node_modules of its own: resolve it from the gateway and maps, the apps that load this file.
+const shared = require(require.resolve('openvibe-shared/egress', { paths: [path.join(__dirname, '..', 'gateway'), path.join(__dirname, '..', 'maps')] }));
const { isPublicAddress, embeddedV4, normalizeHost, isInternalName } = shared;
const { isPublicAddress, embeddedV4, normalizeHost, isInternalName } = shared;
class TargetRefused extends Error {
class TargetRefused extends Error {
@@ -158,12 +156,14 @@ function createEgress({
@@ -158,12 +156,14 @@ function createEgress({
/**
/**
* One HTTP(S) request to a user-chosen URL; redirects are NOT followed.
* One HTTP(S) request to a user-chosen URL; redirects are NOT followed.
* → Promise<{ url, status, statusText, headers, body, truncated }>. body is a Buffer ('' for HEAD
* → Promise<{ url, status, statusText, headers, body, truncated }>. body is a Buffer ('' for HEAD
- * or when maxBytes is 0).
+ * or when maxBytes is 0). `body` is the request body (string or Buffer, default none).
*/
*/
- async function request(rawUrl, { method = 'GET', headers = {}, timeoutMs = 12000, maxBytes = 0 } = {}) {
+ async function request(rawUrl, { method = 'GET', headers = {}, timeoutMs = 12000, maxBytes = 0, body } = {}) {
const url = parseUrl(rawUrl);
const url = parseUrl(rawUrl);
const target = await resolve(url.hostname);
const target = await resolve(url.hostname);
const secure = url.protocol === 'https:';
const secure = url.protocol === 'https:';
+ const outHeaders = body == null || Object.keys(headers).some(h => h.toLowerCase() === 'content-length')
+ ? headers : { ...headers, 'Content-Length': Buffer.byteLength(body) };
return new Promise((ok, fail) => {
return new Promise((ok, fail) => {
let settled = false;
let settled = false;
let deadline = null;
let deadline = null;
@@ -173,7 +173,7 @@ function createEgress({
@@ -173,7 +173,7 @@ function createEgress({
port: url.port || (secure ? 443 : 80),
port: url.port || (secure ? 443 : 80),
path: `${url.pathname}${url.search}`,
path: `${url.pathname}${url.search}`,
method,
method,
- headers,
+ headers: outHeaders,
agent: false,
agent: false,
// Only the checked addresses: no second DNS answer is ever used.
// Only the checked addresses: no second DNS answer is ever used.
lookup: target.literal ? undefined : pinnedLookup(target.addresses),
lookup: target.literal ? undefined : pinnedLookup(target.addresses),
@@ -204,7 +204,7 @@ function createEgress({
@@ -204,7 +204,7 @@ function createEgress({
deadline = setTimeout(timeout, timeoutMs);
deadline = setTimeout(timeout, timeoutMs);
req.on('timeout', timeout);
req.on('timeout', timeout);
req.on('error', (err) => done(fail, err));
req.on('error', (err) => done(fail, err));
- req.end();
+ req.end(body == null ? undefined : body);
});
});
}
}
@@ -103,6 +103,30 @@ const { createEgress, isPublicAddress, embeddedV4, TargetRefused } = require('..
@@ -103,6 +103,30 @@ const { createEgress, isPublicAddress, embeddedV4, TargetRefused } = require('..
const head = await g3.request(`http://127.0.0.1:${port}/`, { method: 'HEAD' });
const head = await g3.request(`http://127.0.0.1:${port}/`, { method: 'HEAD' });
assert.equal(head.status, 200); assert.equal(head.body.length, 0);
assert.equal(head.status, 200); assert.equal(head.body.length, 0);
+ // A request body is written verbatim, with its Content-Length.
+ let sent = null;
+ const { EventEmitter: EE } = require('events');
+ const poster = createEgress({
+ lookup: (h, o, cb) => cb(null, [{ address: '127.0.0.1', family: 4 }]),
+ isAllowed: (a) => a === '127.0.0.1' || isPublicAddress(a),
+ httpRequest: (opts, cb) => {
+ const req = new EE();
+ req.end = (b) => { sent = { headers: opts.headers, body: b }; };
+ req.destroy = () => {};
+ process.nextTick(() => {
+ const res = new EE();
+ res.statusCode = 200; res.statusMessage = 'OK'; res.headers = {}; res.destroy = () => {};
+ cb(res);
+ process.nextTick(() => res.emit('end'));
+ });
+ return req;
+ },
+ });
+ const posted = await poster.request('http://site.example/post', { method: 'POST', body: 'data=x', maxBytes: 100 });
+ assert.equal(posted.status, 200);
+ assert.equal(sent.body, 'data=x');
+ assert.equal(sent.headers['Content-Length'], 6);
+
// request() never follows; follow() re-checks each hop and refuses before dialling.
// request() never follows; follow() re-checks each hop and refuses before dialling.
const one = await g3.request(`http://127.0.0.1:${port}/to-private`);
const one = await g3.request(`http://127.0.0.1:${port}/to-private`);
assert.equal(one.status, 302);
assert.equal(one.status, 302);
@@ -51,7 +51,10 @@ function mergeForm(files, job) {
@@ -51,7 +51,10 @@ function mergeForm(files, job) {
const merging = fetch(`${app.base}/api/v1/tools/mergepdf/run?wait_ms=60000`, { method: 'POST', body: mergeForm([big, big]), headers: { cookie } }).then(async r => ({ status: r.status, body: await r.json(), at: Date.now() }));
const merging = fetch(`${app.base}/api/v1/tools/mergepdf/run?wait_ms=60000`, { method: 'POST', body: mergeForm([big, big]), headers: { cookie } }).then(async r => ({ status: r.status, body: await r.json(), at: Date.now() }));
const latencies = [];
const latencies = [];
let done = null;
let done = null;
- merging.then(r => { done = r; });
+ let sawBusy = false;
+ // Settle `done` either way: an unhandled rejection here would end the process before the
+ // real assertion failure is printed.
+ merging.then(r => { done = r; }, err => { done = { error: err }; });
await sleep(150); // the upload is in and pdf-lib is working
await sleep(150); // the upload is in and pdf-lib is working
while (!done) {
while (!done) {
const s = Date.now();
const s = Date.now();
@@ -59,9 +62,13 @@ function mergeForm(files, job) {
@@ -59,9 +62,13 @@ function mergeForm(files, job) {
assert.strictEqual(h.status, 200);
assert.strictEqual(h.status, 200);
const body = await h.json();
const body = await h.json();
latencies.push(Date.now() - s);
latencies.push(Date.now() - s);
- if (!done && latencies.length === 3) assert.strictEqual(body.workers.busy, 1, 'the merge is in a worker thread');
+ if (!done && body.workers.busy === 1) sawBusy = true;
await sleep(40);
await sleep(40);
}
}
+ if (done.error) throw done.error;
+ // On a loaded machine the upload can outlast the first few health calls, so look for the busy
+ // worker at any point during the merge rather than at a fixed call.
+ assert.ok(sawBusy, 'the merge is in a worker thread');
const took = done.at - t0;
const took = done.at - t0;
assert.strictEqual(done.status, 200, JSON.stringify(done.body));
assert.strictEqual(done.status, 200, JSON.stringify(done.body));
assert.strictEqual(done.body.state, 'succeeded', JSON.stringify(done.body.error));
assert.strictEqual(done.body.state, 'succeeded', JSON.stringify(done.body.error));
@@ -111,12 +111,12 @@ function rawGet(port, p, headers = {}) {
@@ -111,12 +111,12 @@ function rawGet(port, p, headers = {}) {
if (e.isDirectory()) walk(f);
if (e.isDirectory()) walk(f);
else if (e.name.endsWith('.js')) {
else if (e.name.endsWith('.js')) {
const src = fs.readFileSync(f, 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1');
const src = fs.readFileSync(f, 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1');
- if (/(^|[^.\w])(fetch|fetchImpl)\(|\bhttps?\.(get|request)\(|\bmod\.(get|request)\(|new WebSocket\(|require\(['"](axios|got|node-fetch|undici)['"]\)/m.test(src)) found.push(path.relative(APPS, f));
+ if (/(^|[^.\w])(fetch|fetchImpl)\(|\bhttps?\.(get|post|request)\(|\bmod\.(get|request)\(|new WebSocket\(|require\(['"](axios|got|node-fetch|undici)['"]\)/m.test(src)) found.push(path.relative(APPS, f));
}
}
}
}
};
};
for (const app of fs.readdirSync(APPS)) {
for (const app of fs.readdirSync(APPS)) {
- if (app === '_shared') { for (const e of fs.readdirSync(path.join(APPS, '_shared'), { withFileTypes: true })) { if (e.isFile() && e.name.endsWith('.js')) { const f = path.join(APPS, '_shared', e.name); const src = fs.readFileSync(f, 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1'); if (/(^|[^.\w])(fetch|fetchImpl)\(|\bhttps?\.(get|request)\(|\bmod\.(get|request)\(|new WebSocket\(|require\(['"](axios|got|node-fetch|undici)['"]\)/m.test(src)) found.push(path.relative(APPS, f)); } else if (e.isDirectory() && !['node_modules', 'test'].includes(e.name)) walk(path.join(APPS, '_shared', e.name)); } continue; }
+ if (app === '_shared') { for (const e of fs.readdirSync(path.join(APPS, '_shared'), { withFileTypes: true })) { if (e.isFile() && e.name.endsWith('.js')) { const f = path.join(APPS, '_shared', e.name); const src = fs.readFileSync(f, 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1'); if (/(^|[^.\w])(fetch|fetchImpl)\(|\bhttps?\.(get|post|request)\(|\bmod\.(get|request)\(|new WebSocket\(|require\(['"](axios|got|node-fetch|undici)['"]\)/m.test(src)) found.push(path.relative(APPS, f)); } else if (e.isDirectory() && !['node_modules', 'test'].includes(e.name)) walk(path.join(APPS, '_shared', e.name)); } continue; }
if (fs.existsSync(path.join(APPS, app, 'server'))) walk(path.join(APPS, app, 'server'));
if (fs.existsSync(path.join(APPS, app, 'server'))) walk(path.join(APPS, app, 'server'));
}
}
assert.ok(found.length >= 20, `the scan finds the known sites (${found.length})`);
assert.ok(found.length >= 20, `the scan finds the known sites (${found.length})`);
new file mode 100644
new file mode 100644
@@ -0,0 +1,151 @@
@@ -0,0 +1,151 @@
+'use strict';
+/**
+ * The one door out of apps/maps. Every outbound HTTP request from this server goes through `get` /
+ * `post` here, and this file reaches the network only through the shared SSRF guard
+ * (apps/_shared/egress.js): the hostname is resolved, every answer must be a public address, and the
+ * connection is pinned to the checked address. On top of that guard this adapter enforces:
+ *
+ * - HTTPS only, to the allowlisted source hosts below and nothing else. The check runs before
+ * every request, redirect hops included (a redirect to a new host is re-checked), so a source
+ * answering `Location: http://…` or a non-allowlisted name is refused before any connection.
+ * - a per-host response-size cap (the number after each host). A body over the cap is refused as
+ * tools.maps.response_too_large, whether the server sent it raw (truncated at the socket) or
+ * gzip/deflate/brotli (the decompressor is capped at the same number).
+ * - at most 3 redirects, GET only; a redirect in answer to a POST is refused.
+ * - host and pathname only in errors: source API keys ride in query strings (NPS, OpenChargeMap)
+ * and headers (RIDB), so a message that echoed the URL would leak a key into logs.
+ *
+ * Allowlist (host -> max response bytes):
+ * nominatim.openstreetmap.org 256 KB — small JSON, a handful of geocode hits
+ * overpass-api.de 32 MB — Overpass QL can answer a whole OSM extract
+ * ridb.recreation.gov 2 MB — facility/recarea JSON lists
+ * developer.nps.gov 8 MB — campground, visitor-centre and parking JSON
+ * api.openchargemap.io 4 MB — charge-point POI pages
+ * api.weather.gov 2 MB — points and active-alerts JSON
+ * api.open-meteo.com 1 MB — forecast and elevation JSON
+ * apps.fs.usda.gov 16 MB — ArcGIS USFS recreation features (GeoJSON)
+ * services.arcgis.com 16 MB — ArcGIS National Bridge Inventory features
+ * www.refugerestrooms.org 1 MB — restroom JSON, small
+ * www.ioverlander.com 4 MB — scraped place pages (HTML)
+ * freecampsites.net 8 MB — scraped WordPress pages (HTML)
+ * www.campendium.com 2 MB — campground API JSON
+ * edits.nationalmap.gov 2 MB — gazetteer search JSON
+ * www.walmart.com 4 MB — scraped store-search page (HTML)
+ */
+const zlib = require('zlib');
+const { createEgress } = require('../../_shared/egress');
+
+const KB = 1024;
+
+const ALLOW = Object.freeze({
+ 'nominatim.openstreetmap.org': 256 * KB,
+ 'overpass-api.de': 32 * KB * KB,
+ 'ridb.recreation.gov': 2 * KB * KB,
+ 'developer.nps.gov': 8 * KB * KB,
+ 'api.openchargemap.io': 4 * KB * KB,
+ 'api.weather.gov': 2 * KB * KB,
+ 'api.open-meteo.com': 1 * KB * KB,
+ 'apps.fs.usda.gov': 16 * KB * KB,
+ 'services.arcgis.com': 16 * KB * KB,
+ 'www.refugerestrooms.org': 1 * KB * KB,
+ 'www.ioverlander.com': 4 * KB * KB,
+ 'freecampsites.net': 8 * KB * KB,
+ 'www.campendium.com': 2 * KB * KB,
+ 'edits.nationalmap.gov': 2 * KB * KB,
+ 'www.walmart.com': 4 * KB * KB,
+});
+
+let egress = createEgress();
+
+/** Replace the guard; tests inject createEgress({ lookup, httpsRequest }) mocks. */
+function setEgress(e) { egress = e; }
+
+class MapsEgressError extends Error {
+ constructor(message, code) {
+ super(message);
+ this.name = 'MapsEgressError';
+ this.code = code;
+ this.status = 502;
+ }
+}
+
+const REDIRECTS = [301, 302, 303, 307, 308];
+const tooBig = (host) => `${host} answered more than ${ALLOW[host]} bytes`;
+
+async function send(method, rawUrl, { params, headers = {}, timeout = 10000, body } = {}) {
+ let url = new URL(rawUrl);
+ if (params) {
+ for (const [k, v] of Object.entries(params)) {
+ if (v !== undefined && v !== null) url.searchParams.append(k, String(v));
+ }
+ }
+
+ // The caller's headers (an API key rides in RIDB's apikey) belong to the origin the caller asked
+ // for; a redirect to another allowlisted host must not carry them across.
+ const origin = url.origin;
+
+ for (let redirects = 0; ;) {
+ const host = url.hostname;
+ if (url.protocol !== 'https:' || !Object.prototype.hasOwnProperty.call(ALLOW, host)) {
+ throw new MapsEgressError(`${host} is not an allowed maps source`, 'tools.maps.host_not_allowed');
+ }
+
+ let r;
+ try {
+ r = await egress.request(url.href, {
+ method,
+ headers: { 'Accept-Encoding': 'identity', ...(url.origin === origin ? headers : {}) },
+ timeoutMs: timeout,
+ maxBytes: ALLOW[host],
+ body,
+ });
+ } catch (err) {
+ if (err && (err.name === 'TimeoutError' || err.message === 'Timeout')) {
+ const e = new Error(`timeout of ${timeout}ms exceeded`);
+ e.code = 'ECONNABORTED';
+ throw e;
+ }
+ throw err;
+ }
+
+ const location = r.headers.location;
+ if (REDIRECTS.includes(r.status) && location) {
+ if (method !== 'GET') throw new MapsEgressError(`${host}${url.pathname} redirected a POST`, 'tools.maps.redirect_refused');
+ if (redirects >= 3) throw new MapsEgressError(`too many redirects from ${host}`, 'tools.maps.too_many_redirects');
+ redirects++;
+ url = new URL(location, url);
+ continue;
+ }
+
+ if (r.truncated === true) throw new MapsEgressError(tooBig(host), 'tools.maps.response_too_large');
+
+ let buf = r.body;
+ const encoding = String(r.headers['content-encoding'] || '').toLowerCase();
+ if (encoding === 'gzip' || encoding === 'deflate' || encoding === 'br') {
+ try {
+ const opts = { maxOutputLength: ALLOW[host] };
+ buf = encoding === 'gzip' ? zlib.gunzipSync(buf, opts)
+ : encoding === 'deflate' ? zlib.inflateSync(buf, opts)
+ : zlib.brotliDecompressSync(buf, opts);
+ } catch (err) {
+ if (err instanceof RangeError) throw new MapsEgressError(tooBig(host), 'tools.maps.response_too_large');
+ throw err;
+ }
+ }
+
+ let data = buf.toString('utf8');
+ try { data = JSON.parse(data); } catch { /* not JSON: keep the string, cheerio callers want HTML */ }
+
+ if (r.status < 200 || r.status > 299) {
+ const err = new Error(`Request failed with status code ${r.status}`);
+ err.response = { status: r.status, headers: r.headers, data };
+ throw err;
+ }
+ return { status: r.status, headers: r.headers, data };
+ }
+}
+
+const get = (url, opts) => send('GET', url, opts);
+const post = (url, body, opts) => send('POST', url, { ...opts, body });
+
+module.exports = { get, post, ALLOW, setEgress, MapsEgressError };
@@ -16,6 +16,7 @@ const NodeCache = require('node-cache');
@@ -16,6 +16,7 @@ const NodeCache = require('node-cache');
const cookieParser = require('cookie-parser');
const cookieParser = require('cookie-parser');
const config = require('./config');
const config = require('./config');
+const http = require('./egress');
// ── Analytics ────────────────────────────────────────────────
// ── Analytics ────────────────────────────────────────────────
const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recorded
const { AnalyticsTrackerPg } = require('openvibe-shared/analytics/pg'); // ADR-021: no IP/user id, route templates, raw rows pruned after 30 days, Sec-GPC/DNT not recorded
@@ -164,8 +165,7 @@ app.get('/api/geocode', async (req, res) => {
@@ -164,8 +165,7 @@ app.get('/api/geocode', async (req, res) => {
if (cached) return res.json(cached);
if (cached) return res.json(cached);
try {
try {
- const axios = require('axios');
- const resp = await nominatim.run(() => axios.get('https://nominatim.openstreetmap.org/search', {
+ const resp = await nominatim.run(() => http.get('https://nominatim.openstreetmap.org/search', {
params: { q, format: 'json', limit: 5, countrycodes: 'us,ca,mx' },
params: { q, format: 'json', limit: 5, countrycodes: 'us,ca,mx' },
headers: { 'User-Agent': 'Maps.OpenVibe/1.0 (maps.openvibe.tools)' },
headers: { 'User-Agent': 'Maps.OpenVibe/1.0 (maps.openvibe.tools)' },
timeout: 8000,
timeout: 8000,
@@ -11,7 +11,7 @@
@@ -11,7 +11,7 @@
*
*
* Combined → deduplicated, enriched with distance/walking time, scored for stealth-camper utility.
* Combined → deduplicated, enriched with distance/walking time, scored for stealth-camper utility.
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');
const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');
// ─── Constants ─────────────────────────────────────────────────────
// ─── Constants ─────────────────────────────────────────────────────
@@ -105,7 +105,7 @@ async function fetchRefugeRestrooms(lat, lon, perPage = 30) {
@@ -105,7 +105,7 @@ async function fetchRefugeRestrooms(lat, lon, perPage = 30) {
for (let attempt = 0; attempt < maxAttempts; attempt++) {
for (let attempt = 0; attempt < maxAttempts; attempt++) {
try {
try {
const timeout = attempt === 0 ? 15000 : 25000;
const timeout = attempt === 0 ? 15000 : 25000;
- const resp = await axios.get(REFUGE_API, {
+ const resp = await http.get(REFUGE_API, {
params: { lat, lng: lon, per_page: perPage },
params: { lat, lng: lon, per_page: perPage },
timeout,
timeout,
headers: { 'Accept': 'application/json' },
headers: { 'Accept': 'application/json' },
@@ -16,7 +16,7 @@
@@ -16,7 +16,7 @@
* - Finer-grained local bridge data with name, surface, layer info
* - Finer-grained local bridge data with name, surface, layer info
* - Can determine what a bridge crosses by checking nearby waterways
* - Can determine what a bridge crosses by checking nearby waterways
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');
const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');
// ─── Constants ─────────────────────────────────────────────────────
// ─── Constants ─────────────────────────────────────────────────────
@@ -177,7 +177,7 @@ async function fetchNbiBridges(lat, lon, radiusMiles = 5) {
@@ -177,7 +177,7 @@ async function fetchNbiBridges(lat, lon, radiusMiles = 5) {
f: 'json',
f: 'json',
};
};
- const resp = await axios.get(NBI_API, { params, timeout: 15000 });
+ const resp = await http.get(NBI_API, { params, timeout: 15000 });
if (!resp.data || !resp.data.features) return [];
if (!resp.data || !resp.data.features) return [];
return resp.data.features.map(f => {
return resp.data.features.map(f => {
@@ -5,7 +5,7 @@
@@ -5,7 +5,7 @@
*
*
* We fetch their API-like endpoints that serve map data.
* We fetch their API-like endpoints that serve map data.
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const cheerio = require('cheerio');
const cheerio = require('cheerio');
const { haversine } = require('./utils');
const { haversine } = require('./utils');
@@ -23,7 +23,7 @@ async function search(lat, lon, radiusMiles) {
@@ -23,7 +23,7 @@ async function search(lat, lon, radiusMiles) {
try {
try {
// Try the WordPress REST API that many WP sites expose
// Try the WordPress REST API that many WP sites expose
- const { data } = await axios.get(`${BASE_URL}/wp-json/wp/v2/posts`, {
+ const { data } = await http.get(`${BASE_URL}/wp-json/wp/v2/posts`, {
params: {
params: {
per_page: 50,
per_page: 50,
search: 'camping',
search: 'camping',
@@ -68,7 +68,7 @@ async function search(lat, lon, radiusMiles) {
@@ -68,7 +68,7 @@ async function search(lat, lon, radiusMiles) {
// Also try scraping their map page for WA state campsite markers
// Also try scraping their map page for WA state campsite markers
try {
try {
- const { data: html } = await axios.get(`${BASE_URL}/?s=washington+camping`, {
+ const { data: html } = await http.get(`${BASE_URL}/?s=washington+camping`, {
headers: {
headers: {
'User-Agent': 'WA-StealthCampLocator/1.0 (research-project)',
'User-Agent': 'WA-StealthCampLocator/1.0 (research-project)',
},
},
@@ -12,7 +12,7 @@
@@ -12,7 +12,7 @@
* - Fred Meyer / Kroger (API + built-in pricing)
* - Fred Meyer / Kroger (API + built-in pricing)
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const cheerio = require('cheerio');
const cheerio = require('cheerio');
const { haversine } = require('./utils');
const { haversine } = require('./utils');
@@ -821,7 +821,7 @@ async function findNearbyStores(lat, lon, radiusMeters = 8000) {
@@ -821,7 +821,7 @@ async function findNearbyStores(lat, lon, radiusMeters = 8000) {
const query = `[out:json][timeout:15];(\n${filters}\n);out body;`;
const query = `[out:json][timeout:15];(\n${filters}\n);out body;`;
try {
try {
- const resp = await axios.post('https://overpass-api.de/api/interpreter', `data=${encodeURIComponent(query)}`, {
+ const resp = await http.post('https://overpass-api.de/api/interpreter', `data=${encodeURIComponent(query)}`, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
timeout: 15000,
timeout: 15000,
});
});
@@ -894,7 +894,7 @@ out skel qt;
@@ -894,7 +894,7 @@ out skel qt;
`;
`;
try {
try {
- const resp = await axios.post('https://overpass-api.de/api/interpreter',
+ const resp = await http.post('https://overpass-api.de/api/interpreter',
`data=${encodeURIComponent(query)}`,
`data=${encodeURIComponent(query)}`,
{ headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, timeout: 30000 }
{ headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, timeout: 30000 }
);
);
@@ -985,7 +985,7 @@ out skel qt;
@@ -985,7 +985,7 @@ out skel qt;
async function scrapeWalmartPrices(searchTerm) {
async function scrapeWalmartPrices(searchTerm) {
try {
try {
const url = `https://www.walmart.com/search?q=${encodeURIComponent(searchTerm)}`;
const url = `https://www.walmart.com/search?q=${encodeURIComponent(searchTerm)}`;
- const resp = await axios.get(url, {
+ const resp = await http.get(url, {
headers: {
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
@@ -3,7 +3,7 @@
@@ -3,7 +3,7 @@
* Scrapes camping/overlanding spots from iOverlander.com for Washington State.
* Scrapes camping/overlanding spots from iOverlander.com for Washington State.
* iOverlander has a web interface with location data we can extract.
* iOverlander has a web interface with location data we can extract.
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const cheerio = require('cheerio');
const cheerio = require('cheerio');
const { haversine } = require('./utils');
const { haversine } = require('./utils');
@@ -45,7 +45,7 @@ async function search(lat, lon, radiusMiles = 25) {
@@ -45,7 +45,7 @@ async function search(lat, lon, radiusMiles = 25) {
// Try iOverlander's search page
// Try iOverlander's search page
const url = `https://www.ioverlander.com/places?lat=${lat}&lng=${lon}&zoom=10`;
const url = `https://www.ioverlander.com/places?lat=${lat}&lng=${lon}&zoom=10`;
- const resp = await axios.get(url, {
+ const resp = await http.get(url, {
headers: { 'User-Agent': UA },
headers: { 'User-Agent': UA },
timeout: 12000,
timeout: 12000,
});
});
@@ -9,7 +9,7 @@
@@ -9,7 +9,7 @@
* Free API key required: https://www.nps.gov/subjects/developer/get-started.htm
* Free API key required: https://www.nps.gov/subjects/developer/get-started.htm
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const { haversine } = require('./utils');
const { haversine } = require('./utils');
const BASE_URL = 'https://developer.nps.gov/api/v1';
const BASE_URL = 'https://developer.nps.gov/api/v1';
@@ -73,7 +73,7 @@ async function searchCampgrounds(lat, lon, radiusMiles, apiKey) {
@@ -73,7 +73,7 @@ async function searchCampgrounds(lat, lon, radiusMiles, apiKey) {
stateCode,
stateCode,
};
};
- const resp = await axios.get(`${BASE_URL}/campgrounds`, {
+ const resp = await http.get(`${BASE_URL}/campgrounds`, {
params,
params,
timeout: 15000,
timeout: 15000,
headers: { 'User-Agent': 'OpenVibeApp/2.0' },
headers: { 'User-Agent': 'OpenVibeApp/2.0' },
@@ -145,7 +145,7 @@ async function searchVisitorCenters(lat, lon, radiusMiles, apiKey) {
@@ -145,7 +145,7 @@ async function searchVisitorCenters(lat, lon, radiusMiles, apiKey) {
if (!apiKey) return [];
if (!apiKey) return [];
try {
try {
- const resp = await axios.get(`${BASE_URL}/visitorcenters`, {
+ const resp = await http.get(`${BASE_URL}/visitorcenters`, {
params: { api_key: apiKey, limit: 100, stateCode: getNearbyCodes(lat, lon, radiusMiles) || 'WA' },
params: { api_key: apiKey, limit: 100, stateCode: getNearbyCodes(lat, lon, radiusMiles) || 'WA' },
timeout: 10000,
timeout: 10000,
headers: { 'User-Agent': 'OpenVibeApp/2.0' },
headers: { 'User-Agent': 'OpenVibeApp/2.0' },
@@ -197,7 +197,7 @@ async function searchParkingLots(lat, lon, radiusMiles, apiKey) {
@@ -197,7 +197,7 @@ async function searchParkingLots(lat, lon, radiusMiles, apiKey) {
if (!apiKey) return [];
if (!apiKey) return [];
try {
try {
- const resp = await axios.get(`${BASE_URL}/parkinglots`, {
+ const resp = await http.get(`${BASE_URL}/parkinglots`, {
params: { api_key: apiKey, limit: 100, stateCode: getNearbyCodes(lat, lon, radiusMiles) || 'WA' },
params: { api_key: apiKey, limit: 100, stateCode: getNearbyCodes(lat, lon, radiusMiles) || 'WA' },
timeout: 10000,
timeout: 10000,
headers: { 'User-Agent': 'OpenVibeApp/2.0' },
headers: { 'User-Agent': 'OpenVibeApp/2.0' },
@@ -8,11 +8,10 @@
@@ -8,11 +8,10 @@
* Free & open — works without key but key gives higher rate limits
* Free & open — works without key but key gives higher rate limits
*/
*/
-const { createEgress } = require('../../../_shared/egress');
+const http = require('../egress');
const { haversine } = require('./utils');
const { haversine } = require('./utils');
const BASE_URL = 'https://api.openchargemap.io/v3/poi';
const BASE_URL = 'https://api.openchargemap.io/v3/poi';
-const egress = createEgress();
/**
/**
* Search for EV charging stations near a location
* Search for EV charging stations near a location
@@ -33,19 +32,15 @@ async function search(lat, lon, radiusMiles, apiKey) {
@@ -33,19 +32,15 @@ async function search(lat, lon, radiusMiles, apiKey) {
if (apiKey) params.key = apiKey;
if (apiKey) params.key = apiKey;
- const url = new URL(BASE_URL);
- for (const [name, value] of Object.entries(params)) url.searchParams.set(name, value);
- const resp = await egress.follow(url.href, {
- timeoutMs: 12000,
- maxBytes: 2 * 1024 * 1024,
- headers: { 'User-Agent': 'OpenVibeApp/2.0', Accept: 'application/json' },
+ const resp = await http.get(BASE_URL, {
+ params,
+ timeout: 12000,
+ headers: { 'User-Agent': 'OpenVibeApp/2.0' },
});
});
- if (resp.status < 200 || resp.status >= 300) throw new Error(`HTTP ${resp.status}`);
- if (resp.truncated) throw new Error('Response too large');
- const data = JSON.parse(resp.body.toString('utf8'));
- if (!Array.isArray(data)) return [];
- return data
+ if (!Array.isArray(resp.data)) return [];
+
+ return resp.data
.filter(s => s.AddressInfo?.Latitude && s.AddressInfo?.Longitude)
.filter(s => s.AddressInfo?.Latitude && s.AddressInfo?.Longitude)
.map(s => {
.map(s => {
const addr = s.AddressInfo;
const addr = s.AddressInfo;
@@ -11,7 +11,6 @@
@@ -11,7 +11,6 @@
*
*
* Overpass API: https://overpass-api.de/api/interpreter
* Overpass API: https://overpass-api.de/api/interpreter
*/
*/
-const axios = require('axios');
const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');
const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');
/**
/**
@@ -11,7 +11,6 @@
@@ -11,7 +11,6 @@
*
*
* All data sourced from OpenStreetMap via Overpass API.
* All data sourced from OpenStreetMap via Overpass API.
*/
*/
-const axios = require('axios');
const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');
const { haversine, OVERPASS_URL, overpassQuery } = require('./utils');
// ═══════════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════════
@@ -6,7 +6,7 @@
@@ -6,7 +6,7 @@
* NOTE: RIDB requires an API key; the caller passes it in (server/config.js, RIDB_API_KEY).
* NOTE: RIDB requires an API key; the caller passes it in (server/config.js, RIDB_API_KEY).
* Register at https://ridb.recreation.gov/ for one. Without it RIDB is skipped.
* Register at https://ridb.recreation.gov/ for one. Without it RIDB is skipped.
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const { haversine } = require('./utils');
const { haversine } = require('./utils');
// The RIDB key is passed in by the caller (server/config.js); never a key in the code.
// The RIDB key is passed in by the caller (server/config.js); never a key in the code.
@@ -33,7 +33,7 @@ async function search(lat, lon, radiusMiles, apiKey) {
@@ -33,7 +33,7 @@ async function search(lat, lon, radiusMiles, apiKey) {
try {
try {
// Search facilities by lat/lon with radius
// Search facilities by lat/lon with radius
- const { data } = await axios.get(`${BASE_URL}/facilities`, {
+ const { data } = await http.get(`${BASE_URL}/facilities`, {
params: {
params: {
latitude: lat,
latitude: lat,
longitude: lon,
longitude: lon,
@@ -89,7 +89,7 @@ async function search(lat, lon, radiusMiles, apiKey) {
@@ -89,7 +89,7 @@ async function search(lat, lon, radiusMiles, apiKey) {
// Also try RecAreas (larger tracts like national forests)
// Also try RecAreas (larger tracts like national forests)
if (_ridbDisabled) return results;
if (_ridbDisabled) return results;
try {
try {
- const { data } = await axios.get(`${BASE_URL}/recareas`, {
+ const { data } = await http.get(`${BASE_URL}/recareas`, {
params: {
params: {
latitude: lat,
latitude: lat,
longitude: lon,
longitude: lon,
@@ -144,7 +144,7 @@ async function search(lat, lon, radiusMiles, apiKey) {
@@ -144,7 +144,7 @@ async function search(lat, lon, radiusMiles, apiKey) {
* Get detailed facility info from RIDB
* Get detailed facility info from RIDB
*/
*/
async function getDetail(facilityId, apiKey) {
async function getDetail(facilityId, apiKey) {
- const { data } = await axios.get(`${BASE_URL}/facilities/${facilityId}`, {
+ const { data } = await http.get(`${BASE_URL}/facilities/${facilityId}`, {
headers: { apikey: apiKey, Accept: 'application/json' },
headers: { apikey: apiKey, Accept: 'application/json' },
timeout: 10000,
timeout: 10000,
});
});
@@ -12,7 +12,7 @@
@@ -12,7 +12,7 @@
* - USGS Waterfall database
* - USGS Waterfall database
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const cheerio = require('cheerio');
const cheerio = require('cheerio');
const { haversine, overpassQuery } = require('./utils');
const { haversine, overpassQuery } = require('./utils');
@@ -27,7 +27,7 @@ const HEADERS = {
@@ -27,7 +27,7 @@ const HEADERS = {
*/
*/
async function scrapeCampendium(lat, lon, radiusMiles) {
async function scrapeCampendium(lat, lon, radiusMiles) {
try {
try {
- const resp = await axios.get('https://www.campendium.com/api/v2/campgrounds', {
+ const resp = await http.get('https://www.campendium.com/api/v2/campgrounds', {
params: {
params: {
lat,
lat,
lng: lon,
lng: lon,
@@ -91,7 +91,7 @@ async function scrapeCampendium(lat, lon, radiusMiles) {
@@ -91,7 +91,7 @@ async function scrapeCampendium(lat, lon, radiusMiles) {
async function scrapeUSGSFeatures(lat, lon, radiusMiles) {
async function scrapeUSGSFeatures(lat, lon, radiusMiles) {
try {
try {
// GNIS Feature Search API
// GNIS Feature Search API
- const resp = await axios.get('https://edits.nationalmap.gov/apps/gaz-domestic/api/search', {
+ const resp = await http.get('https://edits.nationalmap.gov/apps/gaz-domestic/api/search', {
params: {
params: {
lat,
lat,
lon,
lon,
@@ -5,7 +5,7 @@
@@ -5,7 +5,7 @@
* - OpenStreetMap land use data via Overpass
* - OpenStreetMap land use data via Overpass
* - Slope/terrain classification
* - Slope/terrain classification
*/
*/
-const axios = require('axios');
+const http = require('../egress');
/**
/**
* Get elevation data for a point.
* Get elevation data for a point.
@@ -13,7 +13,7 @@ const axios = require('axios');
@@ -13,7 +13,7 @@ const axios = require('axios');
async function getElevation(lat, lon) {
async function getElevation(lat, lon) {
try {
try {
const url = `https://api.open-meteo.com/v1/elevation?latitude=${lat}&longitude=${lon}`;
const url = `https://api.open-meteo.com/v1/elevation?latitude=${lat}&longitude=${lon}`;
- const resp = await axios.get(url, { timeout: 5000 });
+ const resp = await http.get(url, { timeout: 5000 });
return resp.data?.elevation?.[0] ?? null;
return resp.data?.elevation?.[0] ?? null;
} catch (e) {
} catch (e) {
return null;
return null;
@@ -29,7 +29,7 @@ async function getBulkElevation(coords) {
@@ -29,7 +29,7 @@ async function getBulkElevation(coords) {
const lons = coords.map(c => c.lon).join(',');
const lons = coords.map(c => c.lon).join(',');
try {
try {
const url = `https://api.open-meteo.com/v1/elevation?latitude=${lats}&longitude=${lons}`;
const url = `https://api.open-meteo.com/v1/elevation?latitude=${lats}&longitude=${lons}`;
- const resp = await axios.get(url, { timeout: 8000 });
+ const resp = await http.get(url, { timeout: 8000 });
return resp.data?.elevation || [];
return resp.data?.elevation || [];
} catch (e) {
} catch (e) {
return coords.map(() => null);
return coords.map(() => null);
@@ -62,7 +62,7 @@ async function analyzeLandUse(lat, lon, radiusMeters = 500) {
@@ -62,7 +62,7 @@ async function analyzeLandUse(lat, lon, radiusMeters = 500) {
);
);
out tags;
out tags;
`;
`;
- const resp = await axios.post('https://overpass-api.de/api/interpreter', `data=${encodeURIComponent(query)}`, {
+ const resp = await http.post('https://overpass-api.de/api/interpreter', `data=${encodeURIComponent(query)}`, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
timeout: 10000,
timeout: 10000,
});
});
@@ -12,7 +12,7 @@
@@ -12,7 +12,7 @@
* OPERATIONAL_HOURS, FORESTNAME, RECAREAURL, OPENSTATUS,
* OPERATIONAL_HOURS, FORESTNAME, RECAREAURL, OPENSTATUS,
* RESERVATION_INFO, RESTRICTIONS, ACCESSIBILITY
* RESERVATION_INFO, RESTRICTIONS, ACCESSIBILITY
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const { haversine } = require('./utils');
const { haversine } = require('./utils');
const USFS_API = 'https://apps.fs.usda.gov/arcx/rest/services/EDW/EDW_RecreationOpportunities_01/MapServer/0/query';
const USFS_API = 'https://apps.fs.usda.gov/arcx/rest/services/EDW/EDW_RecreationOpportunities_01/MapServer/0/query';
@@ -109,7 +109,7 @@ async function search(lat, lon, radiusMiles = 25) {
@@ -109,7 +109,7 @@ async function search(lat, lon, radiusMiles = 25) {
resultRecordCount: '200',
resultRecordCount: '200',
});
});
- const { data } = await axios.get(`${USFS_API}?${params.toString()}`, {
+ const { data } = await http.get(`${USFS_API}?${params.toString()}`, {
timeout: 30000,
timeout: 30000,
headers: {
headers: {
'User-Agent': 'OpenVibeApp/2.0 (stealth camping tool)',
'User-Agent': 'OpenVibeApp/2.0 (stealth camping tool)',
@@ -3,7 +3,7 @@
@@ -3,7 +3,7 @@
* Common functions used across multiple modules.
* Common functions used across multiple modules.
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const OVERPASS_URL = 'https://overpass-api.de/api/interpreter';
const OVERPASS_URL = 'https://overpass-api.de/api/interpreter';
@@ -29,7 +29,7 @@ async function _processOverpassQueue() {
@@ -29,7 +29,7 @@ async function _processOverpassQueue() {
async function _runOverpassJob({ query, timeout, resolve, reject, attempt = 0 }) {
async function _runOverpassJob({ query, timeout, resolve, reject, attempt = 0 }) {
try {
try {
- const resp = await axios.post(OVERPASS_URL, `data=${encodeURIComponent(query)}`, {
+ const resp = await http.post(OVERPASS_URL, `data=${encodeURIComponent(query)}`, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
timeout,
timeout,
});
});
@@ -5,7 +5,7 @@
@@ -5,7 +5,7 @@
*
*
* Returns unified: { current, daily[], hourly[], alerts[], sun, moon }
* Returns unified: { current, daily[], hourly[], alerts[], sun, moon }
*/
*/
-const axios = require('axios');
+const http = require('../egress');
const UA = 'GhostCamp-WA/2.0 (stealth-camping-locator)';
const UA = 'GhostCamp-WA/2.0 (stealth-camping-locator)';
@@ -82,7 +82,7 @@ async function getWeather(lat, lon) {
@@ -82,7 +82,7 @@ async function getWeather(lat, lon) {
'forecast_hours=48',
'forecast_hours=48',
].join('&');
].join('&');
- const resp = await axios.get(`https://api.open-meteo.com/v1/forecast?${params}`, { timeout: 10000 });
+ const resp = await http.get(`https://api.open-meteo.com/v1/forecast?${params}`, { timeout: 10000 });
const d = resp.data;
const d = resp.data;
// Current conditions
// Current conditions
@@ -176,14 +176,14 @@ async function getWeather(lat, lon) {
@@ -176,14 +176,14 @@ async function getWeather(lat, lon) {
// ─── NWS Alerts only ───
// ─── NWS Alerts only ───
try {
try {
- const ptResp = await axios.get(`https://api.weather.gov/points/${lat},${lon}`, {
+ const ptResp = await http.get(`https://api.weather.gov/points/${lat},${lon}`, {
headers: { 'User-Agent': UA, Accept: 'application/geo+json' },
headers: { 'User-Agent': UA, Accept: 'application/geo+json' },
timeout: 5000,
timeout: 5000,
});
});
const zone = ptResp.data?.properties?.forecastZone;
const zone = ptResp.data?.properties?.forecastZone;
if (zone) {
if (zone) {
const zoneId = zone.split('/').pop();
const zoneId = zone.split('/').pop();
- const alertResp = await axios.get(`https://api.weather.gov/alerts/active?zone=${zoneId}`, {
+ const alertResp = await http.get(`https://api.weather.gov/alerts/active?zone=${zoneId}`, {
headers: { 'User-Agent': UA },
headers: { 'User-Agent': UA },
timeout: 5000,
timeout: 5000,
});
});
new file mode 100644
new file mode 100644
@@ -0,0 +1,194 @@
@@ -0,0 +1,194 @@
+'use strict';
+// The maps egress adapter (apps/maps/server/egress.js): one allowlisted HTTPS door for every
+// outbound request. Every transport is a mock — no DNS, no socket — so any real network use would
+// fail the test. Covers the allowlist and redirect hops, the pinned-DNS rule from the shared guard,
+// size caps, key handling, and the grep guard that no other maps file reaches the network itself.
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { EventEmitter } = require('events');
+const { createEgress, TargetRefused } = require('../../_shared/egress');
+
+const maps = require('../server/egress');
+const { setEgress, MapsEgressError } = maps;
+
+const PUBLIC = '93.184.216.34';
+const RIDB_KEY = 'ridb-sentinel-key';
+const NPS_KEY = 'nps-sentinel-key';
+const OCM_KEY = 'ocm-sentinel-key';
+
+// A guard whose httpsRequest records the request and answers with `respond(rec)` (or a fixed answer).
+function makeMock(respond) {
+ const seen = [];
+ const lookups = [];
+ const guard = createEgress({
+ lookup: (host, opts, cb) => { lookups.push(host); cb(null, [{ address: PUBLIC, family: 4 }]); },
+ httpsRequest: (opts, cb) => {
+ const rec = { method: opts.method, hostname: opts.hostname, path: opts.path, headers: opts.headers, body: null };
+ seen.push(rec);
+ const req = new EventEmitter();
+ req.destroy = () => {};
+ req.end = (b) => {
+ rec.body = b;
+ process.nextTick(() => {
+ const answer = (typeof respond === 'function' ? respond(rec) : respond) || {};
+ const res = new EventEmitter();
+ res.statusCode = answer.status || 200;
+ res.statusMessage = 'OK';
+ res.headers = answer.headers || {};
+ res.destroy = () => {};
+ cb(res);
+ process.nextTick(() => {
+ res.emit('data', Buffer.from(answer.body != null ? answer.body : '{}'));
+ res.emit('end');
+ });
+ });
+ };
+ return req;
+ },
+ });
+ return { guard, seen, lookups };
+}
+
+(async () => {
+ let failures = 0;
+ const check = async (name, fn) => {
+ try { await fn(); console.log(` ✓ ${name}`); }
+ catch (e) { failures++; console.log(` ✗ ${name}\n ${String(e.stack || e.message).split('\n').slice(0, 8).join('\n ')}`); }
+ };
+
+ const ridb = require('../server/sources/ridb');
+ const nps = require('../server/sources/nps');
+ const ocm = require('../server/sources/openchargemap');
+ const terrain = require('../server/sources/terrain');
+ const { overpassQuery } = require('../server/sources/utils');
+
+ await check('injected RIDB, NPS and OpenChargeMap keys reach their requests and never a log', async () => {
+ const mock = makeMock({ status: 200, body: '{"data":[],"RECDATA":[]}' });
+ setEgress(mock.guard);
+ const logs = [];
+ const real = { log: console.log, warn: console.warn, error: console.error };
+ console.log = console.warn = console.error = (...a) => logs.push(a.map(String).join(' '));
+ let results;
+ try {
+ const r1 = await ridb.search(47.6062, -122.3321, 25, RIDB_KEY);
+ const r2 = await nps.search(47.6062, -122.3321, 25, NPS_KEY);
+ const r3 = await ocm.search(47.6062, -122.3321, 25, OCM_KEY);
+ results = JSON.stringify([r1, r2, r3]);
+ } finally { Object.assign(console, real); }
+
+ const ridbReqs = mock.seen.filter(r => r.hostname === 'ridb.recreation.gov');
+ assert.ok(ridbReqs.length > 0, 'RIDB sent a request');
+ for (const r of ridbReqs) assert.strictEqual(r.headers.apikey, RIDB_KEY, 'RIDB key is the apikey header');
+ assert.ok(mock.seen.some(r => r.hostname === 'developer.nps.gov' && r.path.includes(`api_key=${NPS_KEY}`)), 'NPS key is the api_key query param');
+ assert.ok(mock.seen.some(r => r.hostname === 'api.openchargemap.io' && r.path.includes(`key=${OCM_KEY}`)), 'OpenChargeMap key is the key query param');
+ const all = results + '\n' + logs.join('\n');
+ for (const key of [RIDB_KEY, NPS_KEY, OCM_KEY]) assert.ok(!all.includes(key), `${key} leaked into results or logs`);
+ });
+
+ await check('with an empty key RIDB and NPS send nothing and still resolve', async () => {
+ const mock = makeMock({ status: 200, body: '{"data":[],"RECDATA":[]}' });
+ setEgress(mock.guard);
+ const r1 = await ridb.search(47.6062, -122.3321, 25, '');
+ const r2 = await nps.search(47.6062, -122.3321, 25, '');
+ assert.deepStrictEqual(mock.seen.length, 0, 'no request was sent');
+ for (const r of [r1, r2]) assert.deepStrictEqual(r, [], 'search resolves empty');
+ });
+
+ await check('a redirect to a non-allowlisted host is refused before any DNS lookup', async () => {
+ const mock = makeMock(() => ({ status: 302, headers: { location: 'https://evil.example/' } }));
+ setEgress(mock.guard);
+ await assert.rejects(maps.get('https://api.weather.gov/evil'),
+ (e) => e instanceof MapsEgressError && e.code === 'tools.maps.host_not_allowed');
+ assert.ok(!mock.lookups.includes('evil.example'), 'the refused host was never resolved');
+ });
+
+ await check('a redirect that drops to http is refused', async () => {
+ const mock = makeMock(() => ({ status: 302, headers: { location: 'http://api.weather.gov/plain' } }));
+ setEgress(mock.guard);
+ await assert.rejects(maps.get('https://api.weather.gov/start'),
+ (e) => e instanceof MapsEgressError && e.code === 'tools.maps.host_not_allowed');
+ });
+
+ await check('a fourth redirect between allowlisted hosts is refused', async () => {
+ const mock = makeMock(() => ({ status: 302, headers: { location: 'https://api.weather.gov/next' } }));
+ setEgress(mock.guard);
+ await assert.rejects(maps.get('https://api.weather.gov/start'),
+ (e) => e instanceof MapsEgressError && e.code === 'tools.maps.too_many_redirects');
+ });
+
+ await check('a redirect to another allowlisted host drops the caller headers', async () => {
+ const mock = makeMock((rec) => rec.hostname === 'ridb.recreation.gov'
+ ? { status: 302, headers: { location: 'https://api.weather.gov/next' } }
+ : { status: 200, body: '{}' });
+ setEgress(mock.guard);
+ await maps.get('https://ridb.recreation.gov/api/v1/facilities', { headers: { apikey: 'k' } });
+ assert.strictEqual(mock.seen.length, 2, 'both hops were made');
+ assert.strictEqual(mock.seen[0].headers.apikey, 'k', 'the key goes to the host the caller asked for');
+ assert.strictEqual(mock.seen[1].hostname, 'api.weather.gov');
+ assert.strictEqual(mock.seen[1].headers.apikey, undefined, 'the key is not resent cross-origin');
+ assert.strictEqual(mock.seen[1].headers['Accept-Encoding'], 'identity');
+ });
+
+ await check('a same-origin redirect keeps the caller headers', async () => {
+ const mock = makeMock((rec) => rec.path.endsWith('/start')
+ ? { status: 302, headers: { location: 'https://ridb.recreation.gov/api/v1/facilities' } }
+ : { status: 200, body: '{}' });
+ setEgress(mock.guard);
+ await maps.get('https://ridb.recreation.gov/api/v1/start', { headers: { apikey: 'k' } });
+ assert.strictEqual(mock.seen.length, 2, 'both hops were made');
+ assert.strictEqual(mock.seen[1].headers.apikey, 'k', 'a same-origin redirect keeps the key');
+ });
+
+ await check('a host resolving to a private address is refused by the pinned guard', async () => {
+ setEgress(createEgress({ lookup: (h, o, cb) => cb(null, [{ address: '10.0.0.5', family: 4 }]) }));
+ await assert.rejects(maps.get('https://api.weather.gov/points/47,-122'), (e) => e instanceof TargetRefused);
+ });
+
+ await check('an oversized answer is refused and the terrain source falls back to null', async () => {
+ const mock = makeMock({ status: 200, body: 'x'.repeat(1.5 * 1024 * 1024) });
+ setEgress(mock.guard);
+ await assert.rejects(maps.get('https://api.open-meteo.com/v1/elevation?latitude=47&longitude=-122'),
+ (e) => e instanceof MapsEgressError && e.code === 'tools.maps.response_too_large');
+ assert.strictEqual(await terrain.getElevation(47, -122), null);
+ });
+
+ await check('overpassQuery POSTs the encoded QL as a form body', async () => {
+ const mock = makeMock({ status: 200, body: '{"elements":[]}' });
+ setEgress(mock.guard);
+ await overpassQuery('node(1);');
+ const rec = mock.seen.find(r => r.hostname === 'overpass-api.de');
+ assert.ok(rec, 'Overpass was asked');
+ assert.strictEqual(rec.method, 'POST');
+ assert.strictEqual(rec.body, `data=${encodeURIComponent('node(1);')}`);
+ assert.strictEqual(rec.headers['Content-Type'], 'application/x-www-form-urlencoded');
+ });
+
+ await check('a non-2xx answer surfaces err.response.status', async () => {
+ const mock = makeMock({ status: 429, body: '{"error":"slow down"}' });
+ setEgress(mock.guard);
+ await assert.rejects(maps.get('https://api.weather.gov/alerts/active'),
+ (e) => e.response && e.response.status === 429);
+ });
+
+ await check('no maps server file outside egress.js reaches the network itself', async () => {
+ const serverDir = path.join(__dirname, '..', 'server');
+ const moduleRequire = /require\(\s*['"](axios|node-fetch|got|undici|node:https?|https?)['"]\s*\)/;
+ const bad = [];
+ (function walk(dir) {
+ for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+ const p = path.join(dir, e.name);
+ if (e.isDirectory()) walk(p);
+ else if (e.name.endsWith('.js') && e.name !== 'egress.js') {
+ const src = fs.readFileSync(p, 'utf8');
+ if (moduleRequire.test(src) || /\bfetch\s*\(/.test(src) || /\baxios\b/.test(src)) bad.push(path.relative(serverDir, p));
+ }
+ }
+ })(serverDir);
+ assert.deepStrictEqual(bad, [], `files still reaching the network: ${bad.join(', ')}`);
+ });
+
+ if (failures) { console.log(`\n${failures} failure(s)`); process.exit(1); }
+ console.log('\nmaps egress adapter: all checks passed');
+ process.exit(0);
+})().catch((e) => { console.error(e); process.exit(1); });
@@ -1,63 +1,88 @@
@@ -1,63 +1,88 @@
'use strict';
'use strict';
+// OpenChargeMap leaves the process only through the maps egress adapter
+// (apps/maps/server/egress.js): the request carries the configured key, a key-less source still
+// works (free tier), and a failed answer resolves empty instead of throwing. Every transport is a
+// mock — no DNS, no socket — so any real network use fails the test.
const assert = require('assert');
const assert = require('assert');
-const Module = require('module');
+const { EventEmitter } = require('events');
+const { createEgress } = require('../../_shared/egress');
+const maps = require('../server/egress');
+const { setEgress } = maps;
+const openchargemap = require('../server/sources/openchargemap');
+
+const PUBLIC = '93.184.216.34';
const requests = [];
const requests = [];
-let response = { status: 200, truncated: false, body: Buffer.from('[]') };
-const egress = {
- follow: async (url, options) => {
- requests.push({ url, options });
- return response;
- },
-};
-
-// Keep the upstream offline. utils loads axios for other sources, but this source must use egress.
-const originalLoad = Module._load;
-Module._load = function (request, parent, isMain) {
- if (request === '../../../_shared/egress') return { createEgress: () => egress };
- if (request === 'axios') return {};
- return originalLoad.call(this, request, parent, isMain);
-};
-let openchargemap;
-try {
- openchargemap = require('../server/sources/openchargemap');
-} finally {
- Module._load = originalLoad;
+let answer = { status: 200, body: '[]' };
+
+function install() {
+ const guard = createEgress({
+ lookup: (host, opts, cb) => cb(null, [{ address: PUBLIC, family: 4 }]),
+ httpsRequest: (opts, cb) => {
+ const rec = { method: opts.method, hostname: opts.hostname, path: opts.path, headers: opts.headers };
+ requests.push(rec);
+ const req = new EventEmitter();
+ req.destroy = () => {};
+ req.end = () => {
+ process.nextTick(() => {
+ const res = new EventEmitter();
+ res.statusCode = answer.status;
+ res.statusMessage = 'OK';
+ res.headers = {};
+ res.destroy = () => {};
+ cb(res);
+ process.nextTick(() => {
+ res.emit('data', Buffer.from(answer.body));
+ res.emit('end');
+ });
+ });
+ };
+ return req;
+ },
+ });
+ setEgress(guard);
}
}
(async () => {
(async () => {
+ install();
const station = {
const station = {
ID: 42,
ID: 42,
AddressInfo: { Title: 'Test charger', Latitude: 47.61, Longitude: -122.33 },
AddressInfo: { Title: 'Test charger', Latitude: 47.61, Longitude: -122.33 },
UsageCost: 'Free',
UsageCost: 'Free',
};
};
- response = { status: 200, truncated: false, body: Buffer.from(JSON.stringify([station])) };
const key = 'sentinel-ocm-key-not-a-secret';
const key = 'sentinel-ocm-key-not-a-secret';
+
+ answer = { status: 200, body: JSON.stringify([station]) };
const configured = await openchargemap.search(47.61, -122.33, 25, key);
const configured = await openchargemap.search(47.61, -122.33, 25, key);
- assert.equal(requests.length, 1);
- const url = new URL(requests[0].url);
- assert.equal(url.origin, 'https://api.openchargemap.io');
+ assert.equal(requests.length, 1, 'one request');
+ const req = requests[0];
+ assert.equal(req.method, 'GET');
+ assert.equal(req.hostname, 'api.openchargemap.io');
+ const url = new URL(req.path, 'https://api.openchargemap.io');
assert.equal(url.pathname, '/v3/poi');
assert.equal(url.pathname, '/v3/poi');
assert.equal(url.searchParams.get('key'), key);
assert.equal(url.searchParams.get('key'), key);
assert.equal(url.searchParams.get('latitude'), '47.61');
assert.equal(url.searchParams.get('latitude'), '47.61');
assert.equal(url.searchParams.get('longitude'), '-122.33');
assert.equal(url.searchParams.get('longitude'), '-122.33');
assert.equal(url.searchParams.get('distance'), '25');
assert.equal(url.searchParams.get('distance'), '25');
assert.equal(url.searchParams.get('maxresults'), '50');
assert.equal(url.searchParams.get('maxresults'), '50');
- assert.equal(requests[0].options.timeoutMs, 12000);
- assert.ok(requests[0].options.maxBytes > 0);
- assert.equal(requests[0].options.headers['User-Agent'], 'OpenVibeApp/2.0');
+ assert.equal(req.headers['User-Agent'], 'OpenVibeApp/2.0');
+ assert.ok(req.headers['Accept-Encoding'], 'the adapter asks for identity encoding');
assert.equal(configured[0].id, 'ocm-42');
assert.equal(configured[0].id, 'ocm-42');
assert.ok(!JSON.stringify(configured).includes(key));
assert.ok(!JSON.stringify(configured).includes(key));
+ // Free tier: no key configured still sends (without a key) and still maps.
requests.length = 0;
requests.length = 0;
const unconfigured = await openchargemap.search(47.61, -122.33, 25, '');
const unconfigured = await openchargemap.search(47.61, -122.33, 25, '');
assert.equal(requests.length, 1);
assert.equal(requests.length, 1);
- assert.equal(new URL(requests[0].url).searchParams.has('key'), false);
+ assert.equal(new URL(requests[0].path, 'https://api.openchargemap.io').searchParams.has('key'), false);
assert.equal(unconfigured[0].id, 'ocm-42');
assert.equal(unconfigured[0].id, 'ocm-42');
- response = { status: 503, truncated: false, body: Buffer.from('[]') };
- assert.deepEqual(await openchargemap.search(47.61, -122.33, 25, key), []);
- response = { status: 200, truncated: true, body: Buffer.from('[]') };
- assert.deepEqual(await openchargemap.search(47.61, -122.33, 25, key), []);
- console.log('OpenChargeMap egress: configured, unconfigured, and failed responses passed');
+ // A failed upstream resolves empty: the adapter throws, the source catches.
+ for (const status of [401, 503]) {
+ requests.length = 0;
+ answer = { status, body: '[]' };
+ assert.deepEqual(await openchargemap.search(47.61, -122.33, 25, key), []);
+ }
+
+ console.log('OpenChargeMap egress: configured, key-less and failed responses passed');
})().catch(err => { console.error(err); process.exitCode = 1; });
})().catch(err => { console.error(err); process.exitCode = 1; });
@@ -1,7 +1,7 @@
@@ -1,7 +1,7 @@
'use strict';
'use strict';
// Provider keys live in the satellite's config (apps/maps/server/config.js) and are handed to each
// Provider keys live in the satellite's config (apps/maps/server/config.js) and are handed to each
// source — never read from the environment inside a source module, never a key in the code.
// source — never read from the environment inside a source module, never a key in the code.
-// Here the transport the sources use (axios) is stubbed, so nothing touches the network:
+// The transport is mocked (setEgress), so nothing touches the network:
// 1. the RIDB key the caller passes reaches the RIDB request,
// 1. the RIDB key the caller passes reaches the RIDB request,
// 2. that key never appears in the search results a caller gets back,
// 2. that key never appears in the search results a caller gets back,
// 3. with no RIDB key, RIDB sends nothing and search still answers (an empty list),
// 3. with no RIDB key, RIDB sends nothing and search still answers (an empty list),
@@ -9,16 +9,32 @@
@@ -9,16 +9,32 @@
const assert = require('assert');
const assert = require('assert');
const fs = require('fs');
const fs = require('fs');
const path = require('path');
const path = require('path');
+const { EventEmitter } = require('events');
+const { createEgress } = require('../../_shared/egress');
+const { setEgress } = require('../server/egress');
-// Stub axios before the source module is loaded: both hold the same cached module object.
-const axios = require('axios');
+const RIDB_KEY = 'sentinel-ridb-key-not-a-secret';
+
+// A guard whose transport records every request and answers 200 with a small JSON body.
const requests = [];
const requests = [];
-axios.get = async (url, opts = {}) => { requests.push({ url, opts }); return { data: {} }; };
+setEgress(createEgress({
+ lookup: (host, opts, cb) => cb(null, [{ address: '93.184.216.34', family: 4 }]),
+ httpsRequest: (opts, cb) => {
+ requests.push({ hostname: opts.hostname, path: opts.path, opts: { headers: opts.headers } });
+ const req = new EventEmitter();
+ req.destroy = () => {};
+ req.end = () => process.nextTick(() => {
+ const res = new EventEmitter();
+ res.statusCode = 200; res.statusMessage = 'OK'; res.headers = {}; res.destroy = () => {};
+ cb(res);
+ process.nextTick(() => { res.emit('data', Buffer.from('{"RECDATA":[]}')); res.emit('end'); });
+ });
+ return req;
+ },
+}));
const ridb = require('../server/sources/ridb');
const ridb = require('../server/sources/ridb');
-const RIDB_KEY = 'sentinel-ridb-key-not-a-secret';
-
(async () => {
(async () => {
let failures = 0;
let failures = 0;
const check = async (name, fn) => {
const check = async (name, fn) => {
@@ -31,8 +47,8 @@ const RIDB_KEY = 'sentinel-ridb-key-not-a-secret';
@@ -31,8 +47,8 @@ const RIDB_KEY = 'sentinel-ridb-key-not-a-secret';
await ridb.search(47.6062, -122.3321, 25, RIDB_KEY);
await ridb.search(47.6062, -122.3321, 25, RIDB_KEY);
assert.ok(requests.length > 0, 'RIDB sent a request');
assert.ok(requests.length > 0, 'RIDB sent a request');
for (const r of requests) {
for (const r of requests) {
- assert.ok(/recreation\.gov/.test(r.url), `request goes to RIDB: ${r.url}`);
- assert.strictEqual(r.opts.headers && r.opts.headers.apikey, RIDB_KEY, `apikey header on ${r.url}`);
+ assert.ok(/recreation\.gov/.test(`https://${r.hostname}${r.path}`), `request goes to RIDB: ${r.hostname}${r.path}`);
+ assert.strictEqual(r.opts.headers && r.opts.headers.apikey, RIDB_KEY, `apikey header on ${r.hostname}${r.path}`);
}
}
});
});