Rent robots.txt: live search queries (/everywhere?, /housing?) are not for crawlers; their landing pages and the city pages stay open (#71)

ddiff --git a/server/http/discovery.js b/server/http/discovery.js
iindex 8e09d5a..efd5679 100644
---- a/server/http/discovery.js
++++ b/server/http/discovery.js
@@@ -21,7 +21,10 @@ const listings = require('../listings/listings');
  
  const SITE_NAME = 'OpenVibe.Rent';
  const DESCRIPTION = 'OpenVibe.Rent — every rental site, one search: apartments, houses, rooms, furnished stays, commercial space, parking, storage and cars across Craigslist, Facebook Marketplace, Zillow, Apartments.com, Zumper and more, with typical rents for 885 US cities, one board for listings from anywhere, and listings people post here.';
--const DISALLOW = ['/auth/', '/api/', '/post', '/mine', '/saved', '/board', '/staff'];
++// /everywhere?… and /housing?… are live searches that ask other rental sites on each visit: crawlers get the landing
++// pages and the city pages (/in/…), not every query. GPTBot crawled 1,500 such queries on 2026-10-16 and was
++// refused by the per-address limit for about 1,000 of them.
++const DISALLOW = ['/auth/', '/api/', '/post', '/mine', '/saved', '/board', '/staff', '/everywhere?', '/housing?'];
  // A sitemap floor and a ceiling that keeps llms-full.txt inside its own byte budget, so the two stay in step.
  const LISTINGS_MAX = 1000;
  
ddiff --git a/test/discovery.test.js b/test/discovery.test.js
iindex 8a7560b..cef2d4f 100644
---- a/test/discovery.test.js
++++ b/test/discovery.test.js
@@@ -24,6 +24,9 @@ const PRIVATE = ['/auth/', '/api/'];
              assert.match(r.headers.get('cache-control') || '', /public, max-age=\d+/);
              for (const d of ['/auth/', '/api/']) assert.ok(r.text.includes(`Disallow: ${d}`), `missing Disallow: ${d}`);
              for (const d of ['/updates']) assert.ok(!r.text.includes(`Disallow: ${d}`), `${d} must stay crawlable`);
++            // Live searches of other rental sites are not crawled per query; their landing pages and the city pages are.
++            for (const d of ['/everywhere?', '/housing?']) assert.ok(r.text.includes(`Disallow: ${d}`), `missing Disallow: ${d}`);
++            for (const d of ['/everywhere', '/housing', '/in/']) assert.ok(!new RegExp(`^Disallow: ${d.replace(/\//g, '\\/')}$`, 'm').test(r.text), `${d} must stay crawlable`);
              assert.ok(r.text.includes('Sitemap: https://openvibe.rent/sitemap.xml'), 'sitemap not named');
              assert.ok(/^User-agent: \*$/m.test(r.text), 'no User-agent: * group');
          });