openvibers/openvibe.rent Commits
Rent: city pages (Rent and OpenVibe.Homes) are swept apart from listings, at start and daily; Homes city pages are Search documents (homes_city, contracts 0.198.0) (#69)
Changes
@@@ -31,7 +31,8 @@ Census Bureau's keyless ACS 5-year table files and gazetteer (see the script for(Census ACS B25077, with its margin; the $2,000,001 top code reads "$2,000,000 or more") for the same 885 cities asRent by city, the FHFA trend for the area, the calculator prefilled with the median price (the rate stays theperson's), what renting there costs with a link to the Rent city page (and back), every for-sale site, the county-- records and nearby cities. In the Homes sitemap and llms.txt.++ records and nearby cities. In the Homes sitemap and llms.txt, and OpenVibe.Search documents (type `homes_city`, sent++ by the same sweep through a second search feed on the Homes origin; contracts 0.198.0).- **Links back to Rent:** renting and the board stay on OpenVibe.Rent.## What it is@@@ -71,6 +71,24 @@ function describeCity(c) {};}++/** An OpenVibe.Homes city page (server/homes/city-pages.js) as a Search document (type homes_city, contracts 0.198.0). */++function describeHomesCity(c) {++ const where = `${c.name}, ${c.state}`;++ const value = cities.valueText(c.value);++ const two = cities.rentText(c.rent.b2);++ return {++ listed: true,++ title: `Buying a home in ${where}`,++ summary: `${value ? `Median home value in ${where}: ${value}. ` : ''}Every for-sale site's search for ${c.name} in one place${two ? `; renting a typical 2-bedroom costs ${two} a month` : ''}.`,++ body: [`Buying a home in ${where}, ${c.stateName}.`, value ? `Median value of owner-occupied homes (Census ACS ${cities.SOURCE.years}): ${value}.` : '',++ `Every for-sale site's search for ${c.name}: Zillow, Realtor.com, Homes.com, Trulia and more, government and bank homes, county records and the mortgage on a typical home.`].filter(Boolean).join('\n'),++ facets: { state: c.state, population: c.population, home_value: c.value },++ authorship: { mode: 'human' },++ publishedAt: cities.SOURCE.built ? `${cities.SOURCE.built}T00:00:00.000Z` : null,++ updatedAt: cities.SOURCE.built ? `${cities.SOURCE.built}T00:00:00.000Z` : null,++ };++}++function createSearchIndex({ config, s, log = console, outbox = null, indexnow = null }) {const out = outbox || createServiceOutbox({db: s.db, source: 'rent', eventsUrl: config.events.url || null, networkInternalUrl: config.networkInternalUrl,@@@ -85,6 +103,14 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =rows: (after, limit) => s.db.many('SELECT * FROM rent_listings WHERE id > $1 ORDER BY id LIMIT $2', [after, limit]),exists: async (ids) => (await s.db.many('SELECT id FROM rent_listings WHERE id = ANY($1)', [ids])).map((r) => r.id),},++++ },++ });++ // Rent by city's pages change only when the city data is rebuilt: their own feed, swept at start and once a day, so++ // the listing sweeps (every expiry tick) stay small.++ const cityFeed = createSearchFeed({++ owner: 'rent', db: s.db, outbox: out, baseUrl: config.baseUrl, now: s.now, log, indexnow,++ types: {city: {page: (c) => c.path,document: (c) => describeCity(c),@@@ -93,6 +119,19 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =},},});++ // OpenVibe.Homes' city pages live on its own host: another feed with that origin, the same owner, outbox and++ // revision table (their type, homes_city, keeps them apart).++ const homesFeed = config.homes && config.homes.url ? createSearchFeed({++ owner: 'rent', db: s.db, outbox: out, baseUrl: config.homes.url, now: s.now, log,++ types: {++ homes_city: {++ page: (c) => c.path,++ document: (c) => describeHomesCity(c),++ rows: async (after, limit) => CITY_ROWS.filter((c) => c.id > after).slice(0, limit),++ exists: async (ids) => ids.filter((id) => CITY_IDS.has(String(id))),++ },++ },++ }) : null;const pending = new Set();const timers = [];let flushTimer = null;@@@ -134,11 +173,20 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =const res = await feed.sweep();const l = res.listing;if (l.sent || l.removed || l.failed) log.log(`[Search] listings: ${l.sent} sent, ${l.removed} removed, ${l.failed} failed of ${l.seen}`);-- const c = res.city;-- if (c && (c.sent || c.removed || c.failed)) log.log(`[Search] cities: ${c.sent} sent, ${c.removed} removed, ${c.failed} failed of ${c.seen}`);++++ if (out.kick) out.kick().catch(() => {});++ return res;++ }++ /** The city pages of Rent and Homes (cityFeed, homesFeed): at start and once a day. */++ async function sweepCities() {++ const res = { city: (await cityFeed.sweep()).city };++ if (homesFeed) res.homes_city = (await homesFeed.sweep()).homes_city;++ const parts = Object.entries(res).filter(([, v]) => v && (v.sent || v.removed || v.failed)).map(([k, v]) => `${k} ${v.sent} sent, ${v.removed} removed, ${v.failed} failed of ${v.seen}`);++ if (parts.length) log.log(`[Search] ${parts.join('; ')}`);if (out.kick) out.kick().catch(() => {});return res;}++ const citiesQuietly = () => { sweepCities().catch((err) => log.warn(`[Search] city sweep failed: ${(err && err.message) || err}`)); };const quietly = () => { sweep().catch((err) => log.warn(`[Search] sweep failed: ${(err && err.message) || err}`)); };function start() {@@@ -148,7 +196,11 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =if (kick.unref) kick.unref();const tick = setInterval(quietly, INTERVAL_MS);if (tick.unref) tick.unref();-- timers.push(kick, tick);++ const cityKick = setTimeout(citiesQuietly, START_DELAY_MS + 30_000);++ if (cityKick.unref) cityKick.unref();++ const cityTick = setInterval(citiesQuietly, 24 * 3_600_000);++ if (cityTick.unref) cityTick.unref();++ timers.push(kick, tick, cityKick, cityTick);return true;}@@@ -164,7 +216,7 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =if (flushing) await flushing;}-- return { feed, outbox: out, describe, touch, syncId, sweep, afterExpiry: quietly, settle, start, stop, status: () => out.status() };++ return { feed, outbox: out, describe, touch, syncId, sweep, sweepCities, afterExpiry: quietly, settle, start, stop, status: () => out.status() };}module.exports = { createSearchIndex, describe, EVENT_TYPES };@@@ -59,7 +59,7 @@ const DAY = 86_400_000;await check('Rent by city: the first sweep sends a city document per city page, exactly the contract; the next sends none', async () => {const before = (await outbox()).length;-- const res = await search.sweep();++ const res = await search.sweepCities();assert.ok(res.city.sent >= 850, JSON.stringify(res.city));const cityEnvs = (await outbox()).slice(before).filter((e) => e.payload && e.payload.type === 'city');assert.strictEqual(cityEnvs.length, res.city.sent);@@@ -68,7 +68,16 @@ const DAY = 86_400_000;assert.match(austin.payload.title, /^Renting in Austin, TX$/);assert.match(austin.payload.summary, /2 bedrooms \$1,845/);assert.deepStrictEqual([austin.payload.facets.state, austin.payload.facets.rent_2br], ['TX', 1845]);-- assert.strictEqual((await search.sweep()).city.sent, 0, 'unchanged cities send nothing');++ const homes = (await outbox()).slice(before).filter((e) => e.payload && e.payload.type === 'homes_city');++ assert.strictEqual(homes.length, res.homes_city.sent, 'OpenVibe.Homes\' city pages too');++ const hAustin = valid(homes.find((e) => e.payload.id === 'tx-austin'));++ assert.match(hAustin.payload.canonical_url, /\/in\/tx\/austin$/);++ assert.ok(!hAustin.payload.canonical_url.startsWith('https://openvibe.rent'), 'on the Homes host');++ assert.match(hAustin.payload.title, /^Buying a home in Austin, TX$/);++ assert.match(hAustin.payload.summary, /Median home value in Austin, TX: \$555,300/);++ const again = await search.sweepCities();++ assert.deepStrictEqual([again.city.sent, again.homes_city.sent], [0, 0], 'unchanged cities send nothing');++ assert.ok(!('city' in (await search.sweep())), 'a listing sweep leaves the cities alone');});await check('an edit is the next revision; a sweep then sends nothing', async () => {
@@ -31,7 +31,8 @@ Census Bureau's keyless ACS 5-year table files and gazetteer (see the script for
@@ -31,7 +31,8 @@ Census Bureau's keyless ACS 5-year table files and gazetteer (see the script for
(Census ACS B25077, with its margin; the $2,000,001 top code reads "$2,000,000 or more") for the same 885 cities as
(Census ACS B25077, with its margin; the $2,000,001 top code reads "$2,000,000 or more") for the same 885 cities as
Rent by city, the FHFA trend for the area, the calculator prefilled with the median price (the rate stays the
Rent by city, the FHFA trend for the area, the calculator prefilled with the median price (the rate stays the
person's), what renting there costs with a link to the Rent city page (and back), every for-sale site, the county
person's), what renting there costs with a link to the Rent city page (and back), every for-sale site, the county
- records and nearby cities. In the Homes sitemap and llms.txt.
+ records and nearby cities. In the Homes sitemap and llms.txt, and OpenVibe.Search documents (type `homes_city`, sent
+ by the same sweep through a second search feed on the Homes origin; contracts 0.198.0).
- **Links back to Rent:** renting and the board stay on OpenVibe.Rent.
- **Links back to Rent:** renting and the board stay on OpenVibe.Rent.
## What it is
## What it is
@@ -71,6 +71,24 @@ function describeCity(c) {
@@ -71,6 +71,24 @@ function describeCity(c) {
};
};
}
}
+/** An OpenVibe.Homes city page (server/homes/city-pages.js) as a Search document (type homes_city, contracts 0.198.0). */
+function describeHomesCity(c) {
+ const where = `${c.name}, ${c.state}`;
+ const value = cities.valueText(c.value);
+ const two = cities.rentText(c.rent.b2);
+ return {
+ listed: true,
+ title: `Buying a home in ${where}`,
+ summary: `${value ? `Median home value in ${where}: ${value}. ` : ''}Every for-sale site's search for ${c.name} in one place${two ? `; renting a typical 2-bedroom costs ${two} a month` : ''}.`,
+ body: [`Buying a home in ${where}, ${c.stateName}.`, value ? `Median value of owner-occupied homes (Census ACS ${cities.SOURCE.years}): ${value}.` : '',
+ `Every for-sale site's search for ${c.name}: Zillow, Realtor.com, Homes.com, Trulia and more, government and bank homes, county records and the mortgage on a typical home.`].filter(Boolean).join('\n'),
+ facets: { state: c.state, population: c.population, home_value: c.value },
+ authorship: { mode: 'human' },
+ publishedAt: cities.SOURCE.built ? `${cities.SOURCE.built}T00:00:00.000Z` : null,
+ updatedAt: cities.SOURCE.built ? `${cities.SOURCE.built}T00:00:00.000Z` : null,
+ };
+}
+
function createSearchIndex({ config, s, log = console, outbox = null, indexnow = null }) {
function createSearchIndex({ config, s, log = console, outbox = null, indexnow = null }) {
const out = outbox || createServiceOutbox({
const out = outbox || createServiceOutbox({
db: s.db, source: 'rent', eventsUrl: config.events.url || null, networkInternalUrl: config.networkInternalUrl,
db: s.db, source: 'rent', eventsUrl: config.events.url || null, networkInternalUrl: config.networkInternalUrl,
@@ -85,6 +103,14 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
@@ -85,6 +103,14 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
rows: (after, limit) => s.db.many('SELECT * FROM rent_listings WHERE id > $1 ORDER BY id LIMIT $2', [after, limit]),
rows: (after, limit) => s.db.many('SELECT * FROM rent_listings WHERE id > $1 ORDER BY id LIMIT $2', [after, limit]),
exists: async (ids) => (await s.db.many('SELECT id FROM rent_listings WHERE id = ANY($1)', [ids])).map((r) => r.id),
exists: async (ids) => (await s.db.many('SELECT id FROM rent_listings WHERE id = ANY($1)', [ids])).map((r) => r.id),
},
},
+
+ },
+ });
+ // Rent by city's pages change only when the city data is rebuilt: their own feed, swept at start and once a day, so
+ // the listing sweeps (every expiry tick) stay small.
+ const cityFeed = createSearchFeed({
+ owner: 'rent', db: s.db, outbox: out, baseUrl: config.baseUrl, now: s.now, log, indexnow,
+ types: {
city: {
city: {
page: (c) => c.path,
page: (c) => c.path,
document: (c) => describeCity(c),
document: (c) => describeCity(c),
@@ -93,6 +119,19 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
@@ -93,6 +119,19 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
},
},
},
},
});
});
+ // OpenVibe.Homes' city pages live on its own host: another feed with that origin, the same owner, outbox and
+ // revision table (their type, homes_city, keeps them apart).
+ const homesFeed = config.homes && config.homes.url ? createSearchFeed({
+ owner: 'rent', db: s.db, outbox: out, baseUrl: config.homes.url, now: s.now, log,
+ types: {
+ homes_city: {
+ page: (c) => c.path,
+ document: (c) => describeHomesCity(c),
+ rows: async (after, limit) => CITY_ROWS.filter((c) => c.id > after).slice(0, limit),
+ exists: async (ids) => ids.filter((id) => CITY_IDS.has(String(id))),
+ },
+ },
+ }) : null;
const pending = new Set();
const pending = new Set();
const timers = [];
const timers = [];
let flushTimer = null;
let flushTimer = null;
@@ -134,11 +173,20 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
@@ -134,11 +173,20 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
const res = await feed.sweep();
const res = await feed.sweep();
const l = res.listing;
const l = res.listing;
if (l.sent || l.removed || l.failed) log.log(`[Search] listings: ${l.sent} sent, ${l.removed} removed, ${l.failed} failed of ${l.seen}`);
if (l.sent || l.removed || l.failed) log.log(`[Search] listings: ${l.sent} sent, ${l.removed} removed, ${l.failed} failed of ${l.seen}`);
- const c = res.city;
- if (c && (c.sent || c.removed || c.failed)) log.log(`[Search] cities: ${c.sent} sent, ${c.removed} removed, ${c.failed} failed of ${c.seen}`);
+
+ if (out.kick) out.kick().catch(() => {});
+ return res;
+ }
+ /** The city pages of Rent and Homes (cityFeed, homesFeed): at start and once a day. */
+ async function sweepCities() {
+ const res = { city: (await cityFeed.sweep()).city };
+ if (homesFeed) res.homes_city = (await homesFeed.sweep()).homes_city;
+ const parts = Object.entries(res).filter(([, v]) => v && (v.sent || v.removed || v.failed)).map(([k, v]) => `${k} ${v.sent} sent, ${v.removed} removed, ${v.failed} failed of ${v.seen}`);
+ if (parts.length) log.log(`[Search] ${parts.join('; ')}`);
if (out.kick) out.kick().catch(() => {});
if (out.kick) out.kick().catch(() => {});
return res;
return res;
}
}
+ const citiesQuietly = () => { sweepCities().catch((err) => log.warn(`[Search] city sweep failed: ${(err && err.message) || err}`)); };
const quietly = () => { sweep().catch((err) => log.warn(`[Search] sweep failed: ${(err && err.message) || err}`)); };
const quietly = () => { sweep().catch((err) => log.warn(`[Search] sweep failed: ${(err && err.message) || err}`)); };
function start() {
function start() {
@@ -148,7 +196,11 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
@@ -148,7 +196,11 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
if (kick.unref) kick.unref();
if (kick.unref) kick.unref();
const tick = setInterval(quietly, INTERVAL_MS);
const tick = setInterval(quietly, INTERVAL_MS);
if (tick.unref) tick.unref();
if (tick.unref) tick.unref();
- timers.push(kick, tick);
+ const cityKick = setTimeout(citiesQuietly, START_DELAY_MS + 30_000);
+ if (cityKick.unref) cityKick.unref();
+ const cityTick = setInterval(citiesQuietly, 24 * 3_600_000);
+ if (cityTick.unref) cityTick.unref();
+ timers.push(kick, tick, cityKick, cityTick);
return true;
return true;
}
}
@@ -164,7 +216,7 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
@@ -164,7 +216,7 @@ function createSearchIndex({ config, s, log = console, outbox = null, indexnow =
if (flushing) await flushing;
if (flushing) await flushing;
}
}
- return { feed, outbox: out, describe, touch, syncId, sweep, afterExpiry: quietly, settle, start, stop, status: () => out.status() };
+ return { feed, outbox: out, describe, touch, syncId, sweep, sweepCities, afterExpiry: quietly, settle, start, stop, status: () => out.status() };
}
}
module.exports = { createSearchIndex, describe, EVENT_TYPES };
module.exports = { createSearchIndex, describe, EVENT_TYPES };
@@ -59,7 +59,7 @@ const DAY = 86_400_000;
@@ -59,7 +59,7 @@ const DAY = 86_400_000;
await check('Rent by city: the first sweep sends a city document per city page, exactly the contract; the next sends none', async () => {
await check('Rent by city: the first sweep sends a city document per city page, exactly the contract; the next sends none', async () => {
const before = (await outbox()).length;
const before = (await outbox()).length;
- const res = await search.sweep();
+ const res = await search.sweepCities();
assert.ok(res.city.sent >= 850, JSON.stringify(res.city));
assert.ok(res.city.sent >= 850, JSON.stringify(res.city));
const cityEnvs = (await outbox()).slice(before).filter((e) => e.payload && e.payload.type === 'city');
const cityEnvs = (await outbox()).slice(before).filter((e) => e.payload && e.payload.type === 'city');
assert.strictEqual(cityEnvs.length, res.city.sent);
assert.strictEqual(cityEnvs.length, res.city.sent);
@@ -68,7 +68,16 @@ const DAY = 86_400_000;
@@ -68,7 +68,16 @@ const DAY = 86_400_000;
assert.match(austin.payload.title, /^Renting in Austin, TX$/);
assert.match(austin.payload.title, /^Renting in Austin, TX$/);
assert.match(austin.payload.summary, /2 bedrooms \$1,845/);
assert.match(austin.payload.summary, /2 bedrooms \$1,845/);
assert.deepStrictEqual([austin.payload.facets.state, austin.payload.facets.rent_2br], ['TX', 1845]);
assert.deepStrictEqual([austin.payload.facets.state, austin.payload.facets.rent_2br], ['TX', 1845]);
- assert.strictEqual((await search.sweep()).city.sent, 0, 'unchanged cities send nothing');
+ const homes = (await outbox()).slice(before).filter((e) => e.payload && e.payload.type === 'homes_city');
+ assert.strictEqual(homes.length, res.homes_city.sent, 'OpenVibe.Homes\' city pages too');
+ const hAustin = valid(homes.find((e) => e.payload.id === 'tx-austin'));
+ assert.match(hAustin.payload.canonical_url, /\/in\/tx\/austin$/);
+ assert.ok(!hAustin.payload.canonical_url.startsWith('https://openvibe.rent'), 'on the Homes host');
+ assert.match(hAustin.payload.title, /^Buying a home in Austin, TX$/);
+ assert.match(hAustin.payload.summary, /Median home value in Austin, TX: \$555,300/);
+ const again = await search.sweepCities();
+ assert.deepStrictEqual([again.city.sent, again.homes_city.sent], [0, 0], 'unchanged cities send nothing');
+ assert.ok(!('city' in (await search.sweep())), 'a listing sweep leaves the cities alone');
});
});
await check('an edit is the next revision; a sweep then sends nothing', async () => {
await check('an edit is the next revision; a sweep then sends nothing', async () => {