openvibers/openvibe.rent Commits

Show HUD's Small Area Fair Market Rent beside the housing in a known ZIP (#149)

OP OpenVibers committed 14 days ago · a113d1b121 · 5 files, +193 −1

Changes

ddiff --git a/server/app.js b/server/app.js
iindex e53eefa..9fe5fa3 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -85,6 +85,8 @@ async function createApp(opts = {}) {
// HUD's Fair Market Rents from OpenVibe.Sources (server/housing/fair-market-rents.js), for the city pages; a test
// passes a stand-in source through opts.fmrSource.
ctx.fmr = require('./housing/fair-market-rents').createFairMarketRents({ config, s, fetchImpl, log, source: opts.fmrSource || null });
++ // HUD's Small Area Fair Market Rents, read from OpenVibe.Sources (source hud-safmr) and kept in memory.
++ ctx.safmr = require('./housing/safmr').createSafmr({ config, s, fetchImpl, log, source: opts.safmrSource || null });
// Account export and deletion (ADR-033, ./identity/account-data.js): the three tables that hold a person's rows.
// The sender posts to Network's internal export/deletion routes with this service's own client-credentials token;
ddiff --git a/server/housing/safmr.js b/server/housing/safmr.js
nnew file mode 100644
iindex 0000000..db854d0
---- /dev/null
++++ b/server/housing/safmr.js
@@@ -0,0 +1,133 @@
++'use strict';
++
++/**
++ * HUD's Small Area Fair Market Rents, read from OpenVibe.Sources (source hud-safmr, category housing), never from HUD:
++ * Sources reads HUD's own SAFMR layer under its robots and terms checks, and Rent follows Sources' change feed
++ * (openvibe-publishing/ingest) into memory — no table of its own, because the figure is shown next to the housing a
++ * reader is looking at, not stored as a listing.
++ *
++ * A Small Area Fair Market Rent is HUD's estimate of what a modest rental costs in a ZIP code (and, where HUD has
++ * several, in each HUD metro area inside it), from a studio to four bedrooms. HUD sets them each October, so every
++ * figure is shown with the fiscal year it carries: the layer holds the years 2018 and 2024 today.
++ *
++ * A ZIP can appear under two HUD metro codes (the same figures twice) and under more than one fiscal year: the newest
++ * year for a ZIP is the one shown, and a ZIP is shown once.
++ *
++ * Off without OV_SOURCES_INTERNAL_URL and the OAuth client secret (a test passes its own source).
++ */
++const ingest = require('openvibe-publishing/ingest');
++
++const SOURCE_KEY = 'hud-safmr';
++const CURSOR = 'sources.housing.safmr.after';
++const EVERY_MS = 6 * 60 * 60 * 1000;
++const FIRST_MS = 30 * 1000;
++const HUD_URL = 'https://www.huduser.gov/portal/datasets/fmr.html';
++
++// The sizes HUD states, in the words a reader sees: a studio, then one to four bedrooms.
++const SIZES = [['studio', 'safmr_0br'], ['b1', 'safmr_1br'], ['b2', 'safmr_2br'], ['b3', 'safmr_3br'], ['b4', 'safmr_4br']];
++
++const amount = (v) => {
++ const n = typeof v === 'number' ? v : (typeof v === 'string' && /^\d{1,6}$/.test(v.trim()) ? Number(v.trim()) : NaN);
++ return Number.isFinite(n) && n > 0 && n < 100000 ? n : null;
++};
++const zipOf = (v) => {
++ const s = String(v == null ? '' : v).trim();
++ return /^\d{5}$/.test(s) ? s : null;
++};
++const yearOf = (v) => {
++ const n = Number(v);
++ return Number.isInteger(n) && n >= 1990 && n <= 2100 ? n : null;
++};
++
++/** A Sources item → the SAFMR figures Rent keeps in memory, or null when the item is not one of them. */
++function rowOf(item) {
++ const f = (item && item.fields) || {};
++ const zip = zipOf(f.zip);
++ const year = yearOf(f.fiscal_year);
++ if (!zip || !year) return null;
++ const row = { zip, year, metro: typeof f.metro_name === 'string' ? f.metro_name.replace(/\s+/g, ' ').trim().slice(0, 200) : null };
++ let any = false;
++ for (const [, col] of SIZES) { row[col] = amount(f[col]); if (row[col] != null) any = true; }
++ if (!any) return null;
++ return row;
++}
++
++function createSafmr({ config, s, fetchImpl = globalThis.fetch, log = console, source = null }) {
++ const url = String((config.sources && config.sources.internalUrl) || '').replace(/\/+$/, '');
++ const client = source || (url && config.oauth.clientSecret
++ ? ingest.createSourcesClient({ config: { ...config, sources: { internalUrl: url, category: 'housing' } }, fetchImpl })
++ : null);
++ const cursor = ingest.createChangeCursor(s.db, { prefix: 'rent', now: s.now });
++ const stats = { pulls: 0, applied: 0, removed: 0, last_at: null, last_error: null };
++ let byZip = new Map();
++ let timer = null;
++ let first = null;
++ let running = false;
++
++ async function apply(item, t) {
++ if (!item || item.source_key !== SOURCE_KEY || item.kind !== 'safmr') return 'hold';
++ const row = rowOf(item);
++ if (!row) return 'hold';
++ const old = byZip.get(row.zip);
++ // A ZIP is shown once, with the newest fiscal year HUD gives it; the same figures under a second HUD metro
++ // code change nothing.
++ if (!old || row.year > old.year) byZip.set(row.zip, row);
++ return 'applied';
++ }
++
++ async function pull() {
++ if (!client || running) return { skipped: true };
++ running = true;
++ try {
++ const out = await ingest.pullChanges({ db: s.db, cursor, source: client, name: CURSOR, apply });
++ stats.pulls++;
++ stats.applied += out.applied;
++ stats.removed += out.removed;
++ stats.last_at = s.iso();
++ stats.last_error = null;
++ if (out.applied || out.removed) log.log(`[SAFMR] ${out.applied} ZIP figures updated from OpenVibe.Sources`);
++ return out;
++ } catch (err) {
++ stats.last_error = String((err && err.message) || err).slice(0, 300);
++ log.warn(`[SAFMR] pull from OpenVibe.Sources failed: ${stats.last_error}`);
++ return { error: stats.last_error };
++ } finally { running = false; }
++ }
++
++ return {
++ enabled: Boolean(client),
++ stats,
++ pull,
++ apply,
++ size: () => byZip.size,
++ /** HUD's figures for a ZIP, by size, with the fiscal year they carry; null when the ZIP is not known. */
++ forZip(zip) {
++ const key = zipOf(zip);
++ const row = key ? byZip.get(key) : null;
++ if (!row) return null;
++ return { zip: row.zip, year: row.year, metro: row.metro, rents: Object.fromEntries(SIZES.map(([k, col]) => [k, row[col]])) };
++ },
++ /** The one-line reference shown next to a place in a known ZIP: never a listing, always dated. */
++ reference(zip, beds = 2) {
++ const row = this.forZip(zip);
++ if (!row) return null;
++ const col = SIZES.find(([k]) => k === (beds === 0 ? 'studio' : `b${beds}`));
++ const value = col ? row.rents[col[0]] : null;
++ if (value == null) return null;
++ const size = col[0] === 'studio' ? 'studio' : `${beds}-bedroom`;
++ return { amount: value, year: row.year, metro: row.metro, size, text: `HUD fair market rent for a ${size} here: $${value.toLocaleString('en-US')} (FY${row.year})` };
++ },
++ start() {
++ if (timer) return false;
++ if (!client) return false;
++ timer = setInterval(() => { pull().catch(() => {}); }, EVERY_MS);
++ timer.unref();
++ first = setTimeout(() => { pull().catch(() => {}); }, FIRST_MS);
++ first.unref();
++ return true;
++ },
++ stop() { clearInterval(timer); clearTimeout(first); timer = null; first = null; },
++ };
++}
++
++module.exports = { createSafmr, rowOf, SOURCE_KEY, SIZES, HUD_URL };
ddiff --git a/server/http/pages.js b/server/http/pages.js
iindex b3d23f9..0eda319 100644
---- a/server/http/pages.js
++++ b/server/http/pages.js
@@@ -381,10 +381,13 @@ ${housingPlaces.KINDS.map((k) => html`<label class="housing-kind"><input type="c
const details = housingDetails(r);
const units = housingUnits(r);
const map = r.lat != null && r.lon != null ? `https://www.openstreetmap.org/?mlat=${r.lat}&mlon=${r.lon}#map=17/${r.lat}/${r.lon}` : null;
++ // HUD's Small Area Fair Market Rent for this ZIP, as a dated reference beside the place: never a listing.
++ const safmr = ctx.safmr && r.zip ? ctx.safmr.reference(r.zip, 2) : null;
return html`<li class="housing-card" id="${r.id}">
<div class="housing-card-head"><h4>${titleCase(r.name) || '—'}</h4><span class="housing-dist">${housingMi(r.distance_km)}</span></div>
${where ? html`<p class="housing-addr">${where}</p>` : ''}
${units ? html`<p class="housing-units">${units}</p>` : ''}
++${safmr ? html`<p class="housing-fmr muted small">${safmr.text}</p>` : ''}
${details.length ? html`<ul class="housing-facts">${details.map((d) => html`<li>${d}</li>`)}</ul>` : ''}
<p class="housing-actions">${r.phone ? html`<a href="tel:${tel}">${phoneText(r.phone)}</a>` : ''}${r.extra && r.extra.email ? html`<a href="mailto:${r.extra.email}">Email</a>` : ''}${r.extra && r.extra.website ? html`<a href="${r.extra.website}" rel="nofollow noopener external">Website</a>` : ''}${r.extra && r.extra.homestore_url ? html`<a href="${r.extra.homestore_url}" rel="noopener external">Check it on HUD Homestore</a>` : ''}${map ? html`<a href="${map}" rel="noopener">Map</a>` : ''}</p>
</li>`;
ddiff --git a/server/index.js b/server/index.js
iindex e3aa69a..c3a42cc 100644
---- a/server/index.js
++++ b/server/index.js
@@@ -43,6 +43,7 @@ async function start() {
ctx.search.start();
ctx.screen.start(); // the AI listing screen's sweep (off without OpenVibe.AI)
ctx.fmr.start(); // HUD's Fair Market Rents: loaded now, then followed from OpenVibe.Sources every six hours
++ ctx.safmr.start(); // HUD's Small Area Fair Market Rents: followed from OpenVibe.Sources every six hours
// One sweep at boot (a restart must not leave a stale listing up a moment longer than it should), then a timer.
const expiry = startExpiryTimer(ctx.s);
@@@ -55,7 +56,7 @@ async function start() {
// Subscribe to the two ADR-033 topics at OpenVibe.Events (idempotent; off without RENT_EVENTS_URL and
// RENT_EVENTS_SECRET). The consumer itself is mounted in server/app.js.
const subscriptions = require('./events-consumer').startSubscriptions({ config, port: config.port, secret: config.events.secrets[0] || '' });
-- const extra = [() => { if (subscriptions) subscriptions.stop(); }, () => ctx.fmr.stop()];
++ const extra = [() => { if (subscriptions) subscriptions.stop(); }, () => ctx.fmr.stop(), () => ctx.safmr.stop()];
createLifecycle({ server, ctx, timers: [expiry.timer, ctx.housingPruneTimer, ...(hpiJob ? [hpiJob.timer] : [])], extra });
return { server, ctx, expiry };
}
ddiff --git a/test/safmr.test.js b/test/safmr.test.js
nnew file mode 100644
iindex 0000000..7f58b59
---- /dev/null
++++ b/test/safmr.test.js
@@@ -0,0 +1,53 @@
++'use strict';
++/**
++ * HUD's Small Area Fair Market Rents (server/housing/safmr.js), from a recorded item of the Sources item (no live call
++ * here): the figures a ZIP carries, shown as a dated reference, and only where a ZIP is known.
++ */
++const assert = require('assert');
++const { createSafmr, rowOf } = require('../server/housing/safmr');
++const { check, done } = require('./helpers/boot');
++const { testDb } = require('./helpers/db');
++
++// A recorded item of the Sources `hud-safmr` feed: the shape the api adapter stores (a HUD row, Indianapolis-Carmel).
++const item = (over = {}) => ({
++ id: 'itm_1', source_key: 'hud-safmr', category: 'housing', kind: 'safmr', identity: '1',
++ title: 'Indianapolis-Carmel, IN HUD Metro FMR Area', summary: null, authors: [], published_at: null,
++ fields: { zip: 47435, hud_code: 'METRO26900M26900', metro_name: 'Indianapolis-Carmel, IN HUD Metro FMR Area', fiscal_year: 2024, safmr_0br: 850, safmr_1br: 990, safmr_2br: 1170, safmr_3br: 1430, safmr_4br: 1690 },
++ provenance: { retrieved_at: '2026-10-30T06:00:00.000Z' }, removed: null, ...over,
++});
++
++(async () => {
++ const fresh = await testDb();
++ const s = { db: fresh.db, now: () => Date.parse('2026-10-30T08:00:00Z'), iso: () => '2026-10-30T08:00:00.000Z' };
++ const safmr = createSafmr({ config: { sources: { internalUrl: '' } }, s, log: { log() {}, warn() {} } });
++ try {
++ await check('a HUD row becomes the reference for its ZIP, with the fiscal year it carries', async () => {
++ await safmr.apply(item());
++ const ref = safmr.reference('47435', 2);
++ assert.strictEqual(ref.text, 'HUD fair market rent for a 2-bedroom here: $1,170 (FY2024)');
++ assert.strictEqual(ref.amount, 1170);
++ assert.strictEqual(ref.year, 2024);
++ assert.strictEqual(safmr.reference('47435', 0).text, 'HUD fair market rent for a studio here: $850 (FY2024)');
++ });
++
++ await check('the reference appears only where a ZIP is known', async () => {
++ assert.strictEqual(safmr.reference('99999', 2), null, 'a ZIP HUD does not know has no figure');
++ assert.strictEqual(safmr.reference(null, 2), null, 'no ZIP, no figure');
++ assert.strictEqual(safmr.reference('', 2), null);
++ });
++
++ await check('the newest fiscal year for a ZIP is the one shown, and a second metro row changes nothing', async () => {
++ await safmr.apply(item({ id: 'itm_2', identity: '2', fields: { ...item().fields, fiscal_year: 2018, safmr_2br: 900 } }));
++ assert.strictEqual(safmr.reference('47435', 2).amount, 1170, 'the older year does not win');
++ await safmr.apply(item({ id: 'itm_3', identity: '3', fields: { ...item().fields, hud_code: 'METRO26900MM8960' } }));
++ assert.strictEqual(safmr.size(), 1, 'a ZIP is shown once');
++ });
++
++ await check('a row that is not a HUD figure is not kept', () => {
++ assert.strictEqual(rowOf(item({ fields: { ...item().fields, zip: 'not a zip' } })), null);
++ assert.strictEqual(rowOf(item({ fields: { ...item().fields, fiscal_year: null } })), null);
++ assert.strictEqual(rowOf(item({ fields: { ...item().fields, safmr_0br: null, safmr_1br: null, safmr_2br: null, safmr_3br: null, safmr_4br: null } })), null);
++ });
++ } finally { await fresh.close(); }
++ done();
++})().catch((err) => { console.error(err); process.exit(1); });