openvibers/openvibe.work Commits

One city, however a board spells it: München finds Munich, and the home counts each city once (#68)

OP OpenVibers committed 21 days ago · 39038fdeed · 4 files, +175 −10

Changes

ddiff --git a/server/jobs/companies.js b/server/jobs/companies.js
iindex a10c08e..1d0a175 100644
---- a/server/jobs/companies.js
++++ b/server/jobs/companies.js
@@@ -10,6 +10,8 @@
* A name with no letter or digit left (only CJK, say) has no slug and no page; its listings stay in the search.
*/
++const { mergePlaces } = require('./places');
++
// translate() pairs, one character each side: what the slug folds an accented letter to.
const FOLD = [['äàáâãåā', 'a'], ['çćč', 'c'], ['ďđ', 'd'], ['ëèéêē', 'e'], ['ğ', 'g'], ['ïìíîī', 'i'], ['ł', 'l'], ['ñńň', 'n'],
['öòóôõøō', 'o'], ['řŕ', 'r'], ['ßśšş', 's'], ['ťţ', 't'], ['üùúûůū', 'u'], ['ýÿ', 'y'], ['źżž', 'z'], ['æ', 'a'], ['œ', 'o']];
@@@ -58,13 +60,14 @@ async function bySlug(s, slug, { limit = 60 } = {}) {
if (!totals || !num(totals.n)) return null;
const [places, bySource, rows] = await Promise.all([
s.db.many(`SELECT trim(split_part(location, ',', 1)) AS p, count(*) AS n FROM work_listings WHERE ${where} AND coalesce(location, '') <> ''
-- GROUP BY 1 ORDER BY n DESC, p ASC LIMIT 12`, args),
++ GROUP BY 1 ORDER BY n DESC, p ASC LIMIT 40`, args),
s.db.many(`SELECT source, count(*) AS n FROM work_listings WHERE ${where} GROUP BY 1 ORDER BY n DESC, source ASC`, args),
s.db.many(`SELECT * FROM work_listings WHERE ${where} ORDER BY sort_at DESC, id DESC LIMIT ${Math.min(200, Math.max(1, Number(limit) || 60))}`, args),
]);
return {
slug: String(slug), company: totals.company, count: num(totals.n), remote: num(totals.remote), salaried: num(totals.salaried),
-- places: places.filter((r) => r.p).map((r) => ({ place: r.p, count: num(r.n) })),
++ // One city once, however the boards spell it (server/jobs/places.js).
++ places: mergePlaces(places).slice(0, 12),
sources: bySource.map((r) => ({ source: r.source, count: num(r.n) })),
rows,
};
ddiff --git a/server/jobs/places.js b/server/jobs/places.js
nnew file mode 100644
iindex 0000000..b56b8bb
---- /dev/null
++++ b/server/jobs/places.js
@@@ -0,0 +1,88 @@
++'use strict';
++
++/**
++ * One city, however a board spells it. The boards write the same place many ways: "München" and "Munich", "Köln" and
++ * "Cologne", "Zürich" and "Zurich", "London Office" (production on 2026-10-23: 467 Munich + 159 München, 157 Zürich +
++ * 56 Zurich). The listing keeps the board's own words; these helpers make a search for one spelling find all of them
++ * and let the home and company pages count one city once.
++ *
++ * patternsOf('münchen') → ['%munich%', '%münchen%', '%muenchen%'] the location filter's LIKE patterns
++ * placeOf('München') → { key: 'munich', name: 'Munich' } a place as the pages name it
++ * placeOf('London Office') → { key: 'london', name: 'London' }
++ */
++
++// Each city: the name the pages show, then the spellings boards use (lower case; the name's own included).
++const CITIES = [
++ ['Munich', 'münchen', 'muenchen'],
++ ['Cologne', 'köln', 'koeln'],
++ ['Zurich', 'zürich', 'zuerich'],
++ ['Nuremberg', 'nürnberg', 'nuernberg'],
++ ['Düsseldorf', 'dusseldorf', 'duesseldorf'],
++ ['Frankfurt', 'frankfurt am main'],
++ ['Hanover', 'hannover'],
++ ['Vienna', 'wien'],
++ ['Geneva', 'genève', 'geneve', 'genf'],
++ ['Prague', 'praha'],
++ ['Warsaw', 'warszawa'],
++ ['Milan', 'milano'],
++ ['Lisbon', 'lisboa'],
++ ['The Hague', 'den haag', "'s-gravenhage"],
++ ['Brussels', 'bruxelles', 'brussel'],
++ ['Copenhagen', 'københavn', 'kobenhavn'],
++ ['Gothenburg', 'göteborg', 'goteborg'],
++ ['Malmö', 'malmo'],
++ ['Kraków', 'krakow', 'cracow'],
++ ['Wrocław', 'wroclaw'],
++ ['Athens', 'athína'],
++ ['Bucharest', 'bucurești', 'bucuresti'],
++];
++const BY_SPELLING = new Map();
++for (const [name, ...others] of CITIES) {
++ const entry = { key: name.toLowerCase(), name, spellings: [name.toLowerCase(), ...others] };
++ for (const sp of entry.spellings) BY_SPELLING.set(sp, entry);
++}
++
++// Words a board adds to a city that are not part of it: "London Office", "Berlin (Hybrid)", "Paris HQ". (Not "Area":
++// "Bay Area" is a place.) A spelling that is part of other names stays out of CITIES ("roma" is in "Romania").
++const NOISE = /\s*(?:\((?:hybrid|remote|on-?site)\)|\b(?:office|hq|headquarters)\b)\s*$/i;
++
++/** % and _ in a place are characters, not wildcards (backslash is LIKE's default escape). */
++const literal = (s) => s.replace(/[\\%_]/g, '\\$&');
++
++/** The LIKE patterns a location filter applies: every spelling of a known city, or the term itself. [] for none. */
++function patternsOf(term) {
++ const t = String(term == null ? '' : term).trim().toLowerCase().slice(0, 120);
++ if (!t) return [];
++ const bare = t.replace(NOISE, '').trim() || t;
++ const city = BY_SPELLING.get(bare);
++ return (city ? city.spellings : [bare]).map((sp) => `%${literal(sp)}%`);
++}
++
++/** A board's place (the first part of its location) → { key, name }: one key and one name per city. */
++function placeOf(text) {
++ const raw = String(text == null ? '' : text).split(',')[0].trim().replace(NOISE, '').trim();
++ const city = BY_SPELLING.get(raw.toLowerCase());
++ if (city) return { key: city.key, name: city.name };
++ return { key: raw.toLowerCase(), name: raw };
++}
++
++/**
++ * Rows of { p: place text, n: count } → [{ place, count }], one per city, its name the most common spelling of an
++ * unknown place (the known cities use their own name), the largest first.
++ */
++function mergePlaces(rows) {
++ const byKey = new Map();
++ for (const r of rows) {
++ if (!r || !r.p) continue;
++ const { key, name } = placeOf(r.p);
++ if (!key) continue;
++ const n = Number(r.n) || 0;
++ const e = byKey.get(key) || { place: name, count: 0, top: 0 };
++ if (!BY_SPELLING.has(key) && n > e.top) { e.place = name; e.top = n; }
++ e.count += n;
++ byKey.set(key, e);
++ }
++ return [...byKey.values()].map(({ place, count }) => ({ place, count })).sort((a, b) => b.count - a.count || a.place.localeCompare(b.place));
++}
++
++module.exports = { patternsOf, placeOf, mergePlaces, CITIES };
ddiff --git a/server/jobs/store.js b/server/jobs/store.js
iindex 0e93e22..35728e5 100644
---- a/server/jobs/store.js
++++ b/server/jobs/store.js
@@@ -9,6 +9,7 @@
* index, and the LIKE pattern is a bound parameter, never built from what a caller typed.
*/
const crypto = require('crypto');
++const places = require('./places');
const PAGE_SIZE = 30;
const MAX_PAGE = 100;
@@@ -80,14 +81,15 @@ function toWire(row, { source } = {}) {
}
// The filters a caller set, as bound parameters — never string-built. Everything is a LIKE on a lowercased column,
--// and the term's own % and _ are literal characters.
++// and the term's own % and _ are literal characters. The location is a LIKE over every spelling of the place
++// (server/jobs/places.js: "munich" also finds "München"), an empty array for none.
const PARAM = {
q: (n) => `($${n} = '' OR search_text LIKE $${n} ESCAPE '\\')`,
remote: (n) => `($${n} = '' OR remote = ($${n} = 'true'))`,
type: (n) => `($${n} = '' OR lower(coalesce(job_type, '')) LIKE $${n} ESCAPE '\\')`,
-- location: (n) => `($${n} = '' OR lower(coalesce(location, '')) LIKE $${n} ESCAPE '\\')`,
++ location: (n) => `(cardinality($${n}::text[]) = 0 OR lower(coalesce(location, '')) LIKE ANY ($${n}::text[]))`,
};
--const valueOf = { q: (f) => (f.q ? like(f.q) : ''), remote: (f) => f.remote, type: (f) => (f.type ? like(f.type) : ''), location: (f) => (f.location ? like(f.location) : '') };
++const valueOf = { q: (f) => (f.q ? like(f.q) : ''), remote: (f) => f.remote, type: (f) => (f.type ? like(f.type) : ''), location: (f) => places.patternsOf(f.location) };
/**
* One page of listings, newest first, plus the facets for the current scope. The cursor is the position of the last
@@@ -166,7 +168,7 @@ const listRecent = (s, limit = 5, { remote = false } = {}) => s.db.many(
const SUMMARY_TTL_MS = 10 * 60 * 1000;
const summaries = new WeakMap();
const NOT_A_PLACE = /^(remote|remote job|anywhere|worldwide|hybrid|global|europe|emea|usa|united states|united kingdom|germany|france|uk)$/i;
--async function homeSummary(s, { now = Date.now(), tags = 12, places = 12, companies = 12 } = {}) {
++async function homeSummary(s, { now = Date.now(), tags = 12, places: placeCount = 12, companies = 12 } = {}) {
const hit = summaries.get(s);
if (hit && now - hit.at < SUMMARY_TTL_MS) return hit.value;
const iso = (ms) => new Date(ms).toISOString();
@@@ -178,13 +180,15 @@ async function homeSummary(s, { now = Date.now(), tags = 12, places = 12, compan
const tagRows = await s.db.many(
`SELECT t, count(*) AS n FROM (SELECT lower(trim(unnest(string_to_array(tags, ',')))) AS t FROM work_listings WHERE expired = false) x
WHERE length(t) >= 3 AND t <> 'remote' GROUP BY t ORDER BY n DESC, t ASC LIMIT $1`, [tags]);
-- const placeNames = (await s.db.many(
++ const placeGroups = (await s.db.many(
`SELECT trim(split_part(location, ',', 1)) AS p, count(*) AS n FROM work_listings
-- WHERE expired = false AND coalesce(location, '') <> '' GROUP BY 1 ORDER BY n DESC, p ASC LIMIT $1`, [places * 2]))
-- .map((r) => r.p).filter((p) => p && p.length >= 3 && p.length <= 40 && !NOT_A_PLACE.test(p) && !/remote/i.test(p)).slice(0, places);
++ WHERE expired = false AND coalesce(location, '') <> '' GROUP BY 1 ORDER BY n DESC, p ASC LIMIT $1`, [placeCount * 4]))
++ .filter((r) => r.p && r.p.length >= 3 && r.p.length <= 40 && !NOT_A_PLACE.test(r.p) && !/remote|home ?office/i.test(r.p));
++ // One city once, however the boards spell it, counted with the patterns its link's search applies.
++ const placeNames = places.mergePlaces(placeGroups).map((r) => r.place).slice(0, placeCount);
const placeRows = [];
for (const p of placeNames) {
-- placeRows.push({ place: p, count: num(await s.db.value(`SELECT count(*) AS n FROM work_listings WHERE expired = false AND ${PARAM.location(1)}`, [like(lower(p))])) });
++ placeRows.push({ place: p, count: num(await s.db.value(`SELECT count(*) AS n FROM work_listings WHERE expired = false AND ${PARAM.location(1)}`, [places.patternsOf(p)])) });
}
placeRows.sort((a, b) => b.count - a.count || a.place.localeCompare(b.place));
const companyRows = await s.db.many(
ddiff --git a/test/places.test.js b/test/places.test.js
nnew file mode 100644
iindex 0000000..17f65d8
---- /dev/null
++++ b/test/places.test.js
@@@ -0,0 +1,70 @@
++'use strict';
++/**
++ * One city, however a board spells it (server/jobs/places.js): a search for one spelling finds every spelling, the
++ * home page names the city once with the count its link's search finds, and a company page counts it once. A place
++ * that only contains a city's name ("Romania", "Bay Area") is not mistaken for one.
++ */
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++const places = require('../server/jobs/places');
++const store = require('../server/jobs/store');
++
++(async () => {
++ await check('the helpers: spellings, noise words, literal % and _', async () => {
++ assert.deepStrictEqual(places.patternsOf('MÜNCHEN'), ['%munich%', '%münchen%', '%muenchen%']);
++ assert.deepStrictEqual(places.patternsOf('Munich'), places.patternsOf('muenchen'));
++ assert.deepStrictEqual(places.patternsOf('London Office'), ['%london%']);
++ assert.deepStrictEqual(places.patternsOf('50%_off'), ['%50\\%\\_off%']);
++ assert.deepStrictEqual(places.patternsOf(''), []);
++ assert.deepStrictEqual(places.placeOf('Zürich, Zürich, Switzerland'), { key: 'zurich', name: 'Zurich' });
++ assert.deepStrictEqual(places.placeOf('Berlin (Hybrid)'), { key: 'berlin', name: 'Berlin' });
++ assert.deepStrictEqual(places.placeOf('San Francisco Bay Area'), { key: 'san francisco bay area', name: 'San Francisco Bay Area' });
++ assert.deepStrictEqual(places.mergePlaces([{ p: 'Munich', n: 4 }, { p: 'München', n: 2 }, { p: 'London Office', n: 1 }, { p: 'London', n: 3 }]),
++ [{ place: 'Munich', count: 6 }, { place: 'London', count: 4 }]);
++ });
++
++ const t = await boot();
++ try {
++ const s = t.ctx.s;
++ const now = new Date().toISOString();
++ const row = (sourceId, company, location) => ({
++ id: store.idOf('probe', sourceId), source: 'probe', source_id: sourceId, url: `https://jobs.example/${sourceId}`, title: `Engineer ${sourceId}`,
++ company, location, remote: false, tags: '', job_type: 'full time', salary: null, excerpt: 'x', search_text: `engineer ${company}`.toLowerCase(),
++ posted_at: now, sort_at: now,
++ });
++ await store.upsertListings(s, [
++ row('m1', 'Alpenwerk', 'München'), row('m2', 'Alpenwerk', 'Munich, Bavaria'), row('m3', 'Isar Labs', 'Muenchen'),
++ row('r1', 'Carpati', 'Bucharest, Romania'), row('l1', 'Thames Ltd', 'London Office'), row('l2', 'Thames Ltd', 'London'),
++ ], { sourceId: 'probe', fetchedAt: now });
++
++ await check('a search for one spelling finds every spelling, and only them', async () => {
++ const ids = async (q) => (await (await t.get(`/api/v1/jobs?location=${encodeURIComponent(q)}&limit=100`)).json()).jobs.filter((j) => j.source === 'probe').map((j) => j.title).sort();
++ assert.deepStrictEqual(await ids('munich'), ['Engineer m1', 'Engineer m2', 'Engineer m3']);
++ assert.deepStrictEqual(await ids('München'), ['Engineer m1', 'Engineer m2', 'Engineer m3']);
++ assert.deepStrictEqual(await ids('Romania'), ['Engineer r1'], 'a country still finds its places');
++ assert.deepStrictEqual(await ids('london office'), ['Engineer l1', 'Engineer l2']);
++ });
++
++ await check('a saved search counts new listings with the same spellings', async () => {
++ const n = await store.newCount(s, { query: '', remote: '', type: '', location: 'münchen', last_seen_at: '1970-01-01T00:00:00.000Z' });
++ assert.ok(n >= 3, `new since: ${n}`);
++ });
++
++ await check('the home names the city once, with the count its search finds', async () => {
++ const sum = await store.homeSummary(s, { now: Date.now() + 3600e3, places: 40 });
++ const munich = sum.places.filter((p) => /m(u|ü|ue)nchen|munich/i.test(p.place));
++ assert.deepStrictEqual(munich.map((p) => p.place), ['Munich']);
++ const found = (await (await t.get(`/api/v1/jobs?location=Munich&limit=100`)).json()).count;
++ assert.strictEqual(munich[0].count, found);
++ });
++
++ await check('a company page counts its city once', async () => {
++ const companies = require('../server/jobs/companies');
++ const c = await companies.bySlug(s, 'alpenwerk');
++ assert.deepStrictEqual(c.places, [{ place: 'Munich', count: 2 }]);
++ });
++ } finally {
++ await t.close();
++ }
++ done();
++})();