openvibers/openvibe.work Commits
One city, however a board spells it: München finds Munich, and the home counts each city once (#68)
Changes
@@@ -10,6 +10,8 @@* A name with no letter or digit left (only CJK, say) has no slug and no page; its listings stay in the search.*/++const { mergePlaces } = require('./places');++// translate() pairs, one character each side: what the slug folds an accented letter to.const FOLD = [['äàáâãåā', 'a'], ['çćč', 'c'], ['ďđ', 'd'], ['ëèéêē', 'e'], ['ğ', 'g'], ['ïìíîī', 'i'], ['ł', 'l'], ['ñńň', 'n'],['öòóôõøō', 'o'], ['řŕ', 'r'], ['ßśšş', 's'], ['ťţ', 't'], ['üùúûůū', 'u'], ['ýÿ', 'y'], ['źżž', 'z'], ['æ', 'a'], ['œ', 'o']];@@@ -58,13 +60,14 @@ async function bySlug(s, slug, { limit = 60 } = {}) {if (!totals || !num(totals.n)) return null;const [places, bySource, rows] = await Promise.all([s.db.many(`SELECT trim(split_part(location, ',', 1)) AS p, count(*) AS n FROM work_listings WHERE ${where} AND coalesce(location, '') <> ''-- GROUP BY 1 ORDER BY n DESC, p ASC LIMIT 12`, args),++ GROUP BY 1 ORDER BY n DESC, p ASC LIMIT 40`, args),s.db.many(`SELECT source, count(*) AS n FROM work_listings WHERE ${where} GROUP BY 1 ORDER BY n DESC, source ASC`, args),s.db.many(`SELECT * FROM work_listings WHERE ${where} ORDER BY sort_at DESC, id DESC LIMIT ${Math.min(200, Math.max(1, Number(limit) || 60))}`, args),]);return {slug: String(slug), company: totals.company, count: num(totals.n), remote: num(totals.remote), salaried: num(totals.salaried),-- places: places.filter((r) => r.p).map((r) => ({ place: r.p, count: num(r.n) })),++ // One city once, however the boards spell it (server/jobs/places.js).++ places: mergePlaces(places).slice(0, 12),sources: bySource.map((r) => ({ source: r.source, count: num(r.n) })),rows,};nnew file mode 100644@@@ -0,0 +1,88 @@++'use strict';++++/**++ * One city, however a board spells it. The boards write the same place many ways: "München" and "Munich", "Köln" and++ * "Cologne", "Zürich" and "Zurich", "London Office" (production on 2026-10-23: 467 Munich + 159 München, 157 Zürich +++ * 56 Zurich). The listing keeps the board's own words; these helpers make a search for one spelling find all of them++ * and let the home and company pages count one city once.++ *++ * patternsOf('münchen') → ['%munich%', '%münchen%', '%muenchen%'] the location filter's LIKE patterns++ * placeOf('München') → { key: 'munich', name: 'Munich' } a place as the pages name it++ * placeOf('London Office') → { key: 'london', name: 'London' }++ */++++// Each city: the name the pages show, then the spellings boards use (lower case; the name's own included).++const CITIES = [++ ['Munich', 'münchen', 'muenchen'],++ ['Cologne', 'köln', 'koeln'],++ ['Zurich', 'zürich', 'zuerich'],++ ['Nuremberg', 'nürnberg', 'nuernberg'],++ ['Düsseldorf', 'dusseldorf', 'duesseldorf'],++ ['Frankfurt', 'frankfurt am main'],++ ['Hanover', 'hannover'],++ ['Vienna', 'wien'],++ ['Geneva', 'genève', 'geneve', 'genf'],++ ['Prague', 'praha'],++ ['Warsaw', 'warszawa'],++ ['Milan', 'milano'],++ ['Lisbon', 'lisboa'],++ ['The Hague', 'den haag', "'s-gravenhage"],++ ['Brussels', 'bruxelles', 'brussel'],++ ['Copenhagen', 'københavn', 'kobenhavn'],++ ['Gothenburg', 'göteborg', 'goteborg'],++ ['Malmö', 'malmo'],++ ['Kraków', 'krakow', 'cracow'],++ ['Wrocław', 'wroclaw'],++ ['Athens', 'athína'],++ ['Bucharest', 'bucurești', 'bucuresti'],++];++const BY_SPELLING = new Map();++for (const [name, ...others] of CITIES) {++ const entry = { key: name.toLowerCase(), name, spellings: [name.toLowerCase(), ...others] };++ for (const sp of entry.spellings) BY_SPELLING.set(sp, entry);++}++++// Words a board adds to a city that are not part of it: "London Office", "Berlin (Hybrid)", "Paris HQ". (Not "Area":++// "Bay Area" is a place.) A spelling that is part of other names stays out of CITIES ("roma" is in "Romania").++const NOISE = /\s*(?:\((?:hybrid|remote|on-?site)\)|\b(?:office|hq|headquarters)\b)\s*$/i;++++/** % and _ in a place are characters, not wildcards (backslash is LIKE's default escape). */++const literal = (s) => s.replace(/[\\%_]/g, '\\$&');++++/** The LIKE patterns a location filter applies: every spelling of a known city, or the term itself. [] for none. */++function patternsOf(term) {++ const t = String(term == null ? '' : term).trim().toLowerCase().slice(0, 120);++ if (!t) return [];++ const bare = t.replace(NOISE, '').trim() || t;++ const city = BY_SPELLING.get(bare);++ return (city ? city.spellings : [bare]).map((sp) => `%${literal(sp)}%`);++}++++/** A board's place (the first part of its location) → { key, name }: one key and one name per city. */++function placeOf(text) {++ const raw = String(text == null ? '' : text).split(',')[0].trim().replace(NOISE, '').trim();++ const city = BY_SPELLING.get(raw.toLowerCase());++ if (city) return { key: city.key, name: city.name };++ return { key: raw.toLowerCase(), name: raw };++}++++/**++ * Rows of { p: place text, n: count } → [{ place, count }], one per city, its name the most common spelling of an++ * unknown place (the known cities use their own name), the largest first.++ */++function mergePlaces(rows) {++ const byKey = new Map();++ for (const r of rows) {++ if (!r || !r.p) continue;++ const { key, name } = placeOf(r.p);++ if (!key) continue;++ const n = Number(r.n) || 0;++ const e = byKey.get(key) || { place: name, count: 0, top: 0 };++ if (!BY_SPELLING.has(key) && n > e.top) { e.place = name; e.top = n; }++ e.count += n;++ byKey.set(key, e);++ }++ return [...byKey.values()].map(({ place, count }) => ({ place, count })).sort((a, b) => b.count - a.count || a.place.localeCompare(b.place));++}++++module.exports = { patternsOf, placeOf, mergePlaces, CITIES };@@@ -9,6 +9,7 @@* index, and the LIKE pattern is a bound parameter, never built from what a caller typed.*/const crypto = require('crypto');++const places = require('./places');const PAGE_SIZE = 30;const MAX_PAGE = 100;@@@ -80,14 +81,15 @@ function toWire(row, { source } = {}) {}// The filters a caller set, as bound parameters — never string-built. Everything is a LIKE on a lowercased column,--// and the term's own % and _ are literal characters.++// and the term's own % and _ are literal characters. The location is a LIKE over every spelling of the place++// (server/jobs/places.js: "munich" also finds "München"), an empty array for none.const PARAM = {q: (n) => `($${n} = '' OR search_text LIKE $${n} ESCAPE '\\')`,remote: (n) => `($${n} = '' OR remote = ($${n} = 'true'))`,type: (n) => `($${n} = '' OR lower(coalesce(job_type, '')) LIKE $${n} ESCAPE '\\')`,-- location: (n) => `($${n} = '' OR lower(coalesce(location, '')) LIKE $${n} ESCAPE '\\')`,++ location: (n) => `(cardinality($${n}::text[]) = 0 OR lower(coalesce(location, '')) LIKE ANY ($${n}::text[]))`,};--const valueOf = { q: (f) => (f.q ? like(f.q) : ''), remote: (f) => f.remote, type: (f) => (f.type ? like(f.type) : ''), location: (f) => (f.location ? like(f.location) : '') };++const valueOf = { q: (f) => (f.q ? like(f.q) : ''), remote: (f) => f.remote, type: (f) => (f.type ? like(f.type) : ''), location: (f) => places.patternsOf(f.location) };/*** One page of listings, newest first, plus the facets for the current scope. The cursor is the position of the last@@@ -166,7 +168,7 @@ const listRecent = (s, limit = 5, { remote = false } = {}) => s.db.many(const SUMMARY_TTL_MS = 10 * 60 * 1000;const summaries = new WeakMap();const NOT_A_PLACE = /^(remote|remote job|anywhere|worldwide|hybrid|global|europe|emea|usa|united states|united kingdom|germany|france|uk)$/i;--async function homeSummary(s, { now = Date.now(), tags = 12, places = 12, companies = 12 } = {}) {++async function homeSummary(s, { now = Date.now(), tags = 12, places: placeCount = 12, companies = 12 } = {}) {const hit = summaries.get(s);if (hit && now - hit.at < SUMMARY_TTL_MS) return hit.value;const iso = (ms) => new Date(ms).toISOString();@@@ -178,13 +180,15 @@ async function homeSummary(s, { now = Date.now(), tags = 12, places = 12, companconst tagRows = await s.db.many(`SELECT t, count(*) AS n FROM (SELECT lower(trim(unnest(string_to_array(tags, ',')))) AS t FROM work_listings WHERE expired = false) xWHERE length(t) >= 3 AND t <> 'remote' GROUP BY t ORDER BY n DESC, t ASC LIMIT $1`, [tags]);-- const placeNames = (await s.db.many(++ const placeGroups = (await s.db.many(`SELECT trim(split_part(location, ',', 1)) AS p, count(*) AS n FROM work_listings-- WHERE expired = false AND coalesce(location, '') <> '' GROUP BY 1 ORDER BY n DESC, p ASC LIMIT $1`, [places * 2]))-- .map((r) => r.p).filter((p) => p && p.length >= 3 && p.length <= 40 && !NOT_A_PLACE.test(p) && !/remote/i.test(p)).slice(0, places);++ WHERE expired = false AND coalesce(location, '') <> '' GROUP BY 1 ORDER BY n DESC, p ASC LIMIT $1`, [placeCount * 4]))++ .filter((r) => r.p && r.p.length >= 3 && r.p.length <= 40 && !NOT_A_PLACE.test(r.p) && !/remote|home ?office/i.test(r.p));++ // One city once, however the boards spell it, counted with the patterns its link's search applies.++ const placeNames = places.mergePlaces(placeGroups).map((r) => r.place).slice(0, placeCount);const placeRows = [];for (const p of placeNames) {-- placeRows.push({ place: p, count: num(await s.db.value(`SELECT count(*) AS n FROM work_listings WHERE expired = false AND ${PARAM.location(1)}`, [like(lower(p))])) });++ placeRows.push({ place: p, count: num(await s.db.value(`SELECT count(*) AS n FROM work_listings WHERE expired = false AND ${PARAM.location(1)}`, [places.patternsOf(p)])) });}placeRows.sort((a, b) => b.count - a.count || a.place.localeCompare(b.place));const companyRows = await s.db.many(nnew file mode 100644@@@ -0,0 +1,70 @@++'use strict';++/**++ * One city, however a board spells it (server/jobs/places.js): a search for one spelling finds every spelling, the++ * home page names the city once with the count its link's search finds, and a company page counts it once. A place++ * that only contains a city's name ("Romania", "Bay Area") is not mistaken for one.++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const places = require('../server/jobs/places');++const store = require('../server/jobs/store');++++(async () => {++ await check('the helpers: spellings, noise words, literal % and _', async () => {++ assert.deepStrictEqual(places.patternsOf('MÜNCHEN'), ['%munich%', '%münchen%', '%muenchen%']);++ assert.deepStrictEqual(places.patternsOf('Munich'), places.patternsOf('muenchen'));++ assert.deepStrictEqual(places.patternsOf('London Office'), ['%london%']);++ assert.deepStrictEqual(places.patternsOf('50%_off'), ['%50\\%\\_off%']);++ assert.deepStrictEqual(places.patternsOf(''), []);++ assert.deepStrictEqual(places.placeOf('Zürich, Zürich, Switzerland'), { key: 'zurich', name: 'Zurich' });++ assert.deepStrictEqual(places.placeOf('Berlin (Hybrid)'), { key: 'berlin', name: 'Berlin' });++ assert.deepStrictEqual(places.placeOf('San Francisco Bay Area'), { key: 'san francisco bay area', name: 'San Francisco Bay Area' });++ assert.deepStrictEqual(places.mergePlaces([{ p: 'Munich', n: 4 }, { p: 'München', n: 2 }, { p: 'London Office', n: 1 }, { p: 'London', n: 3 }]),++ [{ place: 'Munich', count: 6 }, { place: 'London', count: 4 }]);++ });++++ const t = await boot();++ try {++ const s = t.ctx.s;++ const now = new Date().toISOString();++ const row = (sourceId, company, location) => ({++ id: store.idOf('probe', sourceId), source: 'probe', source_id: sourceId, url: `https://jobs.example/${sourceId}`, title: `Engineer ${sourceId}`,++ company, location, remote: false, tags: '', job_type: 'full time', salary: null, excerpt: 'x', search_text: `engineer ${company}`.toLowerCase(),++ posted_at: now, sort_at: now,++ });++ await store.upsertListings(s, [++ row('m1', 'Alpenwerk', 'München'), row('m2', 'Alpenwerk', 'Munich, Bavaria'), row('m3', 'Isar Labs', 'Muenchen'),++ row('r1', 'Carpati', 'Bucharest, Romania'), row('l1', 'Thames Ltd', 'London Office'), row('l2', 'Thames Ltd', 'London'),++ ], { sourceId: 'probe', fetchedAt: now });++++ await check('a search for one spelling finds every spelling, and only them', async () => {++ const ids = async (q) => (await (await t.get(`/api/v1/jobs?location=${encodeURIComponent(q)}&limit=100`)).json()).jobs.filter((j) => j.source === 'probe').map((j) => j.title).sort();++ assert.deepStrictEqual(await ids('munich'), ['Engineer m1', 'Engineer m2', 'Engineer m3']);++ assert.deepStrictEqual(await ids('München'), ['Engineer m1', 'Engineer m2', 'Engineer m3']);++ assert.deepStrictEqual(await ids('Romania'), ['Engineer r1'], 'a country still finds its places');++ assert.deepStrictEqual(await ids('london office'), ['Engineer l1', 'Engineer l2']);++ });++++ await check('a saved search counts new listings with the same spellings', async () => {++ const n = await store.newCount(s, { query: '', remote: '', type: '', location: 'münchen', last_seen_at: '1970-01-01T00:00:00.000Z' });++ assert.ok(n >= 3, `new since: ${n}`);++ });++++ await check('the home names the city once, with the count its search finds', async () => {++ const sum = await store.homeSummary(s, { now: Date.now() + 3600e3, places: 40 });++ const munich = sum.places.filter((p) => /m(u|ü|ue)nchen|munich/i.test(p.place));++ assert.deepStrictEqual(munich.map((p) => p.place), ['Munich']);++ const found = (await (await t.get(`/api/v1/jobs?location=Munich&limit=100`)).json()).count;++ assert.strictEqual(munich[0].count, found);++ });++++ await check('a company page counts its city once', async () => {++ const companies = require('../server/jobs/companies');++ const c = await companies.bySlug(s, 'alpenwerk');++ assert.deepStrictEqual(c.places, [{ place: 'Munich', count: 2 }]);++ });++ } finally {++ await t.close();++ }++ done();++})();
@@ -10,6 +10,8 @@
@@ -10,6 +10,8 @@
* A name with no letter or digit left (only CJK, say) has no slug and no page; its listings stay in the search.
* A name with no letter or digit left (only CJK, say) has no slug and no page; its listings stay in the search.
*/
*/
+const { mergePlaces } = require('./places');
+
// translate() pairs, one character each side: what the slug folds an accented letter to.
// translate() pairs, one character each side: what the slug folds an accented letter to.
const FOLD = [['äàáâãåā', 'a'], ['çćč', 'c'], ['ďđ', 'd'], ['ëèéêē', 'e'], ['ğ', 'g'], ['ïìíîī', 'i'], ['ł', 'l'], ['ñńň', 'n'],
const FOLD = [['äàáâãåā', 'a'], ['çćč', 'c'], ['ďđ', 'd'], ['ëèéêē', 'e'], ['ğ', 'g'], ['ïìíîī', 'i'], ['ł', 'l'], ['ñńň', 'n'],
['öòóôõøō', 'o'], ['řŕ', 'r'], ['ßśšş', 's'], ['ťţ', 't'], ['üùúûůū', 'u'], ['ýÿ', 'y'], ['źżž', 'z'], ['æ', 'a'], ['œ', 'o']];
['öòóôõøō', 'o'], ['řŕ', 'r'], ['ßśšş', 's'], ['ťţ', 't'], ['üùúûůū', 'u'], ['ýÿ', 'y'], ['źżž', 'z'], ['æ', 'a'], ['œ', 'o']];
@@ -58,13 +60,14 @@ async function bySlug(s, slug, { limit = 60 } = {}) {
@@ -58,13 +60,14 @@ async function bySlug(s, slug, { limit = 60 } = {}) {
if (!totals || !num(totals.n)) return null;
if (!totals || !num(totals.n)) return null;
const [places, bySource, rows] = await Promise.all([
const [places, bySource, rows] = await Promise.all([
s.db.many(`SELECT trim(split_part(location, ',', 1)) AS p, count(*) AS n FROM work_listings WHERE ${where} AND coalesce(location, '') <> ''
s.db.many(`SELECT trim(split_part(location, ',', 1)) AS p, count(*) AS n FROM work_listings WHERE ${where} AND coalesce(location, '') <> ''
- GROUP BY 1 ORDER BY n DESC, p ASC LIMIT 12`, args),
+ GROUP BY 1 ORDER BY n DESC, p ASC LIMIT 40`, args),
s.db.many(`SELECT source, count(*) AS n FROM work_listings WHERE ${where} GROUP BY 1 ORDER BY n DESC, source ASC`, args),
s.db.many(`SELECT source, count(*) AS n FROM work_listings WHERE ${where} GROUP BY 1 ORDER BY n DESC, source ASC`, args),
s.db.many(`SELECT * FROM work_listings WHERE ${where} ORDER BY sort_at DESC, id DESC LIMIT ${Math.min(200, Math.max(1, Number(limit) || 60))}`, args),
s.db.many(`SELECT * FROM work_listings WHERE ${where} ORDER BY sort_at DESC, id DESC LIMIT ${Math.min(200, Math.max(1, Number(limit) || 60))}`, args),
]);
]);
return {
return {
slug: String(slug), company: totals.company, count: num(totals.n), remote: num(totals.remote), salaried: num(totals.salaried),
slug: String(slug), company: totals.company, count: num(totals.n), remote: num(totals.remote), salaried: num(totals.salaried),
- places: places.filter((r) => r.p).map((r) => ({ place: r.p, count: num(r.n) })),
+ // One city once, however the boards spell it (server/jobs/places.js).
+ places: mergePlaces(places).slice(0, 12),
sources: bySource.map((r) => ({ source: r.source, count: num(r.n) })),
sources: bySource.map((r) => ({ source: r.source, count: num(r.n) })),
rows,
rows,
};
};
new file mode 100644
new file mode 100644
@@ -0,0 +1,88 @@
@@ -0,0 +1,88 @@
+'use strict';
+
+/**
+ * One city, however a board spells it. The boards write the same place many ways: "München" and "Munich", "Köln" and
+ * "Cologne", "Zürich" and "Zurich", "London Office" (production on 2026-10-23: 467 Munich + 159 München, 157 Zürich +
+ * 56 Zurich). The listing keeps the board's own words; these helpers make a search for one spelling find all of them
+ * and let the home and company pages count one city once.
+ *
+ * patternsOf('münchen') → ['%munich%', '%münchen%', '%muenchen%'] the location filter's LIKE patterns
+ * placeOf('München') → { key: 'munich', name: 'Munich' } a place as the pages name it
+ * placeOf('London Office') → { key: 'london', name: 'London' }
+ */
+
+// Each city: the name the pages show, then the spellings boards use (lower case; the name's own included).
+const CITIES = [
+ ['Munich', 'münchen', 'muenchen'],
+ ['Cologne', 'köln', 'koeln'],
+ ['Zurich', 'zürich', 'zuerich'],
+ ['Nuremberg', 'nürnberg', 'nuernberg'],
+ ['Düsseldorf', 'dusseldorf', 'duesseldorf'],
+ ['Frankfurt', 'frankfurt am main'],
+ ['Hanover', 'hannover'],
+ ['Vienna', 'wien'],
+ ['Geneva', 'genève', 'geneve', 'genf'],
+ ['Prague', 'praha'],
+ ['Warsaw', 'warszawa'],
+ ['Milan', 'milano'],
+ ['Lisbon', 'lisboa'],
+ ['The Hague', 'den haag', "'s-gravenhage"],
+ ['Brussels', 'bruxelles', 'brussel'],
+ ['Copenhagen', 'københavn', 'kobenhavn'],
+ ['Gothenburg', 'göteborg', 'goteborg'],
+ ['Malmö', 'malmo'],
+ ['Kraków', 'krakow', 'cracow'],
+ ['Wrocław', 'wroclaw'],
+ ['Athens', 'athína'],
+ ['Bucharest', 'bucurești', 'bucuresti'],
+];
+const BY_SPELLING = new Map();
+for (const [name, ...others] of CITIES) {
+ const entry = { key: name.toLowerCase(), name, spellings: [name.toLowerCase(), ...others] };
+ for (const sp of entry.spellings) BY_SPELLING.set(sp, entry);
+}
+
+// Words a board adds to a city that are not part of it: "London Office", "Berlin (Hybrid)", "Paris HQ". (Not "Area":
+// "Bay Area" is a place.) A spelling that is part of other names stays out of CITIES ("roma" is in "Romania").
+const NOISE = /\s*(?:\((?:hybrid|remote|on-?site)\)|\b(?:office|hq|headquarters)\b)\s*$/i;
+
+/** % and _ in a place are characters, not wildcards (backslash is LIKE's default escape). */
+const literal = (s) => s.replace(/[\\%_]/g, '\\$&');
+
+/** The LIKE patterns a location filter applies: every spelling of a known city, or the term itself. [] for none. */
+function patternsOf(term) {
+ const t = String(term == null ? '' : term).trim().toLowerCase().slice(0, 120);
+ if (!t) return [];
+ const bare = t.replace(NOISE, '').trim() || t;
+ const city = BY_SPELLING.get(bare);
+ return (city ? city.spellings : [bare]).map((sp) => `%${literal(sp)}%`);
+}
+
+/** A board's place (the first part of its location) → { key, name }: one key and one name per city. */
+function placeOf(text) {
+ const raw = String(text == null ? '' : text).split(',')[0].trim().replace(NOISE, '').trim();
+ const city = BY_SPELLING.get(raw.toLowerCase());
+ if (city) return { key: city.key, name: city.name };
+ return { key: raw.toLowerCase(), name: raw };
+}
+
+/**
+ * Rows of { p: place text, n: count } → [{ place, count }], one per city, its name the most common spelling of an
+ * unknown place (the known cities use their own name), the largest first.
+ */
+function mergePlaces(rows) {
+ const byKey = new Map();
+ for (const r of rows) {
+ if (!r || !r.p) continue;
+ const { key, name } = placeOf(r.p);
+ if (!key) continue;
+ const n = Number(r.n) || 0;
+ const e = byKey.get(key) || { place: name, count: 0, top: 0 };
+ if (!BY_SPELLING.has(key) && n > e.top) { e.place = name; e.top = n; }
+ e.count += n;
+ byKey.set(key, e);
+ }
+ return [...byKey.values()].map(({ place, count }) => ({ place, count })).sort((a, b) => b.count - a.count || a.place.localeCompare(b.place));
+}
+
+module.exports = { patternsOf, placeOf, mergePlaces, CITIES };
@@ -9,6 +9,7 @@
@@ -9,6 +9,7 @@
* index, and the LIKE pattern is a bound parameter, never built from what a caller typed.
* index, and the LIKE pattern is a bound parameter, never built from what a caller typed.
*/
*/
const crypto = require('crypto');
const crypto = require('crypto');
+const places = require('./places');
const PAGE_SIZE = 30;
const PAGE_SIZE = 30;
const MAX_PAGE = 100;
const MAX_PAGE = 100;
@@ -80,14 +81,15 @@ function toWire(row, { source } = {}) {
@@ -80,14 +81,15 @@ function toWire(row, { source } = {}) {
}
}
// The filters a caller set, as bound parameters — never string-built. Everything is a LIKE on a lowercased column,
// The filters a caller set, as bound parameters — never string-built. Everything is a LIKE on a lowercased column,
-// and the term's own % and _ are literal characters.
+// and the term's own % and _ are literal characters. The location is a LIKE over every spelling of the place
+// (server/jobs/places.js: "munich" also finds "München"), an empty array for none.
const PARAM = {
const PARAM = {
q: (n) => `($${n} = '' OR search_text LIKE $${n} ESCAPE '\\')`,
q: (n) => `($${n} = '' OR search_text LIKE $${n} ESCAPE '\\')`,
remote: (n) => `($${n} = '' OR remote = ($${n} = 'true'))`,
remote: (n) => `($${n} = '' OR remote = ($${n} = 'true'))`,
type: (n) => `($${n} = '' OR lower(coalesce(job_type, '')) LIKE $${n} ESCAPE '\\')`,
type: (n) => `($${n} = '' OR lower(coalesce(job_type, '')) LIKE $${n} ESCAPE '\\')`,
- location: (n) => `($${n} = '' OR lower(coalesce(location, '')) LIKE $${n} ESCAPE '\\')`,
+ location: (n) => `(cardinality($${n}::text[]) = 0 OR lower(coalesce(location, '')) LIKE ANY ($${n}::text[]))`,
};
};
-const valueOf = { q: (f) => (f.q ? like(f.q) : ''), remote: (f) => f.remote, type: (f) => (f.type ? like(f.type) : ''), location: (f) => (f.location ? like(f.location) : '') };
+const valueOf = { q: (f) => (f.q ? like(f.q) : ''), remote: (f) => f.remote, type: (f) => (f.type ? like(f.type) : ''), location: (f) => places.patternsOf(f.location) };
/**
/**
* One page of listings, newest first, plus the facets for the current scope. The cursor is the position of the last
* One page of listings, newest first, plus the facets for the current scope. The cursor is the position of the last
@@ -166,7 +168,7 @@ const listRecent = (s, limit = 5, { remote = false } = {}) => s.db.many(
@@ -166,7 +168,7 @@ const listRecent = (s, limit = 5, { remote = false } = {}) => s.db.many(
const SUMMARY_TTL_MS = 10 * 60 * 1000;
const SUMMARY_TTL_MS = 10 * 60 * 1000;
const summaries = new WeakMap();
const summaries = new WeakMap();
const NOT_A_PLACE = /^(remote|remote job|anywhere|worldwide|hybrid|global|europe|emea|usa|united states|united kingdom|germany|france|uk)$/i;
const NOT_A_PLACE = /^(remote|remote job|anywhere|worldwide|hybrid|global|europe|emea|usa|united states|united kingdom|germany|france|uk)$/i;
-async function homeSummary(s, { now = Date.now(), tags = 12, places = 12, companies = 12 } = {}) {
+async function homeSummary(s, { now = Date.now(), tags = 12, places: placeCount = 12, companies = 12 } = {}) {
const hit = summaries.get(s);
const hit = summaries.get(s);
if (hit && now - hit.at < SUMMARY_TTL_MS) return hit.value;
if (hit && now - hit.at < SUMMARY_TTL_MS) return hit.value;
const iso = (ms) => new Date(ms).toISOString();
const iso = (ms) => new Date(ms).toISOString();
@@ -178,13 +180,15 @@ async function homeSummary(s, { now = Date.now(), tags = 12, places = 12, compan
@@ -178,13 +180,15 @@ async function homeSummary(s, { now = Date.now(), tags = 12, places = 12, compan
const tagRows = await s.db.many(
const tagRows = await s.db.many(
`SELECT t, count(*) AS n FROM (SELECT lower(trim(unnest(string_to_array(tags, ',')))) AS t FROM work_listings WHERE expired = false) x
`SELECT t, count(*) AS n FROM (SELECT lower(trim(unnest(string_to_array(tags, ',')))) AS t FROM work_listings WHERE expired = false) x
WHERE length(t) >= 3 AND t <> 'remote' GROUP BY t ORDER BY n DESC, t ASC LIMIT $1`, [tags]);
WHERE length(t) >= 3 AND t <> 'remote' GROUP BY t ORDER BY n DESC, t ASC LIMIT $1`, [tags]);
- const placeNames = (await s.db.many(
+ const placeGroups = (await s.db.many(
`SELECT trim(split_part(location, ',', 1)) AS p, count(*) AS n FROM work_listings
`SELECT trim(split_part(location, ',', 1)) AS p, count(*) AS n FROM work_listings
- WHERE expired = false AND coalesce(location, '') <> '' GROUP BY 1 ORDER BY n DESC, p ASC LIMIT $1`, [places * 2]))
- .map((r) => r.p).filter((p) => p && p.length >= 3 && p.length <= 40 && !NOT_A_PLACE.test(p) && !/remote/i.test(p)).slice(0, places);
+ WHERE expired = false AND coalesce(location, '') <> '' GROUP BY 1 ORDER BY n DESC, p ASC LIMIT $1`, [placeCount * 4]))
+ .filter((r) => r.p && r.p.length >= 3 && r.p.length <= 40 && !NOT_A_PLACE.test(r.p) && !/remote|home ?office/i.test(r.p));
+ // One city once, however the boards spell it, counted with the patterns its link's search applies.
+ const placeNames = places.mergePlaces(placeGroups).map((r) => r.place).slice(0, placeCount);
const placeRows = [];
const placeRows = [];
for (const p of placeNames) {
for (const p of placeNames) {
- placeRows.push({ place: p, count: num(await s.db.value(`SELECT count(*) AS n FROM work_listings WHERE expired = false AND ${PARAM.location(1)}`, [like(lower(p))])) });
+ placeRows.push({ place: p, count: num(await s.db.value(`SELECT count(*) AS n FROM work_listings WHERE expired = false AND ${PARAM.location(1)}`, [places.patternsOf(p)])) });
}
}
placeRows.sort((a, b) => b.count - a.count || a.place.localeCompare(b.place));
placeRows.sort((a, b) => b.count - a.count || a.place.localeCompare(b.place));
const companyRows = await s.db.many(
const companyRows = await s.db.many(
new file mode 100644
new file mode 100644
@@ -0,0 +1,70 @@
@@ -0,0 +1,70 @@
+'use strict';
+/**
+ * One city, however a board spells it (server/jobs/places.js): a search for one spelling finds every spelling, the
+ * home page names the city once with the count its link's search finds, and a company page counts it once. A place
+ * that only contains a city's name ("Romania", "Bay Area") is not mistaken for one.
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const places = require('../server/jobs/places');
+const store = require('../server/jobs/store');
+
+(async () => {
+ await check('the helpers: spellings, noise words, literal % and _', async () => {
+ assert.deepStrictEqual(places.patternsOf('MÜNCHEN'), ['%munich%', '%münchen%', '%muenchen%']);
+ assert.deepStrictEqual(places.patternsOf('Munich'), places.patternsOf('muenchen'));
+ assert.deepStrictEqual(places.patternsOf('London Office'), ['%london%']);
+ assert.deepStrictEqual(places.patternsOf('50%_off'), ['%50\\%\\_off%']);
+ assert.deepStrictEqual(places.patternsOf(''), []);
+ assert.deepStrictEqual(places.placeOf('Zürich, Zürich, Switzerland'), { key: 'zurich', name: 'Zurich' });
+ assert.deepStrictEqual(places.placeOf('Berlin (Hybrid)'), { key: 'berlin', name: 'Berlin' });
+ assert.deepStrictEqual(places.placeOf('San Francisco Bay Area'), { key: 'san francisco bay area', name: 'San Francisco Bay Area' });
+ assert.deepStrictEqual(places.mergePlaces([{ p: 'Munich', n: 4 }, { p: 'München', n: 2 }, { p: 'London Office', n: 1 }, { p: 'London', n: 3 }]),
+ [{ place: 'Munich', count: 6 }, { place: 'London', count: 4 }]);
+ });
+
+ const t = await boot();
+ try {
+ const s = t.ctx.s;
+ const now = new Date().toISOString();
+ const row = (sourceId, company, location) => ({
+ id: store.idOf('probe', sourceId), source: 'probe', source_id: sourceId, url: `https://jobs.example/${sourceId}`, title: `Engineer ${sourceId}`,
+ company, location, remote: false, tags: '', job_type: 'full time', salary: null, excerpt: 'x', search_text: `engineer ${company}`.toLowerCase(),
+ posted_at: now, sort_at: now,
+ });
+ await store.upsertListings(s, [
+ row('m1', 'Alpenwerk', 'München'), row('m2', 'Alpenwerk', 'Munich, Bavaria'), row('m3', 'Isar Labs', 'Muenchen'),
+ row('r1', 'Carpati', 'Bucharest, Romania'), row('l1', 'Thames Ltd', 'London Office'), row('l2', 'Thames Ltd', 'London'),
+ ], { sourceId: 'probe', fetchedAt: now });
+
+ await check('a search for one spelling finds every spelling, and only them', async () => {
+ const ids = async (q) => (await (await t.get(`/api/v1/jobs?location=${encodeURIComponent(q)}&limit=100`)).json()).jobs.filter((j) => j.source === 'probe').map((j) => j.title).sort();
+ assert.deepStrictEqual(await ids('munich'), ['Engineer m1', 'Engineer m2', 'Engineer m3']);
+ assert.deepStrictEqual(await ids('München'), ['Engineer m1', 'Engineer m2', 'Engineer m3']);
+ assert.deepStrictEqual(await ids('Romania'), ['Engineer r1'], 'a country still finds its places');
+ assert.deepStrictEqual(await ids('london office'), ['Engineer l1', 'Engineer l2']);
+ });
+
+ await check('a saved search counts new listings with the same spellings', async () => {
+ const n = await store.newCount(s, { query: '', remote: '', type: '', location: 'münchen', last_seen_at: '1970-01-01T00:00:00.000Z' });
+ assert.ok(n >= 3, `new since: ${n}`);
+ });
+
+ await check('the home names the city once, with the count its search finds', async () => {
+ const sum = await store.homeSummary(s, { now: Date.now() + 3600e3, places: 40 });
+ const munich = sum.places.filter((p) => /m(u|ü|ue)nchen|munich/i.test(p.place));
+ assert.deepStrictEqual(munich.map((p) => p.place), ['Munich']);
+ const found = (await (await t.get(`/api/v1/jobs?location=Munich&limit=100`)).json()).count;
+ assert.strictEqual(munich[0].count, found);
+ });
+
+ await check('a company page counts its city once', async () => {
+ const companies = require('../server/jobs/companies');
+ const c = await companies.bySlug(s, 'alpenwerk');
+ assert.deepStrictEqual(c.places, [{ place: 'Munich', count: 2 }]);
+ });
+ } finally {
+ await t.close();
+ }
+ done();
+})();