openvibers/openvibe.work Commits
Work: split a one-line job post into title, place, terms and pay (#267)
Changes
@@@ -75,7 +75,9 @@ Its own PostgreSQL tables, created by [migrations/](migrations/) and written by- **Event delivery** — OpenVibe.Events: Work receives `network.account.export_requested` and `network.account.deleted`and creates its two subscriptions at boot; it does not own the topics or the delivery.- **The listings themselves** — the boards (Arbeitnow, Remotive, RemoteOK and Hacker News "Who is hiring"): Work keeps a short excerpt and a link back,-- never the full description, and reads each board only on the terms it publishes.++ never the full description, and reads each board only on the terms it publishes. A board that writes a listing as++ one pipe-separated line (Hacker News) has that line split into company, role(s), place, terms and pay++ (`server/jobs/hiring-line.js`); the line itself stays at the front of the excerpt.## Depends onnnew file mode 100644@@@ -0,0 +1,5 @@++-- phase: expand++-- The rule that split a one-line board post into the listing's title (server/jobs/hiring-line.js). A listing is++-- re-derived once (scripts/rederive-hn-titles.js) and the marker means a later run never reads its own new title back++-- as if it were the board's original line. NULL for every board that never needed a split.++ALTER TABLE work_listings ADD COLUMN IF NOT EXISTS title_rule text;nnew file mode 100644@@@ -0,0 +1,33 @@++#!/usr/bin/env node++'use strict';++++/**++ * Re-derive the stored Hacker News "Who is hiring" listings whose title is still the board's whole pipe-separated++ * line (server/jobs/rederive-hn-titles.js). One-off: the ingest writes the split title from now on.++ *++ * node scripts/rederive-hn-titles.js show what would change (first 20), write nothing++ * node scripts/rederive-hn-titles.js --apply write it, then sweep Search with the changed documents++ *++ * It runs with the service's own environment and connection, like scripts/ingest.js. It reads only source++ * 'hn-whoishiring' and only rows without the title_rule marker, so running it twice is harmless.++ */++const configLib = require('../server/config');++const { openStore } = require('../server/db');++const { rederiveHnTitles } = require('../server/jobs/rederive-hn-titles');++++(async () => {++ const apply = process.argv.includes('--apply');++ const config = configLib.load();++ const s = await openStore(config, { log: console });++ try {++ const out = await rederiveHnTitles({ s, config, apply, log: console });++ for (const sample of out.samples) {++ process.stdout.write(` ${sample.id}\n was: ${sample.before}\n now: ${sample.after}\n`);++ }++ if (out.sent) process.stdout.write(` Search: ${out.sent.sent} sent, ${out.sent.removed} removed, ${out.sent.failed} failed of ${out.sent.seen}\n`);++ process.stdout.write(`${apply ? 'Applied' : 'Dry run'}: ${out.changed} of ${out.scanned} listing(s) change, ${out.skipped} left alone (no company)\n`);++ if (!apply) process.stdout.write('Nothing was written. Run with --apply to write it.\n');++ } finally {++ await s.close();++ }++})().catch((err) => { console.error('[OpenVibe.Work] re-derive failed:', err && err.message ? err.message : err); process.exit(1); });@@@ -285,7 +285,7 @@ ${raw(foldBlock({const l = listing.meta(row);const at = isoOf(l.posted_at) || isoOf(row.fetched_at) || new Date().toISOString();const where = [l.location, l.remote && !saysRemote(l.location) ? 'Remote' : '', listing.jobTypeLabel(l.job_type)].filter(Boolean).join(' · ');-- return `<entry><id>${xml(`${config.baseUrl}/jobs/${row.id}`)}</id><title>${xml(`${l.title} at ${l.company}`)}</title>`++ return `<entry><id>${xml(`${config.baseUrl}/jobs/${row.id}`)}</id><title>${xml(listing.fullTitle(l))}</title>`+ `<link rel="alternate" type="text/html" href="${xml(`${config.baseUrl}/jobs/${row.id}`)}"/><updated>${at}</updated>`+ `<author><name>${xml(sourceOf(row).name)}</name></author>`+ `<summary>${xml([where, listing.plainExcerpt(row).slice(0, 300)].filter(Boolean).join(' — '))}</summary></entry>`;@@@ -355,15 +355,15 @@ ${nextUrl ? html`<a class="sc-btn sc-primary" href="${nextUrl}">Older listings</styles: [showcase.STYLESHEET],body: html`<h1>No longer on the board</h1><p class="job-gone"><span class="badge bad">Gone</span> This listing is no longer on the board.${lastSeen ? html` Last seen on ${src.name} on ${lastSeen}.` : ' We did not record the day we last saw it.'}</p>--<p>${l.title} at ${l.company} is not in our listings any more, so it is not in the search either. The board's own listing is still <a href="${l.url}" rel="nofollow noopener external" target="_blank">here</a>, but the board may have taken it down.</p>++<p>${listing.fullTitle(l)} is not in our listings any more, so it is not in the search either. The board's own listing is still <a href="${l.url}" rel="nofollow noopener external" target="_blank">here</a>, but the board may have taken it down.</p><p><a class="sc-btn" href="/jobs">Search the listings</a></p>`,}, 404);}const tracked = signedIn(req) ? await applications.applicationByListing(s, subjectOf(req), row.id) : null;page(req, res, {index: true, cache: PUBLIC_CACHE,-- title: `${l.title} at ${l.company}`,-- description: listing.plainExcerpt(row).slice(0, 300) || `${l.title} at ${l.company}, from ${src.name}.`,++ title: listing.fullTitle(l),++ description: listing.plainExcerpt(row).slice(0, 300) || `${listing.fullTitle(l)}, from ${src.name}.`,crumbs: [{ label: 'Jobs', href: '/jobs' }, { label: l.title }],styles: [showcase.STYLESHEET, 'kit.css'],jsonLd: [listing.jobPosting(row, { siteUrl: config.baseUrl })],@@@ -461,7 +461,7 @@ ${!q && all.length > shown.length ? html`<p class="muted small">The ${n(shown.letitle: `Jobs at ${c.company}`,description: `${n(c.count)} open listing${c.count === 1 ? '' : 's'} at ${c.company}${c.remote ? `, ${n(c.remote)} remote` : ''}${c.places.length ? `, in ${c.places.slice(0, 3).map((p) => p.place).join(', ')}` : ''}, from the boards OpenVibe.Work reads, each linking back to the original listing.`.slice(0, 300),crumbs: [{ label: 'Companies', href: '/companies' }, { label: c.company }], styles: [showcase.STYLESHEET],-- jsonLd: [seo.jsonLd.itemList(`Jobs at ${c.company}`, c.rows.slice(0, 50).map((row) => ({ name: `${row.title} at ${row.company}`, url: `${base}/jobs/${row.id}` })), { url: `${base}/companies/${c.slug}` })],++ jsonLd: [seo.jsonLd.itemList(`Jobs at ${c.company}`, c.rows.slice(0, 50).map((row) => ({ name: listing.fullTitle(row), url: `${base}/jobs/${row.id}` })), { url: `${base}/companies/${c.slug}` })],body: html`<h1>Jobs at ${c.company}</h1><p class="lede">${n(c.count)} open listing${c.count === 1 ? '' : 's'}${c.remote ? html`, ${n(c.remote)} of them remote` : ''}${c.salaried ? html`, ${n(c.salaried)} stating the pay` : ''}. Listed on ${boards.length > 1 ? html`${boards.slice(0, -1).map((b, i) => html`${i ? ', ' : ''}${b}`)} and ${boards[boards.length - 1]}` : boards[0]}.</p>${c.places.length ? html`<p class="small">Where: ${c.places.map((p, i) => html`${i ? ', ' : ''}<a href="/jobs?${new URLSearchParams({ q: c.company, location: p.place })}">${p.place}</a> (${n(p.count)})`)}</p>` : ''}@@@ -556,7 +556,7 @@ ${notice('Give the search at least one thing to look for — words, a location,<p class="muted small">Or press “Track this job” on any <a href="/jobs">listing</a>.</p></section>`;const renderApplication = (req, res, a, { events = [], problem = null, assisted = false } = {}) => page(req, res, {-- title: `${a.title} at ${a.company}`, styles: [showcase.STYLESHEET], localStyles: APP_STYLES,++ title: listing.fullTitle(a), styles: [showcase.STYLESHEET], localStyles: APP_STYLES,crumbs: [{ label: 'Applications', href: '/applications' }, { label: a.title }],body: html`<div class="app-head"><h1>${a.title}</h1>${statusBadge(a.status)}</div><p class="app-company">${a.company}${a.url ? html` · <a href="${a.url}" rel="nofollow noopener external" target="_blank">the original listing</a>` : ''}${a.listing_id ? html` · <a href="/jobs/${a.listing_id}">this listing on OpenVibe.Work</a>` : ''}</p>@@@ -21,6 +21,7 @@*/const ingest = require('openvibe-publishing/ingest');const sources = require('../jobs/sources');++const { parseHiringLine, TITLE_RULE } = require('../jobs/hiring-line');const store = require('../jobs/store');const SOURCE_ID = 'arbeitnow';@@@ -103,42 +104,41 @@ function rawOfWwr(item) {/*** A Sources item → the row a Hacker News "Who is hiring" comment makes. The resolver's title is the comment's first-- * line, which posters usually write as "Company | Role | Place" or "Company is hiring a role": the company comes from-- * that, and a comment that does not name one is held by the same thin-listing gate as any other board. The excerpt is-- * Sources' plain-text excerpt, already stripped of email addresses and phone numbers; the full comment is its link.++ * line, which posters usually write as "Company | Role | Place | Terms | Pay" (or "Company is hiring a role"). Work++ * splits that line into the company, the role(s), the place, the terms and the pay (server/jobs/hiring-line.js), and++ * keeps the line at the front of the description so nothing the poster wrote is lost. A comment that names no company++ * is held by the same thin-listing gate as any other board. The excerpt is Sources' plain-text excerpt, already++ * stripped of email addresses and phone numbers; the full comment is its link.*/function rawOfHn(item) {const f = (item && item.fields) || {};const first = String((item && item.title) || '').replace(/\s+/g, ' ').trim();-- const parts = first.split(/\s*\|\s*/);-- let company = '';-- let title = first;-- if (parts.length >= 2 && parts[0].trim()) {-- company = parts[0].trim();-- title = parts.slice(1).join(' | ').trim() || first;-- } else {-- const m = /^(.+?)\s+(?:is|are)\s+hiring\b/i.exec(first);-- if (m && m[1].trim()) {-- company = m[1].trim();-- title = first.slice(m[0].length).replace(/^[\s:,-]+/, '').trim() || first;-- }-- }++ const parsed = parseHiringLine(first);const summary = String((item && item.summary) || '');++ // The original line is the listing's description too: Sources starts its excerpt with it, but an excerpt that was++ // cut before the first newline, or a board that sends the fields apart, must not lose it.++ const plain = excerptLines(summary);++ const description = plain.startsWith(first) ? summary : `${first}\n${summary}`;return {source_id: item.identity,url: item.canonical_url,-- title,-- company,-- location: null, // the comment rarely states one; the title may, in its own words-- remote: /\bremote\b/i.test(`${first} ${summary}`),-- tags: list(f.tags),-- job_type: null,-- salary: null,++ title: parsed.title,++ company: parsed.company,++ location: parsed.location,++ remote: parsed.remote,++ tags: list([...list(f.tags), ...parsed.tags]),++ job_type: parsed.job_type,++ salary: parsed.salary,posted_at: item.published_at || null,-- description_html: summary,++ description_html: description,++ title_rule: TITLE_RULE,};}++/** The start of a plain-text summary, with whitespace collapsed, for comparing it with the comment's first line. */++const excerptLines = (text) => String(text || '').replace(/\s+/g, ' ').trim();++++/*** The boards Work reads through OpenVibe.Sources: the Sources key an item arrives under, the id the listing is* stored with, the kind of item that source keeps, and how the item becomes a listing.@@@ -239,4 +239,4 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = consreturn { enabled: Boolean(client), pull, start, stop, apply, SOURCE_ID, sourceIds: active.map(([, e]) => e.source_id) };}--module.exports = { createFromSources, rawOf, rawOfWwr, rawOfHn, SOURCES, SOURCE_ID, SOURCE_KEY };++module.exports = { createFromSources, rawOf, rawOfWwr, rawOfHn, SOURCES, SOURCE_ID, SOURCE_KEY, TITLE_RULE };nnew file mode 100644@@@ -0,0 +1,439 @@++'use strict';++++/**++ * A job board post written as one pipe-separated line, split into the fields a listing has.++ *++ * Hacker News "Who is hiring" comments (and boards that copy its style) keep a whole listing in the first line of a++ * post:++ *++ * https://www.monumental.co/ | Amsterdam, The Netherlands | Full Time | Onsite at MONUMENTAL++ * Mechanical Engineer, Electrical Engineer, Robotic Controls Engineer | New York, NY | ONSITE | Full-time | $220K-$300K at Nirvana Robotics++ *++ * Kept whole, that line is not a title: a URL, a place, the terms and the pay all end up in it, and the card says the++ * same things twice in the wrong places. This module reads the line the way a person does:++ *++ * company from "at X", the "X is hiring …" opening, the first segment that is a name, or a URL's host++ * roles the segments that name a role; several are kept ("the first +N more roles")++ * place a location segment (a city, a country, "Remote (US)", a time zone), never a role or a term++ * terms on-site / remote / hybrid, full / part time, contract, visa and relocation++ * pay a salary segment, in the poster's own words++ *++ * The title is "Role at Company", or "First role +N more roles at Company"; with no role named it is the company++ * alone, and a URL is never a title. The line itself is returned so the description can keep it — nothing is lost.++ *++ * It is deliberately conservative: a token that names none of these is dropped rather than shown in the wrong field,++ * and a line with no company at all is not a listing (server/jobs/sources.js's thin-listing gate holds it). No I/O.++ */++++/** The rule's name, stored on the listing (work_listings.title_rule) so a re-derive never reads its own output back. */++const TITLE_RULE = 'hn-line@1';++++const clip = (s, n) => String(s == null ? '' : s).replace(/\s+/g, ' ').trim().slice(0, n);++const letters = (s) => (String(s == null ? '' : s).match(/\p{L}/gu) || []).length;++const uniq = (a) => [...new Set(a.filter(Boolean))];++++/** The job types the boards name, in the words this module reports (server/jobs/job-types.js maps them for display). */++const TYPES = [++ { label: 'Full time', re: /\bfull[\s-]?time\b|\bfulltime\b|\bft\b/i },++ { label: 'Part time', re: /\bpart[\s-]?time\b|\bparttime\b|\bpt\b/i },++ { label: 'Apprenticeship', re: /\bapprentic(?:e|eship)\b/i },++ { label: 'Internship', re: /\bintern(?:ship)?s?\b|\bco-?ops?\b/i },++ { label: 'Freelance', re: /\bfreelanc(?:e|er)\b|\bfractional\b/i },++ { label: 'Contract', re: /\bcontract(?:or|s)?\b|\bcontract-to-hire\b|\bb2b\b|\b1099\b|\bfixed[\s-]?term\b/i },++ { label: 'Permanent', re: /\bpermanent\b|\bcdi\b/i },++ { label: 'Temporary', re: /\btemporary\b|\binterim\b/i },++];++++const REMOTE_RE = /\bremote(?:[-\s]?first)?\b|\banywhere\b|\bworldwide\b|\bwork\s+(?:from|at)\s+home\b|\bdistributed\s+team\b/i;++const NO_REMOTE_RE = /\bno\s+remote\b|\bnot\s+remote\b|\bremote\s+not\s+(?:possible|available)\b/i;++const ONSITE_RE = /\bon[\s-]?site\b|\bin[\s-]?person\b|\bin[\s-]?office\b|\bon[\s-]?prem(?:ise|ises)?\b/i;++const HYBRID_RE = /\bhybrid\b/i;++const VISA_RE = /\bvisa\b|\bsponsor(?:ship|s)?\b/i;++const NO_VISA_RE = /\bno\s+(?:visa\s+)?sponsor|\bnot\s+(?:able|offer|offering)\b[^.]*\bsponsor|\bno\s+sponsorship\b|\bwithout\s+sponsorship\b/i;++const AUTH_RE = /\bcitizenship\b|\bwork\s+authori[sz]ation\b|\bsecurity\s+clearance\b|\bclearance\s+required\b|\bgreen\s+card\b/i;++const RELOCATION_RE = /\brelocation\b|\brelocate\b/i;++++/**++ * A role's own words. Two strengths: a segment is a role when it names any of these, and several comma/plus-separated++ * roles are only split apart when every part names a strong one — so "Senior Java, C# and React engineers" stays one++ * role, while "Mechanical Engineer, Electrical Engineer" becomes two.++ */++const ROLE_NOUN = /\b(?:engineers?|developers?|designers?|managers?|scientists?|analysts?|architects?|leads?|directors?|administrators?|admins?|consultants?|specialists?|executives?|officers?|recruiters?|marketers?|writers?|editors?|researchers?|interns?|founders?|swe|sde|pm|tpm|em|qa|sre|devops|cto|ceo|cfo|coo|vp|president|heads?|coordinators?|strategists?|accountants?|attorneys?|lawyers?|professors?|technicians?|operators?|producers?|artists?|assistants?|associates?|representatives?|reps?|hires?|ba|po|sales|growth|marketing|product|operations|finance|legal|people|talent|security|infrastructure|platform|backend|frontend|full[\s-]?stack|mobile|cloud|data|ml|ai|robotics?|mechanical|electrical|controls?|embedded|firmware|hardware|software|systems?|network|database|devrel|advocate|evangelist|support|success|customer|partnerships?|revenue|gtm|go[\s-]?to[\s-]?market|agents?)\b/i;++const ROLE_STRONG = /\b(?:engineers?|developers?|designers?|managers?|scientists?|analysts?|architects?|leads?|directors?|administrators?|admins?|consultants?|specialists?|executives?|officers?|recruiters?|marketers?|writers?|editors?|researchers?|interns?|founders?|swe|sde|pm|tpm|em|qa|sre|devops|cto|ceo|cfo|coo|vp|president|heads?|coordinators?|strategists?|accountants?|attorneys?|lawyers?|professors?|technicians?|operators?|producers?|artists?|assistants?|associates?|representatives?|reps?|hires?|ba|po|go[\s-]?to[\s-]?market|advocate|evangelist)\b/i;++++/** A company often names what it does; that suffix keeps "Charge Robotics" a name even though "robotics" is a role word. */++const COMPANY_SUFFIX = /\b(?:inc|ltd|limited|llc|corp|corporation|company|co|labs?|systems?|technologies|technology|tech|ai|studio|studios|health|capital|ventures?|group|solutions|aerospace|pharmacy|trading|partners|foundation|institute|university|bank|media|networks?|robotics|science|sciences|design|consulting|agency|division|startup|holdings|finance|markets?|commodit(?:y|ies)|software)\b/i;++++/** A first segment that opens a sentence or asks a question is not a company name. */++const NOT_A_COMPANY_START = /^(?:no|not|none|without|unknown|various|multiple|several|looking|seeking|hiring|we|our|this|the|about|backed|join|apply|role|job|location|remote|onsite|hybrid|full|part|contract|intern)\b/i;++++/** A role phrase opens with a level or a quantity ("Senior Backend Engineer", "Multiple roles"), never a company. */++const ROLE_TITLE_START = /^(?:senior|staff|lead|principal|junior|sr\.?|jr\.?|entry|mid|chief|head|vice|vp|founding|multiple|several|various)\b/i;++const NOT_A_ROLE_START = /^(?:closed|stop|apply|email|url|http|www|see|read|more|details|salary|compensation|equity|visa|relocation|seed|series|yc|stealth|profitable|pmf|fintech|healthtech|edtech|ai\/ml)\b/i;++++const MONEY_RE = /(?:[$€£]\s?\d|\b\d[\d,.]*\s?[kK]\b|\b\d{2,3}[,.]\d{3}\b|\b(?:usd|eur|gbp|cad|aud|chf)\b)/i;++const COMP_RE = /\b(?:salary|compensation|comp|ote|equity|stock|share\s+options|options|bonus|commission|hourly|per\s+hour|\/\s?hr|annuali[sz]ed|revenue\s+share)\b/i;++const PAY_AMOUNT_RE = /\b(?:ote|equity|stock|share\s+options|options|bonus|commission|revenue\s+share)\b/i;++++const PLACE_RE = /\b(?:remote|hybrid|on[\s-]?site|in[\s-]?person|in[\s-]?office|office|worldwide|anywhere|global|international|united\s+states|uk|u\.k\.|united\s+kingdom|europe|european|emea|apac|latam|americas|north\s+america|south\s+america|asia|africa|australia|canada|mexico|brazil|argentina|colombia|chile|peru|uruguay|germany|france|netherlands|belgium|spain|portugal|italy|austria|switzerland|sweden|norway|denmark|finland|iceland|ireland|poland|romania|bulgaria|czech|slovakia|hungary|ukraine|estonia|latvia|lithuania|serbia|croatia|slovenia|greece|turkey|israel|india|pakistan|bangladesh|philippines|indonesia|malaysia|singapore|vietnam|thailand|japan|korea|china|taiwan|hong\s+kong|australia|new\s+zealand|south\s+africa|nigeria|kenya|egypt|morocco|uae|dubai|saudi|qatar|time\s*zones?|hours|overlap|bay\s+area|sf\s+bay|silicon\s+valley|peninsula)\b/i;++const PLACE_CODE_RE = /\b(?:US|UK|EU|USA|NYC|SF|LA|DC)\b/;++const TIMEZONE_RE = /\b(?:CET|CEST|PST|PDT|EST|EDT|MST|MDT|CST|CDT|UTC|GMT|PT|ET|CT|MT)(?:[+-]\d{1,2})?\b/;++const TIME_CONTEXT_RE = /\b(?:remote|time\s*zones?|hours|overlap|from|to)\b|[±]/i;++const PLACE_COMMA_RE = /,\s*(?:[A-Z]{2}|[A-Z][a-z]+)(?:\s|,|$)/;++const PLACE_CITY_RE = /\b(?:london|berlin|paris|amsterdam|rotterdam|utrecht|munich|münchen|zurich|zürich|geneva|stockholm|copenhagen|oslo|helsinki|dublin|lisbon|barcelona|madrid|milan|rome|vienna|prague|warsaw|krakow|bucharest|budapest|athens|istanbul|tel\s+aviv|bangalore|bengaluru|mumbai|delhi|singapore|tokyo|seoul|sydney|melbourne|auckland|toronto|vancouver|montreal|new\s+york|brooklyn|san\s+francisco|san\s+diego|san\s+jose|palo\s+alto|mountain\s+view|redwood\s+city|berkeley|oakland|seattle|boston|chicago|denver|boulder|atlanta|miami|los\s+angeles|austin|dallas|houston|phoenix|portland|pittsburgh|philadelphia|raleigh|durham|washington|arlington|salt\s+lake\s+city|provo|mexico\s+city|são\s+paulo|sao\s+paulo|buenos\s+aires|bogotá|bogota|santiago)\b/i;++++/** A URL's host, from an http(s) URL or a bare domain ("k2space.com", "startuphakk.com/jobs"). */++function hostOf(value) {++ const s = String(value == null ? '' : value).trim();++ try {++ const u = new URL(s);++ if (u.protocol === 'http:' || u.protocol === 'https:') return u.hostname.toLowerCase().replace(/^www\./, '');++ } catch { /* a bare domain is not a URL until the pattern below says it is */ }++ const m = /^((?:[a-z0-9-]+\.)+[a-z]{2,})(?::\d+)?(?:\/\S*)?$/i.exec(s);++ return m ? m[1].toLowerCase().replace(/^www\./, '') : null;++}++++/** The URLs a segment carries, whose text is removed before the segment is read as anything else. */++function stripUrls(text) {++ return String(text)++ .replace(/\((?:https?:\/\/|www\.)[^)]*\)/gi, ' ')++ .replace(/https?:\/\/[^\s)]+/gi, ' ')++ .replace(/\bwww\.[^\s)]+/gi, ' ')++ .replace(/\s+/g, ' ')++ .trim();++}++++/** Whether the whole segment is an http(s) URL (so it is a link, not a name, place or role). A bare domain can be a++ * company's name ("Planlab.ai") and is left to the caller to read as one. */++function isUrlSegment(text) {++ const s = String(text == null ? '' : text).trim();++ return /^https?:\/\/\S+$/i.test(s) || /^www\.\S+$/i.test(s);++}++++/** A segment that is only a pay amount / compensation line. "Compensation in the link below" is neither. */++function looksPay(text) {++ const s = String(text == null ? '' : text);++ return MONEY_RE.test(s) || (PAY_AMOUNT_RE.test(s) && !ROLE_STRONG.test(s));++}++++/** The pay a segment carries, and what is left of it ("Full time, $150k" → "$150k"). */++function takePay(text) {++ const s = String(text == null ? '' : text).trim();++ if (!looksPay(s)) return null;++ // A segment can mix terms and pay ("Full-time, $150K"): keep the parts that name money or compensation.++ const parts = s.replace(/\s*\/\s*(hour|hr|month|year|annum|week|day)\b/gi, ' per $1')++ .split(/\s*;\s*|,(?!\d{3}\b)|\s+\/\s+/).filter(Boolean);++ const kept = parts.filter((p) => MONEY_RE.test(p) || (COMP_RE.test(p) && !TYPES.some((t) => t.re.test(p))));++ const pay = kept.length && kept.join(', ').toLowerCase() !== s.toLowerCase() ? kept.join(', ') : s;++ if (pay === s && TYPES.some((t) => t.re.test(s)) && !MONEY_RE.test(s)) return null; // "Equity, full-time" is not a pay line++ const rest = kept.length && kept.join(', ').toLowerCase() !== s.toLowerCase()++ ? parts.filter((p) => !kept.includes(p)).join(', ')++ : s;++ return { text: clip(pay, 160), rest: rest === s ? '' : rest };++}++++/** The job types a segment names, in the order they appear. */++function typesOf(text) {++ const s = String(text == null ? '' : text);++ // "(PT to ET overlap preferred)" is Pacific Time, not part time; the same for FT when it is a time zone.++ const timeZone = TIMEZONE_RE.test(s) && TIME_CONTEXT_RE.test(s) || /\b(?:PT|FT)\s+(?:to|and|or)\s+(?:ET|CT|MT|PT)\b/i.test(s);++ const probe = timeZone ? s.replace(/\b(?:FT|PT)\b/gi, ' ') : s;++ return TYPES.filter((t) => t.re.test(probe)).map((t) => t.label);++}++++/** The place a segment names, cleaned of the terms around it, or null when it names none. */++function placeOf(text) {++ let s = stripUrls(text);++ if (!s || looksPay(s)) return null;++ if ((VISA_RE.test(s) || AUTH_RE.test(s)) && !PLACE_COMMA_RE.test(s) && !PLACE_CITY_RE.test(s)) return null;++ const comma = PLACE_COMMA_RE.test(s) && !ROLE_NOUN.test(s);++ const city = PLACE_CITY_RE.test(s);++ const zone = TIMEZONE_RE.test(s) && (TIME_CONTEXT_RE.test(s) || s.replace(TIMEZONE_RE, '').replace(/[\s(),./+-]/g, '').length < 2);++ const named = PLACE_RE.test(s) || PLACE_CODE_RE.test(s) || comma || city || zone;++ if (!named) return null;++ // "Mechanical Engineer, Electrical Engineer" and "Founding Account Executive (US)": a comma or a work-authorisation++ // code between roles is not a place.++ if (ROLE_STRONG.test(s) && !PLACE_RE.test(s) && !city) return null;++ s = s++ .replace(/\b(?:full[\s-]?time|part[\s-]?time|fulltime|parttime|contract(?:or)?|freelance|fractional|permanent|temporary|internship|interns?|co-?ops?)\b/gi, ' ')++ .replace(/\b(?:visa\s+sponsorship|no\s+visa\s+sponsorship|visa|sponsorship|relocation)\b/gi, ' ')++ .replace(/\b(?:citizenship\s+required|work\s+authori[sz]ation\s+required|security\s+clearance\s+required|clearance\s+required)\b/gi, ' ')++ .replace(/\bhybrid\b/gi, ' ')++ .replace(/\b(?:on[\s-]?site|in[\s-]?person|in[\s-]?office)\b/gi, ' ')++ .replace(/\b(?:no\s+remote|not\s+remote)\b/gi, ' ')++ .replace(/\(\s*(?:\d+\s*days?[^)]*|in[\s-]?office[^)]*|some\s+crossover[^)]*)\)/gi, ' ')++ .replace(/\b\d+\s*days?\b(?:\s*(?:a|\/|per)\s*(?:week|wk))?/gi, ' ')++ .replace(/\bin\s+office\b/gi, ' ')++ .replace(/\b(?:based|located)\s+in\b/gi, ' ')++ .replace(/\s+/g, ' ')++ .replace(/^[\s,;:/|.·•–—-]+|[\s,;:/|.·•–—-]+$/g, '')++ .replace(/^(?:in|at)\s+/i, '')++ .replace(/\s+(?:or|and)\s*$/i, '')++ .replace(/\s*\/\s*(?:or|and)\s+/gi, ' / ')++ .replace(/^(?:or|and)\s+/i, ' ')++ .replace(/\(\s*\)/g, ' ')++ .replace(/\((?:[^)]*\b(?:require[sd]?|preferred|only|hours|days?|week)\b[^)]*)\)/gi, ' ')++ .replace(/\b(?:preferred|required|only)\b/gi, ' ')++ .replace(/^(?:or|and)\s+/i, ' ')++ .replace(/\s+(?:or|and)\s*$/i, '')++ .replace(/\s*,\s*(?=,|$)/g, '')++ .replace(/\s*,\s*/g, ', ')++ .replace(/^[\s,;:/|.·•–—-]+|[\s,;:/|.·•–—-]+$/g, '')++ .trim();++ s = s.replace(/^\(([^()]+)\)$/, '$1').replace(/^(?:or|and)\s+/i, '').trim();++ if (letters(s) < 2 || s.length > 100) return null;++ if (/\b(?:looking|apply|description|we\s+are)\b/i.test(s)) return null;++ return clip(s, 160);++}++++/** Whether a cleaned string reaches this module's bar for a company name. */++function looksLikeCompany(text) {++ const s = clip(text, 160);++ if (letters(s) < 2 || s.length > 120) return false;++ if (isUrlSegment(s) || NOT_A_COMPANY_START.test(s)) return false;++ if (TYPES.some((t) => t.re.test(s))) return false;++ if (VISA_RE.test(s) || AUTH_RE.test(s)) return false;++ if (looksPay(s)) return false;++ if (placeOf(s)) return false;++ if (/\b(?:is|are|was|were)\b/i.test(s) && s.split(/\s+/).length > 6) return false;++ if (ROLE_NOUN.test(s) && ROLE_TITLE_START.test(s)) return false;++ if (ROLE_NOUN.test(s) && s.split(/\s+/).length <= 8 && !COMPANY_SUFFIX.test(s)) return false;++ // A bare time zone is a place, not a company, however it is punctuated.++ if (/[<>]/.test(s) && TIMEZONE_RE.test(s)) return false;++ return true;++}++++/** A company name, without a trailing parenthetical URL or a leading "at". */++function cleanCompany(text) {++ return clip(String(text == null ? '' : text)++ .replace(/^[\s\-–—,:]+|[\s\-–—,:]+$/g, '')++ .replace(/\s*\((?:https?:\/\/|www\.)[^)]*\)\s*$/gi, '')++ // A descriptive bracket ("(stealth, regulated commodity markets infrastructure)") is not part of the name;++ // a short one ("(Series B)", "(YC S24)") is how the poster identifies the company.++ .replace(/\s*\(([^)]*)\)\s*$/g, (m, inner) => (inner.includes(',') || inner.split(/\s+/).length > 3 ? '' : m))++ .replace(/\s+/g, ' '), 120);++}++++/**++ * The company a line names: "at X" (the last such segment), the "X is hiring …" opening, the first segment that is a++ * name, or a URL's host. The segments array is mutated: the company's text is removed from the segment it was in, so++ * the rest of that segment (the pay before "at X", the role before "at X") is still read.++ */++function takeCompany(segments) {++ // "X is hiring …": the company is the subject, the rest is the role.++ const opening = /^(.+?)\s+(?:is|are|was|were)\s+hiring\b(?:\s+(?:a|an|for|the)\b)?\s*(.*)$/i.exec(segments[0] || '');++ if (opening) {++ const company = cleanCompany(opening[1]);++ if (looksLikeCompany(company)) {++ segments[0] = opening[2].trim();++ return company;++ }++ }++ // "… | $220K-$300K at Nirvana Robotics" or "… | Onsite at MONUMENTAL": the last "at X" names the company.++ for (let i = segments.length - 1; i >= 0; i--) {++ const m = /(?:^|\s)at\s+([^|]+?)\s*$/i.exec(segments[i]);++ if (!m) continue;++ const company = cleanCompany(m[1]);++ if (!looksLikeCompany(company)) continue;++ segments[i] = segments[i].slice(0, m.index).trim();++ return company;++ }++ // The first segment that is a name.++ for (let i = 0; i < segments.length; i++) {++ const cleaned = cleanCompany(stripUrls(segments[i]));++ if (!cleaned) continue;++ if (looksLikeCompany(cleaned)) {++ segments[i] = '';++ return cleaned;++ }++ }++ // A URL's host, when the line names no company but links one.++ for (let i = 0; i < segments.length; i++) {++ const host = hostOf(segments[i]) || hostOf(stripUrls(segments[i]));++ if (host) {++ segments[i] = stripUrls(segments[i]) && !isUrlSegment(stripUrls(segments[i])) ? stripUrls(segments[i]) : '';++ return host;++ }++ }++ return '';++}++++/** Whether a cleaned segment names a role, rather than a tagline, a place, a term or a pay line. */++function looksLikeRole(text) {++ const s = clip(text, 200);++ if (letters(s) < 2 || s.length > 160) return false;++ if (NOT_A_ROLE_START.test(s)) return false;++ if (looksPay(s) || typesOf(s).length || VISA_RE.test(s) || AUTH_RE.test(s)) return false;++ if (/\b(?:is|are|we|our|you|your)\b/i.test(s) && s.split(/\s+/).length > 5) return false;++ if (/^(?:building|making|shipping|helping|join|work(?:ing)?\s+with|about|the\s+team|series|seed)\b/i.test(s)) return false;++ if (ROLE_NOUN.test(s)) return true;++ return /^(?:multiple|various|several)\b/i.test(s) && /\broles?\b/i.test(s);++}++++/** The role text with the leading "hiring a …" / "looking for …" / separators cleaned off. */++function cleanRole(text) {++ let s = stripUrls(text)++ .replace(/^[\s,;:+/&|–—-]+|[\s,;:+/&|–—-]+$/g, '')++ .replace(/^(?:we(?:'re| are)\s+)?(?:hiring|looking\s+for|seeking|recruiting|searching\s+for)\b\s*(?:a|an|the)?\s*/i, '')++ .replace(/^[\s,;:+/&|–—-]+|[\s,;:+/&|–—-]+$/g, '')++ .replace(/\.$/, '')++ .replace(/\.\s.*$/, '')++ .replace(/\s+/g, ' ')++ .trim();++ // "Engineers to work on event-driven, distributed systems …" is a description of the job, not its title.++ s = s.replace(/^((?:[\w/&.+-]+\s+){0,4}(?:engineers?|developers?|designers?|managers?|scientists?|architects?|leads?|directors?))\s+(?:to|who|that)\s+.*$/i, '$1');++ if (letters(s) < 2) return '';++ return clip(s, 140);++}++++/** Several roles in one segment, split on a comma, semicolon, plus or "&" only between parenthesised groups. */++function splitRoles(text) {++ const s = String(text || '').trim();++ if (!s || /^(?:multiple|various|several)\b/i.test(s)) return [s];++ const parts = [];++ let depth = 0;++ let cur = '';++ for (let i = 0; i < s.length; i += 1) {++ const ch = s[i];++ if (ch === '(') depth += 1;++ if (ch === ')') depth = Math.max(0, depth - 1);++ if (depth === 0) {++ if (ch === ',' || ch === ';') { parts.push(cur); cur = ''; continue; }++ if ((ch === '+' || ch === '&') && /\s/.test(s[i - 1] || '') && /\s/.test(s[i + 1] || '')) { parts.push(cur); cur = ''; continue; }++ if (ch === ' ' && /^ and $/i.test(s.slice(i - 1, i + 4))) { parts.push(cur); cur = ''; i += 3; continue; }++ }++ cur += ch;++ }++ parts.push(cur);++ const cleaned = parts.map((p) => cleanRole(p)).filter(Boolean);++ if (cleaned.length < 2) return [s];++ // Only split when every part is a role of its own; "Senior Java, C# and React engineers" is one role.++ if (!cleaned.every((p) => looksLikeRole(p) && ROLE_STRONG.test(p))) return [s];++ return cleaned;++}++++/** "Role at Company", "First role +N more roles at Company", or the company alone. */++function titleOf(roles, company) {++ const at = company ? ` at ${company}` : '';++ const first = roles.length ? roles[0].replace(/^[a-z]/, (c) => c.toUpperCase()) : '';++ if (!roles.length) return company;++ if (roles.length === 1) return `${first}${at}`;++ return `${first} +${roles.length - 1} more role${roles.length - 1 === 1 ? '' : 's'}${at}`;++}++++/**++ * A one-line job post → { company, title, location, remote, job_type, salary, tags, roles, line }.++ * company is '' when the line names none: the caller's thin-listing gate holds it.++ */++function parseHiringLine(line) {++ const original = clip(line, 500);++ const segments = String(line == null ? '' : line)++ .replace(/^[-–—*•\s]+/, '')++ .split(/\s*\|\s*/)++ .map((s) => s.trim())++ .filter(Boolean);++ const company = takeCompany(segments);++++ const roles = [];++ const places = [];++ const typeLabels = [];++ const tags = [];++ let salary = null;++ let remote = false;++ let saidOnsite = false;++ let saidHybrid = false;++ let saidVisa = false;++ let noVisa = false;++ let saidAuth = false;++ let saidRelocation = false;++++ for (const raw of segments) {++ let seg = stripUrls(raw);++ if (!seg) continue;++ // A pay segment (or the pay part of one) is the salary, and nothing else in it is a role.++ const pay = takePay(seg);++ if (pay) {++ salary = salary || pay.text;++ seg = pay.rest;++ if (!seg) continue;++ }++ const visaHere = VISA_RE.test(seg);++ const authHere = AUTH_RE.test(seg);++ const types = typesOf(seg);++ if (types.length) { for (const t of types) typeLabels.push(t); }++ if (REMOTE_RE.test(seg) && !NO_REMOTE_RE.test(seg)) remote = true;++ if (ONSITE_RE.test(seg)) saidOnsite = true;++ if (HYBRID_RE.test(seg)) saidHybrid = true;++ if (visaHere) { saidVisa = true; if (NO_VISA_RE.test(seg)) noVisa = true; }++ if (authHere) saidAuth = true;++ if (RELOCATION_RE.test(seg)) saidRelocation = true;++++ // Remove the terms from the segment; what is left is a place or a role, or nothing.++ let rest = seg++ .replace(REMOTE_RE, ' ')++ .replace(NO_REMOTE_RE, ' ')++ .replace(ONSITE_RE, ' ')++ .replace(HYBRID_RE, ' ')++ .replace(VISA_RE, ' ')++ .replace(NO_VISA_RE, ' ')++ .replace(AUTH_RE, ' ')++ .replace(RELOCATION_RE, ' ')++ .replace(/\b(?:citizenship|authori[sz]ation|clearance|required|preferred|only)\b/gi, ' ')++ .replace(/\b(?:full[\s-]?time|part[\s-]?time|fulltime|parttime|permanent|temporary|internship|apprenticeship)\b/gi, ' ')++ .replace(/\s+/g, ' ')++ .replace(/^[\s,;:/|.·•–—-]+|[\s,;:/|.·•–—-]+$/g, '')++ .trim();++ const place = (visaHere || authHere) ? null : (placeOf(seg) || placeOf(rest));++ if (place) {++ const known = places.some((p) => p.toLowerCase() === place.toLowerCase());++ const covered = /^remote$/i.test(place) && places.some((p) => /\bremote\b/i.test(p));++ if (!known && !covered) places.push(place);++ }++ // A segment that was nothing but terms, visa wording or pay is done (a bare "Remote" already went in as a place).++ if (letters(rest) < 2) continue;++ if (visaHere || authHere) {++ const residual = rest.replace(/[\s.,()/&-]*(?:u\.?s\.?a?|uk|eu)?[\s.,()/&-]*/gi, '').trim();++ if (letters(residual) < 2) continue;++ }++ let role = cleanRole(rest);++ // "Full-stack Engineer (Elixir) / Helsinki, Finland": the place is in the same segment as the role.++ if (role && place) {++ const parts = role.split(/\s+\/\s+/);++ const roleParts = parts.filter((p) => looksLikeRole(p));++ const placeParts = parts.filter((p) => !looksLikeRole(p) && placeOf(p));++ if (roleParts.length && placeParts.length) {++ role = roleParts.join(' / ');++ const combined = placeOf(placeParts.join(' / '));++ if (combined) {++ const at = places.indexOf(place);++ if (at >= 0) places[at] = combined;++ else place = combined;++ }++ }++ }++ if (role && looksLikeRole(role) && (!place || ROLE_STRONG.test(role))) roles.push(...splitRoles(role));++ }++++ // Terms, in the service's own existing fields: the workplace boolean, the type line and the chips.++ if (saidHybrid) tags.push('Hybrid');++ else if (saidOnsite) tags.push('Onsite');++ if (saidVisa) tags.push(noVisa ? 'No visa sponsorship' : 'Visa sponsorship');++ if (saidAuth) tags.push('Work authorization required');++ if (saidRelocation) tags.push('Relocation');++ const jobType = uniq(typeLabels).join(', ') || null;++ const title = titleOf(uniq(roles.map((r) => clip(r, 120))), company);++++ return {++ company,++ title: clip(title, 240),++ location: uniq(places).join(' / ').slice(0, 200) || null,++ remote,++ job_type: jobType,++ salary: salary || null,++ tags: uniq(tags).slice(0, 6),++ roles: uniq(roles),++ line: original,++ };++}++++module.exports = { parseHiringLine, hostOf, splitRoles, looksLikeRole, looksLikeCompany, titleOf, TITLE_RULE };nnew file mode 100644@@@ -0,0 +1,107 @@++'use strict';++++/**++ * Re-derive the titles of Hacker News "Who is hiring" listings stored before server/jobs/hiring-line.js split the++ * first line into company, roles, place, terms and pay (scripts/rederive-hn-titles.js runs it once).++ *++ * The old row kept the first segment as the company and the rest of the pipe-separated line as the title, so the++ * board's line can be put back together from the two: "Company | Role | Place" (or "Company | | Role" when the line++ * had an empty segment, which is why a title starting with "|" is joined without another separator). That line is++ * parsed by the same module the ingest uses, and the row's own title, company, place, remote flag, type, pay and tags++ * are replaced with what it says. The excerpt is never touched: it already starts with the original line.++ *++ * Only rows without the title_rule marker are read, so running it twice is harmless and a second run can never read a++ * title this job wrote back as if it were the board's line. A line that names no company is left alone (the same++ * thin-listing gate the ingest applies) and reported.++ */++const { parseHiringLine, TITLE_RULE } = require('./hiring-line');++++const SOURCE = 'hn-whoishiring';++const clip = (s, n) => String(s == null ? '' : s).replace(/\s+/g, ' ').trim().slice(0, n);++const uniq = (a) => [...new Set(a.filter(Boolean))];++++/** The board's first line, as the old ingest stored it: the company segment and the rest of the line. */++function legacyLine(row) {++ const company = clip(row && row.company, 120);++ const title = clip(row && row.title, 300);++ if (!company || !title) return title || company || '';++ // rawOfHn joined the title's own parts with " | "; an empty first part left the title starting with "|".++ return title.startsWith('|') ? `${company} ${title}` : `${company} | ${title}`;++}++++/** The row's search text, exactly as server/jobs/sources.js's toRow builds it. */++function searchText(row) {++ return [row.title, row.company, row.tags, row.location, row.excerpt, row.salary].filter(Boolean).join(' ').toLowerCase().slice(0, 4000);++}++++/**++ * Re-derive every old HN title. Dry by default: with { apply: true } it writes the rows in one transaction and, when++ * a search index (or a config to build one) is given, sweeps it so the changed documents are re-sent.++ * → { scanned, changed, skipped, samples, sent }++ */++async function rederiveHnTitles({ s, config = null, log = console, apply = false, search = null, limit = 0 } = {}) {++ if (!s || !s.db) throw new TypeError('rederiveHnTitles needs the store (s)');++ const rows = await s.db.many(++ `SELECT * FROM work_listings WHERE source = $1 AND (title_rule IS NULL OR title_rule <> $2) ORDER BY id${limit ? ` LIMIT ${Number(limit)}` : ''}`,++ [SOURCE, TITLE_RULE]);++ const changes = [];++ const skipped = [];++ for (const row of rows) {++ const parsed = parseHiringLine(legacyLine(row));++ if (!parsed.company || !parsed.title) { skipped.push(row); continue; }++ const tags = uniq([...String(row.tags || '').split(',').map((t) => t.trim()).filter(Boolean), ...parsed.tags]);++ const next = {++ id: row.id,++ title: parsed.title,++ company: parsed.company,++ location: parsed.location,++ remote: parsed.remote,++ tags: tags.join(', '),++ job_type: parsed.job_type,++ salary: parsed.salary,++ title_rule: TITLE_RULE,++ };++ next.search_text = searchText({ ...row, ...next });++ if (next.title === row.title && next.company === row.company && next.location === (row.location || null)++ && next.remote === row.remote && next.tags === row.tags && next.job_type === (row.job_type || null)++ && next.salary === (row.salary || null) && row.title_rule === TITLE_RULE) continue;++ changes.push({ row, next });++ }++++ let sent = null;++ if (apply && changes.length) {++ await s.tx(async () => {++ for (const { next } of changes) {++ await s.db.exec(++ `UPDATE work_listings SET title = $2, company = $3, location = $4, remote = $5, tags = $6,++ job_type = $7, salary = $8, search_text = $9, title_rule = $10 WHERE id = $1`,++ [next.id, next.title, next.company, next.location, next.remote, next.tags, next.job_type,++ next.salary, next.search_text, next.title_rule]);++ }++ });++ // The rows changed, so Search's sequencer sends their documents again on the next sweep; do it now rather than++ // an hour later. The relay is off in a test and the rows wait there, which is the service's own behaviour.++ const index = search || (config ? require('../search-index').createSearchIndex({ config, s, log }) : null);++ if (index) {++ const out = await index.sweep();++ sent = out && out.job ? out.job : out;++ if (index.outbox && index.outbox.kick) await index.outbox.kick().catch(() => {});++ if (!search && index.stop) index.stop();++ }++ }++ if (log && log.log) {++ log.log(`[Redrive] HN titles: ${rows.length} old listing(s), ${changes.length} to change, ${skipped.length} with no company${apply ? ' (applied)' : ' (dry run)'}`);++ }++ return {++ scanned: rows.length,++ changed: changes.length,++ skipped: skipped.length,++ samples: changes.slice(0, 12).map(({ row, next }) => ({++ id: row.id, before: `${row.title} — ${row.company}`, after: next.title, company: next.company,++ location: next.location, job_type: next.job_type, salary: next.salary, tags: next.tags,++ })),++ sent,++ };++}++++module.exports = { rederiveHnTitles, legacyLine, SOURCE, TITLE_RULE };@@@ -208,6 +208,7 @@ function toRow(raw, { sourceId, fetchedAt, idOf }) {excerpt: excerpt(raw.description_html).slice(0, 400),posted_at: raw.posted_at || null,sort_at: raw.posted_at || fetchedAt,++ title_rule: raw.title_rule || null,fetched_at: fetchedAt,};row.search_text = [row.title, row.company, row.tags, row.location, row.excerpt, row.salary].filter(Boolean).join(' ').toLowerCase().slice(0, 4000);@@@ -307,16 +307,16 @@ const recentIds = (s, limit = 1000) => s.db.many(/** One listing in, through `q` (the database or a transaction). → true when it is new. */async function upsertListing(q, row, fetchedAt) {const out = await q.maybe(-- `INSERT INTO work_listings (id, source, source_id, url, title, company, location, remote, tags, job_type, salary, excerpt, search_text, posted_at, sort_at, fetched_at, first_seen_at, last_seen_at, expired)-- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $16, $16, false)++ `INSERT INTO work_listings (id, source, source_id, url, title, company, location, remote, tags, job_type, salary, excerpt, search_text, posted_at, sort_at, fetched_at, first_seen_at, last_seen_at, expired, title_rule)++ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $16, $16, false, $17)ON CONFLICT (source, source_id) DO UPDATE SETurl = EXCLUDED.url, title = EXCLUDED.title, company = EXCLUDED.company, location = EXCLUDED.location,remote = EXCLUDED.remote, tags = EXCLUDED.tags, job_type = EXCLUDED.job_type, salary = EXCLUDED.salary,excerpt = EXCLUDED.excerpt, search_text = EXCLUDED.search_text, posted_at = EXCLUDED.posted_at,sort_at = EXCLUDED.sort_at, fetched_at = EXCLUDED.fetched_at, last_seen_at = EXCLUDED.last_seen_at,-- expired = false++ expired = false, title_rule = EXCLUDED.title_ruleRETURNING (xmax = 0) AS inserted`,-- [row.id, row.source, row.source_id, row.url, row.title, row.company, row.location, row.remote, row.tags, row.job_type, row.salary, row.excerpt, row.search_text, row.posted_at, row.sort_at, fetchedAt]);++ [row.id, row.source, row.source_id, row.url, row.title, row.company, row.location, row.remote, row.tags, row.job_type, row.salary, row.excerpt, row.search_text, row.posted_at, row.sort_at, fetchedAt, row.title_rule || null]);return Boolean(out && out.inserted);}@@@ -22,6 +22,21 @@ function dateWords(iso) {return `${d.getUTCDate()} ${MONTHS[d.getUTCMonth()]} ${d.getUTCFullYear()}`;}++/**++ * The listing's title with its company: "Role at Company". A title the ingest already wrote that way (a one-line++ * board post split by server/jobs/hiring-line.js) is not made to say the company twice, and a title that is only the++ * company is not followed by it either.++ */++function fullTitle(l) {++ const title = String((l && l.title) || '').trim();++ const company = String((l && l.company) || '').trim();++ if (!title) return company;++ if (!company) return title;++ if (title.toLowerCase() === company.toLowerCase()) return title;++ if (title.toLowerCase().endsWith(` at ${company.toLowerCase()}`)) return title;++ return `${title} at ${company}`;++}++const meta = (row) => ({id: row.id,url: row.url,@@@ -138,4 +153,5 @@ function jobPosting(row, { siteUrl }) {/** The excerpt as plain text, for the meta description and the API. */const plainExcerpt = (row) => stripHtml(row.excerpt || '');--module.exports = { listingItem, listingList, sourceLink, provenance, jobPosting, dateWords, plainExcerpt, meta, salaryLine, jobTypeLabel };++module.exports = {++ fullTitle, listingItem, listingList, sourceLink, provenance, jobPosting, dateWords, plainExcerpt, meta, salaryLine, jobTypeLabel };@@@ -32,7 +32,7 @@ function describe(row) {const facts = [where, l.salary].filter(Boolean).join(' · ');return {listed: !row.expired,-- title: `${l.title} at ${l.company}`,++ title: listing.fullTitle(l),summary: facts || excerpt.slice(0, 200) || null,// The body holds what the title and summary do not (the snippet reads both, and repeated the title and place).body: [l.tags.join(', '), facts ? excerpt : excerpt.slice(200).replace(/^\S*\s*/, '')].filter(Boolean).join('\n'),nnew file mode 100644@@@ -0,0 +1,88 @@++'use strict';++/**++ * A one-line board post split into a listing (server/jobs/hiring-line.js): the two lines from the brief, a URL first,++ * several roles, a role after the place and terms, visa and relocation, no company, and the title helper that stops++ * "Role at Company at Company" on the pages and in Search.++ */++const assert = require('assert');++const { check, done } = require('./helpers/boot');++const { parseHiringLine } = require('../server/jobs/hiring-line');++const { fullTitle } = require('../server/render/listing');++++(async () => {++ await check('a URL first, a place, terms and "at COMPANY": title is the company alone', async () => {++ const p = parseHiringLine('https://www.monumental.co/ | Amsterdam, The Netherlands | Full Time | Onsite at MONUMENTAL');++ assert.deepStrictEqual(++ [p.company, p.title, p.location, p.remote, p.job_type, p.salary, p.tags],++ ['MONUMENTAL', 'MONUMENTAL', 'Amsterdam, The Netherlands', false, 'Full time', null, ['Onsite']]);++ assert.ok(!p.title.includes('http'), 'a URL is never a title');++ assert.strictEqual(p.line, 'https://www.monumental.co/ | Amsterdam, The Netherlands | Full Time | Onsite at MONUMENTAL');++ });++++ await check('several roles: the first and "+N more roles", with the place, terms and pay in their fields', async () => {++ const p = parseHiringLine('Mechanical Engineer, Electrical Engineer, Robotic Controls Engineer | New York, NY | ONSITE | Full-time | $220K-$300K at Nirvana Robotics');++ assert.strictEqual(p.company, 'Nirvana Robotics');++ assert.strictEqual(p.title, 'Mechanical Engineer +2 more roles at Nirvana Robotics');++ assert.deepStrictEqual(p.roles, ['Mechanical Engineer', 'Electrical Engineer', 'Robotic Controls Engineer']);++ assert.deepStrictEqual([p.location, p.remote, p.job_type, p.salary, p.tags],++ ['New York, NY', false, 'Full time', '$220K-$300K', ['Onsite']]);++ });++++ await check('a URL with no "at X": the company is the URL\'s host, and the host is not a title URL', async () => {++ const p = parseHiringLine('https://www.monumental.co/ | Amsterdam, The Netherlands | Full Time | Onsite');++ assert.deepStrictEqual([p.company, p.title, p.location], ['monumental.co', 'monumental.co', 'Amsterdam, The Netherlands']);++ assert.ok(!p.title.includes('http') && !p.title.includes('www.'));++ });++++ await check('roles separated by a pipe: "+1 more role"', async () => {++ const p = parseHiringLine('Acme | Software Engineer | Platform Engineer | Remote (US) | Full-time');++ assert.strictEqual(p.title, 'Software Engineer +1 more role at Acme');++ assert.deepStrictEqual(p.roles, ['Software Engineer', 'Platform Engineer']);++ assert.deepStrictEqual([p.location, p.remote, p.job_type], ['Remote (US)', true, 'Full time']);++ });++++ await check('a role after the place, terms and pay; visa and relocation are chips', async () => {++ const p = parseHiringLine('KSC Trading | Remote | Full-Time | Senior Backend Software Engineer | $200k to $250k USD Total Compensation + Benefits');++ assert.deepStrictEqual([p.company, p.title, p.location, p.remote, p.job_type, p.salary, p.tags],++ ['KSC Trading', 'Senior Backend Software Engineer at KSC Trading', 'Remote', true, 'Full time',++ '$200k to $250k USD Total Compensation + Benefits', []]);++++ const flow = parseHiringLine('Flow Traders | Research Engineer | Hong Kong | Full Time | Visa | Relocation');++ assert.deepStrictEqual([flow.title, flow.location, flow.job_type, flow.tags],++ ['Research Engineer at Flow Traders', 'Hong Kong', 'Full time', ['Visa sponsorship', 'Relocation']]);++ });++++ await check('no company: the line is not a listing (the thin-listing gate holds it)', async () => {++ const p = parseHiringLine('Remote (US) | Full-time | Senior Software Engineer');++ assert.strictEqual(p.company, '');++ assert.strictEqual(p.title, 'Senior Software Engineer', 'a role can be read, but there is no company to publish it under');++ });++++ await check('real lines: multiple roles, a place list, salary with punctuation, no invented fields', async () => {++ const charge = parseHiringLine('Charge Robotics | Multiple Roles | Full-time | Onsite | SF Bay Area');++ assert.deepStrictEqual([charge.title, charge.company, charge.location, charge.job_type, charge.tags],++ ['Multiple Roles at Charge Robotics', 'Charge Robotics', 'SF Bay Area', 'Full time', ['Onsite']]);++++ const stoke = parseHiringLine('Stoke Space | ONSITE in Seattle, WA / Moses Lake, WA / Cape Canaveral, FL | Software Engineer | Full-Time, Internship | Salary: $98k-$289k USD | https://www.stokespace.com');++ assert.deepStrictEqual([stoke.title, stoke.company, stoke.location, stoke.remote, stoke.job_type, stoke.salary, stoke.tags],++ ['Software Engineer at Stoke Space', 'Stoke Space', 'Seattle, WA / Moses Lake, WA / Cape Canaveral, FL', false,++ 'Full time, Internship', 'Salary: $98k-$289k USD', ['Onsite']]);++++ const discord = parseHiringLine('Discord | Senior Software Engineer, Application Security | REMOTE (US West Coast) or San Francisco, CA | $196K-$245K base + equity');++ assert.deepStrictEqual([discord.title, discord.location, discord.remote, discord.salary],++ ['Senior Software Engineer, Application Security at Discord', 'REMOTE (US West Coast) or San Francisco, CA', true, '$196K-$245K base + equity']);++++ const odoo = parseHiringLine('Odoo | Software Developer | Belgium | Full Time | Hybrid');++ assert.deepStrictEqual([odoo.title, odoo.location, odoo.job_type, odoo.tags],++ ['Software Developer at Odoo', 'Belgium', 'Full time', ['Hybrid']]);++ });++++ await check('"Role at Company" is not made to say the company twice', async () => {++ assert.strictEqual(fullTitle({ title: 'Senior Engineer at Acme', company: 'Acme' }), 'Senior Engineer at Acme');++ assert.strictEqual(fullTitle({ title: 'Acme', company: 'Acme' }), 'Acme');++ assert.strictEqual(fullTitle({ title: 'Senior Engineer', company: 'Acme' }), 'Senior Engineer at Acme');++ assert.strictEqual(fullTitle({ title: 'Senior Engineer', company: '' }), 'Senior Engineer');++ });++++ done();++})().catch((err) => { console.error(err); process.exit(1); });nnew file mode 100644@@@ -0,0 +1,95 @@++'use strict';++/**++ * Re-deriving the Hacker News titles stored before the one-line split (server/jobs/rederive-hn-titles.js): an old row++ * is read back from its company and title, the same parser the ingest uses writes the title, place, terms and pay,++ * Search is swept with the changed documents, and a second run reads nothing (the title_rule marker).++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const store = require('../server/jobs/store');++const { rederiveHnTitles, legacyLine } = require('../server/jobs/rederive-hn-titles');++++const legacy = (s, sourceId, company, title, over = {}) => ({++ id: store.idOf('hn-whoishiring', sourceId), source: 'hn-whoishiring', source_id: sourceId,++ url: `https://news.ycombinator.com/item?id=${sourceId}`, title, company, location: null, remote: false,++ tags: '', job_type: null, salary: null,++ excerpt: `${company}${title ? ` | ${title}` : ''} The comment's own words, kept.`,++ search_text: `${title} ${company}`.toLowerCase(), posted_at: '2026-10-01T15:00:00.000Z',++ sort_at: '2026-10-01T15:00:00.000Z', fetched_at: '2026-10-30T06:00:00.000Z',++ first_seen_at: '2026-10-30T06:00:00.000Z', last_seen_at: '2026-10-30T06:00:00.000Z',++ expired: false, title_rule: null, ...over,++});++++(async () => {++ const t = await boot();++ const { s, search } = t.ctx;++ try {++ const now = '2026-10-30T06:00:00.000Z';++ const rows = [++ legacy(s, 'hn-301', 'Acme', 'Senior Software Engineer | REMOTE (US) | Full-Time'),++ legacy(s, 'hn-302', 'Planlab.ai', '| Product Engineer | ONSITE in London, UK'),++ legacy(s, 'hn-303', '', 'Remote (US) | Full-time | Senior Software Engineer'),++ ];++ for (const row of rows) {++ await s.db.exec(++ `INSERT INTO work_listings (id, source, source_id, url, title, company, location, remote, tags, job_type, salary, excerpt,++ search_text, posted_at, sort_at, fetched_at, first_seen_at, last_seen_at, expired, title_rule)++ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$16,$16,false,NULL)`,++ [row.id, row.source, row.source_id, row.url, row.title, row.company, row.location, row.remote, row.tags,++ row.job_type, row.salary, row.excerpt, row.search_text, row.posted_at, row.sort_at, row.fetched_at]);++ }++++ await check('a legacy line is put back together from its company and title', async () => {++ assert.strictEqual(legacyLine(rows[0]), 'Acme | Senior Software Engineer | REMOTE (US) | Full-Time');++ assert.strictEqual(legacyLine(rows[1]), 'Planlab.ai | Product Engineer | ONSITE in London, UK');++ });++++ let out;++ await check('the old titles are re-derived, and the row keeps the original line in its excerpt', async () => {++ out = await rederiveHnTitles({ s, config: t.config, apply: true, search, log: { log() {} } });++ assert.deepStrictEqual([out.scanned, out.changed, out.skipped], [3, 2, 1]);++ const acme = await s.db.one('SELECT * FROM work_listings WHERE id = $1', [rows[0].id]);++ assert.deepStrictEqual([acme.title, acme.company, acme.location, acme.remote, acme.job_type, acme.title_rule],++ ['Senior Software Engineer at Acme', 'Acme', 'REMOTE (US)', true, 'Full time', 'hn-line@1']);++ const planlab = await s.db.one('SELECT * FROM work_listings WHERE id = $1', [rows[1].id]);++ assert.deepStrictEqual([planlab.title, planlab.company, planlab.location, planlab.tags],++ ['Product Engineer at Planlab.ai', 'Planlab.ai', 'London, UK', 'Onsite']);++ assert.ok(acme.excerpt.startsWith('Acme | Senior Software Engineer | REMOTE (US) | Full-Time'), 'nothing is lost');++ assert.strictEqual((await s.db.one('SELECT title_rule FROM work_listings WHERE id = $1', [rows[2].id])).title_rule, null,++ 'a line with no company is left alone');++ });++++ await check('the listing page and its feed say "Role at Company" once, not twice', async () => {++ const page = await t.get(`/jobs/${rows[0].id}`);++ assert.strictEqual(page.status, 200);++ assert.ok(page.text.includes('<h1>Senior Software Engineer at Acme</h1>'));++ assert.ok(!page.text.includes('at Acme at Acme'), 'the company is not appended to a title that already ends with it');++ const feed = await t.get('/jobs.atom');++ assert.strictEqual(feed.status, 200);++ assert.ok(feed.text.includes('<title>Senior Software Engineer at Acme</title>'));++ assert.ok(!feed.text.includes('at Acme at Acme'));++ });++++ await check('Search is swept with the changed documents, and the title is not doubled in the document', async () => {++ assert.ok(out.sent && out.sent.sent >= 2, `swept: ${JSON.stringify(out.sent)}`);++ const envs = (await s.db.many('SELECT envelope FROM event_outbox')).map((r) => r.envelope);++ const doc = envs.find((e) => e.payload && e.payload.id === rows[0].id);++ assert.ok(doc, 'the changed listing is in the outbox');++ assert.strictEqual(doc.payload.title, 'Senior Software Engineer at Acme');++ const plan = envs.find((e) => e.payload && e.payload.id === rows[1].id);++ assert.strictEqual(plan.payload.title, 'Product Engineer at Planlab.ai');++ });++++ await check('a second run reads nothing: the marker means its own title is never parsed as the board line', async () => {++ const again = await rederiveHnTitles({ s, config: t.config, apply: false, search, log: { log() {} } });++ // The one row with no company is read again and skipped again; every row it could re-title is marked.++ assert.deepStrictEqual([again.scanned, again.changed, again.skipped], [1, 0, 1]);++ const acme = await s.db.one('SELECT title FROM work_listings WHERE id = $1', [rows[0].id]);++ assert.strictEqual(acme.title, 'Senior Software Engineer at Acme');++ });++ } finally {++ t.ctx.search.stop();++ await t.close();++ }++ done();++})().catch((err) => { console.error(err); process.exit(1); });@@@ -43,16 +43,21 @@ const hn = (identity, title, summary, over = {}) => ({push(hn('hn-203', 'Beta & Co is hiring a product designer', 'Beta & Co is hiring a product designer. Contact [email].'));push(hn('hn-205', 'No company here', 'A comment with no company in its first line.'));push(hn('hn-206', 'Z | Q', 'A board test post with one-letter fields.'));++ push(hn('hn-207', 'Mechanical Engineer, Electrical Engineer, Robotic Controls Engineer | New York, NY | ONSITE | Full-time | $220K-$300K at Nirvana Robotics',++ 'Three roles on one line. Email [email].'));const out = await from.pull();-- assert.deepStrictEqual([out.applied, out.hold, out.removed], [2, 2, 0]);++ assert.deepStrictEqual([out.applied, out.hold, out.removed], [3, 2, 0]);const acme = await s.db.maybe('SELECT * FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-201')]);assert.deepStrictEqual([acme.source, acme.title, acme.company, acme.remote, acme.url],-- ['hn-whoishiring', 'Senior Backend Engineer | Remote (US)', 'Acme Corp', true, 'https://news.ycombinator.com/item?id=201']);++ ['hn-whoishiring', 'Senior Backend Engineer at Acme Corp', 'Acme Corp', true, 'https://news.ycombinator.com/item?id=201']);++ assert.strictEqual(acme.location, 'Remote (US)', 'the place is its own field, not part of the title');++ assert.strictEqual(acme.title_rule, 'hn-line@1', 'the split is marked so a re-derive never reads it back');++ assert.ok(acme.excerpt.startsWith('Acme Corp | Senior Backend Engineer | Remote (US)'), 'the original line stays in the description');assert.ok(acme.excerpt.includes('[email]') && acme.excerpt.includes('[phone]'), 'the excerpt is Sources\' privacy-stripped text');const beta = await s.db.maybe('SELECT * FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-203')]);-- assert.deepStrictEqual([beta.title, beta.company, beta.remote], ['a product designer', 'Beta & Co', false]);++ assert.deepStrictEqual([beta.title, beta.company, beta.remote], ['Product designer at Beta & Co', 'Beta & Co', false]);assert.strictEqual(await s.db.maybe('SELECT id FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-205')]), null);assert.strictEqual(await s.db.maybe('SELECT id FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-206')]), null);});@@@ -63,6 +68,11 @@ const hn = (identity, title, summary, over = {}) => ({assert.strictEqual(list.status, 200);assert.ok(list.text.includes('Hacker News: Who is hiring'));assert.ok(list.text.includes('https://news.ycombinator.com/item?id=201'));++ // The place, terms and pay are the card's meta line, and the title is the roles at the company.++ assert.ok(list.text.includes('Mechanical Engineer +2 more roles at Nirvana Robotics'));++ assert.ok(list.text.includes('New York, NY'));++ assert.ok(list.text.includes('$220K-$300K'));++ assert.ok(list.text.includes('Onsite'));const page = await t.get(`/jobs/${id}`);assert.strictEqual(page.status, 200);assert.ok(page.text.includes('Hacker News: Who is hiring'));
@@ -75,7 +75,9 @@ Its own PostgreSQL tables, created by [migrations/](migrations/) and written by
@@ -75,7 +75,9 @@ Its own PostgreSQL tables, created by [migrations/](migrations/) and written by
- **Event delivery** — OpenVibe.Events: Work receives `network.account.export_requested` and `network.account.deleted`
- **Event delivery** — OpenVibe.Events: Work receives `network.account.export_requested` and `network.account.deleted`
and creates its two subscriptions at boot; it does not own the topics or the delivery.
and creates its two subscriptions at boot; it does not own the topics or the delivery.
- **The listings themselves** — the boards (Arbeitnow, Remotive, RemoteOK and Hacker News "Who is hiring"): Work keeps a short excerpt and a link back,
- **The listings themselves** — the boards (Arbeitnow, Remotive, RemoteOK and Hacker News "Who is hiring"): Work keeps a short excerpt and a link back,
- never the full description, and reads each board only on the terms it publishes.
+ never the full description, and reads each board only on the terms it publishes. A board that writes a listing as
+ one pipe-separated line (Hacker News) has that line split into company, role(s), place, terms and pay
+ (`server/jobs/hiring-line.js`); the line itself stays at the front of the excerpt.
## Depends on
## Depends on
new file mode 100644
new file mode 100644
@@ -0,0 +1,5 @@
@@ -0,0 +1,5 @@
+-- phase: expand
+-- The rule that split a one-line board post into the listing's title (server/jobs/hiring-line.js). A listing is
+-- re-derived once (scripts/rederive-hn-titles.js) and the marker means a later run never reads its own new title back
+-- as if it were the board's original line. NULL for every board that never needed a split.
+ALTER TABLE work_listings ADD COLUMN IF NOT EXISTS title_rule text;
new file mode 100644
new file mode 100644
@@ -0,0 +1,33 @@
@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Re-derive the stored Hacker News "Who is hiring" listings whose title is still the board's whole pipe-separated
+ * line (server/jobs/rederive-hn-titles.js). One-off: the ingest writes the split title from now on.
+ *
+ * node scripts/rederive-hn-titles.js show what would change (first 20), write nothing
+ * node scripts/rederive-hn-titles.js --apply write it, then sweep Search with the changed documents
+ *
+ * It runs with the service's own environment and connection, like scripts/ingest.js. It reads only source
+ * 'hn-whoishiring' and only rows without the title_rule marker, so running it twice is harmless.
+ */
+const configLib = require('../server/config');
+const { openStore } = require('../server/db');
+const { rederiveHnTitles } = require('../server/jobs/rederive-hn-titles');
+
+(async () => {
+ const apply = process.argv.includes('--apply');
+ const config = configLib.load();
+ const s = await openStore(config, { log: console });
+ try {
+ const out = await rederiveHnTitles({ s, config, apply, log: console });
+ for (const sample of out.samples) {
+ process.stdout.write(` ${sample.id}\n was: ${sample.before}\n now: ${sample.after}\n`);
+ }
+ if (out.sent) process.stdout.write(` Search: ${out.sent.sent} sent, ${out.sent.removed} removed, ${out.sent.failed} failed of ${out.sent.seen}\n`);
+ process.stdout.write(`${apply ? 'Applied' : 'Dry run'}: ${out.changed} of ${out.scanned} listing(s) change, ${out.skipped} left alone (no company)\n`);
+ if (!apply) process.stdout.write('Nothing was written. Run with --apply to write it.\n');
+ } finally {
+ await s.close();
+ }
+})().catch((err) => { console.error('[OpenVibe.Work] re-derive failed:', err && err.message ? err.message : err); process.exit(1); });
@@ -285,7 +285,7 @@ ${raw(foldBlock({
@@ -285,7 +285,7 @@ ${raw(foldBlock({
const l = listing.meta(row);
const l = listing.meta(row);
const at = isoOf(l.posted_at) || isoOf(row.fetched_at) || new Date().toISOString();
const at = isoOf(l.posted_at) || isoOf(row.fetched_at) || new Date().toISOString();
const where = [l.location, l.remote && !saysRemote(l.location) ? 'Remote' : '', listing.jobTypeLabel(l.job_type)].filter(Boolean).join(' · ');
const where = [l.location, l.remote && !saysRemote(l.location) ? 'Remote' : '', listing.jobTypeLabel(l.job_type)].filter(Boolean).join(' · ');
- return `<entry><id>${xml(`${config.baseUrl}/jobs/${row.id}`)}</id><title>${xml(`${l.title} at ${l.company}`)}</title>`
+ return `<entry><id>${xml(`${config.baseUrl}/jobs/${row.id}`)}</id><title>${xml(listing.fullTitle(l))}</title>`
+ `<link rel="alternate" type="text/html" href="${xml(`${config.baseUrl}/jobs/${row.id}`)}"/><updated>${at}</updated>`
+ `<link rel="alternate" type="text/html" href="${xml(`${config.baseUrl}/jobs/${row.id}`)}"/><updated>${at}</updated>`
+ `<author><name>${xml(sourceOf(row).name)}</name></author>`
+ `<author><name>${xml(sourceOf(row).name)}</name></author>`
+ `<summary>${xml([where, listing.plainExcerpt(row).slice(0, 300)].filter(Boolean).join(' — '))}</summary></entry>`;
+ `<summary>${xml([where, listing.plainExcerpt(row).slice(0, 300)].filter(Boolean).join(' — '))}</summary></entry>`;
@@ -355,15 +355,15 @@ ${nextUrl ? html`<a class="sc-btn sc-primary" href="${nextUrl}">Older listings</
@@ -355,15 +355,15 @@ ${nextUrl ? html`<a class="sc-btn sc-primary" href="${nextUrl}">Older listings</
styles: [showcase.STYLESHEET],
styles: [showcase.STYLESHEET],
body: html`<h1>No longer on the board</h1>
body: html`<h1>No longer on the board</h1>
<p class="job-gone"><span class="badge bad">Gone</span> This listing is no longer on the board.${lastSeen ? html` Last seen on ${src.name} on ${lastSeen}.` : ' We did not record the day we last saw it.'}</p>
<p class="job-gone"><span class="badge bad">Gone</span> This listing is no longer on the board.${lastSeen ? html` Last seen on ${src.name} on ${lastSeen}.` : ' We did not record the day we last saw it.'}</p>
-<p>${l.title} at ${l.company} is not in our listings any more, so it is not in the search either. The board's own listing is still <a href="${l.url}" rel="nofollow noopener external" target="_blank">here</a>, but the board may have taken it down.</p>
+<p>${listing.fullTitle(l)} is not in our listings any more, so it is not in the search either. The board's own listing is still <a href="${l.url}" rel="nofollow noopener external" target="_blank">here</a>, but the board may have taken it down.</p>
<p><a class="sc-btn" href="/jobs">Search the listings</a></p>`,
<p><a class="sc-btn" href="/jobs">Search the listings</a></p>`,
}, 404);
}, 404);
}
}
const tracked = signedIn(req) ? await applications.applicationByListing(s, subjectOf(req), row.id) : null;
const tracked = signedIn(req) ? await applications.applicationByListing(s, subjectOf(req), row.id) : null;
page(req, res, {
page(req, res, {
index: true, cache: PUBLIC_CACHE,
index: true, cache: PUBLIC_CACHE,
- title: `${l.title} at ${l.company}`,
- description: listing.plainExcerpt(row).slice(0, 300) || `${l.title} at ${l.company}, from ${src.name}.`,
+ title: listing.fullTitle(l),
+ description: listing.plainExcerpt(row).slice(0, 300) || `${listing.fullTitle(l)}, from ${src.name}.`,
crumbs: [{ label: 'Jobs', href: '/jobs' }, { label: l.title }],
crumbs: [{ label: 'Jobs', href: '/jobs' }, { label: l.title }],
styles: [showcase.STYLESHEET, 'kit.css'],
styles: [showcase.STYLESHEET, 'kit.css'],
jsonLd: [listing.jobPosting(row, { siteUrl: config.baseUrl })],
jsonLd: [listing.jobPosting(row, { siteUrl: config.baseUrl })],
@@ -461,7 +461,7 @@ ${!q && all.length > shown.length ? html`<p class="muted small">The ${n(shown.le
@@ -461,7 +461,7 @@ ${!q && all.length > shown.length ? html`<p class="muted small">The ${n(shown.le
title: `Jobs at ${c.company}`,
title: `Jobs at ${c.company}`,
description: `${n(c.count)} open listing${c.count === 1 ? '' : 's'} at ${c.company}${c.remote ? `, ${n(c.remote)} remote` : ''}${c.places.length ? `, in ${c.places.slice(0, 3).map((p) => p.place).join(', ')}` : ''}, from the boards OpenVibe.Work reads, each linking back to the original listing.`.slice(0, 300),
description: `${n(c.count)} open listing${c.count === 1 ? '' : 's'} at ${c.company}${c.remote ? `, ${n(c.remote)} remote` : ''}${c.places.length ? `, in ${c.places.slice(0, 3).map((p) => p.place).join(', ')}` : ''}, from the boards OpenVibe.Work reads, each linking back to the original listing.`.slice(0, 300),
crumbs: [{ label: 'Companies', href: '/companies' }, { label: c.company }], styles: [showcase.STYLESHEET],
crumbs: [{ label: 'Companies', href: '/companies' }, { label: c.company }], styles: [showcase.STYLESHEET],
- jsonLd: [seo.jsonLd.itemList(`Jobs at ${c.company}`, c.rows.slice(0, 50).map((row) => ({ name: `${row.title} at ${row.company}`, url: `${base}/jobs/${row.id}` })), { url: `${base}/companies/${c.slug}` })],
+ jsonLd: [seo.jsonLd.itemList(`Jobs at ${c.company}`, c.rows.slice(0, 50).map((row) => ({ name: listing.fullTitle(row), url: `${base}/jobs/${row.id}` })), { url: `${base}/companies/${c.slug}` })],
body: html`<h1>Jobs at ${c.company}</h1>
body: html`<h1>Jobs at ${c.company}</h1>
<p class="lede">${n(c.count)} open listing${c.count === 1 ? '' : 's'}${c.remote ? html`, ${n(c.remote)} of them remote` : ''}${c.salaried ? html`, ${n(c.salaried)} stating the pay` : ''}. Listed on ${boards.length > 1 ? html`${boards.slice(0, -1).map((b, i) => html`${i ? ', ' : ''}${b}`)} and ${boards[boards.length - 1]}` : boards[0]}.</p>
<p class="lede">${n(c.count)} open listing${c.count === 1 ? '' : 's'}${c.remote ? html`, ${n(c.remote)} of them remote` : ''}${c.salaried ? html`, ${n(c.salaried)} stating the pay` : ''}. Listed on ${boards.length > 1 ? html`${boards.slice(0, -1).map((b, i) => html`${i ? ', ' : ''}${b}`)} and ${boards[boards.length - 1]}` : boards[0]}.</p>
${c.places.length ? html`<p class="small">Where: ${c.places.map((p, i) => html`${i ? ', ' : ''}<a href="/jobs?${new URLSearchParams({ q: c.company, location: p.place })}">${p.place}</a> (${n(p.count)})`)}</p>` : ''}
${c.places.length ? html`<p class="small">Where: ${c.places.map((p, i) => html`${i ? ', ' : ''}<a href="/jobs?${new URLSearchParams({ q: c.company, location: p.place })}">${p.place}</a> (${n(p.count)})`)}</p>` : ''}
@@ -556,7 +556,7 @@ ${notice('Give the search at least one thing to look for — words, a location,
@@ -556,7 +556,7 @@ ${notice('Give the search at least one thing to look for — words, a location,
<p class="muted small">Or press “Track this job” on any <a href="/jobs">listing</a>.</p></section>`;
<p class="muted small">Or press “Track this job” on any <a href="/jobs">listing</a>.</p></section>`;
const renderApplication = (req, res, a, { events = [], problem = null, assisted = false } = {}) => page(req, res, {
const renderApplication = (req, res, a, { events = [], problem = null, assisted = false } = {}) => page(req, res, {
- title: `${a.title} at ${a.company}`, styles: [showcase.STYLESHEET], localStyles: APP_STYLES,
+ title: listing.fullTitle(a), styles: [showcase.STYLESHEET], localStyles: APP_STYLES,
crumbs: [{ label: 'Applications', href: '/applications' }, { label: a.title }],
crumbs: [{ label: 'Applications', href: '/applications' }, { label: a.title }],
body: html`<div class="app-head"><h1>${a.title}</h1>${statusBadge(a.status)}</div>
body: html`<div class="app-head"><h1>${a.title}</h1>${statusBadge(a.status)}</div>
<p class="app-company">${a.company}${a.url ? html` · <a href="${a.url}" rel="nofollow noopener external" target="_blank">the original listing</a>` : ''}${a.listing_id ? html` · <a href="/jobs/${a.listing_id}">this listing on OpenVibe.Work</a>` : ''}</p>
<p class="app-company">${a.company}${a.url ? html` · <a href="${a.url}" rel="nofollow noopener external" target="_blank">the original listing</a>` : ''}${a.listing_id ? html` · <a href="/jobs/${a.listing_id}">this listing on OpenVibe.Work</a>` : ''}</p>
@@ -21,6 +21,7 @@
@@ -21,6 +21,7 @@
*/
*/
const ingest = require('openvibe-publishing/ingest');
const ingest = require('openvibe-publishing/ingest');
const sources = require('../jobs/sources');
const sources = require('../jobs/sources');
+const { parseHiringLine, TITLE_RULE } = require('../jobs/hiring-line');
const store = require('../jobs/store');
const store = require('../jobs/store');
const SOURCE_ID = 'arbeitnow';
const SOURCE_ID = 'arbeitnow';
@@ -103,42 +104,41 @@ function rawOfWwr(item) {
@@ -103,42 +104,41 @@ function rawOfWwr(item) {
/**
/**
* A Sources item → the row a Hacker News "Who is hiring" comment makes. The resolver's title is the comment's first
* A Sources item → the row a Hacker News "Who is hiring" comment makes. The resolver's title is the comment's first
- * line, which posters usually write as "Company | Role | Place" or "Company is hiring a role": the company comes from
- * that, and a comment that does not name one is held by the same thin-listing gate as any other board. The excerpt is
- * Sources' plain-text excerpt, already stripped of email addresses and phone numbers; the full comment is its link.
+ * line, which posters usually write as "Company | Role | Place | Terms | Pay" (or "Company is hiring a role"). Work
+ * splits that line into the company, the role(s), the place, the terms and the pay (server/jobs/hiring-line.js), and
+ * keeps the line at the front of the description so nothing the poster wrote is lost. A comment that names no company
+ * is held by the same thin-listing gate as any other board. The excerpt is Sources' plain-text excerpt, already
+ * stripped of email addresses and phone numbers; the full comment is its link.
*/
*/
function rawOfHn(item) {
function rawOfHn(item) {
const f = (item && item.fields) || {};
const f = (item && item.fields) || {};
const first = String((item && item.title) || '').replace(/\s+/g, ' ').trim();
const first = String((item && item.title) || '').replace(/\s+/g, ' ').trim();
- const parts = first.split(/\s*\|\s*/);
- let company = '';
- let title = first;
- if (parts.length >= 2 && parts[0].trim()) {
- company = parts[0].trim();
- title = parts.slice(1).join(' | ').trim() || first;
- } else {
- const m = /^(.+?)\s+(?:is|are)\s+hiring\b/i.exec(first);
- if (m && m[1].trim()) {
- company = m[1].trim();
- title = first.slice(m[0].length).replace(/^[\s:,-]+/, '').trim() || first;
- }
- }
+ const parsed = parseHiringLine(first);
const summary = String((item && item.summary) || '');
const summary = String((item && item.summary) || '');
+ // The original line is the listing's description too: Sources starts its excerpt with it, but an excerpt that was
+ // cut before the first newline, or a board that sends the fields apart, must not lose it.
+ const plain = excerptLines(summary);
+ const description = plain.startsWith(first) ? summary : `${first}\n${summary}`;
return {
return {
source_id: item.identity,
source_id: item.identity,
url: item.canonical_url,
url: item.canonical_url,
- title,
- company,
- location: null, // the comment rarely states one; the title may, in its own words
- remote: /\bremote\b/i.test(`${first} ${summary}`),
- tags: list(f.tags),
- job_type: null,
- salary: null,
+ title: parsed.title,
+ company: parsed.company,
+ location: parsed.location,
+ remote: parsed.remote,
+ tags: list([...list(f.tags), ...parsed.tags]),
+ job_type: parsed.job_type,
+ salary: parsed.salary,
posted_at: item.published_at || null,
posted_at: item.published_at || null,
- description_html: summary,
+ description_html: description,
+ title_rule: TITLE_RULE,
};
};
}
}
+/** The start of a plain-text summary, with whitespace collapsed, for comparing it with the comment's first line. */
+const excerptLines = (text) => String(text || '').replace(/\s+/g, ' ').trim();
+
+
/**
/**
* The boards Work reads through OpenVibe.Sources: the Sources key an item arrives under, the id the listing is
* The boards Work reads through OpenVibe.Sources: the Sources key an item arrives under, the id the listing is
* stored with, the kind of item that source keeps, and how the item becomes a listing.
* stored with, the kind of item that source keeps, and how the item becomes a listing.
@@ -239,4 +239,4 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
@@ -239,4 +239,4 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
return { enabled: Boolean(client), pull, start, stop, apply, SOURCE_ID, sourceIds: active.map(([, e]) => e.source_id) };
return { enabled: Boolean(client), pull, start, stop, apply, SOURCE_ID, sourceIds: active.map(([, e]) => e.source_id) };
}
}
-module.exports = { createFromSources, rawOf, rawOfWwr, rawOfHn, SOURCES, SOURCE_ID, SOURCE_KEY };
+module.exports = { createFromSources, rawOf, rawOfWwr, rawOfHn, SOURCES, SOURCE_ID, SOURCE_KEY, TITLE_RULE };
new file mode 100644
new file mode 100644
@@ -0,0 +1,439 @@
@@ -0,0 +1,439 @@
+'use strict';
+
+/**
+ * A job board post written as one pipe-separated line, split into the fields a listing has.
+ *
+ * Hacker News "Who is hiring" comments (and boards that copy its style) keep a whole listing in the first line of a
+ * post:
+ *
+ * https://www.monumental.co/ | Amsterdam, The Netherlands | Full Time | Onsite at MONUMENTAL
+ * Mechanical Engineer, Electrical Engineer, Robotic Controls Engineer | New York, NY | ONSITE | Full-time | $220K-$300K at Nirvana Robotics
+ *
+ * Kept whole, that line is not a title: a URL, a place, the terms and the pay all end up in it, and the card says the
+ * same things twice in the wrong places. This module reads the line the way a person does:
+ *
+ * company from "at X", the "X is hiring …" opening, the first segment that is a name, or a URL's host
+ * roles the segments that name a role; several are kept ("the first +N more roles")
+ * place a location segment (a city, a country, "Remote (US)", a time zone), never a role or a term
+ * terms on-site / remote / hybrid, full / part time, contract, visa and relocation
+ * pay a salary segment, in the poster's own words
+ *
+ * The title is "Role at Company", or "First role +N more roles at Company"; with no role named it is the company
+ * alone, and a URL is never a title. The line itself is returned so the description can keep it — nothing is lost.
+ *
+ * It is deliberately conservative: a token that names none of these is dropped rather than shown in the wrong field,
+ * and a line with no company at all is not a listing (server/jobs/sources.js's thin-listing gate holds it). No I/O.
+ */
+
+/** The rule's name, stored on the listing (work_listings.title_rule) so a re-derive never reads its own output back. */
+const TITLE_RULE = 'hn-line@1';
+
+const clip = (s, n) => String(s == null ? '' : s).replace(/\s+/g, ' ').trim().slice(0, n);
+const letters = (s) => (String(s == null ? '' : s).match(/\p{L}/gu) || []).length;
+const uniq = (a) => [...new Set(a.filter(Boolean))];
+
+/** The job types the boards name, in the words this module reports (server/jobs/job-types.js maps them for display). */
+const TYPES = [
+ { label: 'Full time', re: /\bfull[\s-]?time\b|\bfulltime\b|\bft\b/i },
+ { label: 'Part time', re: /\bpart[\s-]?time\b|\bparttime\b|\bpt\b/i },
+ { label: 'Apprenticeship', re: /\bapprentic(?:e|eship)\b/i },
+ { label: 'Internship', re: /\bintern(?:ship)?s?\b|\bco-?ops?\b/i },
+ { label: 'Freelance', re: /\bfreelanc(?:e|er)\b|\bfractional\b/i },
+ { label: 'Contract', re: /\bcontract(?:or|s)?\b|\bcontract-to-hire\b|\bb2b\b|\b1099\b|\bfixed[\s-]?term\b/i },
+ { label: 'Permanent', re: /\bpermanent\b|\bcdi\b/i },
+ { label: 'Temporary', re: /\btemporary\b|\binterim\b/i },
+];
+
+const REMOTE_RE = /\bremote(?:[-\s]?first)?\b|\banywhere\b|\bworldwide\b|\bwork\s+(?:from|at)\s+home\b|\bdistributed\s+team\b/i;
+const NO_REMOTE_RE = /\bno\s+remote\b|\bnot\s+remote\b|\bremote\s+not\s+(?:possible|available)\b/i;
+const ONSITE_RE = /\bon[\s-]?site\b|\bin[\s-]?person\b|\bin[\s-]?office\b|\bon[\s-]?prem(?:ise|ises)?\b/i;
+const HYBRID_RE = /\bhybrid\b/i;
+const VISA_RE = /\bvisa\b|\bsponsor(?:ship|s)?\b/i;
+const NO_VISA_RE = /\bno\s+(?:visa\s+)?sponsor|\bnot\s+(?:able|offer|offering)\b[^.]*\bsponsor|\bno\s+sponsorship\b|\bwithout\s+sponsorship\b/i;
+const AUTH_RE = /\bcitizenship\b|\bwork\s+authori[sz]ation\b|\bsecurity\s+clearance\b|\bclearance\s+required\b|\bgreen\s+card\b/i;
+const RELOCATION_RE = /\brelocation\b|\brelocate\b/i;
+
+/**
+ * A role's own words. Two strengths: a segment is a role when it names any of these, and several comma/plus-separated
+ * roles are only split apart when every part names a strong one — so "Senior Java, C# and React engineers" stays one
+ * role, while "Mechanical Engineer, Electrical Engineer" becomes two.
+ */
+const ROLE_NOUN = /\b(?:engineers?|developers?|designers?|managers?|scientists?|analysts?|architects?|leads?|directors?|administrators?|admins?|consultants?|specialists?|executives?|officers?|recruiters?|marketers?|writers?|editors?|researchers?|interns?|founders?|swe|sde|pm|tpm|em|qa|sre|devops|cto|ceo|cfo|coo|vp|president|heads?|coordinators?|strategists?|accountants?|attorneys?|lawyers?|professors?|technicians?|operators?|producers?|artists?|assistants?|associates?|representatives?|reps?|hires?|ba|po|sales|growth|marketing|product|operations|finance|legal|people|talent|security|infrastructure|platform|backend|frontend|full[\s-]?stack|mobile|cloud|data|ml|ai|robotics?|mechanical|electrical|controls?|embedded|firmware|hardware|software|systems?|network|database|devrel|advocate|evangelist|support|success|customer|partnerships?|revenue|gtm|go[\s-]?to[\s-]?market|agents?)\b/i;
+const ROLE_STRONG = /\b(?:engineers?|developers?|designers?|managers?|scientists?|analysts?|architects?|leads?|directors?|administrators?|admins?|consultants?|specialists?|executives?|officers?|recruiters?|marketers?|writers?|editors?|researchers?|interns?|founders?|swe|sde|pm|tpm|em|qa|sre|devops|cto|ceo|cfo|coo|vp|president|heads?|coordinators?|strategists?|accountants?|attorneys?|lawyers?|professors?|technicians?|operators?|producers?|artists?|assistants?|associates?|representatives?|reps?|hires?|ba|po|go[\s-]?to[\s-]?market|advocate|evangelist)\b/i;
+
+/** A company often names what it does; that suffix keeps "Charge Robotics" a name even though "robotics" is a role word. */
+const COMPANY_SUFFIX = /\b(?:inc|ltd|limited|llc|corp|corporation|company|co|labs?|systems?|technologies|technology|tech|ai|studio|studios|health|capital|ventures?|group|solutions|aerospace|pharmacy|trading|partners|foundation|institute|university|bank|media|networks?|robotics|science|sciences|design|consulting|agency|division|startup|holdings|finance|markets?|commodit(?:y|ies)|software)\b/i;
+
+/** A first segment that opens a sentence or asks a question is not a company name. */
+const NOT_A_COMPANY_START = /^(?:no|not|none|without|unknown|various|multiple|several|looking|seeking|hiring|we|our|this|the|about|backed|join|apply|role|job|location|remote|onsite|hybrid|full|part|contract|intern)\b/i;
+
+/** A role phrase opens with a level or a quantity ("Senior Backend Engineer", "Multiple roles"), never a company. */
+const ROLE_TITLE_START = /^(?:senior|staff|lead|principal|junior|sr\.?|jr\.?|entry|mid|chief|head|vice|vp|founding|multiple|several|various)\b/i;
+const NOT_A_ROLE_START = /^(?:closed|stop|apply|email|url|http|www|see|read|more|details|salary|compensation|equity|visa|relocation|seed|series|yc|stealth|profitable|pmf|fintech|healthtech|edtech|ai\/ml)\b/i;
+
+const MONEY_RE = /(?:[$€£]\s?\d|\b\d[\d,.]*\s?[kK]\b|\b\d{2,3}[,.]\d{3}\b|\b(?:usd|eur|gbp|cad|aud|chf)\b)/i;
+const COMP_RE = /\b(?:salary|compensation|comp|ote|equity|stock|share\s+options|options|bonus|commission|hourly|per\s+hour|\/\s?hr|annuali[sz]ed|revenue\s+share)\b/i;
+const PAY_AMOUNT_RE = /\b(?:ote|equity|stock|share\s+options|options|bonus|commission|revenue\s+share)\b/i;
+
+const PLACE_RE = /\b(?:remote|hybrid|on[\s-]?site|in[\s-]?person|in[\s-]?office|office|worldwide|anywhere|global|international|united\s+states|uk|u\.k\.|united\s+kingdom|europe|european|emea|apac|latam|americas|north\s+america|south\s+america|asia|africa|australia|canada|mexico|brazil|argentina|colombia|chile|peru|uruguay|germany|france|netherlands|belgium|spain|portugal|italy|austria|switzerland|sweden|norway|denmark|finland|iceland|ireland|poland|romania|bulgaria|czech|slovakia|hungary|ukraine|estonia|latvia|lithuania|serbia|croatia|slovenia|greece|turkey|israel|india|pakistan|bangladesh|philippines|indonesia|malaysia|singapore|vietnam|thailand|japan|korea|china|taiwan|hong\s+kong|australia|new\s+zealand|south\s+africa|nigeria|kenya|egypt|morocco|uae|dubai|saudi|qatar|time\s*zones?|hours|overlap|bay\s+area|sf\s+bay|silicon\s+valley|peninsula)\b/i;
+const PLACE_CODE_RE = /\b(?:US|UK|EU|USA|NYC|SF|LA|DC)\b/;
+const TIMEZONE_RE = /\b(?:CET|CEST|PST|PDT|EST|EDT|MST|MDT|CST|CDT|UTC|GMT|PT|ET|CT|MT)(?:[+-]\d{1,2})?\b/;
+const TIME_CONTEXT_RE = /\b(?:remote|time\s*zones?|hours|overlap|from|to)\b|[±]/i;
+const PLACE_COMMA_RE = /,\s*(?:[A-Z]{2}|[A-Z][a-z]+)(?:\s|,|$)/;
+const PLACE_CITY_RE = /\b(?:london|berlin|paris|amsterdam|rotterdam|utrecht|munich|münchen|zurich|zürich|geneva|stockholm|copenhagen|oslo|helsinki|dublin|lisbon|barcelona|madrid|milan|rome|vienna|prague|warsaw|krakow|bucharest|budapest|athens|istanbul|tel\s+aviv|bangalore|bengaluru|mumbai|delhi|singapore|tokyo|seoul|sydney|melbourne|auckland|toronto|vancouver|montreal|new\s+york|brooklyn|san\s+francisco|san\s+diego|san\s+jose|palo\s+alto|mountain\s+view|redwood\s+city|berkeley|oakland|seattle|boston|chicago|denver|boulder|atlanta|miami|los\s+angeles|austin|dallas|houston|phoenix|portland|pittsburgh|philadelphia|raleigh|durham|washington|arlington|salt\s+lake\s+city|provo|mexico\s+city|são\s+paulo|sao\s+paulo|buenos\s+aires|bogotá|bogota|santiago)\b/i;
+
+/** A URL's host, from an http(s) URL or a bare domain ("k2space.com", "startuphakk.com/jobs"). */
+function hostOf(value) {
+ const s = String(value == null ? '' : value).trim();
+ try {
+ const u = new URL(s);
+ if (u.protocol === 'http:' || u.protocol === 'https:') return u.hostname.toLowerCase().replace(/^www\./, '');
+ } catch { /* a bare domain is not a URL until the pattern below says it is */ }
+ const m = /^((?:[a-z0-9-]+\.)+[a-z]{2,})(?::\d+)?(?:\/\S*)?$/i.exec(s);
+ return m ? m[1].toLowerCase().replace(/^www\./, '') : null;
+}
+
+/** The URLs a segment carries, whose text is removed before the segment is read as anything else. */
+function stripUrls(text) {
+ return String(text)
+ .replace(/\((?:https?:\/\/|www\.)[^)]*\)/gi, ' ')
+ .replace(/https?:\/\/[^\s)]+/gi, ' ')
+ .replace(/\bwww\.[^\s)]+/gi, ' ')
+ .replace(/\s+/g, ' ')
+ .trim();
+}
+
+/** Whether the whole segment is an http(s) URL (so it is a link, not a name, place or role). A bare domain can be a
+ * company's name ("Planlab.ai") and is left to the caller to read as one. */
+function isUrlSegment(text) {
+ const s = String(text == null ? '' : text).trim();
+ return /^https?:\/\/\S+$/i.test(s) || /^www\.\S+$/i.test(s);
+}
+
+/** A segment that is only a pay amount / compensation line. "Compensation in the link below" is neither. */
+function looksPay(text) {
+ const s = String(text == null ? '' : text);
+ return MONEY_RE.test(s) || (PAY_AMOUNT_RE.test(s) && !ROLE_STRONG.test(s));
+}
+
+/** The pay a segment carries, and what is left of it ("Full time, $150k" → "$150k"). */
+function takePay(text) {
+ const s = String(text == null ? '' : text).trim();
+ if (!looksPay(s)) return null;
+ // A segment can mix terms and pay ("Full-time, $150K"): keep the parts that name money or compensation.
+ const parts = s.replace(/\s*\/\s*(hour|hr|month|year|annum|week|day)\b/gi, ' per $1')
+ .split(/\s*;\s*|,(?!\d{3}\b)|\s+\/\s+/).filter(Boolean);
+ const kept = parts.filter((p) => MONEY_RE.test(p) || (COMP_RE.test(p) && !TYPES.some((t) => t.re.test(p))));
+ const pay = kept.length && kept.join(', ').toLowerCase() !== s.toLowerCase() ? kept.join(', ') : s;
+ if (pay === s && TYPES.some((t) => t.re.test(s)) && !MONEY_RE.test(s)) return null; // "Equity, full-time" is not a pay line
+ const rest = kept.length && kept.join(', ').toLowerCase() !== s.toLowerCase()
+ ? parts.filter((p) => !kept.includes(p)).join(', ')
+ : s;
+ return { text: clip(pay, 160), rest: rest === s ? '' : rest };
+}
+
+/** The job types a segment names, in the order they appear. */
+function typesOf(text) {
+ const s = String(text == null ? '' : text);
+ // "(PT to ET overlap preferred)" is Pacific Time, not part time; the same for FT when it is a time zone.
+ const timeZone = TIMEZONE_RE.test(s) && TIME_CONTEXT_RE.test(s) || /\b(?:PT|FT)\s+(?:to|and|or)\s+(?:ET|CT|MT|PT)\b/i.test(s);
+ const probe = timeZone ? s.replace(/\b(?:FT|PT)\b/gi, ' ') : s;
+ return TYPES.filter((t) => t.re.test(probe)).map((t) => t.label);
+}
+
+/** The place a segment names, cleaned of the terms around it, or null when it names none. */
+function placeOf(text) {
+ let s = stripUrls(text);
+ if (!s || looksPay(s)) return null;
+ if ((VISA_RE.test(s) || AUTH_RE.test(s)) && !PLACE_COMMA_RE.test(s) && !PLACE_CITY_RE.test(s)) return null;
+ const comma = PLACE_COMMA_RE.test(s) && !ROLE_NOUN.test(s);
+ const city = PLACE_CITY_RE.test(s);
+ const zone = TIMEZONE_RE.test(s) && (TIME_CONTEXT_RE.test(s) || s.replace(TIMEZONE_RE, '').replace(/[\s(),./+-]/g, '').length < 2);
+ const named = PLACE_RE.test(s) || PLACE_CODE_RE.test(s) || comma || city || zone;
+ if (!named) return null;
+ // "Mechanical Engineer, Electrical Engineer" and "Founding Account Executive (US)": a comma or a work-authorisation
+ // code between roles is not a place.
+ if (ROLE_STRONG.test(s) && !PLACE_RE.test(s) && !city) return null;
+ s = s
+ .replace(/\b(?:full[\s-]?time|part[\s-]?time|fulltime|parttime|contract(?:or)?|freelance|fractional|permanent|temporary|internship|interns?|co-?ops?)\b/gi, ' ')
+ .replace(/\b(?:visa\s+sponsorship|no\s+visa\s+sponsorship|visa|sponsorship|relocation)\b/gi, ' ')
+ .replace(/\b(?:citizenship\s+required|work\s+authori[sz]ation\s+required|security\s+clearance\s+required|clearance\s+required)\b/gi, ' ')
+ .replace(/\bhybrid\b/gi, ' ')
+ .replace(/\b(?:on[\s-]?site|in[\s-]?person|in[\s-]?office)\b/gi, ' ')
+ .replace(/\b(?:no\s+remote|not\s+remote)\b/gi, ' ')
+ .replace(/\(\s*(?:\d+\s*days?[^)]*|in[\s-]?office[^)]*|some\s+crossover[^)]*)\)/gi, ' ')
+ .replace(/\b\d+\s*days?\b(?:\s*(?:a|\/|per)\s*(?:week|wk))?/gi, ' ')
+ .replace(/\bin\s+office\b/gi, ' ')
+ .replace(/\b(?:based|located)\s+in\b/gi, ' ')
+ .replace(/\s+/g, ' ')
+ .replace(/^[\s,;:/|.·•–—-]+|[\s,;:/|.·•–—-]+$/g, '')
+ .replace(/^(?:in|at)\s+/i, '')
+ .replace(/\s+(?:or|and)\s*$/i, '')
+ .replace(/\s*\/\s*(?:or|and)\s+/gi, ' / ')
+ .replace(/^(?:or|and)\s+/i, ' ')
+ .replace(/\(\s*\)/g, ' ')
+ .replace(/\((?:[^)]*\b(?:require[sd]?|preferred|only|hours|days?|week)\b[^)]*)\)/gi, ' ')
+ .replace(/\b(?:preferred|required|only)\b/gi, ' ')
+ .replace(/^(?:or|and)\s+/i, ' ')
+ .replace(/\s+(?:or|and)\s*$/i, '')
+ .replace(/\s*,\s*(?=,|$)/g, '')
+ .replace(/\s*,\s*/g, ', ')
+ .replace(/^[\s,;:/|.·•–—-]+|[\s,;:/|.·•–—-]+$/g, '')
+ .trim();
+ s = s.replace(/^\(([^()]+)\)$/, '$1').replace(/^(?:or|and)\s+/i, '').trim();
+ if (letters(s) < 2 || s.length > 100) return null;
+ if (/\b(?:looking|apply|description|we\s+are)\b/i.test(s)) return null;
+ return clip(s, 160);
+}
+
+/** Whether a cleaned string reaches this module's bar for a company name. */
+function looksLikeCompany(text) {
+ const s = clip(text, 160);
+ if (letters(s) < 2 || s.length > 120) return false;
+ if (isUrlSegment(s) || NOT_A_COMPANY_START.test(s)) return false;
+ if (TYPES.some((t) => t.re.test(s))) return false;
+ if (VISA_RE.test(s) || AUTH_RE.test(s)) return false;
+ if (looksPay(s)) return false;
+ if (placeOf(s)) return false;
+ if (/\b(?:is|are|was|were)\b/i.test(s) && s.split(/\s+/).length > 6) return false;
+ if (ROLE_NOUN.test(s) && ROLE_TITLE_START.test(s)) return false;
+ if (ROLE_NOUN.test(s) && s.split(/\s+/).length <= 8 && !COMPANY_SUFFIX.test(s)) return false;
+ // A bare time zone is a place, not a company, however it is punctuated.
+ if (/[<>]/.test(s) && TIMEZONE_RE.test(s)) return false;
+ return true;
+}
+
+/** A company name, without a trailing parenthetical URL or a leading "at". */
+function cleanCompany(text) {
+ return clip(String(text == null ? '' : text)
+ .replace(/^[\s\-–—,:]+|[\s\-–—,:]+$/g, '')
+ .replace(/\s*\((?:https?:\/\/|www\.)[^)]*\)\s*$/gi, '')
+ // A descriptive bracket ("(stealth, regulated commodity markets infrastructure)") is not part of the name;
+ // a short one ("(Series B)", "(YC S24)") is how the poster identifies the company.
+ .replace(/\s*\(([^)]*)\)\s*$/g, (m, inner) => (inner.includes(',') || inner.split(/\s+/).length > 3 ? '' : m))
+ .replace(/\s+/g, ' '), 120);
+}
+
+/**
+ * The company a line names: "at X" (the last such segment), the "X is hiring …" opening, the first segment that is a
+ * name, or a URL's host. The segments array is mutated: the company's text is removed from the segment it was in, so
+ * the rest of that segment (the pay before "at X", the role before "at X") is still read.
+ */
+function takeCompany(segments) {
+ // "X is hiring …": the company is the subject, the rest is the role.
+ const opening = /^(.+?)\s+(?:is|are|was|were)\s+hiring\b(?:\s+(?:a|an|for|the)\b)?\s*(.*)$/i.exec(segments[0] || '');
+ if (opening) {
+ const company = cleanCompany(opening[1]);
+ if (looksLikeCompany(company)) {
+ segments[0] = opening[2].trim();
+ return company;
+ }
+ }
+ // "… | $220K-$300K at Nirvana Robotics" or "… | Onsite at MONUMENTAL": the last "at X" names the company.
+ for (let i = segments.length - 1; i >= 0; i--) {
+ const m = /(?:^|\s)at\s+([^|]+?)\s*$/i.exec(segments[i]);
+ if (!m) continue;
+ const company = cleanCompany(m[1]);
+ if (!looksLikeCompany(company)) continue;
+ segments[i] = segments[i].slice(0, m.index).trim();
+ return company;
+ }
+ // The first segment that is a name.
+ for (let i = 0; i < segments.length; i++) {
+ const cleaned = cleanCompany(stripUrls(segments[i]));
+ if (!cleaned) continue;
+ if (looksLikeCompany(cleaned)) {
+ segments[i] = '';
+ return cleaned;
+ }
+ }
+ // A URL's host, when the line names no company but links one.
+ for (let i = 0; i < segments.length; i++) {
+ const host = hostOf(segments[i]) || hostOf(stripUrls(segments[i]));
+ if (host) {
+ segments[i] = stripUrls(segments[i]) && !isUrlSegment(stripUrls(segments[i])) ? stripUrls(segments[i]) : '';
+ return host;
+ }
+ }
+ return '';
+}
+
+/** Whether a cleaned segment names a role, rather than a tagline, a place, a term or a pay line. */
+function looksLikeRole(text) {
+ const s = clip(text, 200);
+ if (letters(s) < 2 || s.length > 160) return false;
+ if (NOT_A_ROLE_START.test(s)) return false;
+ if (looksPay(s) || typesOf(s).length || VISA_RE.test(s) || AUTH_RE.test(s)) return false;
+ if (/\b(?:is|are|we|our|you|your)\b/i.test(s) && s.split(/\s+/).length > 5) return false;
+ if (/^(?:building|making|shipping|helping|join|work(?:ing)?\s+with|about|the\s+team|series|seed)\b/i.test(s)) return false;
+ if (ROLE_NOUN.test(s)) return true;
+ return /^(?:multiple|various|several)\b/i.test(s) && /\broles?\b/i.test(s);
+}
+
+/** The role text with the leading "hiring a …" / "looking for …" / separators cleaned off. */
+function cleanRole(text) {
+ let s = stripUrls(text)
+ .replace(/^[\s,;:+/&|–—-]+|[\s,;:+/&|–—-]+$/g, '')
+ .replace(/^(?:we(?:'re| are)\s+)?(?:hiring|looking\s+for|seeking|recruiting|searching\s+for)\b\s*(?:a|an|the)?\s*/i, '')
+ .replace(/^[\s,;:+/&|–—-]+|[\s,;:+/&|–—-]+$/g, '')
+ .replace(/\.$/, '')
+ .replace(/\.\s.*$/, '')
+ .replace(/\s+/g, ' ')
+ .trim();
+ // "Engineers to work on event-driven, distributed systems …" is a description of the job, not its title.
+ s = s.replace(/^((?:[\w/&.+-]+\s+){0,4}(?:engineers?|developers?|designers?|managers?|scientists?|architects?|leads?|directors?))\s+(?:to|who|that)\s+.*$/i, '$1');
+ if (letters(s) < 2) return '';
+ return clip(s, 140);
+}
+
+/** Several roles in one segment, split on a comma, semicolon, plus or "&" only between parenthesised groups. */
+function splitRoles(text) {
+ const s = String(text || '').trim();
+ if (!s || /^(?:multiple|various|several)\b/i.test(s)) return [s];
+ const parts = [];
+ let depth = 0;
+ let cur = '';
+ for (let i = 0; i < s.length; i += 1) {
+ const ch = s[i];
+ if (ch === '(') depth += 1;
+ if (ch === ')') depth = Math.max(0, depth - 1);
+ if (depth === 0) {
+ if (ch === ',' || ch === ';') { parts.push(cur); cur = ''; continue; }
+ if ((ch === '+' || ch === '&') && /\s/.test(s[i - 1] || '') && /\s/.test(s[i + 1] || '')) { parts.push(cur); cur = ''; continue; }
+ if (ch === ' ' && /^ and $/i.test(s.slice(i - 1, i + 4))) { parts.push(cur); cur = ''; i += 3; continue; }
+ }
+ cur += ch;
+ }
+ parts.push(cur);
+ const cleaned = parts.map((p) => cleanRole(p)).filter(Boolean);
+ if (cleaned.length < 2) return [s];
+ // Only split when every part is a role of its own; "Senior Java, C# and React engineers" is one role.
+ if (!cleaned.every((p) => looksLikeRole(p) && ROLE_STRONG.test(p))) return [s];
+ return cleaned;
+}
+
+/** "Role at Company", "First role +N more roles at Company", or the company alone. */
+function titleOf(roles, company) {
+ const at = company ? ` at ${company}` : '';
+ const first = roles.length ? roles[0].replace(/^[a-z]/, (c) => c.toUpperCase()) : '';
+ if (!roles.length) return company;
+ if (roles.length === 1) return `${first}${at}`;
+ return `${first} +${roles.length - 1} more role${roles.length - 1 === 1 ? '' : 's'}${at}`;
+}
+
+/**
+ * A one-line job post → { company, title, location, remote, job_type, salary, tags, roles, line }.
+ * company is '' when the line names none: the caller's thin-listing gate holds it.
+ */
+function parseHiringLine(line) {
+ const original = clip(line, 500);
+ const segments = String(line == null ? '' : line)
+ .replace(/^[-–—*•\s]+/, '')
+ .split(/\s*\|\s*/)
+ .map((s) => s.trim())
+ .filter(Boolean);
+ const company = takeCompany(segments);
+
+ const roles = [];
+ const places = [];
+ const typeLabels = [];
+ const tags = [];
+ let salary = null;
+ let remote = false;
+ let saidOnsite = false;
+ let saidHybrid = false;
+ let saidVisa = false;
+ let noVisa = false;
+ let saidAuth = false;
+ let saidRelocation = false;
+
+ for (const raw of segments) {
+ let seg = stripUrls(raw);
+ if (!seg) continue;
+ // A pay segment (or the pay part of one) is the salary, and nothing else in it is a role.
+ const pay = takePay(seg);
+ if (pay) {
+ salary = salary || pay.text;
+ seg = pay.rest;
+ if (!seg) continue;
+ }
+ const visaHere = VISA_RE.test(seg);
+ const authHere = AUTH_RE.test(seg);
+ const types = typesOf(seg);
+ if (types.length) { for (const t of types) typeLabels.push(t); }
+ if (REMOTE_RE.test(seg) && !NO_REMOTE_RE.test(seg)) remote = true;
+ if (ONSITE_RE.test(seg)) saidOnsite = true;
+ if (HYBRID_RE.test(seg)) saidHybrid = true;
+ if (visaHere) { saidVisa = true; if (NO_VISA_RE.test(seg)) noVisa = true; }
+ if (authHere) saidAuth = true;
+ if (RELOCATION_RE.test(seg)) saidRelocation = true;
+
+ // Remove the terms from the segment; what is left is a place or a role, or nothing.
+ let rest = seg
+ .replace(REMOTE_RE, ' ')
+ .replace(NO_REMOTE_RE, ' ')
+ .replace(ONSITE_RE, ' ')
+ .replace(HYBRID_RE, ' ')
+ .replace(VISA_RE, ' ')
+ .replace(NO_VISA_RE, ' ')
+ .replace(AUTH_RE, ' ')
+ .replace(RELOCATION_RE, ' ')
+ .replace(/\b(?:citizenship|authori[sz]ation|clearance|required|preferred|only)\b/gi, ' ')
+ .replace(/\b(?:full[\s-]?time|part[\s-]?time|fulltime|parttime|permanent|temporary|internship|apprenticeship)\b/gi, ' ')
+ .replace(/\s+/g, ' ')
+ .replace(/^[\s,;:/|.·•–—-]+|[\s,;:/|.·•–—-]+$/g, '')
+ .trim();
+ const place = (visaHere || authHere) ? null : (placeOf(seg) || placeOf(rest));
+ if (place) {
+ const known = places.some((p) => p.toLowerCase() === place.toLowerCase());
+ const covered = /^remote$/i.test(place) && places.some((p) => /\bremote\b/i.test(p));
+ if (!known && !covered) places.push(place);
+ }
+ // A segment that was nothing but terms, visa wording or pay is done (a bare "Remote" already went in as a place).
+ if (letters(rest) < 2) continue;
+ if (visaHere || authHere) {
+ const residual = rest.replace(/[\s.,()/&-]*(?:u\.?s\.?a?|uk|eu)?[\s.,()/&-]*/gi, '').trim();
+ if (letters(residual) < 2) continue;
+ }
+ let role = cleanRole(rest);
+ // "Full-stack Engineer (Elixir) / Helsinki, Finland": the place is in the same segment as the role.
+ if (role && place) {
+ const parts = role.split(/\s+\/\s+/);
+ const roleParts = parts.filter((p) => looksLikeRole(p));
+ const placeParts = parts.filter((p) => !looksLikeRole(p) && placeOf(p));
+ if (roleParts.length && placeParts.length) {
+ role = roleParts.join(' / ');
+ const combined = placeOf(placeParts.join(' / '));
+ if (combined) {
+ const at = places.indexOf(place);
+ if (at >= 0) places[at] = combined;
+ else place = combined;
+ }
+ }
+ }
+ if (role && looksLikeRole(role) && (!place || ROLE_STRONG.test(role))) roles.push(...splitRoles(role));
+ }
+
+ // Terms, in the service's own existing fields: the workplace boolean, the type line and the chips.
+ if (saidHybrid) tags.push('Hybrid');
+ else if (saidOnsite) tags.push('Onsite');
+ if (saidVisa) tags.push(noVisa ? 'No visa sponsorship' : 'Visa sponsorship');
+ if (saidAuth) tags.push('Work authorization required');
+ if (saidRelocation) tags.push('Relocation');
+ const jobType = uniq(typeLabels).join(', ') || null;
+ const title = titleOf(uniq(roles.map((r) => clip(r, 120))), company);
+
+ return {
+ company,
+ title: clip(title, 240),
+ location: uniq(places).join(' / ').slice(0, 200) || null,
+ remote,
+ job_type: jobType,
+ salary: salary || null,
+ tags: uniq(tags).slice(0, 6),
+ roles: uniq(roles),
+ line: original,
+ };
+}
+
+module.exports = { parseHiringLine, hostOf, splitRoles, looksLikeRole, looksLikeCompany, titleOf, TITLE_RULE };
new file mode 100644
new file mode 100644
@@ -0,0 +1,107 @@
@@ -0,0 +1,107 @@
+'use strict';
+
+/**
+ * Re-derive the titles of Hacker News "Who is hiring" listings stored before server/jobs/hiring-line.js split the
+ * first line into company, roles, place, terms and pay (scripts/rederive-hn-titles.js runs it once).
+ *
+ * The old row kept the first segment as the company and the rest of the pipe-separated line as the title, so the
+ * board's line can be put back together from the two: "Company | Role | Place" (or "Company | | Role" when the line
+ * had an empty segment, which is why a title starting with "|" is joined without another separator). That line is
+ * parsed by the same module the ingest uses, and the row's own title, company, place, remote flag, type, pay and tags
+ * are replaced with what it says. The excerpt is never touched: it already starts with the original line.
+ *
+ * Only rows without the title_rule marker are read, so running it twice is harmless and a second run can never read a
+ * title this job wrote back as if it were the board's line. A line that names no company is left alone (the same
+ * thin-listing gate the ingest applies) and reported.
+ */
+const { parseHiringLine, TITLE_RULE } = require('./hiring-line');
+
+const SOURCE = 'hn-whoishiring';
+const clip = (s, n) => String(s == null ? '' : s).replace(/\s+/g, ' ').trim().slice(0, n);
+const uniq = (a) => [...new Set(a.filter(Boolean))];
+
+/** The board's first line, as the old ingest stored it: the company segment and the rest of the line. */
+function legacyLine(row) {
+ const company = clip(row && row.company, 120);
+ const title = clip(row && row.title, 300);
+ if (!company || !title) return title || company || '';
+ // rawOfHn joined the title's own parts with " | "; an empty first part left the title starting with "|".
+ return title.startsWith('|') ? `${company} ${title}` : `${company} | ${title}`;
+}
+
+/** The row's search text, exactly as server/jobs/sources.js's toRow builds it. */
+function searchText(row) {
+ return [row.title, row.company, row.tags, row.location, row.excerpt, row.salary].filter(Boolean).join(' ').toLowerCase().slice(0, 4000);
+}
+
+/**
+ * Re-derive every old HN title. Dry by default: with { apply: true } it writes the rows in one transaction and, when
+ * a search index (or a config to build one) is given, sweeps it so the changed documents are re-sent.
+ * → { scanned, changed, skipped, samples, sent }
+ */
+async function rederiveHnTitles({ s, config = null, log = console, apply = false, search = null, limit = 0 } = {}) {
+ if (!s || !s.db) throw new TypeError('rederiveHnTitles needs the store (s)');
+ const rows = await s.db.many(
+ `SELECT * FROM work_listings WHERE source = $1 AND (title_rule IS NULL OR title_rule <> $2) ORDER BY id${limit ? ` LIMIT ${Number(limit)}` : ''}`,
+ [SOURCE, TITLE_RULE]);
+ const changes = [];
+ const skipped = [];
+ for (const row of rows) {
+ const parsed = parseHiringLine(legacyLine(row));
+ if (!parsed.company || !parsed.title) { skipped.push(row); continue; }
+ const tags = uniq([...String(row.tags || '').split(',').map((t) => t.trim()).filter(Boolean), ...parsed.tags]);
+ const next = {
+ id: row.id,
+ title: parsed.title,
+ company: parsed.company,
+ location: parsed.location,
+ remote: parsed.remote,
+ tags: tags.join(', '),
+ job_type: parsed.job_type,
+ salary: parsed.salary,
+ title_rule: TITLE_RULE,
+ };
+ next.search_text = searchText({ ...row, ...next });
+ if (next.title === row.title && next.company === row.company && next.location === (row.location || null)
+ && next.remote === row.remote && next.tags === row.tags && next.job_type === (row.job_type || null)
+ && next.salary === (row.salary || null) && row.title_rule === TITLE_RULE) continue;
+ changes.push({ row, next });
+ }
+
+ let sent = null;
+ if (apply && changes.length) {
+ await s.tx(async () => {
+ for (const { next } of changes) {
+ await s.db.exec(
+ `UPDATE work_listings SET title = $2, company = $3, location = $4, remote = $5, tags = $6,
+ job_type = $7, salary = $8, search_text = $9, title_rule = $10 WHERE id = $1`,
+ [next.id, next.title, next.company, next.location, next.remote, next.tags, next.job_type,
+ next.salary, next.search_text, next.title_rule]);
+ }
+ });
+ // The rows changed, so Search's sequencer sends their documents again on the next sweep; do it now rather than
+ // an hour later. The relay is off in a test and the rows wait there, which is the service's own behaviour.
+ const index = search || (config ? require('../search-index').createSearchIndex({ config, s, log }) : null);
+ if (index) {
+ const out = await index.sweep();
+ sent = out && out.job ? out.job : out;
+ if (index.outbox && index.outbox.kick) await index.outbox.kick().catch(() => {});
+ if (!search && index.stop) index.stop();
+ }
+ }
+ if (log && log.log) {
+ log.log(`[Redrive] HN titles: ${rows.length} old listing(s), ${changes.length} to change, ${skipped.length} with no company${apply ? ' (applied)' : ' (dry run)'}`);
+ }
+ return {
+ scanned: rows.length,
+ changed: changes.length,
+ skipped: skipped.length,
+ samples: changes.slice(0, 12).map(({ row, next }) => ({
+ id: row.id, before: `${row.title} — ${row.company}`, after: next.title, company: next.company,
+ location: next.location, job_type: next.job_type, salary: next.salary, tags: next.tags,
+ })),
+ sent,
+ };
+}
+
+module.exports = { rederiveHnTitles, legacyLine, SOURCE, TITLE_RULE };
@@ -208,6 +208,7 @@ function toRow(raw, { sourceId, fetchedAt, idOf }) {
@@ -208,6 +208,7 @@ function toRow(raw, { sourceId, fetchedAt, idOf }) {
excerpt: excerpt(raw.description_html).slice(0, 400),
excerpt: excerpt(raw.description_html).slice(0, 400),
posted_at: raw.posted_at || null,
posted_at: raw.posted_at || null,
sort_at: raw.posted_at || fetchedAt,
sort_at: raw.posted_at || fetchedAt,
+ title_rule: raw.title_rule || null,
fetched_at: fetchedAt,
fetched_at: fetchedAt,
};
};
row.search_text = [row.title, row.company, row.tags, row.location, row.excerpt, row.salary].filter(Boolean).join(' ').toLowerCase().slice(0, 4000);
row.search_text = [row.title, row.company, row.tags, row.location, row.excerpt, row.salary].filter(Boolean).join(' ').toLowerCase().slice(0, 4000);
@@ -307,16 +307,16 @@ const recentIds = (s, limit = 1000) => s.db.many(
@@ -307,16 +307,16 @@ const recentIds = (s, limit = 1000) => s.db.many(
/** One listing in, through `q` (the database or a transaction). → true when it is new. */
/** One listing in, through `q` (the database or a transaction). → true when it is new. */
async function upsertListing(q, row, fetchedAt) {
async function upsertListing(q, row, fetchedAt) {
const out = await q.maybe(
const out = await q.maybe(
- `INSERT INTO work_listings (id, source, source_id, url, title, company, location, remote, tags, job_type, salary, excerpt, search_text, posted_at, sort_at, fetched_at, first_seen_at, last_seen_at, expired)
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $16, $16, false)
+ `INSERT INTO work_listings (id, source, source_id, url, title, company, location, remote, tags, job_type, salary, excerpt, search_text, posted_at, sort_at, fetched_at, first_seen_at, last_seen_at, expired, title_rule)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $16, $16, false, $17)
ON CONFLICT (source, source_id) DO UPDATE SET
ON CONFLICT (source, source_id) DO UPDATE SET
url = EXCLUDED.url, title = EXCLUDED.title, company = EXCLUDED.company, location = EXCLUDED.location,
url = EXCLUDED.url, title = EXCLUDED.title, company = EXCLUDED.company, location = EXCLUDED.location,
remote = EXCLUDED.remote, tags = EXCLUDED.tags, job_type = EXCLUDED.job_type, salary = EXCLUDED.salary,
remote = EXCLUDED.remote, tags = EXCLUDED.tags, job_type = EXCLUDED.job_type, salary = EXCLUDED.salary,
excerpt = EXCLUDED.excerpt, search_text = EXCLUDED.search_text, posted_at = EXCLUDED.posted_at,
excerpt = EXCLUDED.excerpt, search_text = EXCLUDED.search_text, posted_at = EXCLUDED.posted_at,
sort_at = EXCLUDED.sort_at, fetched_at = EXCLUDED.fetched_at, last_seen_at = EXCLUDED.last_seen_at,
sort_at = EXCLUDED.sort_at, fetched_at = EXCLUDED.fetched_at, last_seen_at = EXCLUDED.last_seen_at,
- expired = false
+ expired = false, title_rule = EXCLUDED.title_rule
RETURNING (xmax = 0) AS inserted`,
RETURNING (xmax = 0) AS inserted`,
- [row.id, row.source, row.source_id, row.url, row.title, row.company, row.location, row.remote, row.tags, row.job_type, row.salary, row.excerpt, row.search_text, row.posted_at, row.sort_at, fetchedAt]);
+ [row.id, row.source, row.source_id, row.url, row.title, row.company, row.location, row.remote, row.tags, row.job_type, row.salary, row.excerpt, row.search_text, row.posted_at, row.sort_at, fetchedAt, row.title_rule || null]);
return Boolean(out && out.inserted);
return Boolean(out && out.inserted);
}
}
@@ -22,6 +22,21 @@ function dateWords(iso) {
@@ -22,6 +22,21 @@ function dateWords(iso) {
return `${d.getUTCDate()} ${MONTHS[d.getUTCMonth()]} ${d.getUTCFullYear()}`;
return `${d.getUTCDate()} ${MONTHS[d.getUTCMonth()]} ${d.getUTCFullYear()}`;
}
}
+/**
+ * The listing's title with its company: "Role at Company". A title the ingest already wrote that way (a one-line
+ * board post split by server/jobs/hiring-line.js) is not made to say the company twice, and a title that is only the
+ * company is not followed by it either.
+ */
+function fullTitle(l) {
+ const title = String((l && l.title) || '').trim();
+ const company = String((l && l.company) || '').trim();
+ if (!title) return company;
+ if (!company) return title;
+ if (title.toLowerCase() === company.toLowerCase()) return title;
+ if (title.toLowerCase().endsWith(` at ${company.toLowerCase()}`)) return title;
+ return `${title} at ${company}`;
+}
+
const meta = (row) => ({
const meta = (row) => ({
id: row.id,
id: row.id,
url: row.url,
url: row.url,
@@ -138,4 +153,5 @@ function jobPosting(row, { siteUrl }) {
@@ -138,4 +153,5 @@ function jobPosting(row, { siteUrl }) {
/** The excerpt as plain text, for the meta description and the API. */
/** The excerpt as plain text, for the meta description and the API. */
const plainExcerpt = (row) => stripHtml(row.excerpt || '');
const plainExcerpt = (row) => stripHtml(row.excerpt || '');
-module.exports = { listingItem, listingList, sourceLink, provenance, jobPosting, dateWords, plainExcerpt, meta, salaryLine, jobTypeLabel };
+module.exports = {
+ fullTitle, listingItem, listingList, sourceLink, provenance, jobPosting, dateWords, plainExcerpt, meta, salaryLine, jobTypeLabel };
@@ -32,7 +32,7 @@ function describe(row) {
@@ -32,7 +32,7 @@ function describe(row) {
const facts = [where, l.salary].filter(Boolean).join(' · ');
const facts = [where, l.salary].filter(Boolean).join(' · ');
return {
return {
listed: !row.expired,
listed: !row.expired,
- title: `${l.title} at ${l.company}`,
+ title: listing.fullTitle(l),
summary: facts || excerpt.slice(0, 200) || null,
summary: facts || excerpt.slice(0, 200) || null,
// The body holds what the title and summary do not (the snippet reads both, and repeated the title and place).
// The body holds what the title and summary do not (the snippet reads both, and repeated the title and place).
body: [l.tags.join(', '), facts ? excerpt : excerpt.slice(200).replace(/^\S*\s*/, '')].filter(Boolean).join('\n'),
body: [l.tags.join(', '), facts ? excerpt : excerpt.slice(200).replace(/^\S*\s*/, '')].filter(Boolean).join('\n'),
new file mode 100644
new file mode 100644
@@ -0,0 +1,88 @@
@@ -0,0 +1,88 @@
+'use strict';
+/**
+ * A one-line board post split into a listing (server/jobs/hiring-line.js): the two lines from the brief, a URL first,
+ * several roles, a role after the place and terms, visa and relocation, no company, and the title helper that stops
+ * "Role at Company at Company" on the pages and in Search.
+ */
+const assert = require('assert');
+const { check, done } = require('./helpers/boot');
+const { parseHiringLine } = require('../server/jobs/hiring-line');
+const { fullTitle } = require('../server/render/listing');
+
+(async () => {
+ await check('a URL first, a place, terms and "at COMPANY": title is the company alone', async () => {
+ const p = parseHiringLine('https://www.monumental.co/ | Amsterdam, The Netherlands | Full Time | Onsite at MONUMENTAL');
+ assert.deepStrictEqual(
+ [p.company, p.title, p.location, p.remote, p.job_type, p.salary, p.tags],
+ ['MONUMENTAL', 'MONUMENTAL', 'Amsterdam, The Netherlands', false, 'Full time', null, ['Onsite']]);
+ assert.ok(!p.title.includes('http'), 'a URL is never a title');
+ assert.strictEqual(p.line, 'https://www.monumental.co/ | Amsterdam, The Netherlands | Full Time | Onsite at MONUMENTAL');
+ });
+
+ await check('several roles: the first and "+N more roles", with the place, terms and pay in their fields', async () => {
+ const p = parseHiringLine('Mechanical Engineer, Electrical Engineer, Robotic Controls Engineer | New York, NY | ONSITE | Full-time | $220K-$300K at Nirvana Robotics');
+ assert.strictEqual(p.company, 'Nirvana Robotics');
+ assert.strictEqual(p.title, 'Mechanical Engineer +2 more roles at Nirvana Robotics');
+ assert.deepStrictEqual(p.roles, ['Mechanical Engineer', 'Electrical Engineer', 'Robotic Controls Engineer']);
+ assert.deepStrictEqual([p.location, p.remote, p.job_type, p.salary, p.tags],
+ ['New York, NY', false, 'Full time', '$220K-$300K', ['Onsite']]);
+ });
+
+ await check('a URL with no "at X": the company is the URL\'s host, and the host is not a title URL', async () => {
+ const p = parseHiringLine('https://www.monumental.co/ | Amsterdam, The Netherlands | Full Time | Onsite');
+ assert.deepStrictEqual([p.company, p.title, p.location], ['monumental.co', 'monumental.co', 'Amsterdam, The Netherlands']);
+ assert.ok(!p.title.includes('http') && !p.title.includes('www.'));
+ });
+
+ await check('roles separated by a pipe: "+1 more role"', async () => {
+ const p = parseHiringLine('Acme | Software Engineer | Platform Engineer | Remote (US) | Full-time');
+ assert.strictEqual(p.title, 'Software Engineer +1 more role at Acme');
+ assert.deepStrictEqual(p.roles, ['Software Engineer', 'Platform Engineer']);
+ assert.deepStrictEqual([p.location, p.remote, p.job_type], ['Remote (US)', true, 'Full time']);
+ });
+
+ await check('a role after the place, terms and pay; visa and relocation are chips', async () => {
+ const p = parseHiringLine('KSC Trading | Remote | Full-Time | Senior Backend Software Engineer | $200k to $250k USD Total Compensation + Benefits');
+ assert.deepStrictEqual([p.company, p.title, p.location, p.remote, p.job_type, p.salary, p.tags],
+ ['KSC Trading', 'Senior Backend Software Engineer at KSC Trading', 'Remote', true, 'Full time',
+ '$200k to $250k USD Total Compensation + Benefits', []]);
+
+ const flow = parseHiringLine('Flow Traders | Research Engineer | Hong Kong | Full Time | Visa | Relocation');
+ assert.deepStrictEqual([flow.title, flow.location, flow.job_type, flow.tags],
+ ['Research Engineer at Flow Traders', 'Hong Kong', 'Full time', ['Visa sponsorship', 'Relocation']]);
+ });
+
+ await check('no company: the line is not a listing (the thin-listing gate holds it)', async () => {
+ const p = parseHiringLine('Remote (US) | Full-time | Senior Software Engineer');
+ assert.strictEqual(p.company, '');
+ assert.strictEqual(p.title, 'Senior Software Engineer', 'a role can be read, but there is no company to publish it under');
+ });
+
+ await check('real lines: multiple roles, a place list, salary with punctuation, no invented fields', async () => {
+ const charge = parseHiringLine('Charge Robotics | Multiple Roles | Full-time | Onsite | SF Bay Area');
+ assert.deepStrictEqual([charge.title, charge.company, charge.location, charge.job_type, charge.tags],
+ ['Multiple Roles at Charge Robotics', 'Charge Robotics', 'SF Bay Area', 'Full time', ['Onsite']]);
+
+ const stoke = parseHiringLine('Stoke Space | ONSITE in Seattle, WA / Moses Lake, WA / Cape Canaveral, FL | Software Engineer | Full-Time, Internship | Salary: $98k-$289k USD | https://www.stokespace.com');
+ assert.deepStrictEqual([stoke.title, stoke.company, stoke.location, stoke.remote, stoke.job_type, stoke.salary, stoke.tags],
+ ['Software Engineer at Stoke Space', 'Stoke Space', 'Seattle, WA / Moses Lake, WA / Cape Canaveral, FL', false,
+ 'Full time, Internship', 'Salary: $98k-$289k USD', ['Onsite']]);
+
+ const discord = parseHiringLine('Discord | Senior Software Engineer, Application Security | REMOTE (US West Coast) or San Francisco, CA | $196K-$245K base + equity');
+ assert.deepStrictEqual([discord.title, discord.location, discord.remote, discord.salary],
+ ['Senior Software Engineer, Application Security at Discord', 'REMOTE (US West Coast) or San Francisco, CA', true, '$196K-$245K base + equity']);
+
+ const odoo = parseHiringLine('Odoo | Software Developer | Belgium | Full Time | Hybrid');
+ assert.deepStrictEqual([odoo.title, odoo.location, odoo.job_type, odoo.tags],
+ ['Software Developer at Odoo', 'Belgium', 'Full time', ['Hybrid']]);
+ });
+
+ await check('"Role at Company" is not made to say the company twice', async () => {
+ assert.strictEqual(fullTitle({ title: 'Senior Engineer at Acme', company: 'Acme' }), 'Senior Engineer at Acme');
+ assert.strictEqual(fullTitle({ title: 'Acme', company: 'Acme' }), 'Acme');
+ assert.strictEqual(fullTitle({ title: 'Senior Engineer', company: 'Acme' }), 'Senior Engineer at Acme');
+ assert.strictEqual(fullTitle({ title: 'Senior Engineer', company: '' }), 'Senior Engineer');
+ });
+
+ done();
+})().catch((err) => { console.error(err); process.exit(1); });
new file mode 100644
new file mode 100644
@@ -0,0 +1,95 @@
@@ -0,0 +1,95 @@
+'use strict';
+/**
+ * Re-deriving the Hacker News titles stored before the one-line split (server/jobs/rederive-hn-titles.js): an old row
+ * is read back from its company and title, the same parser the ingest uses writes the title, place, terms and pay,
+ * Search is swept with the changed documents, and a second run reads nothing (the title_rule marker).
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const store = require('../server/jobs/store');
+const { rederiveHnTitles, legacyLine } = require('../server/jobs/rederive-hn-titles');
+
+const legacy = (s, sourceId, company, title, over = {}) => ({
+ id: store.idOf('hn-whoishiring', sourceId), source: 'hn-whoishiring', source_id: sourceId,
+ url: `https://news.ycombinator.com/item?id=${sourceId}`, title, company, location: null, remote: false,
+ tags: '', job_type: null, salary: null,
+ excerpt: `${company}${title ? ` | ${title}` : ''} The comment's own words, kept.`,
+ search_text: `${title} ${company}`.toLowerCase(), posted_at: '2026-10-01T15:00:00.000Z',
+ sort_at: '2026-10-01T15:00:00.000Z', fetched_at: '2026-10-30T06:00:00.000Z',
+ first_seen_at: '2026-10-30T06:00:00.000Z', last_seen_at: '2026-10-30T06:00:00.000Z',
+ expired: false, title_rule: null, ...over,
+});
+
+(async () => {
+ const t = await boot();
+ const { s, search } = t.ctx;
+ try {
+ const now = '2026-10-30T06:00:00.000Z';
+ const rows = [
+ legacy(s, 'hn-301', 'Acme', 'Senior Software Engineer | REMOTE (US) | Full-Time'),
+ legacy(s, 'hn-302', 'Planlab.ai', '| Product Engineer | ONSITE in London, UK'),
+ legacy(s, 'hn-303', '', 'Remote (US) | Full-time | Senior Software Engineer'),
+ ];
+ for (const row of rows) {
+ await s.db.exec(
+ `INSERT INTO work_listings (id, source, source_id, url, title, company, location, remote, tags, job_type, salary, excerpt,
+ search_text, posted_at, sort_at, fetched_at, first_seen_at, last_seen_at, expired, title_rule)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$16,$16,false,NULL)`,
+ [row.id, row.source, row.source_id, row.url, row.title, row.company, row.location, row.remote, row.tags,
+ row.job_type, row.salary, row.excerpt, row.search_text, row.posted_at, row.sort_at, row.fetched_at]);
+ }
+
+ await check('a legacy line is put back together from its company and title', async () => {
+ assert.strictEqual(legacyLine(rows[0]), 'Acme | Senior Software Engineer | REMOTE (US) | Full-Time');
+ assert.strictEqual(legacyLine(rows[1]), 'Planlab.ai | Product Engineer | ONSITE in London, UK');
+ });
+
+ let out;
+ await check('the old titles are re-derived, and the row keeps the original line in its excerpt', async () => {
+ out = await rederiveHnTitles({ s, config: t.config, apply: true, search, log: { log() {} } });
+ assert.deepStrictEqual([out.scanned, out.changed, out.skipped], [3, 2, 1]);
+ const acme = await s.db.one('SELECT * FROM work_listings WHERE id = $1', [rows[0].id]);
+ assert.deepStrictEqual([acme.title, acme.company, acme.location, acme.remote, acme.job_type, acme.title_rule],
+ ['Senior Software Engineer at Acme', 'Acme', 'REMOTE (US)', true, 'Full time', 'hn-line@1']);
+ const planlab = await s.db.one('SELECT * FROM work_listings WHERE id = $1', [rows[1].id]);
+ assert.deepStrictEqual([planlab.title, planlab.company, planlab.location, planlab.tags],
+ ['Product Engineer at Planlab.ai', 'Planlab.ai', 'London, UK', 'Onsite']);
+ assert.ok(acme.excerpt.startsWith('Acme | Senior Software Engineer | REMOTE (US) | Full-Time'), 'nothing is lost');
+ assert.strictEqual((await s.db.one('SELECT title_rule FROM work_listings WHERE id = $1', [rows[2].id])).title_rule, null,
+ 'a line with no company is left alone');
+ });
+
+ await check('the listing page and its feed say "Role at Company" once, not twice', async () => {
+ const page = await t.get(`/jobs/${rows[0].id}`);
+ assert.strictEqual(page.status, 200);
+ assert.ok(page.text.includes('<h1>Senior Software Engineer at Acme</h1>'));
+ assert.ok(!page.text.includes('at Acme at Acme'), 'the company is not appended to a title that already ends with it');
+ const feed = await t.get('/jobs.atom');
+ assert.strictEqual(feed.status, 200);
+ assert.ok(feed.text.includes('<title>Senior Software Engineer at Acme</title>'));
+ assert.ok(!feed.text.includes('at Acme at Acme'));
+ });
+
+ await check('Search is swept with the changed documents, and the title is not doubled in the document', async () => {
+ assert.ok(out.sent && out.sent.sent >= 2, `swept: ${JSON.stringify(out.sent)}`);
+ const envs = (await s.db.many('SELECT envelope FROM event_outbox')).map((r) => r.envelope);
+ const doc = envs.find((e) => e.payload && e.payload.id === rows[0].id);
+ assert.ok(doc, 'the changed listing is in the outbox');
+ assert.strictEqual(doc.payload.title, 'Senior Software Engineer at Acme');
+ const plan = envs.find((e) => e.payload && e.payload.id === rows[1].id);
+ assert.strictEqual(plan.payload.title, 'Product Engineer at Planlab.ai');
+ });
+
+ await check('a second run reads nothing: the marker means its own title is never parsed as the board line', async () => {
+ const again = await rederiveHnTitles({ s, config: t.config, apply: false, search, log: { log() {} } });
+ // The one row with no company is read again and skipped again; every row it could re-title is marked.
+ assert.deepStrictEqual([again.scanned, again.changed, again.skipped], [1, 0, 1]);
+ const acme = await s.db.one('SELECT title FROM work_listings WHERE id = $1', [rows[0].id]);
+ assert.strictEqual(acme.title, 'Senior Software Engineer at Acme');
+ });
+ } finally {
+ t.ctx.search.stop();
+ await t.close();
+ }
+ done();
+})().catch((err) => { console.error(err); process.exit(1); });
@@ -43,16 +43,21 @@ const hn = (identity, title, summary, over = {}) => ({
@@ -43,16 +43,21 @@ const hn = (identity, title, summary, over = {}) => ({
push(hn('hn-203', 'Beta & Co is hiring a product designer', 'Beta & Co is hiring a product designer. Contact [email].'));
push(hn('hn-203', 'Beta & Co is hiring a product designer', 'Beta & Co is hiring a product designer. Contact [email].'));
push(hn('hn-205', 'No company here', 'A comment with no company in its first line.'));
push(hn('hn-205', 'No company here', 'A comment with no company in its first line.'));
push(hn('hn-206', 'Z | Q', 'A board test post with one-letter fields.'));
push(hn('hn-206', 'Z | Q', 'A board test post with one-letter fields.'));
+ push(hn('hn-207', 'Mechanical Engineer, Electrical Engineer, Robotic Controls Engineer | New York, NY | ONSITE | Full-time | $220K-$300K at Nirvana Robotics',
+ 'Three roles on one line. Email [email].'));
const out = await from.pull();
const out = await from.pull();
- assert.deepStrictEqual([out.applied, out.hold, out.removed], [2, 2, 0]);
+ assert.deepStrictEqual([out.applied, out.hold, out.removed], [3, 2, 0]);
const acme = await s.db.maybe('SELECT * FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-201')]);
const acme = await s.db.maybe('SELECT * FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-201')]);
assert.deepStrictEqual([acme.source, acme.title, acme.company, acme.remote, acme.url],
assert.deepStrictEqual([acme.source, acme.title, acme.company, acme.remote, acme.url],
- ['hn-whoishiring', 'Senior Backend Engineer | Remote (US)', 'Acme Corp', true, 'https://news.ycombinator.com/item?id=201']);
+ ['hn-whoishiring', 'Senior Backend Engineer at Acme Corp', 'Acme Corp', true, 'https://news.ycombinator.com/item?id=201']);
+ assert.strictEqual(acme.location, 'Remote (US)', 'the place is its own field, not part of the title');
+ assert.strictEqual(acme.title_rule, 'hn-line@1', 'the split is marked so a re-derive never reads it back');
+ assert.ok(acme.excerpt.startsWith('Acme Corp | Senior Backend Engineer | Remote (US)'), 'the original line stays in the description');
assert.ok(acme.excerpt.includes('[email]') && acme.excerpt.includes('[phone]'), 'the excerpt is Sources\' privacy-stripped text');
assert.ok(acme.excerpt.includes('[email]') && acme.excerpt.includes('[phone]'), 'the excerpt is Sources\' privacy-stripped text');
const beta = await s.db.maybe('SELECT * FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-203')]);
const beta = await s.db.maybe('SELECT * FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-203')]);
- assert.deepStrictEqual([beta.title, beta.company, beta.remote], ['a product designer', 'Beta & Co', false]);
+ assert.deepStrictEqual([beta.title, beta.company, beta.remote], ['Product designer at Beta & Co', 'Beta & Co', false]);
assert.strictEqual(await s.db.maybe('SELECT id FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-205')]), null);
assert.strictEqual(await s.db.maybe('SELECT id FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-205')]), null);
assert.strictEqual(await s.db.maybe('SELECT id FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-206')]), null);
assert.strictEqual(await s.db.maybe('SELECT id FROM work_listings WHERE id = $1', [store.idOf('hn-whoishiring', 'hn-206')]), null);
});
});
@@ -63,6 +68,11 @@ const hn = (identity, title, summary, over = {}) => ({
@@ -63,6 +68,11 @@ const hn = (identity, title, summary, over = {}) => ({
assert.strictEqual(list.status, 200);
assert.strictEqual(list.status, 200);
assert.ok(list.text.includes('Hacker News: Who is hiring'));
assert.ok(list.text.includes('Hacker News: Who is hiring'));
assert.ok(list.text.includes('https://news.ycombinator.com/item?id=201'));
assert.ok(list.text.includes('https://news.ycombinator.com/item?id=201'));
+ // The place, terms and pay are the card's meta line, and the title is the roles at the company.
+ assert.ok(list.text.includes('Mechanical Engineer +2 more roles at Nirvana Robotics'));
+ assert.ok(list.text.includes('New York, NY'));
+ assert.ok(list.text.includes('$220K-$300K'));
+ assert.ok(list.text.includes('Onsite'));
const page = await t.get(`/jobs/${id}`);
const page = await t.get(`/jobs/${id}`);
assert.strictEqual(page.status, 200);
assert.strictEqual(page.status, 200);
assert.ok(page.text.includes('Hacker News: Who is hiring'));
assert.ok(page.text.includes('Hacker News: Who is hiring'));