openvibers/openvibe.work Commits
Read every board through OpenVibe.Sources behind one map, and add We Work Remotely (#134)
Changes
@@@ -7,7 +7,7 @@module.exports = {id: 'work',name: 'OpenVibe.Work',-- purpose: 'Find work on open job boards: listings from Arbeitnow, Remotive and RemoteOK under their terms, searchable with facets and each linking back to the original, plus saved searches and OpenVibe.Watch alerts for a signed-in person.',++ purpose: 'Find work on open job boards: listings from Arbeitnow, We Work Remotely, Remotive and RemoteOK under their terms, searchable with facets and each linking back to the original, plus saved searches and OpenVibe.Watch alerts for a signed-in person.',home: 'https://openvibe.work',site: 'openvibe.work',api: '/api/v1',@@@ -90,6 +90,9 @@ function load(env = process.env) {// OAuth client secret (Network grants Work sources.item.read). Anything else keeps Work's own fetch.internalUrl: trim(env.OV_SOURCES_INTERNAL_URL == null ? 'http://127.0.0.1:4720' : env.OV_SOURCES_INTERNAL_URL),arbeitnowFrom: String(env.WORK_ARBEITNOW_FROM || '').trim().toLowerCase() === 'sources' ? 'sources' : 'board',++ // We Work Remotely reaches Work the same way (WORK_WWR_FROM=sources, the weworkremotely key in Sources);++ // it is off until the board is read through Sources, and Work never fetches the feed itself.++ weworkremotelyFrom: String(env.WORK_WWR_FROM || '').trim().toLowerCase() === 'sources' ? 'sources' : 'board',},// The background ingest (server/ingest/): off unless WORK_INGEST=on, so a test and a development run// never call a job board. Once on, each source is fetched at most every intervalMs (Remotive's terms@@@ -25,7 +25,7 @@ const SITEMAP_JOBS = 1000;const PAGE_TEXT = {'/': ['OpenVibe.Work home', 'OpenVibe.Work gathers job listings from public job boards, names the source of every one and links back to the original listing. Search is free and needs no account.'],-- '/jobs': ['Jobs', 'Search job listings gathered from Arbeitnow, Remotive and RemoteOK, newest first, filterable by words, workplace, job type and location.'],++ '/jobs': ['Jobs', 'Search job listings gathered from Arbeitnow, We Work Remotely, Remotive and RemoteOK, newest first, filterable by words, workplace, job type and location.'],'/companies': ['Companies hiring', 'Every company with open listings on the boards OpenVibe.Work reads, most openings first, each with a page of its jobs, where they are and which boards list them.'],'/sources': ['Sources', 'The public job boards OpenVibe.Work gathers from, the terms each is used under as applied here, and when each was last read.'],'/docs': ['API', 'The OpenVibe.Work API: search the listings, read one with its provenance, list the sources and their terms, keep saved searches.'],@@@ -651,7 +651,7 @@ ${sources.SOURCES.map((src) => {<li>${src.name} listings also carry their source in the JSON-LD on the listing page.</li></ul><h3>What each read does</h3><dl class="job-facts">--<dt>API</dt><dd><code>${src.requests(config)[0]}</code></dd>++<dt>API</dt><dd>${src.viaSources ? html`read through OpenVibe.Sources from <code>${src.requests(config)[0]}</code>` : html`<code>${src.requests(config)[0]}</code>`}</dd><dt>Last read</dt><dd>${f && f.started_at ? html`${time(f.started_at)}${f.ok ? ' — succeeded' : ' — failed'}${f.error ? html` <span class="muted small">(${f.error})</span>` : ''}` : html`<span class="muted">not read yet</span>`}</dd><dt>Reads every</dt><dd>${Math.round(config.ingest.intervalMs / 3_600_000)} hours ${src.id === 'remotive' ? '(its terms allow about four a day)' : ''}</dd><dt>Listings kept</dt><dd>${(counts.get(src.id) || {}).count || 0}, retired when the board stops showing one for ${config.ingest.retentionDays} days</dd>@@@ -1,18 +1,23 @@'use strict';/**-- * Arbeitnow through OpenVibe.Sources (plan 2026-10-22, track F2). With WORK_ARBEITNOW_FROM=sources, Work stops reading-- * Arbeitnow's board itself: Sources fetches it (pages 1-3 every six hours, under its robots and terms checks) and keeps-- * each job as an item (source arbeitnow-job-board, category jobs). Work follows Sources' change feed-- * (openvibe-publishing/ingest: the cursor in work_ingest_cursor, one transaction per page) and keeps one listing per job-- * in work_listings, source 'arbeitnow', the row its own fetch made (the same slug, so the same listing id and URL).++ * Job boards read through OpenVibe.Sources (plan 2026-10-22, track F2; We Work Remotely added 2026-10-30).++ *++ * Sources fetches a board (under its robots and terms checks) and keeps each job as an item. Work follows Sources'++ * change feed (openvibe-publishing/ingest: the cursor in work_ingest_cursor, one transaction per page) and keeps one++ * listing per job in work_listings, the row the board's own fetch made (the same id, so the same listing id and URL).++ *++ * Every board that comes this way is one entry of SOURCES below: the Sources key it arrives under, the id Work stores++ * it with, and how a Sources item becomes the parsed row that board's own parse (server/jobs/sources.js) would make.++ * The thin-listing gate (sources.toRow) is the same for all of them: no id, no title, no company or no http(s) link++ * back means the listing is not published.** A job Sources retires (the board has not shown it for a week: retire_after_sec) or removes is expired here, like a* listing a board stopped showing. Work's own expiry sweep leaves these listings alone: an unchanged job is no change in* Sources' feed, so Work never hears it was seen again, and Sources' retirement is what says it is gone.*-- * Pulls 30 seconds after start and then hourly. Off unless WORK_ARBEITNOW_FROM=sources with OV_SOURCES_INTERNAL_URL and-- * the OAuth client secret (a test passes its own source).++ * Pulls 30 seconds after start and then hourly. Each board has its own switch (WORK_ARBEITNOW_FROM=sources,++ * WORK_WWR_FROM=sources); a board that is not switched on is left to its own fetch, as before.*/const ingest = require('openvibe-publishing/ingest');const sources = require('../jobs/sources');@@@ -24,10 +29,11 @@ const CURSOR = 'sources.jobs.after';const EVERY_MS = 60 * 60 * 1000;const FIRST_MS = 30 * 1000;++const list = (v) => (Array.isArray(v) ? v.filter((x) => typeof x === 'string' && x.trim()).map((x) => x.trim()) : []);++/** A Sources item → the parsed row Work's own Arbeitnow parse makes (server/jobs/sources.js), or null. */function rawOf(item) {const f = (item && item.fields) || {};-- const list = (v) => (Array.isArray(v) ? v.filter((x) => typeof x === 'string' && x.trim()).map((x) => x.trim()) : []);return {source_id: item.identity,url: item.canonical_url,@@@ -44,9 +50,78 @@ function rawOf(item) {};}++/** The labels We Work Remotely writes in a listing's description, in the order they appear. */++const WWR_LABELS = ['Salary/Compensation', 'Compensation', 'Working Hours', 'Job Title', 'Headquarters', 'Duration', 'Location', 'Responsibilities', 'Requirements', 'Qualifications', 'Benefits', 'Salary', 'About', 'Apply', 'URL'];++const escapeRe = (s) => String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');++++/**++ * The value of one labelled line in a We Work Remotely description ("Location: Mexico (Remote)"), or null when the++ * listing does not carry it. Sources keeps a plain-text excerpt of the description, so a line further down may simply++ * not be there: an empty value then means "the board did not say", never a guess.++ */++function labelled(text, label) {++ const body = String(text || '');++ const at = new RegExp(`(?:^|[\\s>])${escapeRe(label)}\\s*:`).exec(body);++ if (!at) return null;++ let rest = body.slice(at.index + at[0].length).replace(/^\s*/, '');++ let cut = rest.length;++ for (const other of WWR_LABELS) {++ const m = new RegExp(`\\s${escapeRe(other)}\\s*:`).exec(rest);++ if (m && m.index > 0 && m.index < cut) cut = m.index;++ }++ const value = rest.slice(0, cut).replace(/\s+/g, ' ').trim().replace(/[.;,]$/, '').trim();++ return value ? value.slice(0, 160) : null;++}++++/**++ * A Sources item → the parsed row Work's own We Work Remotely parse makes, or null. The board's RSS title is++ * "Company: Position", so the company and the position come from it; the description carries a few labelled lines++ * (location, duration, salary). What the feed does not carry is not invented: a listing without a company is not a++ * listing Work may publish.++ */++function rawOfWwr(item) {++ const f = (item && item.fields) || {};++ const title = String((item && item.title) || '').trim();++ const at = title.indexOf(':');++ const company = at > 0 ? title.slice(0, at).trim() : '';++ const position = at > 0 ? title.slice(at + 1).trim() : title;++ const summary = String((item && item.summary) || '');++ return {++ source_id: item.identity,++ url: item.canonical_url,++ title: position,++ company,++ location: labelled(summary, 'Location'),++ remote: true, // the board carries remote work only++ tags: list(f.categories), // the feed's <category>, e.g. "Full-Stack Programming"++ job_type: labelled(summary, 'Duration'),++ salary: labelled(summary, 'Salary/Compensation'),++ posted_at: item.published_at || null,++ description_html: summary,++ };++}++++/**++ * The boards Work reads through OpenVibe.Sources: the Sources key an item arrives under, the id the listing is++ * stored with, the kind of item that source keeps, and how the item becomes a listing.++ */++const SOURCES = {++ [SOURCE_KEY]: { source_id: SOURCE_ID, kind: 'job', rawOf },++ 'weworkremotely': { source_id: 'weworkremotely', kind: 'article', rawOf: rawOfWwr },++};++++/** The switch that says a board comes from Sources instead of from its own fetch. */++const SWITCH = {++ arbeitnow: (config) => (config.sources && config.sources.arbeitnowFrom) === 'sources',++ weworkremotely: (config) => (config.sources && config.sources.weworkremotelyFrom) === 'sources',++};++function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = console, source = null, afterRun = null, indexnow = null }) {const url = String((config.sources && config.sources.internalUrl) || '').replace(/\/+$/, '');-- const on = config.sources && config.sources.arbeitnowFrom === 'sources';++ // The boards that are switched on and therefore read through Sources. Each keeps its own listing id.++ const active = Object.entries(SOURCES).filter(([, e]) => (SWITCH[e.source_id] || (() => false))(config));++ const activeByKey = Object.fromEntries(active);++ const on = active.length > 0;const client = source || (on && url && config.oauth.clientSecret? ingest.createSourcesClient({ config: { ...config, sources: { internalUrl: url, category: 'jobs' } }, fetchImpl }): null);@@@ -57,31 +132,38 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = conslet addedIds = [];async function apply(item, t) {-- if (!item || item.source_key !== SOURCE_KEY || item.kind !== 'job') return 'hold';++ const entry = item && activeByKey[item.source_key];++ if (!entry || item.kind !== entry.kind) return 'hold';if (item.removed) {-- await store.expireListing(t, SOURCE_ID, item.identity);++ await store.expireListing(t, entry.source_id, item.identity);return 'removed';}const fetchedAt = (item.provenance && item.provenance.retrieved_at) || s.iso();-- const row = sources.toRow(rawOf(item), { sourceId: SOURCE_ID, fetchedAt, idOf: store.idOf });++ const row = sources.toRow(entry.rawOf(item), { sourceId: entry.source_id, fetchedAt, idOf: store.idOf });if (!row) return 'hold';if (await store.upsertListing(t, row, fetchedAt)) addedIds.push(row.id);return 'applied';}-- /** Follow Sources' change feed to its end and record it as Arbeitnow's fetch. Never throws. */++ /** Follow Sources' change feed to its end and record it as each board's fetch. Never throws. */async function pull() {if (!client || running) return { skipped: true };running = true;const startedAt = s.iso();addedIds = [];++ const ids = active.map(([, e]) => e.source_id);try {-- await store.beginFetch(s, SOURCE_ID, startedAt);++ for (const id of ids) await store.beginFetch(s, id, startedAt);const out = await ingest.pullChanges({ db: s.db, cursor, source: client, name: CURSOR, apply });-- const listings = await store.liveCount(s, SOURCE_ID);-- await store.finishFetch(s, SOURCE_ID, { ok: true, listings, error: null, finishedAt: s.iso() });++ for (const id of ids) {++ const listings = await store.liveCount(s, id);++ await store.finishFetch(s, id, { ok: true, listings, error: null, finishedAt: s.iso() });++ }if (out.applied || out.removed) {-- log.log(`[Ingest] Arbeitnow through OpenVibe.Sources: ${out.applied} listings updated, ${out.removed} gone (cursor ${out.after})`);++ for (const id of ids) {++ const name = (sources.byId(id) || {}).name || id;++ log.log(`[Ingest] ${name} through OpenVibe.Sources: ${out.applied} listings updated, ${out.removed} gone (cursor ${out.after})`);++ }// As after a board read: search engines hear about new listing pages, and Search catches up.if (indexnow && addedIds.length) indexnow.pingSoon([...addedIds.map((id) => `${config.baseUrl}/jobs/${encodeURIComponent(id)}`), `${config.baseUrl}/sitemap.xml`]);if (afterRun) afterRun();@@@ -89,8 +171,10 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = consreturn out;} catch (err) {const detail = String((err && err.message) || err).slice(0, 300);-- await store.finishFetch(s, SOURCE_ID, { ok: false, listings: await store.liveCount(s, SOURCE_ID).catch(() => 0), error: detail, finishedAt: s.iso() }).catch(() => {});-- log.warn(`[Ingest] Arbeitnow through OpenVibe.Sources failed: ${detail}`);++ for (const id of ids) {++ await store.finishFetch(s, id, { ok: false, listings: await store.liveCount(s, id).catch(() => 0), error: detail, finishedAt: s.iso() }).catch(() => {});++ }++ log.warn(`[Ingest] Jobs through OpenVibe.Sources failed: ${detail}`);return { error: detail };} finally { running = false; }}@@@ -101,7 +185,7 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = consfirst.unref();timer = setInterval(() => { pull().catch(() => {}); }, EVERY_MS);timer.unref();-- log.log('[Ingest] Arbeitnow comes from OpenVibe.Sources (arbeitnow-job-board)');++ log.log(`[Ingest] Jobs come from OpenVibe.Sources (${active.map(([, e]) => e.source_id).join(', ')})`);return true;}@@@ -112,7 +196,7 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = constimer = null;}-- return { enabled: Boolean(client), pull, start, stop, apply, SOURCE_ID };++ return { enabled: Boolean(client), pull, start, stop, apply, SOURCE_ID, sourceIds: active.map(([, e]) => e.source_id) };}--module.exports = { createFromSources, rawOf, SOURCE_ID, SOURCE_KEY };++module.exports = { createFromSources, rawOf, rawOfWwr, SOURCES, SOURCE_ID, SOURCE_KEY };@@@ -39,7 +39,7 @@ const TICK_MS = 3_600_000;function createIngest({ config, s, fetchImpl = globalThis.fetch, log = console, afterRun = null, indexnow = null, fromSources = null }) {const timers = new Map();// A board OpenVibe.Sources feeds (server/ingest/from-sources.js) is not read here, nor swept by the expiry.-- const viaSources = new Set(fromSources && fromSources.enabled ? [fromSources.SOURCE_ID] : []);++ const viaSources = new Set(fromSources && fromSources.enabled ? (fromSources.sourceIds || [fromSources.SOURCE_ID]) : []);let running = false;/** Read one source end to end and record what happened. Never throws: the caller is a timer. */@@@ -92,7 +92,8 @@ function createIngest({ config, s, fetchImpl = globalThis.fetch, log = console,for (const source of sources.SOURCES) {const prev = last.get(source.id);const age = prev && prev.started_at ? s.now() - Date.parse(prev.started_at) : Infinity;-- if (viaSources.has(source.id)) { out.push({ source: source.id, skipped: 'from_sources' }); continue; }++ // A board OpenVibe.Sources feeds is not read here at all, whichever switch is on: Work never fetches it.++ if (viaSources.has(source.id) || source.viaSources) { out.push({ source: source.id, skipped: 'from_sources' }); continue; }if (onlyDue && age < config.ingest.intervalMs) { out.push({ source: source.id, skipped: 'not_due' }); continue; }out.push({ source: source.id, ...(await runSource(source.id)) });}@@@ -88,6 +88,17 @@ const SOURCES = [}));},},++ {++ id: 'weworkremotely',++ name: 'We Work Remotely',++ homepage: 'https://weworkremotely.com/',++ terms: 'We Work Remotely publishes its board as an RSS feed for companies to post in it. OpenVibe.Sources reads that feed under the board\'s own robots and terms checks, and Work follows it there (server/ingest/from-sources.js). We attribute the board by name and link every listing back to its page on weworkremotely.com, and we keep only the headline, the company and a short excerpt.',++ attribution: 'Listing from We Work Remotely',++ remoteDefault: true, // the board carries remote work only++ viaSources: true, // read through OpenVibe.Sources, never by Work itself++ requests() { return ['https://weworkremotely.com/remote-jobs.rss']; },++ parse() { return []; }, // Work never reads this feed itself: the items arrive through Sources++ },{id: 'remotive',name: 'Remotive',@@@ -22,8 +22,10 @@ const DAY = 86_400_000;try {await check('reading the three boards keeps every listing with its source, its id and the original URL', async () => {const out = await ingest.runAll();-- assert.deepStrictEqual(out.map((o) => o.source), ['arbeitnow', 'remotive', 'remoteok']);-- assert.ok(out.every((o) => o.ok), JSON.stringify(out.map((o) => o.error)));++ // We Work Remotely is skipped here: it reaches Work through OpenVibe.Sources, never from its own feed.++ assert.deepStrictEqual(out.map((o) => o.source), ['arbeitnow', 'weworkremotely', 'remotive', 'remoteok']);++ assert.ok(out.filter((o) => o.source !== 'weworkremotely').every((o) => o.ok), JSON.stringify(out.map((o) => o.error)));++ assert.strictEqual(out.find((o) => o.source === 'weworkremotely').skipped, 'from_sources');// Arbeitnow's two pages (2 + 1), Remotive's two jobs, RemoteOK's two listings.const counts = await store.countsBySource(s);assert.strictEqual(counts.get('arbeitnow').count, 3);@@@ -155,7 +157,8 @@ const DAY = 86_400_000;await ingest.runAll();const before = t.boards.requests.length;const out = await ingest.runAll({ onlyDue: true });-- assert.ok(out.every((o) => o.skipped === 'not_due'), 'every source was read just now, so none is due');++ // A board OpenVibe.Sources feeds is never read here at all, so it is skipped rather than due.++ assert.ok(out.every((o) => o.skipped === 'not_due' || o.skipped === 'from_sources'), 'every source was read just now, so none is due');assert.strictEqual(t.boards.requests.length, before, 'no board was called');t.boards.reset();});@@@ -185,8 +185,9 @@ const first = async (t, path) => (await t.get(path)).json();await check('every source is listed with its terms as applied and when it was last read', async () => {const api = await first(t, '/api/v1/sources');-- assert.deepStrictEqual(api.sources.map((s) => s.id).sort(), ['arbeitnow', 'remoteok', 'remotive']);-- for (const src of api.sources) {++ assert.deepStrictEqual(api.sources.map((s) => s.id).sort(), ['arbeitnow', 'remoteok', 'remotive', 'weworkremotely']);++ // A board OpenVibe.Sources feeds has no read of its own here, so it is not asked for one.++ for (const src of api.sources.filter((x) => x.id !== 'weworkremotely')) {assert.ok(src.terms.length > 40, `${src.id} has no terms`);assert.match(src.terms, /link|attribute|mention|name/i, `${src.id}'s terms do not state the attribution rule`);assert.strictEqual(src.last_fetch.ok, true);@@@ -213,7 +214,7 @@ const first = async (t, path) => (await t.get(path)).json();const llms = (await t.get('/llms.txt')).text;assert.ok(llms.includes('(https://openvibe.work/jobs)'));assert.ok(llms.includes('(https://openvibe.work/sources)'));-- assert.ok(llms.includes('Arbeitnow, Remotive, RemoteOK'), 'the sources are not named for a crawler');++ assert.ok(llms.includes('Arbeitnow, We Work Remotely, Remotive, RemoteOK'), 'the sources are not named for a crawler');assert.ok(!llms.includes('/saved'), 'llms.txt names a private page');const robots = (await t.get('/robots.txt')).text;assert.ok(robots.includes('Disallow: /saved'));nnew file mode 100644@@@ -0,0 +1,85 @@++'use strict';++/**++ * We Work Remotely through OpenVibe.Sources (server/ingest/from-sources.js), against a Sources stand-in serving its++ * change feed:++ * - a feed item becomes a listing (the board's "Company: Position" title gives the company and the position, the++ * labelled lines of the description give the location, the duration and the salary, the feed's category a tag);++ * - the thin-listing gate still applies: an item without a company is held, not published;++ * - the listing is attributed to We Work Remotely and links back to the original listing;++ * - Work never reads the feed itself.++ */++const assert = require('assert');++const { boot, check, done } = require('./helpers/boot');++const store = require('../server/jobs/store');++++const feed = [];++let seq = 0;++const push = (item) => feed.push({ ...item, change_seq: ++seq });++const sourcesClient = {++ enabled: true,++ async listItems({ after = 0, limit = 100 } = {}) {++ const page = feed.filter((i) => i.change_seq > after).slice(0, limit);++ return { items: page, next_after: page.length ? page[page.length - 1].change_seq : after, more: false };++ },++};++++// A recorded item of the board's feed, in the shape OpenVibe.Sources keeps it (a recorded fixture: no live call here).++const wwr = (identity, over = {}) => ({++ id: `itm_${identity}`, source_key: 'weworkremotely', category: 'jobs', kind: 'article', identity,++ canonical_url: `https://weworkremotely.com/remote-jobs/${identity}`,++ title: 'Welocalize: Automation Specialist',++ summary: 'Headquarters: Ciudad de Mexico, Mexico URL: https://t.mtrbio.com/WeloData About the Role We are looking for an Automation Specialist. Job Title: Automation Specialist-1 Working Hours: 10:00-18:30 CDT Duration: Contract -indefinite . Salary/Compensation: 480,000 MXN/ yearly Location: Mexico (Remote)',++ authors: [], published_at: '2026-10-29T12:00:00.000Z', source_updated_at: null,++ fields: { categories: ['Full-Stack Programming'] },++ revision: 1, provenance: { retrieved_at: '2026-10-30T06:00:00.000Z' }, removed: null, ...over,++});++++(async () => {++ const clock = { t: Date.parse('2026-10-30T08:00:00Z') };++ const t = await boot({ now: () => clock.t, env: { WORK_WWR_FROM: 'sources' }, sourcesClient });++ const s = t.ctx.s;++ const from = t.ctx.fromSources;++++ try {++ await check('a feed item becomes a listing with the board\'s own words', async () => {++ push(wwr('automation-specialist-welocalize'));++ // The thin-listing gate: no company in the title, so no listing may be published (Work #133).++ push(wwr('automation-specialist', { title: 'Automation Specialist', canonical_url: 'https://weworkremotely.com/remote-jobs/automation-specialist' }));++ const out = await from.pull();++ assert.deepStrictEqual([out.applied, out.hold, out.removed], [1, 1, 0]);++++ const id = store.idOf('weworkremotely', 'automation-specialist-welocalize');++ const row = await s.db.maybe('SELECT * FROM work_listings WHERE id = $1', [id]);++ assert.deepStrictEqual([row.source, row.title, row.company, row.location, row.remote, row.tags, row.job_type, row.salary, row.expired],++ ['weworkremotely', 'Automation Specialist', 'Welocalize', 'Mexico (Remote)', true, 'Full-Stack Programming', 'Contract -indefinite', '480,000 MXN/ yearly', false]);++ assert.strictEqual(row.url, 'https://weworkremotely.com/remote-jobs/automation-specialist-welocalize');++ assert.ok(row.excerpt.includes('Automation Specialist'), 'the excerpt is the board\'s own description');++++ const page = (await t.get('/api/v1/jobs?q=Automation')).json();++ assert.deepStrictEqual(page.jobs.map((j) => [j.id, j.source, j.source_name]), [[id, 'weworkremotely', 'We Work Remotely']]);++ });++++ await check('the listing is attributed to the board and links back to it', async () => {++ const id = store.idOf('weworkremotely', 'automation-specialist-welocalize');++ const list = await t.get('/jobs');++ assert.strictEqual(list.status, 200);++ assert.ok(list.text.includes('Listing from'), 'the listing carries its provenance');++ assert.ok(list.text.includes('We Work Remotely'), 'the board is named');++ assert.ok(list.text.includes('https://weworkremotely.com/remote-jobs/automation-specialist-welocalize'), 'the original listing is linked');++ const page = await t.get(`/jobs/${id}`);++ assert.strictEqual(page.status, 200);++ assert.ok(page.text.includes('We Work Remotely'), 'the board is named on the listing page');++ assert.ok(page.text.includes('https://weworkremotely.com/remote-jobs/automation-specialist-welocalize'), 'the original listing is linked');++ });++++ await check('the fetch record says the listings came through, and Work never reads the feed itself', async () => {++ const rec = (await store.sourceFetches(s)).get('weworkremotely');++ assert.deepStrictEqual([rec.ok, rec.listings, rec.error], [true, 1, null]);++ const out = await t.ctx.ingest.runAll();++ assert.deepStrictEqual(out.find((o) => o.source === 'weworkremotely'), { source: 'weworkremotely', skipped: 'from_sources' });++ });++ } finally {++ await t.close();++ }++ done();++})().catch((err) => { console.error(err); process.exit(1); });
@@ -7,7 +7,7 @@
@@ -7,7 +7,7 @@
module.exports = {
module.exports = {
id: 'work',
id: 'work',
name: 'OpenVibe.Work',
name: 'OpenVibe.Work',
- purpose: 'Find work on open job boards: listings from Arbeitnow, Remotive and RemoteOK under their terms, searchable with facets and each linking back to the original, plus saved searches and OpenVibe.Watch alerts for a signed-in person.',
+ purpose: 'Find work on open job boards: listings from Arbeitnow, We Work Remotely, Remotive and RemoteOK under their terms, searchable with facets and each linking back to the original, plus saved searches and OpenVibe.Watch alerts for a signed-in person.',
home: 'https://openvibe.work',
home: 'https://openvibe.work',
site: 'openvibe.work',
site: 'openvibe.work',
api: '/api/v1',
api: '/api/v1',
@@ -90,6 +90,9 @@ function load(env = process.env) {
@@ -90,6 +90,9 @@ function load(env = process.env) {
// OAuth client secret (Network grants Work sources.item.read). Anything else keeps Work's own fetch.
// OAuth client secret (Network grants Work sources.item.read). Anything else keeps Work's own fetch.
internalUrl: trim(env.OV_SOURCES_INTERNAL_URL == null ? 'http://127.0.0.1:4720' : env.OV_SOURCES_INTERNAL_URL),
internalUrl: trim(env.OV_SOURCES_INTERNAL_URL == null ? 'http://127.0.0.1:4720' : env.OV_SOURCES_INTERNAL_URL),
arbeitnowFrom: String(env.WORK_ARBEITNOW_FROM || '').trim().toLowerCase() === 'sources' ? 'sources' : 'board',
arbeitnowFrom: String(env.WORK_ARBEITNOW_FROM || '').trim().toLowerCase() === 'sources' ? 'sources' : 'board',
+ // We Work Remotely reaches Work the same way (WORK_WWR_FROM=sources, the weworkremotely key in Sources);
+ // it is off until the board is read through Sources, and Work never fetches the feed itself.
+ weworkremotelyFrom: String(env.WORK_WWR_FROM || '').trim().toLowerCase() === 'sources' ? 'sources' : 'board',
},
},
// The background ingest (server/ingest/): off unless WORK_INGEST=on, so a test and a development run
// The background ingest (server/ingest/): off unless WORK_INGEST=on, so a test and a development run
// never call a job board. Once on, each source is fetched at most every intervalMs (Remotive's terms
// never call a job board. Once on, each source is fetched at most every intervalMs (Remotive's terms
@@ -25,7 +25,7 @@ const SITEMAP_JOBS = 1000;
@@ -25,7 +25,7 @@ const SITEMAP_JOBS = 1000;
const PAGE_TEXT = {
const PAGE_TEXT = {
'/': ['OpenVibe.Work home', 'OpenVibe.Work gathers job listings from public job boards, names the source of every one and links back to the original listing. Search is free and needs no account.'],
'/': ['OpenVibe.Work home', 'OpenVibe.Work gathers job listings from public job boards, names the source of every one and links back to the original listing. Search is free and needs no account.'],
- '/jobs': ['Jobs', 'Search job listings gathered from Arbeitnow, Remotive and RemoteOK, newest first, filterable by words, workplace, job type and location.'],
+ '/jobs': ['Jobs', 'Search job listings gathered from Arbeitnow, We Work Remotely, Remotive and RemoteOK, newest first, filterable by words, workplace, job type and location.'],
'/companies': ['Companies hiring', 'Every company with open listings on the boards OpenVibe.Work reads, most openings first, each with a page of its jobs, where they are and which boards list them.'],
'/companies': ['Companies hiring', 'Every company with open listings on the boards OpenVibe.Work reads, most openings first, each with a page of its jobs, where they are and which boards list them.'],
'/sources': ['Sources', 'The public job boards OpenVibe.Work gathers from, the terms each is used under as applied here, and when each was last read.'],
'/sources': ['Sources', 'The public job boards OpenVibe.Work gathers from, the terms each is used under as applied here, and when each was last read.'],
'/docs': ['API', 'The OpenVibe.Work API: search the listings, read one with its provenance, list the sources and their terms, keep saved searches.'],
'/docs': ['API', 'The OpenVibe.Work API: search the listings, read one with its provenance, list the sources and their terms, keep saved searches.'],
@@ -651,7 +651,7 @@ ${sources.SOURCES.map((src) => {
@@ -651,7 +651,7 @@ ${sources.SOURCES.map((src) => {
<li>${src.name} listings also carry their source in the JSON-LD on the listing page.</li></ul>
<li>${src.name} listings also carry their source in the JSON-LD on the listing page.</li></ul>
<h3>What each read does</h3>
<h3>What each read does</h3>
<dl class="job-facts">
<dl class="job-facts">
-<dt>API</dt><dd><code>${src.requests(config)[0]}</code></dd>
+<dt>API</dt><dd>${src.viaSources ? html`read through OpenVibe.Sources from <code>${src.requests(config)[0]}</code>` : html`<code>${src.requests(config)[0]}</code>`}</dd>
<dt>Last read</dt><dd>${f && f.started_at ? html`${time(f.started_at)}${f.ok ? ' — succeeded' : ' — failed'}${f.error ? html` <span class="muted small">(${f.error})</span>` : ''}` : html`<span class="muted">not read yet</span>`}</dd>
<dt>Last read</dt><dd>${f && f.started_at ? html`${time(f.started_at)}${f.ok ? ' — succeeded' : ' — failed'}${f.error ? html` <span class="muted small">(${f.error})</span>` : ''}` : html`<span class="muted">not read yet</span>`}</dd>
<dt>Reads every</dt><dd>${Math.round(config.ingest.intervalMs / 3_600_000)} hours ${src.id === 'remotive' ? '(its terms allow about four a day)' : ''}</dd>
<dt>Reads every</dt><dd>${Math.round(config.ingest.intervalMs / 3_600_000)} hours ${src.id === 'remotive' ? '(its terms allow about four a day)' : ''}</dd>
<dt>Listings kept</dt><dd>${(counts.get(src.id) || {}).count || 0}, retired when the board stops showing one for ${config.ingest.retentionDays} days</dd>
<dt>Listings kept</dt><dd>${(counts.get(src.id) || {}).count || 0}, retired when the board stops showing one for ${config.ingest.retentionDays} days</dd>
@@ -1,18 +1,23 @@
@@ -1,18 +1,23 @@
'use strict';
'use strict';
/**
/**
- * Arbeitnow through OpenVibe.Sources (plan 2026-10-22, track F2). With WORK_ARBEITNOW_FROM=sources, Work stops reading
- * Arbeitnow's board itself: Sources fetches it (pages 1-3 every six hours, under its robots and terms checks) and keeps
- * each job as an item (source arbeitnow-job-board, category jobs). Work follows Sources' change feed
- * (openvibe-publishing/ingest: the cursor in work_ingest_cursor, one transaction per page) and keeps one listing per job
- * in work_listings, source 'arbeitnow', the row its own fetch made (the same slug, so the same listing id and URL).
+ * Job boards read through OpenVibe.Sources (plan 2026-10-22, track F2; We Work Remotely added 2026-10-30).
+ *
+ * Sources fetches a board (under its robots and terms checks) and keeps each job as an item. Work follows Sources'
+ * change feed (openvibe-publishing/ingest: the cursor in work_ingest_cursor, one transaction per page) and keeps one
+ * listing per job in work_listings, the row the board's own fetch made (the same id, so the same listing id and URL).
+ *
+ * Every board that comes this way is one entry of SOURCES below: the Sources key it arrives under, the id Work stores
+ * it with, and how a Sources item becomes the parsed row that board's own parse (server/jobs/sources.js) would make.
+ * The thin-listing gate (sources.toRow) is the same for all of them: no id, no title, no company or no http(s) link
+ * back means the listing is not published.
*
*
* A job Sources retires (the board has not shown it for a week: retire_after_sec) or removes is expired here, like a
* A job Sources retires (the board has not shown it for a week: retire_after_sec) or removes is expired here, like a
* listing a board stopped showing. Work's own expiry sweep leaves these listings alone: an unchanged job is no change in
* listing a board stopped showing. Work's own expiry sweep leaves these listings alone: an unchanged job is no change in
* Sources' feed, so Work never hears it was seen again, and Sources' retirement is what says it is gone.
* Sources' feed, so Work never hears it was seen again, and Sources' retirement is what says it is gone.
*
*
- * Pulls 30 seconds after start and then hourly. Off unless WORK_ARBEITNOW_FROM=sources with OV_SOURCES_INTERNAL_URL and
- * the OAuth client secret (a test passes its own source).
+ * Pulls 30 seconds after start and then hourly. Each board has its own switch (WORK_ARBEITNOW_FROM=sources,
+ * WORK_WWR_FROM=sources); a board that is not switched on is left to its own fetch, as before.
*/
*/
const ingest = require('openvibe-publishing/ingest');
const ingest = require('openvibe-publishing/ingest');
const sources = require('../jobs/sources');
const sources = require('../jobs/sources');
@@ -24,10 +29,11 @@ const CURSOR = 'sources.jobs.after';
@@ -24,10 +29,11 @@ const CURSOR = 'sources.jobs.after';
const EVERY_MS = 60 * 60 * 1000;
const EVERY_MS = 60 * 60 * 1000;
const FIRST_MS = 30 * 1000;
const FIRST_MS = 30 * 1000;
+const list = (v) => (Array.isArray(v) ? v.filter((x) => typeof x === 'string' && x.trim()).map((x) => x.trim()) : []);
+
/** A Sources item → the parsed row Work's own Arbeitnow parse makes (server/jobs/sources.js), or null. */
/** A Sources item → the parsed row Work's own Arbeitnow parse makes (server/jobs/sources.js), or null. */
function rawOf(item) {
function rawOf(item) {
const f = (item && item.fields) || {};
const f = (item && item.fields) || {};
- const list = (v) => (Array.isArray(v) ? v.filter((x) => typeof x === 'string' && x.trim()).map((x) => x.trim()) : []);
return {
return {
source_id: item.identity,
source_id: item.identity,
url: item.canonical_url,
url: item.canonical_url,
@@ -44,9 +50,78 @@ function rawOf(item) {
@@ -44,9 +50,78 @@ function rawOf(item) {
};
};
}
}
+/** The labels We Work Remotely writes in a listing's description, in the order they appear. */
+const WWR_LABELS = ['Salary/Compensation', 'Compensation', 'Working Hours', 'Job Title', 'Headquarters', 'Duration', 'Location', 'Responsibilities', 'Requirements', 'Qualifications', 'Benefits', 'Salary', 'About', 'Apply', 'URL'];
+const escapeRe = (s) => String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+/**
+ * The value of one labelled line in a We Work Remotely description ("Location: Mexico (Remote)"), or null when the
+ * listing does not carry it. Sources keeps a plain-text excerpt of the description, so a line further down may simply
+ * not be there: an empty value then means "the board did not say", never a guess.
+ */
+function labelled(text, label) {
+ const body = String(text || '');
+ const at = new RegExp(`(?:^|[\\s>])${escapeRe(label)}\\s*:`).exec(body);
+ if (!at) return null;
+ let rest = body.slice(at.index + at[0].length).replace(/^\s*/, '');
+ let cut = rest.length;
+ for (const other of WWR_LABELS) {
+ const m = new RegExp(`\\s${escapeRe(other)}\\s*:`).exec(rest);
+ if (m && m.index > 0 && m.index < cut) cut = m.index;
+ }
+ const value = rest.slice(0, cut).replace(/\s+/g, ' ').trim().replace(/[.;,]$/, '').trim();
+ return value ? value.slice(0, 160) : null;
+}
+
+/**
+ * A Sources item → the parsed row Work's own We Work Remotely parse makes, or null. The board's RSS title is
+ * "Company: Position", so the company and the position come from it; the description carries a few labelled lines
+ * (location, duration, salary). What the feed does not carry is not invented: a listing without a company is not a
+ * listing Work may publish.
+ */
+function rawOfWwr(item) {
+ const f = (item && item.fields) || {};
+ const title = String((item && item.title) || '').trim();
+ const at = title.indexOf(':');
+ const company = at > 0 ? title.slice(0, at).trim() : '';
+ const position = at > 0 ? title.slice(at + 1).trim() : title;
+ const summary = String((item && item.summary) || '');
+ return {
+ source_id: item.identity,
+ url: item.canonical_url,
+ title: position,
+ company,
+ location: labelled(summary, 'Location'),
+ remote: true, // the board carries remote work only
+ tags: list(f.categories), // the feed's <category>, e.g. "Full-Stack Programming"
+ job_type: labelled(summary, 'Duration'),
+ salary: labelled(summary, 'Salary/Compensation'),
+ posted_at: item.published_at || null,
+ description_html: summary,
+ };
+}
+
+/**
+ * The boards Work reads through OpenVibe.Sources: the Sources key an item arrives under, the id the listing is
+ * stored with, the kind of item that source keeps, and how the item becomes a listing.
+ */
+const SOURCES = {
+ [SOURCE_KEY]: { source_id: SOURCE_ID, kind: 'job', rawOf },
+ 'weworkremotely': { source_id: 'weworkremotely', kind: 'article', rawOf: rawOfWwr },
+};
+
+/** The switch that says a board comes from Sources instead of from its own fetch. */
+const SWITCH = {
+ arbeitnow: (config) => (config.sources && config.sources.arbeitnowFrom) === 'sources',
+ weworkremotely: (config) => (config.sources && config.sources.weworkremotelyFrom) === 'sources',
+};
+
function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = console, source = null, afterRun = null, indexnow = null }) {
function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = console, source = null, afterRun = null, indexnow = null }) {
const url = String((config.sources && config.sources.internalUrl) || '').replace(/\/+$/, '');
const url = String((config.sources && config.sources.internalUrl) || '').replace(/\/+$/, '');
- const on = config.sources && config.sources.arbeitnowFrom === 'sources';
+ // The boards that are switched on and therefore read through Sources. Each keeps its own listing id.
+ const active = Object.entries(SOURCES).filter(([, e]) => (SWITCH[e.source_id] || (() => false))(config));
+ const activeByKey = Object.fromEntries(active);
+ const on = active.length > 0;
const client = source || (on && url && config.oauth.clientSecret
const client = source || (on && url && config.oauth.clientSecret
? ingest.createSourcesClient({ config: { ...config, sources: { internalUrl: url, category: 'jobs' } }, fetchImpl })
? ingest.createSourcesClient({ config: { ...config, sources: { internalUrl: url, category: 'jobs' } }, fetchImpl })
: null);
: null);
@@ -57,31 +132,38 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
@@ -57,31 +132,38 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
let addedIds = [];
let addedIds = [];
async function apply(item, t) {
async function apply(item, t) {
- if (!item || item.source_key !== SOURCE_KEY || item.kind !== 'job') return 'hold';
+ const entry = item && activeByKey[item.source_key];
+ if (!entry || item.kind !== entry.kind) return 'hold';
if (item.removed) {
if (item.removed) {
- await store.expireListing(t, SOURCE_ID, item.identity);
+ await store.expireListing(t, entry.source_id, item.identity);
return 'removed';
return 'removed';
}
}
const fetchedAt = (item.provenance && item.provenance.retrieved_at) || s.iso();
const fetchedAt = (item.provenance && item.provenance.retrieved_at) || s.iso();
- const row = sources.toRow(rawOf(item), { sourceId: SOURCE_ID, fetchedAt, idOf: store.idOf });
+ const row = sources.toRow(entry.rawOf(item), { sourceId: entry.source_id, fetchedAt, idOf: store.idOf });
if (!row) return 'hold';
if (!row) return 'hold';
if (await store.upsertListing(t, row, fetchedAt)) addedIds.push(row.id);
if (await store.upsertListing(t, row, fetchedAt)) addedIds.push(row.id);
return 'applied';
return 'applied';
}
}
- /** Follow Sources' change feed to its end and record it as Arbeitnow's fetch. Never throws. */
+ /** Follow Sources' change feed to its end and record it as each board's fetch. Never throws. */
async function pull() {
async function pull() {
if (!client || running) return { skipped: true };
if (!client || running) return { skipped: true };
running = true;
running = true;
const startedAt = s.iso();
const startedAt = s.iso();
addedIds = [];
addedIds = [];
+ const ids = active.map(([, e]) => e.source_id);
try {
try {
- await store.beginFetch(s, SOURCE_ID, startedAt);
+ for (const id of ids) await store.beginFetch(s, id, startedAt);
const out = await ingest.pullChanges({ db: s.db, cursor, source: client, name: CURSOR, apply });
const out = await ingest.pullChanges({ db: s.db, cursor, source: client, name: CURSOR, apply });
- const listings = await store.liveCount(s, SOURCE_ID);
- await store.finishFetch(s, SOURCE_ID, { ok: true, listings, error: null, finishedAt: s.iso() });
+ for (const id of ids) {
+ const listings = await store.liveCount(s, id);
+ await store.finishFetch(s, id, { ok: true, listings, error: null, finishedAt: s.iso() });
+ }
if (out.applied || out.removed) {
if (out.applied || out.removed) {
- log.log(`[Ingest] Arbeitnow through OpenVibe.Sources: ${out.applied} listings updated, ${out.removed} gone (cursor ${out.after})`);
+ for (const id of ids) {
+ const name = (sources.byId(id) || {}).name || id;
+ log.log(`[Ingest] ${name} through OpenVibe.Sources: ${out.applied} listings updated, ${out.removed} gone (cursor ${out.after})`);
+ }
// As after a board read: search engines hear about new listing pages, and Search catches up.
// As after a board read: search engines hear about new listing pages, and Search catches up.
if (indexnow && addedIds.length) indexnow.pingSoon([...addedIds.map((id) => `${config.baseUrl}/jobs/${encodeURIComponent(id)}`), `${config.baseUrl}/sitemap.xml`]);
if (indexnow && addedIds.length) indexnow.pingSoon([...addedIds.map((id) => `${config.baseUrl}/jobs/${encodeURIComponent(id)}`), `${config.baseUrl}/sitemap.xml`]);
if (afterRun) afterRun();
if (afterRun) afterRun();
@@ -89,8 +171,10 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
@@ -89,8 +171,10 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
return out;
return out;
} catch (err) {
} catch (err) {
const detail = String((err && err.message) || err).slice(0, 300);
const detail = String((err && err.message) || err).slice(0, 300);
- await store.finishFetch(s, SOURCE_ID, { ok: false, listings: await store.liveCount(s, SOURCE_ID).catch(() => 0), error: detail, finishedAt: s.iso() }).catch(() => {});
- log.warn(`[Ingest] Arbeitnow through OpenVibe.Sources failed: ${detail}`);
+ for (const id of ids) {
+ await store.finishFetch(s, id, { ok: false, listings: await store.liveCount(s, id).catch(() => 0), error: detail, finishedAt: s.iso() }).catch(() => {});
+ }
+ log.warn(`[Ingest] Jobs through OpenVibe.Sources failed: ${detail}`);
return { error: detail };
return { error: detail };
} finally { running = false; }
} finally { running = false; }
}
}
@@ -101,7 +185,7 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
@@ -101,7 +185,7 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
first.unref();
first.unref();
timer = setInterval(() => { pull().catch(() => {}); }, EVERY_MS);
timer = setInterval(() => { pull().catch(() => {}); }, EVERY_MS);
timer.unref();
timer.unref();
- log.log('[Ingest] Arbeitnow comes from OpenVibe.Sources (arbeitnow-job-board)');
+ log.log(`[Ingest] Jobs come from OpenVibe.Sources (${active.map(([, e]) => e.source_id).join(', ')})`);
return true;
return true;
}
}
@@ -112,7 +196,7 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
@@ -112,7 +196,7 @@ function createFromSources({ config, s, fetchImpl = globalThis.fetch, log = cons
timer = null;
timer = null;
}
}
- return { enabled: Boolean(client), pull, start, stop, apply, SOURCE_ID };
+ return { enabled: Boolean(client), pull, start, stop, apply, SOURCE_ID, sourceIds: active.map(([, e]) => e.source_id) };
}
}
-module.exports = { createFromSources, rawOf, SOURCE_ID, SOURCE_KEY };
+module.exports = { createFromSources, rawOf, rawOfWwr, SOURCES, SOURCE_ID, SOURCE_KEY };
@@ -39,7 +39,7 @@ const TICK_MS = 3_600_000;
@@ -39,7 +39,7 @@ const TICK_MS = 3_600_000;
function createIngest({ config, s, fetchImpl = globalThis.fetch, log = console, afterRun = null, indexnow = null, fromSources = null }) {
function createIngest({ config, s, fetchImpl = globalThis.fetch, log = console, afterRun = null, indexnow = null, fromSources = null }) {
const timers = new Map();
const timers = new Map();
// A board OpenVibe.Sources feeds (server/ingest/from-sources.js) is not read here, nor swept by the expiry.
// A board OpenVibe.Sources feeds (server/ingest/from-sources.js) is not read here, nor swept by the expiry.
- const viaSources = new Set(fromSources && fromSources.enabled ? [fromSources.SOURCE_ID] : []);
+ const viaSources = new Set(fromSources && fromSources.enabled ? (fromSources.sourceIds || [fromSources.SOURCE_ID]) : []);
let running = false;
let running = false;
/** Read one source end to end and record what happened. Never throws: the caller is a timer. */
/** Read one source end to end and record what happened. Never throws: the caller is a timer. */
@@ -92,7 +92,8 @@ function createIngest({ config, s, fetchImpl = globalThis.fetch, log = console,
@@ -92,7 +92,8 @@ function createIngest({ config, s, fetchImpl = globalThis.fetch, log = console,
for (const source of sources.SOURCES) {
for (const source of sources.SOURCES) {
const prev = last.get(source.id);
const prev = last.get(source.id);
const age = prev && prev.started_at ? s.now() - Date.parse(prev.started_at) : Infinity;
const age = prev && prev.started_at ? s.now() - Date.parse(prev.started_at) : Infinity;
- if (viaSources.has(source.id)) { out.push({ source: source.id, skipped: 'from_sources' }); continue; }
+ // A board OpenVibe.Sources feeds is not read here at all, whichever switch is on: Work never fetches it.
+ if (viaSources.has(source.id) || source.viaSources) { out.push({ source: source.id, skipped: 'from_sources' }); continue; }
if (onlyDue && age < config.ingest.intervalMs) { out.push({ source: source.id, skipped: 'not_due' }); continue; }
if (onlyDue && age < config.ingest.intervalMs) { out.push({ source: source.id, skipped: 'not_due' }); continue; }
out.push({ source: source.id, ...(await runSource(source.id)) });
out.push({ source: source.id, ...(await runSource(source.id)) });
}
}
@@ -88,6 +88,17 @@ const SOURCES = [
@@ -88,6 +88,17 @@ const SOURCES = [
}));
}));
},
},
},
},
+ {
+ id: 'weworkremotely',
+ name: 'We Work Remotely',
+ homepage: 'https://weworkremotely.com/',
+ terms: 'We Work Remotely publishes its board as an RSS feed for companies to post in it. OpenVibe.Sources reads that feed under the board\'s own robots and terms checks, and Work follows it there (server/ingest/from-sources.js). We attribute the board by name and link every listing back to its page on weworkremotely.com, and we keep only the headline, the company and a short excerpt.',
+ attribution: 'Listing from We Work Remotely',
+ remoteDefault: true, // the board carries remote work only
+ viaSources: true, // read through OpenVibe.Sources, never by Work itself
+ requests() { return ['https://weworkremotely.com/remote-jobs.rss']; },
+ parse() { return []; }, // Work never reads this feed itself: the items arrive through Sources
+ },
{
{
id: 'remotive',
id: 'remotive',
name: 'Remotive',
name: 'Remotive',
@@ -22,8 +22,10 @@ const DAY = 86_400_000;
@@ -22,8 +22,10 @@ const DAY = 86_400_000;
try {
try {
await check('reading the three boards keeps every listing with its source, its id and the original URL', async () => {
await check('reading the three boards keeps every listing with its source, its id and the original URL', async () => {
const out = await ingest.runAll();
const out = await ingest.runAll();
- assert.deepStrictEqual(out.map((o) => o.source), ['arbeitnow', 'remotive', 'remoteok']);
- assert.ok(out.every((o) => o.ok), JSON.stringify(out.map((o) => o.error)));
+ // We Work Remotely is skipped here: it reaches Work through OpenVibe.Sources, never from its own feed.
+ assert.deepStrictEqual(out.map((o) => o.source), ['arbeitnow', 'weworkremotely', 'remotive', 'remoteok']);
+ assert.ok(out.filter((o) => o.source !== 'weworkremotely').every((o) => o.ok), JSON.stringify(out.map((o) => o.error)));
+ assert.strictEqual(out.find((o) => o.source === 'weworkremotely').skipped, 'from_sources');
// Arbeitnow's two pages (2 + 1), Remotive's two jobs, RemoteOK's two listings.
// Arbeitnow's two pages (2 + 1), Remotive's two jobs, RemoteOK's two listings.
const counts = await store.countsBySource(s);
const counts = await store.countsBySource(s);
assert.strictEqual(counts.get('arbeitnow').count, 3);
assert.strictEqual(counts.get('arbeitnow').count, 3);
@@ -155,7 +157,8 @@ const DAY = 86_400_000;
@@ -155,7 +157,8 @@ const DAY = 86_400_000;
await ingest.runAll();
await ingest.runAll();
const before = t.boards.requests.length;
const before = t.boards.requests.length;
const out = await ingest.runAll({ onlyDue: true });
const out = await ingest.runAll({ onlyDue: true });
- assert.ok(out.every((o) => o.skipped === 'not_due'), 'every source was read just now, so none is due');
+ // A board OpenVibe.Sources feeds is never read here at all, so it is skipped rather than due.
+ assert.ok(out.every((o) => o.skipped === 'not_due' || o.skipped === 'from_sources'), 'every source was read just now, so none is due');
assert.strictEqual(t.boards.requests.length, before, 'no board was called');
assert.strictEqual(t.boards.requests.length, before, 'no board was called');
t.boards.reset();
t.boards.reset();
});
});
@@ -185,8 +185,9 @@ const first = async (t, path) => (await t.get(path)).json();
@@ -185,8 +185,9 @@ const first = async (t, path) => (await t.get(path)).json();
await check('every source is listed with its terms as applied and when it was last read', async () => {
await check('every source is listed with its terms as applied and when it was last read', async () => {
const api = await first(t, '/api/v1/sources');
const api = await first(t, '/api/v1/sources');
- assert.deepStrictEqual(api.sources.map((s) => s.id).sort(), ['arbeitnow', 'remoteok', 'remotive']);
- for (const src of api.sources) {
+ assert.deepStrictEqual(api.sources.map((s) => s.id).sort(), ['arbeitnow', 'remoteok', 'remotive', 'weworkremotely']);
+ // A board OpenVibe.Sources feeds has no read of its own here, so it is not asked for one.
+ for (const src of api.sources.filter((x) => x.id !== 'weworkremotely')) {
assert.ok(src.terms.length > 40, `${src.id} has no terms`);
assert.ok(src.terms.length > 40, `${src.id} has no terms`);
assert.match(src.terms, /link|attribute|mention|name/i, `${src.id}'s terms do not state the attribution rule`);
assert.match(src.terms, /link|attribute|mention|name/i, `${src.id}'s terms do not state the attribution rule`);
assert.strictEqual(src.last_fetch.ok, true);
assert.strictEqual(src.last_fetch.ok, true);
@@ -213,7 +214,7 @@ const first = async (t, path) => (await t.get(path)).json();
@@ -213,7 +214,7 @@ const first = async (t, path) => (await t.get(path)).json();
const llms = (await t.get('/llms.txt')).text;
const llms = (await t.get('/llms.txt')).text;
assert.ok(llms.includes('(https://openvibe.work/jobs)'));
assert.ok(llms.includes('(https://openvibe.work/jobs)'));
assert.ok(llms.includes('(https://openvibe.work/sources)'));
assert.ok(llms.includes('(https://openvibe.work/sources)'));
- assert.ok(llms.includes('Arbeitnow, Remotive, RemoteOK'), 'the sources are not named for a crawler');
+ assert.ok(llms.includes('Arbeitnow, We Work Remotely, Remotive, RemoteOK'), 'the sources are not named for a crawler');
assert.ok(!llms.includes('/saved'), 'llms.txt names a private page');
assert.ok(!llms.includes('/saved'), 'llms.txt names a private page');
const robots = (await t.get('/robots.txt')).text;
const robots = (await t.get('/robots.txt')).text;
assert.ok(robots.includes('Disallow: /saved'));
assert.ok(robots.includes('Disallow: /saved'));
new file mode 100644
new file mode 100644
@@ -0,0 +1,85 @@
@@ -0,0 +1,85 @@
+'use strict';
+/**
+ * We Work Remotely through OpenVibe.Sources (server/ingest/from-sources.js), against a Sources stand-in serving its
+ * change feed:
+ * - a feed item becomes a listing (the board's "Company: Position" title gives the company and the position, the
+ * labelled lines of the description give the location, the duration and the salary, the feed's category a tag);
+ * - the thin-listing gate still applies: an item without a company is held, not published;
+ * - the listing is attributed to We Work Remotely and links back to the original listing;
+ * - Work never reads the feed itself.
+ */
+const assert = require('assert');
+const { boot, check, done } = require('./helpers/boot');
+const store = require('../server/jobs/store');
+
+const feed = [];
+let seq = 0;
+const push = (item) => feed.push({ ...item, change_seq: ++seq });
+const sourcesClient = {
+ enabled: true,
+ async listItems({ after = 0, limit = 100 } = {}) {
+ const page = feed.filter((i) => i.change_seq > after).slice(0, limit);
+ return { items: page, next_after: page.length ? page[page.length - 1].change_seq : after, more: false };
+ },
+};
+
+// A recorded item of the board's feed, in the shape OpenVibe.Sources keeps it (a recorded fixture: no live call here).
+const wwr = (identity, over = {}) => ({
+ id: `itm_${identity}`, source_key: 'weworkremotely', category: 'jobs', kind: 'article', identity,
+ canonical_url: `https://weworkremotely.com/remote-jobs/${identity}`,
+ title: 'Welocalize: Automation Specialist',
+ summary: 'Headquarters: Ciudad de Mexico, Mexico URL: https://t.mtrbio.com/WeloData About the Role We are looking for an Automation Specialist. Job Title: Automation Specialist-1 Working Hours: 10:00-18:30 CDT Duration: Contract -indefinite . Salary/Compensation: 480,000 MXN/ yearly Location: Mexico (Remote)',
+ authors: [], published_at: '2026-10-29T12:00:00.000Z', source_updated_at: null,
+ fields: { categories: ['Full-Stack Programming'] },
+ revision: 1, provenance: { retrieved_at: '2026-10-30T06:00:00.000Z' }, removed: null, ...over,
+});
+
+(async () => {
+ const clock = { t: Date.parse('2026-10-30T08:00:00Z') };
+ const t = await boot({ now: () => clock.t, env: { WORK_WWR_FROM: 'sources' }, sourcesClient });
+ const s = t.ctx.s;
+ const from = t.ctx.fromSources;
+
+ try {
+ await check('a feed item becomes a listing with the board\'s own words', async () => {
+ push(wwr('automation-specialist-welocalize'));
+ // The thin-listing gate: no company in the title, so no listing may be published (Work #133).
+ push(wwr('automation-specialist', { title: 'Automation Specialist', canonical_url: 'https://weworkremotely.com/remote-jobs/automation-specialist' }));
+ const out = await from.pull();
+ assert.deepStrictEqual([out.applied, out.hold, out.removed], [1, 1, 0]);
+
+ const id = store.idOf('weworkremotely', 'automation-specialist-welocalize');
+ const row = await s.db.maybe('SELECT * FROM work_listings WHERE id = $1', [id]);
+ assert.deepStrictEqual([row.source, row.title, row.company, row.location, row.remote, row.tags, row.job_type, row.salary, row.expired],
+ ['weworkremotely', 'Automation Specialist', 'Welocalize', 'Mexico (Remote)', true, 'Full-Stack Programming', 'Contract -indefinite', '480,000 MXN/ yearly', false]);
+ assert.strictEqual(row.url, 'https://weworkremotely.com/remote-jobs/automation-specialist-welocalize');
+ assert.ok(row.excerpt.includes('Automation Specialist'), 'the excerpt is the board\'s own description');
+
+ const page = (await t.get('/api/v1/jobs?q=Automation')).json();
+ assert.deepStrictEqual(page.jobs.map((j) => [j.id, j.source, j.source_name]), [[id, 'weworkremotely', 'We Work Remotely']]);
+ });
+
+ await check('the listing is attributed to the board and links back to it', async () => {
+ const id = store.idOf('weworkremotely', 'automation-specialist-welocalize');
+ const list = await t.get('/jobs');
+ assert.strictEqual(list.status, 200);
+ assert.ok(list.text.includes('Listing from'), 'the listing carries its provenance');
+ assert.ok(list.text.includes('We Work Remotely'), 'the board is named');
+ assert.ok(list.text.includes('https://weworkremotely.com/remote-jobs/automation-specialist-welocalize'), 'the original listing is linked');
+ const page = await t.get(`/jobs/${id}`);
+ assert.strictEqual(page.status, 200);
+ assert.ok(page.text.includes('We Work Remotely'), 'the board is named on the listing page');
+ assert.ok(page.text.includes('https://weworkremotely.com/remote-jobs/automation-specialist-welocalize'), 'the original listing is linked');
+ });
+
+ await check('the fetch record says the listings came through, and Work never reads the feed itself', async () => {
+ const rec = (await store.sourceFetches(s)).get('weworkremotely');
+ assert.deepStrictEqual([rec.ok, rec.listings, rec.error], [true, 1, null]);
+ const out = await t.ctx.ingest.runAll();
+ assert.deepStrictEqual(out.find((o) => o.source === 'weworkremotely'), { source: 'weworkremotely', skipped: 'from_sources' });
+ });
+ } finally {
+ await t.close();
+ }
+ done();
+})().catch((err) => { console.error(err); process.exit(1); });