openvibers/openvibe.sources Commits
Sources: RemoteOK's job board, read under its robots and terms (#283)
Changes
@@@ -1918,6 +1918,49 @@"sensitivity": "none","default_indexability": "noindex"},++ {++ "key": "remoteok-job-board",++ "name": "RemoteOK job board API",++ "type": "api",++ "category": "jobs",++ "homepage_url": "https://remoteok.com/",++ "endpoints": [++ {++ "url": "https://remoteok.com/api",++ "format": "json",++ "items_path": "",++ "item_kind": "job",++ "where": { "position": { "pattern": "[A-Za-z]" } },++ "fields": {++ "identity": "id",++ "url": "url",++ "title": "position",++ "summary": "description",++ "published_at": "date"++ },++ "extra": {++ "company": "company",++ "location": "location",++ "job_type": "job_type",++ "salary_min": "salary_min",++ "salary_max": "salary_max"++ },++ "extra_lists": { "tags": "tags" }++ }++ ],++ "robots_note": "robots.txt checked 2026-11-12 from the production host: User-agent * allows /, with Content-Signal search=yes, ai-train=no, use=reference; the named AI training crawlers are disallowed in their own groups, which this source is not. The fetcher re-checks it on every run.",++ "terms_note": "RemoteOK's API response opens with its Terms of Service: link back (follow, not nofollow) to the URL on Remote OK and mention Remote OK as the source; do not use its logo without written permission. OpenVibe.Work keeps a short plain-text excerpt, names RemoteOK and links every listing back to its RemoteOK page. Read every 6 hours, the interval Work already used. The first element is the legal notice, not a listing: the mapping filters it out by requiring a position. Reviewed 2026-11-12 (plan F, the move of Work's own fetch into Sources): keep as noted.",++ "license_note": null,++ "min_interval_ms": 60000,++ "poll_interval_sec": 21600,++ "stale_after_sec": 43200,++ "max_items": 500,++ "retire_after_sec": 2592000,++ "enabled": false,++ "review_required": true,++ "sensitivity": "none",++ "default_indexability": "noindex"++ },{"key": "bls-average-prices","name": "BLS average retail food prices (US city average)",nnew file mode 100644@@@ -0,0 +1,76 @@++'use strict';++/**++ * Job boards in the jobs category (plan 2026-10-22, track F): RemoteOK's API is read by Sources instead of++ * OpenVibe.Work, under its own robots and terms. The response opens with a legal notice, not a listing, so the++ * mapping requires a position and the notice is filtered out. A listing keeps the board's id, its URL, the company,++ * place, tags and salary bounds, exactly the fields Work's own fetch kept.++ *++ * Remotive is deliberately not seeded: its robots.txt disallows /api/* (checked from the production host), and++ * Sources does not fetch a disallowed path. Work keeps reading that API directly, as its terms grant.++ */++const assert = require('assert');++const api = require('../server/adapters/api');++const { validateSource, CATEGORIES } = require('../server/registry');++const { load } = require('../server/config');++const { suite } = require('./helpers');++++const t = suite('job-board-sources');++const seeds = require('../seeds/sources.json').sources;++const remoteok = seeds.find((s) => s.key === 'remoteok-job-board');++++const NOTICE = { last_updated: 1794384008, legal: 'API Terms of Service: Please link back (with follow, and without nofollow!) to the URL on Remote OK and mention Remote OK as a source.' };++const JOB = {++ slug: 'remote-devops-engineer-skyward-1049001',++ id: '1049001',++ date: '2026-11-10T00:00:15+00:00',++ company: 'Skyward Cloud',++ position: 'Remote DevOps Engineer',++ tags: ['devops', 'kubernetes'],++ description: '<h2>About the role</h2><p>Run our Kubernetes fleet.</p>',++ location: 'Worldwide',++ salary_min: 90000,++ salary_max: 130000,++ url: 'https://remoteok.com/remote-jobs/remote-devops-engineer-skyward-1049001',++};++const BODY = JSON.stringify([NOTICE, JOB, { id: 'no-position', company: 'Empty' }]);++++t('the RemoteOK seed validates, is a disabled jobs source, and is read every 6 hours', () => {++ assert.ok(remoteok, 'remoteok-job-board is seeded');++ assert.ok(CATEGORIES.includes('jobs'));++ const rec = validateSource(remoteok);++ assert.strictEqual(rec.enabled, false);++ assert.strictEqual(rec.review_required, true);++ assert.strictEqual(rec.category, 'jobs');++ assert.ok(rec.robots_note && rec.terms_note, 'robots and terms notes');++ assert.strictEqual(new URL(remoteok.endpoints[0].url).host, 'remoteok.com');++ assert.strictEqual(rec.poll_interval_sec, 21600);++ assert.strictEqual(rec.retire_after_sec, 2592000, 'a job the board stopped showing is retired after 30 days');++ assert.ok(rec.max_items <= load({}).items.maxPerFetch);++});++++t('the legal notice is filtered out and a listing keeps the board\'s own id, URL, fields and lists', () => {++ const out = api.parse(BODY, { url: remoteok.endpoints[0].url, endpoint: remoteok.endpoints[0], maxItems: remoteok.max_items });++ assert.strictEqual(out.items.length, 1);++ assert.strictEqual(out.meta.filtered, 2, 'the notice and the record with no position');++ const [job] = out.items;++ assert.strictEqual(job.kind, 'job');++ assert.strictEqual(job.identity, '1049001');++ assert.strictEqual(job.canonical_url, JOB.url);++ assert.strictEqual(job.title, 'Remote DevOps Engineer');++ assert.strictEqual(job.summary, 'About the role Run our Kubernetes fleet.');++ assert.strictEqual(job.published_at, '2026-11-10T00:00:15.000Z');++ assert.deepStrictEqual(job.fields, {++ company: 'Skyward Cloud',++ location: 'Worldwide',++ job_type: null,++ salary_min: 90000,++ salary_max: 130000,++ tags: ['devops', 'kubernetes'],++ });++});++++t('Remotive is not a source: its robots.txt disallows the API path Sources would have to fetch', () => {++ assert.ok(!seeds.some((s) => /remotive/i.test(s.key)), 'Remotive stays outside Sources');++});++++t.run();
@@ -1918,6 +1918,49 @@
@@ -1918,6 +1918,49 @@
"sensitivity": "none",
"sensitivity": "none",
"default_indexability": "noindex"
"default_indexability": "noindex"
},
},
+ {
+ "key": "remoteok-job-board",
+ "name": "RemoteOK job board API",
+ "type": "api",
+ "category": "jobs",
+ "homepage_url": "https://remoteok.com/",
+ "endpoints": [
+ {
+ "url": "https://remoteok.com/api",
+ "format": "json",
+ "items_path": "",
+ "item_kind": "job",
+ "where": { "position": { "pattern": "[A-Za-z]" } },
+ "fields": {
+ "identity": "id",
+ "url": "url",
+ "title": "position",
+ "summary": "description",
+ "published_at": "date"
+ },
+ "extra": {
+ "company": "company",
+ "location": "location",
+ "job_type": "job_type",
+ "salary_min": "salary_min",
+ "salary_max": "salary_max"
+ },
+ "extra_lists": { "tags": "tags" }
+ }
+ ],
+ "robots_note": "robots.txt checked 2026-11-12 from the production host: User-agent * allows /, with Content-Signal search=yes, ai-train=no, use=reference; the named AI training crawlers are disallowed in their own groups, which this source is not. The fetcher re-checks it on every run.",
+ "terms_note": "RemoteOK's API response opens with its Terms of Service: link back (follow, not nofollow) to the URL on Remote OK and mention Remote OK as the source; do not use its logo without written permission. OpenVibe.Work keeps a short plain-text excerpt, names RemoteOK and links every listing back to its RemoteOK page. Read every 6 hours, the interval Work already used. The first element is the legal notice, not a listing: the mapping filters it out by requiring a position. Reviewed 2026-11-12 (plan F, the move of Work's own fetch into Sources): keep as noted.",
+ "license_note": null,
+ "min_interval_ms": 60000,
+ "poll_interval_sec": 21600,
+ "stale_after_sec": 43200,
+ "max_items": 500,
+ "retire_after_sec": 2592000,
+ "enabled": false,
+ "review_required": true,
+ "sensitivity": "none",
+ "default_indexability": "noindex"
+ },
{
{
"key": "bls-average-prices",
"key": "bls-average-prices",
"name": "BLS average retail food prices (US city average)",
"name": "BLS average retail food prices (US city average)",
new file mode 100644
new file mode 100644
@@ -0,0 +1,76 @@
@@ -0,0 +1,76 @@
+'use strict';
+/**
+ * Job boards in the jobs category (plan 2026-10-22, track F): RemoteOK's API is read by Sources instead of
+ * OpenVibe.Work, under its own robots and terms. The response opens with a legal notice, not a listing, so the
+ * mapping requires a position and the notice is filtered out. A listing keeps the board's id, its URL, the company,
+ * place, tags and salary bounds, exactly the fields Work's own fetch kept.
+ *
+ * Remotive is deliberately not seeded: its robots.txt disallows /api/* (checked from the production host), and
+ * Sources does not fetch a disallowed path. Work keeps reading that API directly, as its terms grant.
+ */
+const assert = require('assert');
+const api = require('../server/adapters/api');
+const { validateSource, CATEGORIES } = require('../server/registry');
+const { load } = require('../server/config');
+const { suite } = require('./helpers');
+
+const t = suite('job-board-sources');
+const seeds = require('../seeds/sources.json').sources;
+const remoteok = seeds.find((s) => s.key === 'remoteok-job-board');
+
+const NOTICE = { last_updated: 1794384008, legal: 'API Terms of Service: Please link back (with follow, and without nofollow!) to the URL on Remote OK and mention Remote OK as a source.' };
+const JOB = {
+ slug: 'remote-devops-engineer-skyward-1049001',
+ id: '1049001',
+ date: '2026-11-10T00:00:15+00:00',
+ company: 'Skyward Cloud',
+ position: 'Remote DevOps Engineer',
+ tags: ['devops', 'kubernetes'],
+ description: '<h2>About the role</h2><p>Run our Kubernetes fleet.</p>',
+ location: 'Worldwide',
+ salary_min: 90000,
+ salary_max: 130000,
+ url: 'https://remoteok.com/remote-jobs/remote-devops-engineer-skyward-1049001',
+};
+const BODY = JSON.stringify([NOTICE, JOB, { id: 'no-position', company: 'Empty' }]);
+
+t('the RemoteOK seed validates, is a disabled jobs source, and is read every 6 hours', () => {
+ assert.ok(remoteok, 'remoteok-job-board is seeded');
+ assert.ok(CATEGORIES.includes('jobs'));
+ const rec = validateSource(remoteok);
+ assert.strictEqual(rec.enabled, false);
+ assert.strictEqual(rec.review_required, true);
+ assert.strictEqual(rec.category, 'jobs');
+ assert.ok(rec.robots_note && rec.terms_note, 'robots and terms notes');
+ assert.strictEqual(new URL(remoteok.endpoints[0].url).host, 'remoteok.com');
+ assert.strictEqual(rec.poll_interval_sec, 21600);
+ assert.strictEqual(rec.retire_after_sec, 2592000, 'a job the board stopped showing is retired after 30 days');
+ assert.ok(rec.max_items <= load({}).items.maxPerFetch);
+});
+
+t('the legal notice is filtered out and a listing keeps the board\'s own id, URL, fields and lists', () => {
+ const out = api.parse(BODY, { url: remoteok.endpoints[0].url, endpoint: remoteok.endpoints[0], maxItems: remoteok.max_items });
+ assert.strictEqual(out.items.length, 1);
+ assert.strictEqual(out.meta.filtered, 2, 'the notice and the record with no position');
+ const [job] = out.items;
+ assert.strictEqual(job.kind, 'job');
+ assert.strictEqual(job.identity, '1049001');
+ assert.strictEqual(job.canonical_url, JOB.url);
+ assert.strictEqual(job.title, 'Remote DevOps Engineer');
+ assert.strictEqual(job.summary, 'About the role Run our Kubernetes fleet.');
+ assert.strictEqual(job.published_at, '2026-11-10T00:00:15.000Z');
+ assert.deepStrictEqual(job.fields, {
+ company: 'Skyward Cloud',
+ location: 'Worldwide',
+ job_type: null,
+ salary_min: 90000,
+ salary_max: 130000,
+ tags: ['devops', 'kubernetes'],
+ });
+});
+
+t('Remotive is not a source: its robots.txt disallows the API path Sources would have to fetch', () => {
+ assert.ok(!seeds.some((s) => /remotive/i.test(s.key)), 'Remotive stays outside Sources');
+});
+
+t.run();