openvibers/openvibe.sources Commits

Arbeitnow reads what Work reads (pages 1-3, a plain-text summary); a capped HTML summary keeps no half tag (#91)

OP OpenVibers committed 21 days ago · 1a616afe3a · 3 files, +49 −2

Changes

ddiff --git a/seeds/sources.json b/seeds/sources.json
iindex 5bab631..f500030 100644
---- a/seeds/sources.json
++++ b/seeds/sources.json
@@@ -1753,6 +1753,43 @@
"identity": "slug",
"url": "url",
"title": "title",
++ "summary": "description",
++ "published_at_epoch": "created_at"
++ },
++ "extra": {
++ "company": "company_name",
++ "location": "location",
++ "remote": "remote"
++ }
++ },
++ {
++ "url": "https://www.arbeitnow.com/api/job-board-api?page=2",
++ "format": "json",
++ "items_path": "data",
++ "item_kind": "job",
++ "fields": {
++ "identity": "slug",
++ "url": "url",
++ "title": "title",
++ "summary": "description",
++ "published_at_epoch": "created_at"
++ },
++ "extra": {
++ "company": "company_name",
++ "location": "location",
++ "remote": "remote"
++ }
++ },
++ {
++ "url": "https://www.arbeitnow.com/api/job-board-api?page=3",
++ "format": "json",
++ "items_path": "data",
++ "item_kind": "job",
++ "fields": {
++ "identity": "slug",
++ "url": "url",
++ "title": "title",
++ "summary": "description",
"published_at_epoch": "created_at"
},
"extra": {
@@@ -1763,7 +1800,7 @@
}
],
"robots_note": "robots.txt checked 2026-10-22: User-agent * with an empty Disallow (only ?__hstc tracking URLs excluded). The fetcher re-checks it on every run.",
-- "terms_note": "Arbeitnow publishes its job board API free of charge (the terms OpenVibe.Work records today): attribute Arbeitnow by name and link every listing to its page on arbeitnow.com. The description HTML is deliberately not mapped. The source OpenVibe.Work fetches itself today; Work moves to it here. Re-verify before enabling.",
++ "terms_note": "Arbeitnow publishes its job board API free of charge (the terms OpenVibe.Work records today): attribute Arbeitnow by name and link every listing to its page on arbeitnow.com. The first three pages are read, as Work read them. The description is kept only as a short plain-text excerpt (the summary, as Work shows it); every listing links to arbeitnow.com for the full text. A job the board stops listing is retired after a week. Work moves to this source from its own fetch.",
"license_note": null,
"min_interval_ms": 60000,
"poll_interval_sec": 21600,
ddiff --git a/server/adapters/util.js b/server/adapters/util.js
iindex baaee4a..712678a 100644
---- a/server/adapters/util.js
++++ b/server/adapters/util.js
@@@ -59,7 +59,8 @@ function toText(value, max = 1000) {
s = decodeEntities(String(s))
.replace(/<(script|style|noscript)[\s\S]*?<\/\1>/gi, ' ')
.replace(/<br\s*\/?>|<\/p>|<\/li>|<\/h\d>/gi, ' ')
-- .replace(/<[^>]*>/g, ' ');
++ .replace(/<[^>]*>/g, ' ')
++ .replace(/<[^>]*$/, ' '); // a tag cut off where a capped value ended
s = decodeEntities(s).replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, '').replace(/\s+/g, ' ').trim();
if (!s) return null;
return s.length > max ? `${s.slice(0, max - 1).trimEnd()}…` : s;
ddiff --git a/test/adapters.test.js b/test/adapters.test.js
iindex 06adb82..4354716 100644
---- a/test/adapters.test.js
++++ b/test/adapters.test.js
@@@ -196,4 +196,13 @@ t('helpers: charset decoding, date and URL strictness', () => {
assert.strictEqual(canonicalUrl('HTTPS://Example.COM:443/a?gclid=1&b=2#x'), 'https://example.com/a?b=2');
});
++t('an api summary from HTML longer than the 500-character cap keeps no half tag (Arbeitnow descriptions)', () => {
++ const description = `<p><strong>About us</strong></p><p>${'We build things. '.repeat(26)}</p><p><a href="https://example.org/apply">Apply</a></p>`;
++ const body = JSON.stringify({ data: [{ slug: 'dev-1', url: 'https://www.arbeitnow.com/jobs/dev-1', title: 'Developer', description, created_at: 1761177600 }] });
++ const endpoint = { format: 'json', items_path: 'data', item_kind: 'job', fields: { identity: 'slug', url: 'url', title: 'title', summary: 'description', published_at_epoch: 'created_at' } };
++ const [item] = api.parse(body, { url: 'https://www.arbeitnow.com/api/job-board-api', endpoint }).items;
++ assert.match(item.summary, /^About us We build things\./);
++ assert.ok(!/[<>]/.test(item.summary), `no markup left: ${item.summary.slice(-60)}`);
++});
++
t.run();