openvibers/openvibe.sources Commits
Arbeitnow reads what Work reads (pages 1-3, a plain-text summary); a capped HTML summary keeps no half tag (#91)
Changes
@@@ -1753,6 +1753,43 @@"identity": "slug","url": "url","title": "title",++ "summary": "description",++ "published_at_epoch": "created_at"++ },++ "extra": {++ "company": "company_name",++ "location": "location",++ "remote": "remote"++ }++ },++ {++ "url": "https://www.arbeitnow.com/api/job-board-api?page=2",++ "format": "json",++ "items_path": "data",++ "item_kind": "job",++ "fields": {++ "identity": "slug",++ "url": "url",++ "title": "title",++ "summary": "description",++ "published_at_epoch": "created_at"++ },++ "extra": {++ "company": "company_name",++ "location": "location",++ "remote": "remote"++ }++ },++ {++ "url": "https://www.arbeitnow.com/api/job-board-api?page=3",++ "format": "json",++ "items_path": "data",++ "item_kind": "job",++ "fields": {++ "identity": "slug",++ "url": "url",++ "title": "title",++ "summary": "description","published_at_epoch": "created_at"},"extra": {@@@ -1763,7 +1800,7 @@}],"robots_note": "robots.txt checked 2026-10-22: User-agent * with an empty Disallow (only ?__hstc tracking URLs excluded). The fetcher re-checks it on every run.",-- "terms_note": "Arbeitnow publishes its job board API free of charge (the terms OpenVibe.Work records today): attribute Arbeitnow by name and link every listing to its page on arbeitnow.com. The description HTML is deliberately not mapped. The source OpenVibe.Work fetches itself today; Work moves to it here. Re-verify before enabling.",++ "terms_note": "Arbeitnow publishes its job board API free of charge (the terms OpenVibe.Work records today): attribute Arbeitnow by name and link every listing to its page on arbeitnow.com. The first three pages are read, as Work read them. The description is kept only as a short plain-text excerpt (the summary, as Work shows it); every listing links to arbeitnow.com for the full text. A job the board stops listing is retired after a week. Work moves to this source from its own fetch.","license_note": null,"min_interval_ms": 60000,"poll_interval_sec": 21600,@@@ -59,7 +59,8 @@ function toText(value, max = 1000) {s = decodeEntities(String(s)).replace(/<(script|style|noscript)[\s\S]*?<\/\1>/gi, ' ').replace(/<br\s*\/?>|<\/p>|<\/li>|<\/h\d>/gi, ' ')-- .replace(/<[^>]*>/g, ' ');++ .replace(/<[^>]*>/g, ' ')++ .replace(/<[^>]*$/, ' '); // a tag cut off where a capped value endeds = decodeEntities(s).replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, '').replace(/\s+/g, ' ').trim();if (!s) return null;return s.length > max ? `${s.slice(0, max - 1).trimEnd()}…` : s;@@@ -196,4 +196,13 @@ t('helpers: charset decoding, date and URL strictness', () => {assert.strictEqual(canonicalUrl('HTTPS://Example.COM:443/a?gclid=1&b=2#x'), 'https://example.com/a?b=2');});++t('an api summary from HTML longer than the 500-character cap keeps no half tag (Arbeitnow descriptions)', () => {++ const description = `<p><strong>About us</strong></p><p>${'We build things. '.repeat(26)}</p><p><a href="https://example.org/apply">Apply</a></p>`;++ const body = JSON.stringify({ data: [{ slug: 'dev-1', url: 'https://www.arbeitnow.com/jobs/dev-1', title: 'Developer', description, created_at: 1761177600 }] });++ const endpoint = { format: 'json', items_path: 'data', item_kind: 'job', fields: { identity: 'slug', url: 'url', title: 'title', summary: 'description', published_at_epoch: 'created_at' } };++ const [item] = api.parse(body, { url: 'https://www.arbeitnow.com/api/job-board-api', endpoint }).items;++ assert.match(item.summary, /^About us We build things\./);++ assert.ok(!/[<>]/.test(item.summary), `no markup left: ${item.summary.slice(-60)}`);++});++t.run();
@@ -1753,6 +1753,43 @@
@@ -1753,6 +1753,43 @@
"identity": "slug",
"identity": "slug",
"url": "url",
"url": "url",
"title": "title",
"title": "title",
+ "summary": "description",
+ "published_at_epoch": "created_at"
+ },
+ "extra": {
+ "company": "company_name",
+ "location": "location",
+ "remote": "remote"
+ }
+ },
+ {
+ "url": "https://www.arbeitnow.com/api/job-board-api?page=2",
+ "format": "json",
+ "items_path": "data",
+ "item_kind": "job",
+ "fields": {
+ "identity": "slug",
+ "url": "url",
+ "title": "title",
+ "summary": "description",
+ "published_at_epoch": "created_at"
+ },
+ "extra": {
+ "company": "company_name",
+ "location": "location",
+ "remote": "remote"
+ }
+ },
+ {
+ "url": "https://www.arbeitnow.com/api/job-board-api?page=3",
+ "format": "json",
+ "items_path": "data",
+ "item_kind": "job",
+ "fields": {
+ "identity": "slug",
+ "url": "url",
+ "title": "title",
+ "summary": "description",
"published_at_epoch": "created_at"
"published_at_epoch": "created_at"
},
},
"extra": {
"extra": {
@@ -1763,7 +1800,7 @@
@@ -1763,7 +1800,7 @@
}
}
],
],
"robots_note": "robots.txt checked 2026-10-22: User-agent * with an empty Disallow (only ?__hstc tracking URLs excluded). The fetcher re-checks it on every run.",
"robots_note": "robots.txt checked 2026-10-22: User-agent * with an empty Disallow (only ?__hstc tracking URLs excluded). The fetcher re-checks it on every run.",
- "terms_note": "Arbeitnow publishes its job board API free of charge (the terms OpenVibe.Work records today): attribute Arbeitnow by name and link every listing to its page on arbeitnow.com. The description HTML is deliberately not mapped. The source OpenVibe.Work fetches itself today; Work moves to it here. Re-verify before enabling.",
+ "terms_note": "Arbeitnow publishes its job board API free of charge (the terms OpenVibe.Work records today): attribute Arbeitnow by name and link every listing to its page on arbeitnow.com. The first three pages are read, as Work read them. The description is kept only as a short plain-text excerpt (the summary, as Work shows it); every listing links to arbeitnow.com for the full text. A job the board stops listing is retired after a week. Work moves to this source from its own fetch.",
"license_note": null,
"license_note": null,
"min_interval_ms": 60000,
"min_interval_ms": 60000,
"poll_interval_sec": 21600,
"poll_interval_sec": 21600,
@@ -59,7 +59,8 @@ function toText(value, max = 1000) {
@@ -59,7 +59,8 @@ function toText(value, max = 1000) {
s = decodeEntities(String(s))
s = decodeEntities(String(s))
.replace(/<(script|style|noscript)[\s\S]*?<\/\1>/gi, ' ')
.replace(/<(script|style|noscript)[\s\S]*?<\/\1>/gi, ' ')
.replace(/<br\s*\/?>|<\/p>|<\/li>|<\/h\d>/gi, ' ')
.replace(/<br\s*\/?>|<\/p>|<\/li>|<\/h\d>/gi, ' ')
- .replace(/<[^>]*>/g, ' ');
+ .replace(/<[^>]*>/g, ' ')
+ .replace(/<[^>]*$/, ' '); // a tag cut off where a capped value ended
s = decodeEntities(s).replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, '').replace(/\s+/g, ' ').trim();
s = decodeEntities(s).replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, '').replace(/\s+/g, ' ').trim();
if (!s) return null;
if (!s) return null;
return s.length > max ? `${s.slice(0, max - 1).trimEnd()}…` : s;
return s.length > max ? `${s.slice(0, max - 1).trimEnd()}…` : s;
@@ -196,4 +196,13 @@ t('helpers: charset decoding, date and URL strictness', () => {
@@ -196,4 +196,13 @@ t('helpers: charset decoding, date and URL strictness', () => {
assert.strictEqual(canonicalUrl('HTTPS://Example.COM:443/a?gclid=1&b=2#x'), 'https://example.com/a?b=2');
assert.strictEqual(canonicalUrl('HTTPS://Example.COM:443/a?gclid=1&b=2#x'), 'https://example.com/a?b=2');
});
});
+t('an api summary from HTML longer than the 500-character cap keeps no half tag (Arbeitnow descriptions)', () => {
+ const description = `<p><strong>About us</strong></p><p>${'We build things. '.repeat(26)}</p><p><a href="https://example.org/apply">Apply</a></p>`;
+ const body = JSON.stringify({ data: [{ slug: 'dev-1', url: 'https://www.arbeitnow.com/jobs/dev-1', title: 'Developer', description, created_at: 1761177600 }] });
+ const endpoint = { format: 'json', items_path: 'data', item_kind: 'job', fields: { identity: 'slug', url: 'url', title: 'title', summary: 'description', published_at_epoch: 'created_at' } };
+ const [item] = api.parse(body, { url: 'https://www.arbeitnow.com/api/job-board-api', endpoint }).items;
+ assert.match(item.summary, /^About us We build things\./);
+ assert.ok(!/[<>]/.test(item.summary), `no markup left: ${item.summary.slice(-60)}`);
+});
+
t.run();
t.run();