openvibers/openvibe.sources Commits
An api endpoint may map lists (extra_lists: tags, job types); Arbeitnow maps both (contracts v0.212.0) (#92)
Changes
@@@ -414,7 +414,7 @@ vhost serves it.Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).<!-- versions:start -->--- openvibe-contracts: v0.211.0++- openvibe-contracts: v0.212.0- openvibe-sdk: v0.62.0- openvibe-shared: v4.9.0<!-- versions:end -->@@@ -7,7 +7,7 @@"runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, fast-xml-parser","port": 4720,"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, health and JSON 404s; the API is loopback-only; README \"Public host\")",-- "contracts": "openvibe-contracts v0.211.0",++ "contracts": "openvibe-contracts v0.212.0","sdk": "openvibe-sdk v0.62.0","shared": "openvibe-shared v4.9.0","capabilities": [@@@ -82,8 +82,17 @@"format": {"enum": ["json",-- "xml"-- ]++ "xml",++ "tsv"++ ],++ "description": "json (default), xml, or tsv: a tab-separated file whose first line names the columns, read as an array of records keyed by those names (items_path is \"\"; BLS series files, Sources 2026-10-24)."++ },++ "duplicates": {++ "enum": [++ "first",++ "last"++ ],++ "description": "Two records with the same identity: first (default) keeps the first, last keeps the last in the first one's place (a file listing each series oldest to newest then yields its newest value; Sources, 2026-10-24)."},"items_path": {"type": "string"@@@ -170,6 +179,14 @@},"description": "Extra fields copied as the API gave them (scalars). A path is relative to the record; $key is the map key; $root.<path> reads the response from its root, where a {$key} segment is the record's map key; * steps into an object's single child; key[field=value,...] (one to four conditions) steps into the first element of the array key whose fields equal the values, compared as text, where a value is literal text or $rec.<path> read from the record (Sources#51)."},++ "extra_lists": {++ "type": "object",++ "maxProperties": 10,++ "additionalProperties": {++ "type": "string"++ },++ "description": "Lists the API gives (tags, job types): each field maps to a path whose array of scalars is kept as up to 20 of them; a lone scalar is a list of one and a missing value an empty list. Separate from extra, which reads an array as null (Sources, 2026-10-24)."++ },"extract": {"type": "object","additionalProperties": false,@@@ -12,7 +12,7 @@"dotenv": "^16.4.7","express": "^4.21.2","fast-xml-parser": "^4.5.3",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.211.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.212.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.62.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.9.0","pg": "^8.23.0"@@@ -769,9 +769,9 @@}},"node_modules/openvibe-contracts": {-- "version": "0.211.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.211.0",-- "integrity": "sha512-M/DHcyQcxmwrT65s64D/mJn77ANF6Je5SeDgpWTZKKhxPffKVuhV+SzRZf8grZWvD3flmSqWNTTPOcKUWTjEug==",++ "version": "0.212.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.212.0",++ "integrity": "sha512-qt4iodXw+sZiSSPGA8p3MWgGDr6iXjC9LGyfhLBRpnfwfggGeKajRotlGp0EyGrKko6zu0K8DFVZghK5XDmb/A==","license": "AGPL-3.0-only","dependencies": {"ajv": "^8.17.1",@@@ -27,7 +27,7 @@"dotenv": "^16.4.7","express": "^4.21.2","fast-xml-parser": "^4.5.3",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.211.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.212.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.62.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.9.0","pg": "^8.23.0"@@@ -1760,6 +1760,10 @@"company": "company_name","location": "location","remote": "remote"++ },++ "extra_lists": {++ "tags": "tags",++ "job_types": "job_types"}},{@@@ -1778,6 +1782,10 @@"company": "company_name","location": "location","remote": "remote"++ },++ "extra_lists": {++ "tags": "tags",++ "job_types": "job_types"}},{@@@ -1796,6 +1804,10 @@"company": "company_name","location": "location","remote": "remote"++ },++ "extra_lists": {++ "tags": "tags",++ "job_types": "job_types"}}],@@@ -13,6 +13,10 @@* the array. `extra` values are copied as the API gave them (scalars only, strings capped):* a missing field is null, never a default. A record without an identity is skipped.*++ * `extra_lists` maps a field to a list the API gives (tags, job types): an array of scalars is kept++ * as up to 20 of them (a lone scalar is a list of one, a missing one an empty list). It is separate++ * from `extra` so a mapping that points `extra` at an array keeps reading null, as it always did.++ ** A path segment `*` steps into the single child of an object: when the current value is a* non-array object with exactly one own property it steps into that property, and otherwise the* path resolves to undefined (so the field is null). It may appear anywhere in a path.@@@ -144,6 +148,18 @@ function at(obj, path) {return walk(obj, path, undefined);}++/** A list the API gives → up to 20 of its scalars (a lone scalar → a list of one; nothing → []). */++function scalars(v) {++ const all = Array.isArray(v) ? v : v == null ? [] : [v];++ const out = [];++ for (const x of all) {++ const s = scalar(x);++ if (s != null && s !== '') out.push(s);++ if (out.length >= 20) break;++ }++ return out;++}++function scalar(v) {if (v == null) return null;if (typeof v === 'number' || typeof v === 'boolean') return Number.isFinite(v) || typeof v === 'boolean' ? v : null;@@@ -217,6 +233,15 @@ function checkMapping(ep) {const kerr = keyPathViolation(p, shape);if (kerr) return `extra.${k}: ${kerr}`;}++ const lists = ep.extra_lists == null ? {} : ep.extra_lists;++ if (typeof lists !== 'object' || Array.isArray(lists)) return 'extra_lists must be an object of field -> path';++ if (Object.keys(lists).length > 10) return 'at most 10 extra_lists fields';++ for (const [k, p] of Object.entries(lists)) {++ if (!/^[a-z][a-z0-9_]{0,39}$/.test(k) || typeof p !== 'string' || !PATH_RE.test(p)) return `extra_lists.${k} is malformed`;++ if (Object.prototype.hasOwnProperty.call(extra, k)) return `extra_lists.${k} is also an extra field`;++ const kerr = keyPathViolation(p, shape);++ if (kerr) return `extra_lists.${k}: ${kerr}`;++ }const where = ep.where == null ? {} : ep.where;if (typeof where !== 'object' || Array.isArray(where)) return 'where must be an object of path -> expected';if (Object.keys(where).length > 10) return 'at most 10 where conditions';@@@ -278,6 +303,7 @@ function parse(text, { url, endpoint = {}, summaryMax = 1000, maxItems = 500 } =seen.add(identity);const extra = {};for (const [k, p] of Object.entries(endpoint.extra || {})) extra[k] = scalar(get(p));++ for (const [k, p] of Object.entries(endpoint.extra_lists || {})) extra[k] = scalars(get(p));items.push({identity: String(identity),kind: endpoint.item_kind || 'record',@@@ -79,6 +79,8 @@ function validateSource(input, { maxItemsCap = 12000 } = {}) {Object.assign(out, { format: e.format || 'json', items_shape: e.items_shape || 'array', items_path: e.items_path || '', item_kind: e.item_kind || 'record', fields: e.fields || {}, extra: e.extra || {} });// `where` is only copied when given, so records without conditions stay byte-identical.if (e.where) out.where = e.where;++ // and `extra_lists` (tags, job types) the same way.++ if (e.extra_lists) out.extra_lists = e.extra_lists;if (e.max_bytes !== undefined) out.max_bytes = e.max_bytes;if (e.timeout_ms !== undefined) out.timeout_ms = e.timeout_ms;if (e.request_content_type !== undefined) out.request_content_type = e.request_content_type;@@@ -205,4 +205,22 @@ t('an api summary from HTML longer than the 500-character cap keeps no half tagassert.ok(!/[<>]/.test(item.summary), `no markup left: ${item.summary.slice(-60)}`);});++t('api extra_lists keep an array of scalars as a list (tags, job types); extra still reads an array as null', () => {++ const body = JSON.stringify({ data: [++ { slug: 'a', url: 'https://www.arbeitnow.com/jobs/a', title: 'A', tags: ['Node.js', 'PostgreSQL', { x: 1 }, ''], job_types: ['full time'] },++ { slug: 'b', url: 'https://www.arbeitnow.com/jobs/b', title: 'B', tags: 'Go', job_types: null },++ { slug: 'c', url: 'https://www.arbeitnow.com/jobs/c', title: 'C', tags: Array.from({ length: 30 }, (_, i) => `t${i}`) },++ ] });++ const endpoint = { format: 'json', items_path: 'data', item_kind: 'job', fields: { identity: 'slug', url: 'url', title: 'title' },++ extra: { tags_scalar: 'tags' }, extra_lists: { tags: 'tags', job_types: 'job_types' } };++ assert.strictEqual(api.checkMapping(endpoint), null);++ const [a, b, c] = api.parse(body, { url: 'https://www.arbeitnow.com/api/job-board-api', endpoint }).items;++ assert.deepStrictEqual([a.fields.tags, a.fields.job_types, a.fields.tags_scalar], [['Node.js', 'PostgreSQL'], ['full time'], null]);++ assert.deepStrictEqual([b.fields.tags, b.fields.job_types], [['Go'], []]);++ assert.strictEqual(c.fields.tags.length, 20);++ assert.match(api.checkMapping({ ...endpoint, extra: { tags: 'tags' } }), /also an extra field/);++ assert.match(api.checkMapping({ ...endpoint, extra_lists: Object.fromEntries(Array.from({ length: 11 }, (_, i) => [`l${i}`, 'tags'])) }), /at most 10/);++ assert.match(api.checkMapping({ ...endpoint, extra_lists: ['tags'] }), /must be an object/);++});++t.run();
@@ -414,7 +414,7 @@ vhost serves it.
@@ -414,7 +414,7 @@ vhost serves it.
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
<!-- versions:start -->
<!-- versions:start -->
-- openvibe-contracts: v0.211.0
+- openvibe-contracts: v0.212.0
- openvibe-sdk: v0.62.0
- openvibe-sdk: v0.62.0
- openvibe-shared: v4.9.0
- openvibe-shared: v4.9.0
<!-- versions:end -->
<!-- versions:end -->
@@ -7,7 +7,7 @@
@@ -7,7 +7,7 @@
"runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, fast-xml-parser",
"runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, fast-xml-parser",
"port": 4720,
"port": 4720,
"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, health and JSON 404s; the API is loopback-only; README \"Public host\")",
"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, health and JSON 404s; the API is loopback-only; README \"Public host\")",
- "contracts": "openvibe-contracts v0.211.0",
+ "contracts": "openvibe-contracts v0.212.0",
"sdk": "openvibe-sdk v0.62.0",
"sdk": "openvibe-sdk v0.62.0",
"shared": "openvibe-shared v4.9.0",
"shared": "openvibe-shared v4.9.0",
"capabilities": [
"capabilities": [
@@ -82,8 +82,17 @@
@@ -82,8 +82,17 @@
"format": {
"format": {
"enum": [
"enum": [
"json",
"json",
- "xml"
- ]
+ "xml",
+ "tsv"
+ ],
+ "description": "json (default), xml, or tsv: a tab-separated file whose first line names the columns, read as an array of records keyed by those names (items_path is \"\"; BLS series files, Sources 2026-10-24)."
+ },
+ "duplicates": {
+ "enum": [
+ "first",
+ "last"
+ ],
+ "description": "Two records with the same identity: first (default) keeps the first, last keeps the last in the first one's place (a file listing each series oldest to newest then yields its newest value; Sources, 2026-10-24)."
},
},
"items_path": {
"items_path": {
"type": "string"
"type": "string"
@@ -170,6 +179,14 @@
@@ -170,6 +179,14 @@
},
},
"description": "Extra fields copied as the API gave them (scalars). A path is relative to the record; $key is the map key; $root.<path> reads the response from its root, where a {$key} segment is the record's map key; * steps into an object's single child; key[field=value,...] (one to four conditions) steps into the first element of the array key whose fields equal the values, compared as text, where a value is literal text or $rec.<path> read from the record (Sources#51)."
"description": "Extra fields copied as the API gave them (scalars). A path is relative to the record; $key is the map key; $root.<path> reads the response from its root, where a {$key} segment is the record's map key; * steps into an object's single child; key[field=value,...] (one to four conditions) steps into the first element of the array key whose fields equal the values, compared as text, where a value is literal text or $rec.<path> read from the record (Sources#51)."
},
},
+ "extra_lists": {
+ "type": "object",
+ "maxProperties": 10,
+ "additionalProperties": {
+ "type": "string"
+ },
+ "description": "Lists the API gives (tags, job types): each field maps to a path whose array of scalars is kept as up to 20 of them; a lone scalar is a list of one and a missing value an empty list. Separate from extra, which reads an array as null (Sources, 2026-10-24)."
+ },
"extract": {
"extract": {
"type": "object",
"type": "object",
"additionalProperties": false,
"additionalProperties": false,
@@ -12,7 +12,7 @@
@@ -12,7 +12,7 @@
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"fast-xml-parser": "^4.5.3",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.211.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.212.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.62.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.62.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.9.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.9.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -769,9 +769,9 @@
@@ -769,9 +769,9 @@
}
}
},
},
"node_modules/openvibe-contracts": {
"node_modules/openvibe-contracts": {
- "version": "0.211.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.211.0",
- "integrity": "sha512-M/DHcyQcxmwrT65s64D/mJn77ANF6Je5SeDgpWTZKKhxPffKVuhV+SzRZf8grZWvD3flmSqWNTTPOcKUWTjEug==",
+ "version": "0.212.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.212.0",
+ "integrity": "sha512-qt4iodXw+sZiSSPGA8p3MWgGDr6iXjC9LGyfhLBRpnfwfggGeKajRotlGp0EyGrKko6zu0K8DFVZghK5XDmb/A==",
"license": "AGPL-3.0-only",
"license": "AGPL-3.0-only",
"dependencies": {
"dependencies": {
"ajv": "^8.17.1",
"ajv": "^8.17.1",
@@ -27,7 +27,7 @@
@@ -27,7 +27,7 @@
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"fast-xml-parser": "^4.5.3",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.211.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.212.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.62.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.62.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.9.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.9.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -1760,6 +1760,10 @@
@@ -1760,6 +1760,10 @@
"company": "company_name",
"company": "company_name",
"location": "location",
"location": "location",
"remote": "remote"
"remote": "remote"
+ },
+ "extra_lists": {
+ "tags": "tags",
+ "job_types": "job_types"
}
}
},
},
{
{
@@ -1778,6 +1782,10 @@
@@ -1778,6 +1782,10 @@
"company": "company_name",
"company": "company_name",
"location": "location",
"location": "location",
"remote": "remote"
"remote": "remote"
+ },
+ "extra_lists": {
+ "tags": "tags",
+ "job_types": "job_types"
}
}
},
},
{
{
@@ -1796,6 +1804,10 @@
@@ -1796,6 +1804,10 @@
"company": "company_name",
"company": "company_name",
"location": "location",
"location": "location",
"remote": "remote"
"remote": "remote"
+ },
+ "extra_lists": {
+ "tags": "tags",
+ "job_types": "job_types"
}
}
}
}
],
],
@@ -13,6 +13,10 @@
@@ -13,6 +13,10 @@
* the array. `extra` values are copied as the API gave them (scalars only, strings capped):
* the array. `extra` values are copied as the API gave them (scalars only, strings capped):
* a missing field is null, never a default. A record without an identity is skipped.
* a missing field is null, never a default. A record without an identity is skipped.
*
*
+ * `extra_lists` maps a field to a list the API gives (tags, job types): an array of scalars is kept
+ * as up to 20 of them (a lone scalar is a list of one, a missing one an empty list). It is separate
+ * from `extra` so a mapping that points `extra` at an array keeps reading null, as it always did.
+ *
* A path segment `*` steps into the single child of an object: when the current value is a
* A path segment `*` steps into the single child of an object: when the current value is a
* non-array object with exactly one own property it steps into that property, and otherwise the
* non-array object with exactly one own property it steps into that property, and otherwise the
* path resolves to undefined (so the field is null). It may appear anywhere in a path.
* path resolves to undefined (so the field is null). It may appear anywhere in a path.
@@ -144,6 +148,18 @@ function at(obj, path) {
@@ -144,6 +148,18 @@ function at(obj, path) {
return walk(obj, path, undefined);
return walk(obj, path, undefined);
}
}
+/** A list the API gives → up to 20 of its scalars (a lone scalar → a list of one; nothing → []). */
+function scalars(v) {
+ const all = Array.isArray(v) ? v : v == null ? [] : [v];
+ const out = [];
+ for (const x of all) {
+ const s = scalar(x);
+ if (s != null && s !== '') out.push(s);
+ if (out.length >= 20) break;
+ }
+ return out;
+}
+
function scalar(v) {
function scalar(v) {
if (v == null) return null;
if (v == null) return null;
if (typeof v === 'number' || typeof v === 'boolean') return Number.isFinite(v) || typeof v === 'boolean' ? v : null;
if (typeof v === 'number' || typeof v === 'boolean') return Number.isFinite(v) || typeof v === 'boolean' ? v : null;
@@ -217,6 +233,15 @@ function checkMapping(ep) {
@@ -217,6 +233,15 @@ function checkMapping(ep) {
const kerr = keyPathViolation(p, shape);
const kerr = keyPathViolation(p, shape);
if (kerr) return `extra.${k}: ${kerr}`;
if (kerr) return `extra.${k}: ${kerr}`;
}
}
+ const lists = ep.extra_lists == null ? {} : ep.extra_lists;
+ if (typeof lists !== 'object' || Array.isArray(lists)) return 'extra_lists must be an object of field -> path';
+ if (Object.keys(lists).length > 10) return 'at most 10 extra_lists fields';
+ for (const [k, p] of Object.entries(lists)) {
+ if (!/^[a-z][a-z0-9_]{0,39}$/.test(k) || typeof p !== 'string' || !PATH_RE.test(p)) return `extra_lists.${k} is malformed`;
+ if (Object.prototype.hasOwnProperty.call(extra, k)) return `extra_lists.${k} is also an extra field`;
+ const kerr = keyPathViolation(p, shape);
+ if (kerr) return `extra_lists.${k}: ${kerr}`;
+ }
const where = ep.where == null ? {} : ep.where;
const where = ep.where == null ? {} : ep.where;
if (typeof where !== 'object' || Array.isArray(where)) return 'where must be an object of path -> expected';
if (typeof where !== 'object' || Array.isArray(where)) return 'where must be an object of path -> expected';
if (Object.keys(where).length > 10) return 'at most 10 where conditions';
if (Object.keys(where).length > 10) return 'at most 10 where conditions';
@@ -278,6 +303,7 @@ function parse(text, { url, endpoint = {}, summaryMax = 1000, maxItems = 500 } =
@@ -278,6 +303,7 @@ function parse(text, { url, endpoint = {}, summaryMax = 1000, maxItems = 500 } =
seen.add(identity);
seen.add(identity);
const extra = {};
const extra = {};
for (const [k, p] of Object.entries(endpoint.extra || {})) extra[k] = scalar(get(p));
for (const [k, p] of Object.entries(endpoint.extra || {})) extra[k] = scalar(get(p));
+ for (const [k, p] of Object.entries(endpoint.extra_lists || {})) extra[k] = scalars(get(p));
items.push({
items.push({
identity: String(identity),
identity: String(identity),
kind: endpoint.item_kind || 'record',
kind: endpoint.item_kind || 'record',
@@ -79,6 +79,8 @@ function validateSource(input, { maxItemsCap = 12000 } = {}) {
@@ -79,6 +79,8 @@ function validateSource(input, { maxItemsCap = 12000 } = {}) {
Object.assign(out, { format: e.format || 'json', items_shape: e.items_shape || 'array', items_path: e.items_path || '', item_kind: e.item_kind || 'record', fields: e.fields || {}, extra: e.extra || {} });
Object.assign(out, { format: e.format || 'json', items_shape: e.items_shape || 'array', items_path: e.items_path || '', item_kind: e.item_kind || 'record', fields: e.fields || {}, extra: e.extra || {} });
// `where` is only copied when given, so records without conditions stay byte-identical.
// `where` is only copied when given, so records without conditions stay byte-identical.
if (e.where) out.where = e.where;
if (e.where) out.where = e.where;
+ // and `extra_lists` (tags, job types) the same way.
+ if (e.extra_lists) out.extra_lists = e.extra_lists;
if (e.max_bytes !== undefined) out.max_bytes = e.max_bytes;
if (e.max_bytes !== undefined) out.max_bytes = e.max_bytes;
if (e.timeout_ms !== undefined) out.timeout_ms = e.timeout_ms;
if (e.timeout_ms !== undefined) out.timeout_ms = e.timeout_ms;
if (e.request_content_type !== undefined) out.request_content_type = e.request_content_type;
if (e.request_content_type !== undefined) out.request_content_type = e.request_content_type;
@@ -205,4 +205,22 @@ t('an api summary from HTML longer than the 500-character cap keeps no half tag
@@ -205,4 +205,22 @@ t('an api summary from HTML longer than the 500-character cap keeps no half tag
assert.ok(!/[<>]/.test(item.summary), `no markup left: ${item.summary.slice(-60)}`);
assert.ok(!/[<>]/.test(item.summary), `no markup left: ${item.summary.slice(-60)}`);
});
});
+t('api extra_lists keep an array of scalars as a list (tags, job types); extra still reads an array as null', () => {
+ const body = JSON.stringify({ data: [
+ { slug: 'a', url: 'https://www.arbeitnow.com/jobs/a', title: 'A', tags: ['Node.js', 'PostgreSQL', { x: 1 }, ''], job_types: ['full time'] },
+ { slug: 'b', url: 'https://www.arbeitnow.com/jobs/b', title: 'B', tags: 'Go', job_types: null },
+ { slug: 'c', url: 'https://www.arbeitnow.com/jobs/c', title: 'C', tags: Array.from({ length: 30 }, (_, i) => `t${i}`) },
+ ] });
+ const endpoint = { format: 'json', items_path: 'data', item_kind: 'job', fields: { identity: 'slug', url: 'url', title: 'title' },
+ extra: { tags_scalar: 'tags' }, extra_lists: { tags: 'tags', job_types: 'job_types' } };
+ assert.strictEqual(api.checkMapping(endpoint), null);
+ const [a, b, c] = api.parse(body, { url: 'https://www.arbeitnow.com/api/job-board-api', endpoint }).items;
+ assert.deepStrictEqual([a.fields.tags, a.fields.job_types, a.fields.tags_scalar], [['Node.js', 'PostgreSQL'], ['full time'], null]);
+ assert.deepStrictEqual([b.fields.tags, b.fields.job_types], [['Go'], []]);
+ assert.strictEqual(c.fields.tags.length, 20);
+ assert.match(api.checkMapping({ ...endpoint, extra: { tags: 'tags' } }), /also an extra field/);
+ assert.match(api.checkMapping({ ...endpoint, extra_lists: Object.fromEntries(Array.from({ length: 11 }, (_, i) => [`l${i}`, 'tags'])) }), /at most 10/);
+ assert.match(api.checkMapping({ ...endpoint, extra_lists: ['tags'] }), /must be an object/);
+});
+
t.run();
t.run();