openvibers/openvibe.sources Commits

An api endpoint may map lists (extra_lists: tags, job types); Arbeitnow maps both (contracts v0.212.0) (#92)

OP OpenVibers committed 20 days ago · aea3106f99 · 9 files, +84 −9

* An api endpoint may map lists (extra_lists: tags, job types); Arbeitnow maps both

* openvibe-contracts v0.212.0 (extra_lists, format tsv, duplicates)

* The proposal copy of sources.source@1 matches contracts v0.212.0 (format tsv, duplicates, extra_lists)

Changes

ddiff --git a/README.md b/README.md
iindex 8ae44b1..4c1b362 100644
---- a/README.md
++++ b/README.md
@@@ -414,7 +414,7 @@ vhost serves it.
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
<!-- versions:start -->
--- openvibe-contracts: v0.211.0
++- openvibe-contracts: v0.212.0
- openvibe-sdk: v0.62.0
- openvibe-shared: v4.9.0
<!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex 8c70742..1ff942e 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -7,7 +7,7 @@
"runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, fast-xml-parser",
"port": 4720,
"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, health and JSON 404s; the API is loopback-only; README \"Public host\")",
-- "contracts": "openvibe-contracts v0.211.0",
++ "contracts": "openvibe-contracts v0.212.0",
"sdk": "openvibe-sdk v0.62.0",
"shared": "openvibe-shared v4.9.0",
"capabilities": [
ddiff --git a/docs/contracts-proposal/contracts/sources/source.v1.json b/docs/contracts-proposal/contracts/sources/source.v1.json
iindex fd09659..2165312 100644
---- a/docs/contracts-proposal/contracts/sources/source.v1.json
++++ b/docs/contracts-proposal/contracts/sources/source.v1.json
@@@ -82,8 +82,17 @@
"format": {
"enum": [
"json",
-- "xml"
-- ]
++ "xml",
++ "tsv"
++ ],
++ "description": "json (default), xml, or tsv: a tab-separated file whose first line names the columns, read as an array of records keyed by those names (items_path is \"\"; BLS series files, Sources 2026-10-24)."
++ },
++ "duplicates": {
++ "enum": [
++ "first",
++ "last"
++ ],
++ "description": "Two records with the same identity: first (default) keeps the first, last keeps the last in the first one's place (a file listing each series oldest to newest then yields its newest value; Sources, 2026-10-24)."
},
"items_path": {
"type": "string"
@@@ -170,6 +179,14 @@
},
"description": "Extra fields copied as the API gave them (scalars). A path is relative to the record; $key is the map key; $root.<path> reads the response from its root, where a {$key} segment is the record's map key; * steps into an object's single child; key[field=value,...] (one to four conditions) steps into the first element of the array key whose fields equal the values, compared as text, where a value is literal text or $rec.<path> read from the record (Sources#51)."
},
++ "extra_lists": {
++ "type": "object",
++ "maxProperties": 10,
++ "additionalProperties": {
++ "type": "string"
++ },
++ "description": "Lists the API gives (tags, job types): each field maps to a path whose array of scalars is kept as up to 20 of them; a lone scalar is a list of one and a missing value an empty list. Separate from extra, which reads an array as null (Sources, 2026-10-24)."
++ },
"extract": {
"type": "object",
"additionalProperties": false,
ddiff --git a/package-lock.json b/package-lock.json
iindex d3b3b75..7fa17f5 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -12,7 +12,7 @@
"dotenv": "^16.4.7",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.211.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.212.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.62.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.9.0",
"pg": "^8.23.0"
@@@ -769,9 +769,9 @@
}
},
"node_modules/openvibe-contracts": {
-- "version": "0.211.0",
-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.211.0",
-- "integrity": "sha512-M/DHcyQcxmwrT65s64D/mJn77ANF6Je5SeDgpWTZKKhxPffKVuhV+SzRZf8grZWvD3flmSqWNTTPOcKUWTjEug==",
++ "version": "0.212.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.212.0",
++ "integrity": "sha512-qt4iodXw+sZiSSPGA8p3MWgGDr6iXjC9LGyfhLBRpnfwfggGeKajRotlGp0EyGrKko6zu0K8DFVZghK5XDmb/A==",
"license": "AGPL-3.0-only",
"dependencies": {
"ajv": "^8.17.1",
ddiff --git a/package.json b/package.json
iindex 9786ebe..e2ef91b 100644
---- a/package.json
++++ b/package.json
@@@ -27,7 +27,7 @@
"dotenv": "^16.4.7",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.211.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.212.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.62.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.9.0",
"pg": "^8.23.0"
ddiff --git a/seeds/sources.json b/seeds/sources.json
iindex 718af68..756631b 100644
---- a/seeds/sources.json
++++ b/seeds/sources.json
@@@ -1760,6 +1760,10 @@
"company": "company_name",
"location": "location",
"remote": "remote"
++ },
++ "extra_lists": {
++ "tags": "tags",
++ "job_types": "job_types"
}
},
{
@@@ -1778,6 +1782,10 @@
"company": "company_name",
"location": "location",
"remote": "remote"
++ },
++ "extra_lists": {
++ "tags": "tags",
++ "job_types": "job_types"
}
},
{
@@@ -1796,6 +1804,10 @@
"company": "company_name",
"location": "location",
"remote": "remote"
++ },
++ "extra_lists": {
++ "tags": "tags",
++ "job_types": "job_types"
}
}
],
ddiff --git a/server/adapters/api.js b/server/adapters/api.js
iindex ee6f21f..03240dc 100644
---- a/server/adapters/api.js
++++ b/server/adapters/api.js
@@@ -13,6 +13,10 @@
* the array. `extra` values are copied as the API gave them (scalars only, strings capped):
* a missing field is null, never a default. A record without an identity is skipped.
*
++ * `extra_lists` maps a field to a list the API gives (tags, job types): an array of scalars is kept
++ * as up to 20 of them (a lone scalar is a list of one, a missing one an empty list). It is separate
++ * from `extra` so a mapping that points `extra` at an array keeps reading null, as it always did.
++ *
* A path segment `*` steps into the single child of an object: when the current value is a
* non-array object with exactly one own property it steps into that property, and otherwise the
* path resolves to undefined (so the field is null). It may appear anywhere in a path.
@@@ -144,6 +148,18 @@ function at(obj, path) {
return walk(obj, path, undefined);
}
++/** A list the API gives → up to 20 of its scalars (a lone scalar → a list of one; nothing → []). */
++function scalars(v) {
++ const all = Array.isArray(v) ? v : v == null ? [] : [v];
++ const out = [];
++ for (const x of all) {
++ const s = scalar(x);
++ if (s != null && s !== '') out.push(s);
++ if (out.length >= 20) break;
++ }
++ return out;
++}
++
function scalar(v) {
if (v == null) return null;
if (typeof v === 'number' || typeof v === 'boolean') return Number.isFinite(v) || typeof v === 'boolean' ? v : null;
@@@ -217,6 +233,15 @@ function checkMapping(ep) {
const kerr = keyPathViolation(p, shape);
if (kerr) return `extra.${k}: ${kerr}`;
}
++ const lists = ep.extra_lists == null ? {} : ep.extra_lists;
++ if (typeof lists !== 'object' || Array.isArray(lists)) return 'extra_lists must be an object of field -> path';
++ if (Object.keys(lists).length > 10) return 'at most 10 extra_lists fields';
++ for (const [k, p] of Object.entries(lists)) {
++ if (!/^[a-z][a-z0-9_]{0,39}$/.test(k) || typeof p !== 'string' || !PATH_RE.test(p)) return `extra_lists.${k} is malformed`;
++ if (Object.prototype.hasOwnProperty.call(extra, k)) return `extra_lists.${k} is also an extra field`;
++ const kerr = keyPathViolation(p, shape);
++ if (kerr) return `extra_lists.${k}: ${kerr}`;
++ }
const where = ep.where == null ? {} : ep.where;
if (typeof where !== 'object' || Array.isArray(where)) return 'where must be an object of path -> expected';
if (Object.keys(where).length > 10) return 'at most 10 where conditions';
@@@ -278,6 +303,7 @@ function parse(text, { url, endpoint = {}, summaryMax = 1000, maxItems = 500 } =
seen.add(identity);
const extra = {};
for (const [k, p] of Object.entries(endpoint.extra || {})) extra[k] = scalar(get(p));
++ for (const [k, p] of Object.entries(endpoint.extra_lists || {})) extra[k] = scalars(get(p));
items.push({
identity: String(identity),
kind: endpoint.item_kind || 'record',
ddiff --git a/server/registry.js b/server/registry.js
iindex 598fc20..5b77df7 100644
---- a/server/registry.js
++++ b/server/registry.js
@@@ -79,6 +79,8 @@ function validateSource(input, { maxItemsCap = 12000 } = {}) {
Object.assign(out, { format: e.format || 'json', items_shape: e.items_shape || 'array', items_path: e.items_path || '', item_kind: e.item_kind || 'record', fields: e.fields || {}, extra: e.extra || {} });
// `where` is only copied when given, so records without conditions stay byte-identical.
if (e.where) out.where = e.where;
++ // and `extra_lists` (tags, job types) the same way.
++ if (e.extra_lists) out.extra_lists = e.extra_lists;
if (e.max_bytes !== undefined) out.max_bytes = e.max_bytes;
if (e.timeout_ms !== undefined) out.timeout_ms = e.timeout_ms;
if (e.request_content_type !== undefined) out.request_content_type = e.request_content_type;
ddiff --git a/test/adapters.test.js b/test/adapters.test.js
iindex 4354716..4bf7967 100644
---- a/test/adapters.test.js
++++ b/test/adapters.test.js
@@@ -205,4 +205,22 @@ t('an api summary from HTML longer than the 500-character cap keeps no half tag
assert.ok(!/[<>]/.test(item.summary), `no markup left: ${item.summary.slice(-60)}`);
});
++t('api extra_lists keep an array of scalars as a list (tags, job types); extra still reads an array as null', () => {
++ const body = JSON.stringify({ data: [
++ { slug: 'a', url: 'https://www.arbeitnow.com/jobs/a', title: 'A', tags: ['Node.js', 'PostgreSQL', { x: 1 }, ''], job_types: ['full time'] },
++ { slug: 'b', url: 'https://www.arbeitnow.com/jobs/b', title: 'B', tags: 'Go', job_types: null },
++ { slug: 'c', url: 'https://www.arbeitnow.com/jobs/c', title: 'C', tags: Array.from({ length: 30 }, (_, i) => `t${i}`) },
++ ] });
++ const endpoint = { format: 'json', items_path: 'data', item_kind: 'job', fields: { identity: 'slug', url: 'url', title: 'title' },
++ extra: { tags_scalar: 'tags' }, extra_lists: { tags: 'tags', job_types: 'job_types' } };
++ assert.strictEqual(api.checkMapping(endpoint), null);
++ const [a, b, c] = api.parse(body, { url: 'https://www.arbeitnow.com/api/job-board-api', endpoint }).items;
++ assert.deepStrictEqual([a.fields.tags, a.fields.job_types, a.fields.tags_scalar], [['Node.js', 'PostgreSQL'], ['full time'], null]);
++ assert.deepStrictEqual([b.fields.tags, b.fields.job_types], [['Go'], []]);
++ assert.strictEqual(c.fields.tags.length, 20);
++ assert.match(api.checkMapping({ ...endpoint, extra: { tags: 'tags' } }), /also an extra field/);
++ assert.match(api.checkMapping({ ...endpoint, extra_lists: Object.fromEntries(Array.from({ length: 11 }, (_, i) => [`l${i}`, 'tags'])) }), /at most 10/);
++ assert.match(api.checkMapping({ ...endpoint, extra_lists: ['tags'] }), /must be an object/);
++});
++
t.run();