openvibers/openvibe.sources Commits

Sources: the api adapter reads object maps (items_shape map, $key) (#37)

OP OpenVibers committed 11 Oct 2026 · ad353e072a · 5 files, +148 −14

The `api` adapter can read records from a JSON **object map** as well as an array: new endpoint field `items_shape: "map"`.

- Each own property becomes one record, in sorted key order (deterministic runs), and the key is available as the virtual path `$key` (e.g. `fields.identity: "$key"`).
- A property whose value is not an object is skipped and counted. A shape mismatch (map over an array, or array over an object) is a parse error, never an empty success.
- `checkMapping` and the registry validate `items_shape`, and allow `$key` only as a whole path under `map`.
- Array sources are unchanged.

Why: the hosting-source survey found official price lists keyed by id (AWS Lightsail by SKU, Porkbun TLD pricing) that the adapter could not read. No seed uses `map` yet; the matching Contracts endpoint field comes with the first one.

Test: new `test/api-map.test.js` (sorted records, `$key` in identity and extra, nested paths, skipped non-objects, both mismatches, the `checkMapping` rules). The adapter and seed tests stay green. Drafted by a DeepSeek agent; reviewed.

Changes

ddiff --git a/README.md b/README.md
iindex 051945c..a07fb62 100644
---- a/README.md
++++ b/README.md
@@@ -96,12 +96,26 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled
  | `rss`, `atom` (`feed@1`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars |
  | `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed |
  | `jsonld` (`jsonld@1`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error` |
--| `api` (`api@1`) | JSON or XML official APIs via a registry mapping | `items_path`, `fields`, `extra`; epoch fields only where mapped as such |
++| `api` (`api@1`) | JSON or XML official APIs via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`; epoch fields only where mapped as such |
  | `manual` (`manual@1`) | items entered through the API with an evidence URL | no fetcher |
  
  XML with DTD entity declarations is refused; the parser expands no entities (values are decoded
  per field). Fixtures and expectations: `test/adapters.test.js`.
  
++Some official price lists key their records by id instead of listing them — Porkbun's TLD pricing
++is `{ "pricing": { "com": {…}, "net": {…} } }`. `items_shape: "map"` reads that object: each own
++property becomes one record in sorted key order, and the property's key is available as the
++virtual path `$key` (so `fields.identity: "$key"` makes the id the identity). A property whose
++value is not an object is skipped, and an `items_path` of the wrong shape is a parse error, never
++an empty success.
++
++```json
++{ "url": "https://api.porkbun.com/api/json/v3/pricing/get", "format": "json",
++  "items_path": "pricing", "items_shape": "map", "item_kind": "tld",
++  "fields": { "identity": "$key" },
++  "extra": { "tld": "$key", "registration": "registration", "renewal": "renewal" } }
++```
++
  ### Items
  
  Each item: `source_key`, `category`, `kind`, `identity`, `canonical_url` (tracking parameters and
ddiff --git a/docs/contracts-proposal/contracts/sources/source.v1.json b/docs/contracts-proposal/contracts/sources/source.v1.json
iindex da50cfb..e406e51 100644
---- a/docs/contracts-proposal/contracts/sources/source.v1.json
++++ b/docs/contracts-proposal/contracts/sources/source.v1.json
@@@ -19,6 +19,7 @@
          "properties": {
            "url": { "type": "string", "format": "uri", "pattern": "^https?://" },
            "format": { "enum": ["json", "xml"] },
++          "items_shape": { "enum": ["array", "map"], "description": "array (default): the value at items_path is a list of records. map: it is an object keyed by id, each own property a record, in sorted key order; the key is readable as the virtual path $key." },
            "items_path": { "type": "string" },
            "item_kind": { "type": "string", "pattern": "^[a-z][a-z0-9_]{1,39}$" },
            "fields": { "type": "object", "additionalProperties": { "type": "string" } },
ddiff --git a/server/adapters/api.js b/server/adapters/api.js
iindex 20fc03b..409c08a 100644
---- a/server/adapters/api.js
++++ b/server/adapters/api.js
@@@ -12,6 +12,13 @@
   * Paths are dot-separated keys (array indexes allowed); `items_path` "" means the body itself is
   * the array. `extra` values are copied as the API gave them (scalars only, strings capped):
   * a missing field is null, never a default. A record without an identity is skipped.
++ *
++ * Some price lists key their records by id instead of listing them: Porkbun returns
++ * `{ "pricing": { "com": {…}, "net": {…} } }` and AWS Lightsail keys its price list by SKU.
++ * `"items_shape": "map"` reads such an object: each own property becomes one record, in sorted
++ * key order, and the key is available as the virtual path `$key` (so `fields.identity: "$key"`
++ * makes the id the identity). `"array"` (the default) is today's behaviour. A property whose value
++ * is not an object is skipped; a non-object `items_path` is a parse error, never an empty success.
   */
  const { parseXml, toText, toIsoDate, epochSecondsToIso, canonicalUrl, toId, ParseError } = require('./util');
  
@@@ -36,20 +43,34 @@ function scalar(v) {
      return null;
  }
  
++/** `$key` is the map key: only as a whole path, and only when items_shape is "map". */
++function keyPathViolation(p, shape) {
++    if (!p.includes('$key')) return null;
++    if (p !== '$key') return 'a $key path is exactly "$key"';
++    if (shape !== 'map') return '$key requires items_shape "map"';
++    return null;
++}
++
  /** Validate an endpoint mapping (registry side). Returns an error string or null. */
  function checkMapping(ep) {
      if (!['json', 'xml'].includes(ep.format || 'json')) return 'format must be json or xml';
++    const shape = ep.items_shape || 'array';
++    if (!['array', 'map'].includes(shape)) return 'items_shape must be array or map';
      if (ep.items_path !== undefined && ep.items_path !== '' && !PATH_RE.test(ep.items_path)) return 'items_path is malformed';
      const fields = ep.fields || {};
      if (!fields.identity && !fields.url) return 'fields.identity or fields.url is required';
      for (const [k, p] of Object.entries(fields)) {
          if (!['identity', 'url', 'title', 'summary', 'author', 'published_at', 'published_at_epoch', 'updated_at', 'updated_at_epoch'].includes(k)) return `unknown field ${k}`;
          if (typeof p !== 'string' || !PATH_RE.test(p)) return `fields.${k} path is malformed`;
++        const kerr = keyPathViolation(p, shape);
++        if (kerr) return `fields.${k}: ${kerr}`;
      }
      const extra = ep.extra || {};
      if (Object.keys(extra).length > 30) return 'at most 30 extra fields';
      for (const [k, p] of Object.entries(extra)) {
          if (!/^[a-z][a-z0-9_]{0,39}$/.test(k) || typeof p !== 'string' || !PATH_RE.test(p)) return `extra.${k} is malformed`;
++        const kerr = keyPathViolation(p, shape);
++        if (kerr) return `extra.${k}: ${kerr}`;
      }
      if (ep.item_kind !== undefined && !/^[a-z][a-z0-9_]{1,39}$/.test(ep.item_kind)) return 'item_kind is malformed';
      return null;
@@@ -62,29 +83,39 @@ function parse(text, { url, endpoint = {}, summaryMax = 1000, maxItems = 500 } =
      else {
          try { body = JSON.parse(text); } catch { throw new ParseError('body is not valid JSON'); }
      }
--    const list = at(body, endpoint.items_path || '');
--    if (!Array.isArray(list)) throw new ParseError(`items_path "${endpoint.items_path || ''}" is not an array in the response`);
++    const shape = endpoint.items_shape || 'array';
++    const itemsPath = endpoint.items_path || '';
++    const node = at(body, itemsPath);
++    let records;
++    if (shape === 'map') {
++        if (!node || typeof node !== 'object' || Array.isArray(node)) throw new ParseError(`items_path "${itemsPath}" is not an object in the response`);
++        records = Object.keys(node).sort().map((key) => ({ key, value: node[key] }));
++    } else {
++        if (!Array.isArray(node)) throw new ParseError(`items_path "${itemsPath}" is not an array in the response`);
++        records = node.map((value) => ({ key: undefined, value }));
++    }
      const f = endpoint.fields || {};
      const items = [];
      let skipped = 0;
      const seen = new Set();
--    for (const rec of list) {
--        if (items.length >= maxItems || !rec || typeof rec !== 'object') { skipped++; continue; }
--        const link = f.url ? canonicalUrl(scalar(at(rec, f.url)), url) : null;
--        const identity = (f.identity ? toId(scalar(at(rec, f.identity))) : null) || link;
++    for (const { key, value: rec } of records) {
++        if (items.length >= maxItems || !rec || typeof rec !== 'object' || (shape === 'map' && Array.isArray(rec))) { skipped++; continue; }
++        const get = (p) => (shape === 'map' && p === '$key' ? key : at(rec, p));
++        const link = f.url ? canonicalUrl(scalar(get(f.url)), url) : null;
++        const identity = (f.identity ? toId(scalar(get(f.identity))) : null) || link;
          if (!identity || seen.has(identity)) { skipped++; continue; }
          seen.add(identity);
          const extra = {};
--        for (const [k, p] of Object.entries(endpoint.extra || {})) extra[k] = scalar(at(rec, p));
++        for (const [k, p] of Object.entries(endpoint.extra || {})) extra[k] = scalar(get(p));
          items.push({
              identity: String(identity),
              kind: endpoint.item_kind || 'record',
              canonical_url: link,
--            title: f.title ? toText(scalar(at(rec, f.title)), 500) : null,
--            summary: f.summary ? toText(scalar(at(rec, f.summary)), summaryMax) : null,
--            authors: f.author ? [toText(scalar(at(rec, f.author)), 200)].filter(Boolean) : [],
--            published_at: f.published_at ? toIsoDate(scalar(at(rec, f.published_at))) : f.published_at_epoch ? epochSecondsToIso(scalar(at(rec, f.published_at_epoch))) : null,
--            updated_at: f.updated_at ? toIsoDate(scalar(at(rec, f.updated_at))) : f.updated_at_epoch ? epochSecondsToIso(scalar(at(rec, f.updated_at_epoch))) : null,
++            title: f.title ? toText(scalar(get(f.title)), 500) : null,
++            summary: f.summary ? toText(scalar(get(f.summary)), summaryMax) : null,
++            authors: f.author ? [toText(scalar(get(f.author)), 200)].filter(Boolean) : [],
++            published_at: f.published_at ? toIsoDate(scalar(get(f.published_at))) : f.published_at_epoch ? epochSecondsToIso(scalar(get(f.published_at_epoch))) : null,
++            updated_at: f.updated_at ? toIsoDate(scalar(get(f.updated_at))) : f.updated_at_epoch ? epochSecondsToIso(scalar(get(f.updated_at_epoch))) : null,
              fields: extra,
          });
      }
ddiff --git a/server/registry.js b/server/registry.js
iindex e5f8fdd..3c69e6c 100644
---- a/server/registry.js
++++ b/server/registry.js
@@@ -73,7 +73,7 @@ function validateSource(input, { maxItemsCap = 500 } = {}) {
          if (input.type === 'api') {
              const err = checkApiMapping(e);
              if (err) throw new RegistryError(`endpoints[${i}]: ${err}`);
--            Object.assign(out, { format: e.format || 'json', items_path: e.items_path || '', item_kind: e.item_kind || 'record', fields: e.fields || {}, extra: e.extra || {} });
++            Object.assign(out, { format: e.format || 'json', items_shape: e.items_shape || 'array', items_path: e.items_path || '', item_kind: e.item_kind || 'record', fields: e.fields || {}, extra: e.extra || {} });
          } else {
              for (const k of Object.keys(e)) if (k !== 'url') throw new RegistryError(`endpoints[${i}].${k} is only for api sources`);
          }
ddiff --git a/test/api-map.test.js b/test/api-map.test.js
nnew file mode 100644
iindex 0000000..29a5fb4
---- /dev/null
++++ b/test/api-map.test.js
@@@ -0,0 +1,88 @@
++'use strict';
++/** API adapter: `items_shape: "map"` reads records keyed by id, with the key exposed as `$key`. */
++const assert = require('assert');
++const api = require('../server/adapters/api');
++const { suite } = require('./helpers');
++
++const t = suite('api-map');
++
++// Porkbun-shaped: pricing is an object keyed by TLD, in no particular order in the body.
++const PRICING = JSON.stringify({
++    pricing: {
++        net: { registration: '12.48', renewal: '13.29', meta: { transfer: '12.00' } },
++        com: { registration: '10.29', renewal: '10.29', meta: { transfer: '10.29' } },
++        org: { registration: '9.99', renewal: '12.99', meta: { transfer: '9.99' } },
++        free: 'not a record',
++    },
++});
++
++const mapEndpoint = {
++    format: 'json', items_path: 'pricing', items_shape: 'map', item_kind: 'tld',
++    fields: { identity: '$key' },
++    extra: { tld: '$key', registration: 'registration', transfer: 'meta.transfer' },
++};
++
++t('a map body yields one item per key, sorted, with $key as identity and copied into extra', () => {
++    assert.strictEqual(api.checkMapping(mapEndpoint), null);
++    const r = api.parse(PRICING, { url: 'https://api.porkbun.com/api/json/v3/pricing/get', endpoint: mapEndpoint });
++    assert.strictEqual(r.skipped, 1, 'the non-object value');
++    assert.deepStrictEqual(r.items.map((i) => i.identity), ['com', 'net', 'org'], 'sorted key order, not body order');
++    assert.deepStrictEqual(r.items.map((i) => i.fields.tld), ['com', 'net', 'org']);
++    assert.deepStrictEqual(r.items[0].fields, { tld: 'com', registration: '10.29', transfer: '10.29' });
++});
++
++t('nested paths still resolve inside each mapped record', () => {
++    const r = api.parse(PRICING, { url: 'https://api.porkbun.com/', endpoint: mapEndpoint });
++    assert.deepStrictEqual(r.items.map((i) => i.fields.transfer), ['10.29', '12.00', '9.99']);
++});
++
++t('a non-object value is skipped and counted, never invented', () => {
++    const r = api.parse(PRICING, { url: 'https://api.porkbun.com/', endpoint: mapEndpoint });
++    assert.strictEqual(r.items.length, 3);
++    assert.strictEqual(r.skipped, 1);
++    assert.ok(!r.items.some((i) => i.identity === 'free'));
++    // an array value is not a record either
++    const withArray = JSON.stringify({ pricing: { com: { registration: '10.29' }, bad: ['x'] } });
++    const r2 = api.parse(withArray, { url: 'https://api.porkbun.com/', endpoint: mapEndpoint });
++    assert.deepStrictEqual(r2.items.map((i) => i.identity), ['com']);
++    assert.strictEqual(r2.skipped, 1);
++});
++
++t('the mapped shape must match the body: map over an array and array over an object both fail', () => {
++    assert.throws(
++        () => api.parse('{"pricing": [{"registration":"10.29"}]}', { url: 'https://x.example/', endpoint: mapEndpoint }),
++        (e) => e.code === 'parse_error' && /not an object/.test(e.message),
++    );
++    const arrayEndpoint = { format: 'json', items_path: 'pricing', items_shape: 'array', fields: { identity: 'registration' } };
++    assert.throws(
++        () => api.parse('{"pricing": {"com": {"registration":"10.29"}}}', { url: 'https://x.example/', endpoint: arrayEndpoint }),
++        (e) => e.code === 'parse_error' && /not an array/.test(e.message),
++    );
++});
++
++t('checkMapping refuses $key without items_shape "map" and validates items_shape', () => {
++    assert.match(api.checkMapping({ fields: { identity: '$key' } }), /\$key requires items_shape "map"/);
++    assert.match(api.checkMapping({ fields: { identity: 'a' }, extra: { tld: '$key' } }), /\$key requires items_shape "map"/);
++    assert.match(api.checkMapping({ fields: { identity: '$key' }, items_shape: 'object' }), /items_shape must be array or map/);
++    assert.strictEqual(api.checkMapping({ fields: { identity: 'a' }, items_shape: 'array' }), null);
++    assert.strictEqual(api.checkMapping({ fields: { identity: 'a' }, items_shape: 'map' }), null);
++    // $key as an embedded segment is not the virtual path
++    assert.match(api.checkMapping({ fields: { identity: 'a' }, items_shape: 'map', extra: { tld: 'x.$key' } }), /exactly "\$key"/);
++});
++
++t('existing array sources are unchanged (items_shape defaults to array)', () => {
++    const endpoint = {
++        format: 'json', items_path: 'deals', item_kind: 'offer',
++        fields: { identity: 'dealID', url: 'link' },
++        extra: { sale_price: 'salePrice' },
++    };
++    assert.strictEqual(api.checkMapping(endpoint), null);
++    const r = api.parse('{"deals":[{"dealID":"a","link":"https://shop.example/d/a","salePrice":"4.99"},null,{"dealID":"b"}]}', { url: 'https://api.example.com/deals', endpoint });
++    assert.deepStrictEqual(r.items.map((i) => i.identity), ['a', 'b']);
++    assert.strictEqual(r.skipped, 1, 'the null element');
++    assert.strictEqual(r.items[0].canonical_url, 'https://shop.example/d/a');
++    assert.strictEqual(r.items[0].fields.sale_price, '4.99');
++    assert.strictEqual(r.items[1].canonical_url, null);
++});
++
++t.run();