openvibers/openvibe.sources Commits

Steam's featured lists for OpenVibe.Games: the games category (contracts 0.205.0), three seeds on store.steampowered.com/api/featuredcategories (top sellers, new releases, specials; robots allows /api/, every 3 hours), and $index, an array record's position, so a ranked list keeps its order (#79)

OP OpenVibers committed 26 days ago · 05f98caf08 · 7 files, +182 −12

Changes

ddiff --git a/README.md b/README.md
iindex 5c7d459..cc69904 100644
---- a/README.md
++++ b/README.md
@@@ -115,7 +115,9 @@ is `{ "pricing": { "com": {…}, "net": {…} } }`. `items_shape: "map"` reads t
  property becomes one record in sorted key order, and the property's key is available as the
  virtual path `$key` (so `fields.identity: "$key"` makes the id the identity). A property whose
  value is not an object is skipped, and an `items_path` of the wrong shape is a parse error, never
--an empty success.
++an empty success. With the default `items_shape: "array"`, `$index` is a record's 0-based position in
++the array (as a whole path, e.g. `"extra": { "position": "$index" }`), so a ranked list keeps its
++order (Steam's top sellers).
  
  ```json
  { "url": "https://api.porkbun.com/api/json/v3/pricing/get", "format": "json",
ddiff --git a/docs/contracts-proposal/contracts/sources/item.v1.json b/docs/contracts-proposal/contracts/sources/item.v1.json
iindex 0f9285f..c2e28b1 100644
---- a/docs/contracts-proposal/contracts/sources/item.v1.json
++++ b/docs/contracts-proposal/contracts/sources/item.v1.json
@@@ -9,7 +9,7 @@
    "properties": {
      "id": { "type": "string", "pattern": "^itm_[0-9A-HJKMNP-TV-Z]{26}$" },
      "source_key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" },
--    "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting", "crowdfunding"] },
++    "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting", "crowdfunding", "games"] },
      "kind": { "type": "string", "pattern": "^[a-z][a-z0-9_]{1,39}$", "description": "article | url | sitemap | product | offer | review | review_signal | record | coupon …" },
      "identity": { "type": "string", "minLength": 1, "maxLength": 2048, "description": "The source's own id (guid, @id, API id) or the canonical URL." },
      "canonical_url": { "type": ["string", "null"], "format": "uri" },
ddiff --git a/docs/contracts-proposal/contracts/sources/source.v1.json b/docs/contracts-proposal/contracts/sources/source.v1.json
iindex 9f8fad3..e87f161 100644
---- a/docs/contracts-proposal/contracts/sources/source.v1.json
++++ b/docs/contracts-proposal/contracts/sources/source.v1.json
@@@ -49,7 +49,9 @@
          "deals",
          "coupons",
          "trade",
--        "hosting", "crowdfunding"
++        "hosting",
++        "crowdfunding",
++        "games"
        ]
      },
      "homepage_url": {
@@@ -130,7 +132,7 @@
                  }
                ]
              },
--            "description": "Records kept only when every path equals its value (or one of an array of values), or, for { pattern }, is text the regular expression matches; others are left out before max_items counts. Paths follow the field rules: relative to the record, $key, an absolute $root.<path> into the response (a {$key} segment there is the record's map key), * for an object's single child, and key[field=value,...] selectors (Sources#39, Sources#49)."
++            "description": "Records kept only when every path equals its value (or one of an array of values), or, for { pattern }, is text the regular expression matches; others are left out before max_items counts. Paths follow the field rules: relative to the record, $key, an absolute $root.<path> into the response (a {$key} segment there is the record's map key), * for an object's single child, and key[field=value,...] selectors (Sources#39, Sources#51)."
            },
            "item_kind": {
              "type": "string",
@@@ -144,13 +146,13 @@
              "type": "integer",
              "minimum": 1024,
              "maximum": 33554432,
--            "description": "The body cap for this endpoint, in bytes (1 KiB to 32 MiB); the deployment's cap otherwise. OVHcloud's VPS catalog is about 12 MB (Sources#49)."
++            "description": "The body cap for this endpoint, in bytes (1 KiB to 32 MiB); the deployment's cap otherwise. OVHcloud's VPS catalog is about 12 MB (Sources#51)."
            },
            "timeout_ms": {
              "type": "integer",
              "minimum": 1000,
              "maximum": 120000,
--            "description": "This endpoint's fetch deadline, in milliseconds (1 s to 120 s); the deployment's otherwise. NYC Open Data computes the HPD per-building violation tally in 12 to 60 seconds."
++            "description": "This endpoint's fetch deadline, in milliseconds (1 s to 120 s); the deployment's otherwise. NYC Open Data computes the HPD per-building violation tally in 12 to 60 seconds (Sources#76)."
            },
            "fields": {
              "type": "object",
@@@ -163,7 +165,7 @@
              "additionalProperties": {
                "type": "string"
              },
--            "description": "Extra fields copied as the API gave them (scalars). A path is relative to the record; $key is the map key; $root.<path> reads the response from its root, where a {$key} segment is the record's map key; * steps into an object's single child; key[field=value,...] (one to four conditions) steps into the first element of the array key whose fields equal the values, compared as text, where a value is literal text or $rec.<path> read from the record (Sources#49)."
++            "description": "Extra fields copied as the API gave them (scalars). A path is relative to the record; $key is the map key; $root.<path> reads the response from its root, where a {$key} segment is the record's map key; * steps into an object's single child; key[field=value,...] (one to four conditions) steps into the first element of the array key whose fields equal the values, compared as text, where a value is literal text or $rec.<path> read from the record (Sources#51)."
            },
            "extract": {
              "type": "object",
ddiff --git a/seeds/sources.json b/seeds/sources.json
iindex 5a6d802..06c7092 100644
---- a/seeds/sources.json
++++ b/seeds/sources.json
@@@ -1438,6 +1438,102 @@
        "review_required": true,
        "sensitivity": "none",
        "default_indexability": "noindex"
++    },
++    {
++      "key": "steam-top-sellers",
++      "name": "Steam: top sellers (the store's featured list)",
++      "type": "api",
++      "category": "games",
++      "homepage_url": "https://store.steampowered.com/",
++      "endpoints": [
++        {
++          "url": "https://store.steampowered.com/api/featuredcategories?cc=us&l=english",
++          "format": "json",
++          "items_path": "top_sellers.items",
++          "item_kind": "steam_top_seller",
++          "fields": { "identity": "id", "title": "name" },
++          "extra": {
++            "appid": "id", "position": "$index", "final_price": "final_price", "original_price": "original_price", "discount_percent": "discount_percent",
++            "discounted": "discounted", "currency": "currency", "header_image": "header_image",
++            "windows": "windows_available", "mac": "mac_available", "linux": "linux_available", "discount_expiration": "discount_expiration"
++          }
++        }
++      ],
++      "robots_note": "robots.txt checked 2026-10-18 (https://store.steampowered.com/robots.txt): User-agent * disallows /share/, /news/externalpost/, account and email paths, /login and /join with tokens, /account/ackgift/, /email/ and /widget/; /api/ is not disallowed, so /api/featuredcategories is allowed. It answers 200 to the production host without a key. The fetcher re-checks robots.txt on every run.",
++      "terms_note": "Steam's storefront endpoint is public and unauthenticated (no key, unlike api.steampowered.com's Web API, whose robots.txt disallows everything and which is never fetched). It is read once every 3 hours (the same response for all three lists). OpenVibe.Games shows each game's name, price, discount and Steam's header image as observed at that time, labelled 'From the Steam store, checked <time>', and links every game to its store page; it sells nothing and implies no endorsement by Valve.",
++      "license_note": "Game names, prices and images are Valve's and the publishers' (Steam store data). Shown as dated observations with a link to each game's Steam store page; no licence to republish was found, so only these fields are kept and the images are loaded from Steam's own CDN.",
++      "min_interval_ms": 60000,
++      "poll_interval_sec": 10800,
++      "stale_after_sec": 43200,
++      "max_items": 50,
++      "enabled": false,
++      "review_required": true,
++      "sensitivity": "none",
++      "default_indexability": "noindex"
++    },
++    {
++      "key": "steam-new-releases",
++      "name": "Steam: new releases (the store's featured list)",
++      "type": "api",
++      "category": "games",
++      "homepage_url": "https://store.steampowered.com/",
++      "endpoints": [
++        {
++          "url": "https://store.steampowered.com/api/featuredcategories?cc=us&l=english",
++          "format": "json",
++          "items_path": "new_releases.items",
++          "item_kind": "steam_new_release",
++          "fields": { "identity": "id", "title": "name" },
++          "extra": {
++            "appid": "id", "position": "$index", "final_price": "final_price", "original_price": "original_price", "discount_percent": "discount_percent",
++            "discounted": "discounted", "currency": "currency", "header_image": "header_image",
++            "windows": "windows_available", "mac": "mac_available", "linux": "linux_available", "discount_expiration": "discount_expiration"
++          }
++        }
++      ],
++      "robots_note": "robots.txt checked 2026-10-18 (https://store.steampowered.com/robots.txt): User-agent * disallows /share/, /news/externalpost/, account and email paths, /login and /join with tokens, /account/ackgift/, /email/ and /widget/; /api/ is not disallowed, so /api/featuredcategories is allowed. It answers 200 to the production host without a key. The fetcher re-checks robots.txt on every run.",
++      "terms_note": "Steam's storefront endpoint is public and unauthenticated (no key, unlike api.steampowered.com's Web API, whose robots.txt disallows everything and which is never fetched). It is read once every 3 hours (the same response for all three lists). OpenVibe.Games shows each game's name, price, discount and Steam's header image as observed at that time, labelled 'From the Steam store, checked <time>', and links every game to its store page; it sells nothing and implies no endorsement by Valve.",
++      "license_note": "Game names, prices and images are Valve's and the publishers' (Steam store data). Shown as dated observations with a link to each game's Steam store page; no licence to republish was found, so only these fields are kept and the images are loaded from Steam's own CDN.",
++      "min_interval_ms": 60000,
++      "poll_interval_sec": 10800,
++      "stale_after_sec": 43200,
++      "max_items": 50,
++      "enabled": false,
++      "review_required": true,
++      "sensitivity": "none",
++      "default_indexability": "noindex"
++    },
++    {
++      "key": "steam-specials",
++      "name": "Steam: specials (the store's featured list)",
++      "type": "api",
++      "category": "games",
++      "homepage_url": "https://store.steampowered.com/",
++      "endpoints": [
++        {
++          "url": "https://store.steampowered.com/api/featuredcategories?cc=us&l=english",
++          "format": "json",
++          "items_path": "specials.items",
++          "item_kind": "steam_special",
++          "fields": { "identity": "id", "title": "name" },
++          "extra": {
++            "appid": "id", "position": "$index", "final_price": "final_price", "original_price": "original_price", "discount_percent": "discount_percent",
++            "discounted": "discounted", "currency": "currency", "header_image": "header_image",
++            "windows": "windows_available", "mac": "mac_available", "linux": "linux_available", "discount_expiration": "discount_expiration"
++          }
++        }
++      ],
++      "robots_note": "robots.txt checked 2026-10-18 (https://store.steampowered.com/robots.txt): User-agent * disallows /share/, /news/externalpost/, account and email paths, /login and /join with tokens, /account/ackgift/, /email/ and /widget/; /api/ is not disallowed, so /api/featuredcategories is allowed. It answers 200 to the production host without a key. The fetcher re-checks robots.txt on every run.",
++      "terms_note": "Steam's storefront endpoint is public and unauthenticated (no key, unlike api.steampowered.com's Web API, whose robots.txt disallows everything and which is never fetched). It is read once every 3 hours (the same response for all three lists). OpenVibe.Games shows each game's name, price, discount and Steam's header image as observed at that time, labelled 'From the Steam store, checked <time>', and links every game to its store page; it sells nothing and implies no endorsement by Valve.",
++      "license_note": "Game names, prices and images are Valve's and the publishers' (Steam store data). Shown as dated observations with a link to each game's Steam store page; no licence to republish was found, so only these fields are kept and the images are loaded from Steam's own CDN.",
++      "min_interval_ms": 60000,
++      "poll_interval_sec": 10800,
++      "stale_after_sec": 43200,
++      "max_items": 50,
++      "enabled": false,
++      "review_required": true,
++      "sensitivity": "none",
++      "default_indexability": "noindex"
      }
    ]
  }
ddiff --git a/server/adapters/api.js b/server/adapters/api.js
iindex be0f359..ee6f21f 100644
---- a/server/adapters/api.js
++++ b/server/adapters/api.js
@@@ -59,7 +59,8 @@
   * `"items_shape": "map"` reads such an object: each own property becomes one record, in sorted
   * key order, and the key is available as the virtual path `$key` (so `fields.identity: "$key"`
   * makes the id the identity) or as `{$key}` inside a `$root.` path. `"array"` (the default) is
-- * today's behaviour. A property whose value is not an object is skipped; a non-object
++ * today's behaviour, and there the record's 0-based position in the array is the virtual path `$index` (a ranked
++ * list's order, e.g. Steam's top sellers). A property whose value is not an object is skipped; a non-object
   * `items_path` is a parse error, never an empty success.
   */
  const { parseXml, toText, toIsoDate, epochSecondsToIso, canonicalUrl, toId, ParseError } = require('./util');
@@@ -189,6 +190,8 @@ function keyPathViolation(p, shape) {
      }
      if (hasBrace) return '{$key} is only valid inside a $root. path';
      if (p === '$key') return shape === 'map' ? null : '$key requires items_shape "map"';
++    if (p === '$index') return shape === 'map' ? '$index requires items_shape "array"' : null;
++    if (segs.includes('$index')) return 'an $index path is exactly "$index"';
      if (hasBare) return 'a $key path is exactly "$key"';
      return null;
  }
@@@ -255,7 +258,7 @@ function parse(text, { url, endpoint = {}, summaryMax = 1000, maxItems = 500 } =
          records = Object.keys(node).sort().map((key) => ({ key, value: node[key] }));
      } else {
          if (!Array.isArray(node)) throw new ParseError(`items_path "${itemsPath}" is not an array in the response`);
--        records = node.map((value) => ({ key: undefined, value }));
++        records = node.map((value, index) => ({ key: undefined, index, value }));
      }
      const f = endpoint.fields || {};
      const where = endpoint.where && typeof endpoint.where === 'object' && !Array.isArray(endpoint.where) ? endpoint.where : {};
@@@ -264,11 +267,11 @@ function parse(text, { url, endpoint = {}, summaryMax = 1000, maxItems = 500 } =
      let skipped = 0;
      let filtered = 0;
      const seen = new Set();
--    for (const { key, value: rec } of records) {
++    for (const { key, index, value: rec } of records) {
          if (!rec || typeof rec !== 'object' || (shape === 'map' && Array.isArray(rec))) { skipped++; continue; }
          if (filterKeys.length && !matchesWhere(body, rec, key, where)) { filtered++; continue; }
          if (items.length >= maxItems) { skipped++; continue; }
--        const get = (p) => getField(body, rec, key, p);
++        const get = (p) => (p === '$index' ? index : getField(body, rec, key, p));
          const link = f.url ? canonicalUrl(scalar(get(f.url)), url) : null;
          const identity = (f.identity ? toId(scalar(get(f.identity))) : null) || link;
          if (!identity || seen.has(identity)) { skipped++; continue; }
ddiff --git a/server/registry.js b/server/registry.js
iindex 8e33815..2ebb8cf 100644
---- a/server/registry.js
++++ b/server/registry.js
@@@ -13,7 +13,7 @@ const { checkApiMapping } = require('./adapters');
  const extract = require('./adapters/extract');
  
  const TYPES = ['rss', 'atom', 'sitemap', 'jsonld', 'api', 'manual'];
--const CATEGORIES = ['news', 'blog', 'reviews', 'deals', 'coupons', 'trade', 'hosting', 'crowdfunding'];
++const CATEGORIES = ['news', 'blog', 'reviews', 'deals', 'coupons', 'trade', 'hosting', 'crowdfunding', 'games'];
  const SENSITIVITY = ['none', 'financial', 'health', 'political', 'legal', 'adult'];
  const AUTH_MODES = ['none', 'header', 'bearer', 'query'];
  const KEY_RE = /^[a-z0-9][a-z0-9-]{1,63}$/;
ddiff --git a/test/steam-sources.test.js b/test/steam-sources.test.js
nnew file mode 100644
iindex 0000000..8ad04cc
---- /dev/null
++++ b/test/steam-sources.test.js
@@@ -0,0 +1,67 @@
++'use strict';
++/**
++ * Steam's featured lists for OpenVibe.Games (category games, contracts 0.205.0; the owner's valve.openvibe.games ask):
++ * store.steampowered.com/api/featuredcategories, which robots.txt allows (api.steampowered.com, which disallows
++ * everything, is never fetched), read every 3 hours. One source per list (top sellers, new releases, specials), so a
++ * game on two lists is two items rather than one identity overwritten by the other list. Each item keeps the app id,
++ * the name, the prices in cents, the discount, the currency, the header image and the platforms.
++ */
++const assert = require('assert');
++const api = require('../server/adapters/api');
++const { validateSource, CATEGORIES } = require('../server/registry');
++const { load } = require('../server/config');
++const { suite } = require('./helpers');
++
++const t = suite('steam-sources');
++const seeds = require('../seeds/sources.json').sources;
++const lists = ['steam-top-sellers', 'steam-new-releases', 'steam-specials'].map((k) => seeds.find((s) => s.key === k));
++
++const game = (id, name, extra = {}) => ({
++    id, type: 0, name, discounted: false, discount_percent: 0, original_price: 2999, final_price: 2999, currency: 'USD',
++    large_capsule_image: `https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/${id}/capsule_616x353.jpg`,
++    header_image: `https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/${id}/header.jpg`,
++    windows_available: true, mac_available: false, linux_available: true, controller_support: 'full', ...extra,
++});
++const BODY = JSON.stringify({
++    top_sellers: { id: 'cat_topsellers', name: 'Top Sellers', items: [game(730, 'Counter-Strike 2', { final_price: 0, original_price: 0 }), game(4435490, 'Call of Duty®: Modern Warfare® 4', { final_price: 6999, original_price: 6999 })] },
++    new_releases: { id: 'cat_newreleases', name: 'New Releases', items: [game(3527290, 'A New Game')] },
++    specials: { id: 'cat_specials', name: 'Specials', items: [game(413150, 'Stardew Valley', { discounted: true, discount_percent: 50, final_price: 749, original_price: 1499, discount_expiration: 1792800000 })] },
++    coming_soon: { id: 'cat_comingsoon', name: 'Coming Soon', items: [game(1, 'Not read')] },
++    status: 1,
++});
++
++t('the seeds validate, start disabled, are games sources on the robots-allowed store API, every 3 hours', () => {
++    assert.ok(CATEGORIES.includes('games'));
++    for (const s of lists) {
++        assert.ok(s, 'seeded');
++        const rec = validateSource(s);
++        assert.strictEqual(rec.enabled, false);
++        assert.strictEqual(rec.category, 'games');
++        assert.ok(rec.robots_note && rec.terms_note && rec.license_note, `${s.key}: robots, terms and licence notes`);
++        assert.strictEqual(new URL(s.endpoints[0].url).host, 'store.steampowered.com', 'never api.steampowered.com');
++        assert.strictEqual(s.poll_interval_sec, 10800);
++        assert.ok(rec.max_items <= load({}).items.maxPerFetch);
++    }
++});
++
++t('each list is its own items: id, name, prices, discount, image and platforms', () => {
++    const [top, fresh, specials] = lists.map((s) => api.parse(BODY, { url: s.endpoints[0].url, endpoint: s.endpoints[0], maxItems: s.max_items }).items);
++    assert.deepStrictEqual(top.map((i) => [i.kind, i.identity, i.title]), [['steam_top_seller', '730', 'Counter-Strike 2'], ['steam_top_seller', '4435490', 'Call of Duty®: Modern Warfare® 4']]);
++    assert.deepStrictEqual(fresh.map((i) => i.identity), ['3527290']);
++    const [sale] = specials;
++    assert.strictEqual(sale.kind, 'steam_special');
++    assert.deepStrictEqual(
++        [sale.fields.appid, sale.fields.final_price, sale.fields.original_price, sale.fields.discount_percent, sale.fields.currency, sale.fields.windows, sale.fields.linux, sale.fields.mac, sale.fields.discount_expiration],
++        [413150, 749, 1499, 50, 'USD', true, true, false, 1792800000],
++    );
++    assert.match(sale.fields.header_image, /^https:\/\/shared\.akamai\.steamstatic\.com\/.+header\.jpg$/);
++    assert.ok(![...top, ...fresh, ...specials].some((i) => i.title === 'Not read'), 'coming soon is not one of the lists');
++    assert.deepStrictEqual(top.map((i) => i.fields.position), [0, 1], "a list keeps the store's order ($index)");
++});
++
++t('$index is an array record\'s position, as a whole path, never on a map', () => {
++    const ep = (extra, shape) => ({ url: 'https://example.org/x', items_path: 'list', ...(shape ? { items_shape: shape } : {}), fields: { identity: 'id' }, extra });
++    assert.strictEqual(api.checkMapping(ep({ position: '$index' })), null);
++    assert.match(api.checkMapping(ep({ position: '$index' }, 'map')), /\$index requires items_shape "array"/);
++    assert.match(api.checkMapping(ep({ position: 'a.$index' })), /exactly "\$index"/);
++});