openvibers/openvibe.sources Commits

Categories housing, food and jobs (read from sources.source@1, contracts 0.210.0), with a disabled seed each: HUD Fair Market Rents, openFDA food enforcement, Arbeitnow (#88)

OP OpenVibers committed 22 days ago · ec75808f33 · 9 files, +191 −13

Changes

ddiff --git a/README.md b/README.md
iindex dac886c..983bfee 100644
---- a/README.md
++++ b/README.md
@@@ -54,7 +54,7 @@ start and flushes on stop.
  ## The registry
  
  A source records: `key`, `name`, `type` (`rss|atom|sitemap|jsonld|api|manual`), `category`
--(`news|blog|reviews|deals|coupons|trade|hosting|crowdfunding`), `endpoints` (for `api`: a field mapping, no code per
++(`news|blog|reviews|deals|coupons|trade|hosting|crowdfunding|games|housing|food|jobs`, read from `sources.source@1`; housing, food and jobs since contracts 0.210.0), `endpoints` (for `api`: a field mapping, no code per
  provider), `auth` (`none|header|bearer|query` with the **name** of an environment variable —
  names must start with `SOURCES_CRED_`, so an entry can never point at this service's own
  secrets), `robots_note`, `terms_note`, `license_note`, the rate limit (`min_interval_ms`),
@@@ -408,7 +408,7 @@ vhost serves it.
  Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
  
  <!-- versions:start -->
--- openvibe-contracts: v0.209.0
++- openvibe-contracts: v0.210.0
  - openvibe-sdk: v0.60.0
  - openvibe-shared: v4.4.0
  <!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex dfe1ec7..2e9686d 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -7,7 +7,7 @@
    "runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, fast-xml-parser",
    "port": 4720,
    "domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, health and JSON 404s; the API is loopback-only; README \"Public host\")",
--  "contracts": "openvibe-contracts v0.209.0",
++  "contracts": "openvibe-contracts v0.210.0",
    "sdk": "openvibe-sdk v0.60.0",
    "shared": "openvibe-shared v4.4.0",
    "capabilities": [
@@@ -35,7 +35,7 @@
    "features": [
      "source registry with terms, robots and licence notes",
      "rss, atom, sitemap, jsonld, api and manual adapters",
--    "categories: news, blog, reviews, deals, coupons, trade, hosting, crowdfunding and games",
++    "categories: news, blog, reviews, deals, coupons, trade, hosting, crowdfunding, games, and since contracts 0.210.0 housing, food and jobs (read from sources.source@1)",
      "a deal feed's own offer elements kept as stated (feed@3: DealNews' price, currency, retailer, end date, category and editors' pick)",
      "api mappings: keyed maps, where conditions and patterns, $index for ranked lists; per-endpoint timeout_ms and max_bytes",
      "facts a feed states in prose (extract): coupon codes with the shop and expiry",
ddiff --git a/docs/contracts-proposal/contracts/sources/item.v1.json b/docs/contracts-proposal/contracts/sources/item.v1.json
iindex c2e28b1..9a7ae5e 100644
---- a/docs/contracts-proposal/contracts/sources/item.v1.json
++++ b/docs/contracts-proposal/contracts/sources/item.v1.json
@@@ -9,7 +9,7 @@
    "properties": {
      "id": { "type": "string", "pattern": "^itm_[0-9A-HJKMNP-TV-Z]{26}$" },
      "source_key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" },
--    "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting", "crowdfunding", "games"] },
++    "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting", "crowdfunding", "games", "housing", "food", "jobs"] },
      "kind": { "type": "string", "pattern": "^[a-z][a-z0-9_]{1,39}$", "description": "article | url | sitemap | product | offer | review | review_signal | record | coupon …" },
      "identity": { "type": "string", "minLength": 1, "maxLength": 2048, "description": "The source's own id (guid, @id, API id) or the canonical URL." },
      "canonical_url": { "type": ["string", "null"], "format": "uri" },
ddiff --git a/docs/contracts-proposal/contracts/sources/source.v1.json b/docs/contracts-proposal/contracts/sources/source.v1.json
iindex e87f161..5975e46 100644
---- a/docs/contracts-proposal/contracts/sources/source.v1.json
++++ b/docs/contracts-proposal/contracts/sources/source.v1.json
@@@ -51,7 +51,10 @@
          "trade",
          "hosting",
          "crowdfunding",
--        "games"
++        "games",
++        "housing",
++        "food",
++        "jobs"
        ]
      },
      "homepage_url": {
ddiff --git a/package-lock.json b/package-lock.json
iindex 6b818ce..47b6a18 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -12,7 +12,7 @@
          "dotenv": "^16.4.7",
          "express": "^4.21.2",
          "fast-xml-parser": "^4.5.3",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.209.0",
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.210.0",
          "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.60.0",
          "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.4.0",
          "pg": "^8.23.0"
@@@ -769,9 +769,9 @@
        }
      },
      "node_modules/openvibe-contracts": {
--      "version": "0.209.0",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.209.0",
--      "integrity": "sha512-Ui6obhNLi5HhdH0Jigq+jtBvPuBbv+Vu9Soqatdt+HBXpyWQM7N29yT6hFJM+JsaYip6ys93BAS1Jti4Vxs6Eg==",
++      "version": "0.210.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.210.0",
++      "integrity": "sha512-9PnhWV0DxtqK7eogKfGPLf8A9qEyXjtlZNoAypYIluF6ynYXUJNHtsB/dFAwAwXq6dvqR/ilTJbe+HxgHb5CuQ==",
        "license": "AGPL-3.0-only",
        "dependencies": {
          "ajv": "^8.17.1",
ddiff --git a/package.json b/package.json
iindex fe5641e..6717e61 100644
---- a/package.json
++++ b/package.json
@@@ -27,7 +27,7 @@
      "dotenv": "^16.4.7",
      "express": "^4.21.2",
      "fast-xml-parser": "^4.5.3",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.209.0",
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.210.0",
      "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.60.0",
      "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.4.0",
      "pg": "^8.23.0"
ddiff --git a/seeds/sources.json b/seeds/sources.json
iindex 42ef8a7..bcfb020 100644
---- a/seeds/sources.json
++++ b/seeds/sources.json
@@@ -1,5 +1,5 @@
  {
--  "_about": "One real source per category in the original set, all DISABLED; further news and trade sources were added later, also all DISABLED with review_required. Robots notes were checked by hand on the date each one records (2026-09-22 for the original six, 2026-10-10 for those added later, 2026-10-11 for the two hosting seeds); terms must be re-verified by a person before a source is enabled (PATCH enabled: true). Seeding never overwrites or enables an existing source.",
++  "_about": "One real source per category in the original set, all DISABLED; further news and trade sources were added later, also all DISABLED with review_required. Robots notes were checked by hand on the date each one records (2026-09-22 for the original six, 2026-10-10 for those added later, 2026-10-11 for the two hosting seeds, 2026-10-22 for the housing, food and jobs seeds); terms must be re-verified by a person before a source is enabled (PATCH enabled: true). Seeding never overwrites or enables an existing source.",
    "sources": [
      {
        "key": "nasa-news-releases",
@@@ -1626,6 +1626,152 @@
        "review_required": true,
        "sensitivity": "none",
        "default_indexability": "noindex"
++    },
++    {
++      "key": "hud-fair-market-rents",
++      "name": "HUD Fair Market Rents by area (0 to 4 bedrooms)",
++      "type": "api",
++      "category": "housing",
++      "homepage_url": "https://www.huduser.gov/portal/datasets/fmr.html",
++      "endpoints": [
++        {
++          "url": "https://services.arcgis.com/VTyQ9soqVukalItT/arcgis/rest/services/Fair_Market_Rents/FeatureServer/0/query?where=1%3D1&outFields=FMR_CODE,FMR_AREANAME,FMR_0BDR,FMR_1BDR,FMR_2BDR,FMR_3BDR,FMR_4BDR&orderByFields=OBJECTID&resultOffset=0&resultRecordCount=1000&returnGeometry=false&f=json",
++          "format": "json",
++          "items_path": "features",
++          "item_kind": "rent_area",
++          "fields": {
++            "identity": "attributes.FMR_CODE",
++            "title": "attributes.FMR_AREANAME"
++          },
++          "extra": {
++            "fmr_0br": "attributes.FMR_0BDR",
++            "fmr_1br": "attributes.FMR_1BDR",
++            "fmr_2br": "attributes.FMR_2BDR",
++            "fmr_3br": "attributes.FMR_3BDR",
++            "fmr_4br": "attributes.FMR_4BDR"
++          }
++        },
++        {
++          "url": "https://services.arcgis.com/VTyQ9soqVukalItT/arcgis/rest/services/Fair_Market_Rents/FeatureServer/0/query?where=1%3D1&outFields=FMR_CODE,FMR_AREANAME,FMR_0BDR,FMR_1BDR,FMR_2BDR,FMR_3BDR,FMR_4BDR&orderByFields=OBJECTID&resultOffset=1000&resultRecordCount=1000&returnGeometry=false&f=json",
++          "format": "json",
++          "items_path": "features",
++          "item_kind": "rent_area",
++          "fields": {
++            "identity": "attributes.FMR_CODE",
++            "title": "attributes.FMR_AREANAME"
++          },
++          "extra": {
++            "fmr_0br": "attributes.FMR_0BDR",
++            "fmr_1br": "attributes.FMR_1BDR",
++            "fmr_2br": "attributes.FMR_2BDR",
++            "fmr_3br": "attributes.FMR_3BDR",
++            "fmr_4br": "attributes.FMR_4BDR"
++          }
++        },
++        {
++          "url": "https://services.arcgis.com/VTyQ9soqVukalItT/arcgis/rest/services/Fair_Market_Rents/FeatureServer/0/query?where=1%3D1&outFields=FMR_CODE,FMR_AREANAME,FMR_0BDR,FMR_1BDR,FMR_2BDR,FMR_3BDR,FMR_4BDR&orderByFields=OBJECTID&resultOffset=2000&resultRecordCount=1000&returnGeometry=false&f=json",
++          "format": "json",
++          "items_path": "features",
++          "item_kind": "rent_area",
++          "fields": {
++            "identity": "attributes.FMR_CODE",
++            "title": "attributes.FMR_AREANAME"
++          },
++          "extra": {
++            "fmr_0br": "attributes.FMR_0BDR",
++            "fmr_1br": "attributes.FMR_1BDR",
++            "fmr_2br": "attributes.FMR_2BDR",
++            "fmr_3br": "attributes.FMR_3BDR",
++            "fmr_4br": "attributes.FMR_4BDR"
++          }
++        }
++      ],
++      "robots_note": "services.arcgis.com serves no robots.txt (checked 2026-10-22); the fetcher re-checks it on every run. HUD's own ArcGIS layer, the one OpenVibe.Rent queries by area today.",
++      "terms_note": "HUD publishes Fair Market Rents as open data (hudgis-hud.opendata.arcgis.com); a US federal work. The layer allows 1000 records a request, so three pages cover its 2,622 areas. FMRs change once a year (October): a weekly poll is plenty. Re-verify before enabling.",
++      "license_note": "US federal government work (public domain in the US).",
++      "min_interval_ms": 60000,
++      "poll_interval_sec": 604800,
++      "max_items": 3000,
++      "enabled": false,
++      "review_required": true,
++      "sensitivity": "none",
++      "default_indexability": "noindex"
++    },
++    {
++      "key": "openfda-food-enforcement",
++      "name": "FDA food recall enforcement reports (openFDA)",
++      "type": "api",
++      "category": "food",
++      "homepage_url": "https://open.fda.gov/apis/food/enforcement/",
++      "endpoints": [
++        {
++          "url": "https://api.fda.gov/food/enforcement.json?sort=report_date:desc&limit=100",
++          "format": "json",
++          "items_path": "results",
++          "item_kind": "recall",
++          "fields": {
++            "identity": "recall_number",
++            "title": "product_description",
++            "summary": "reason_for_recall"
++          },
++          "extra": {
++            "recalling_firm": "recalling_firm",
++            "classification": "classification",
++            "status": "status",
++            "state": "state",
++            "distribution_pattern": "distribution_pattern",
++            "voluntary_mandated": "voluntary_mandated",
++            "recall_initiation_date": "recall_initiation_date",
++            "report_date": "report_date"
++          }
++        }
++      ],
++      "robots_note": "api.fda.gov serves no robots.txt (404, checked 2026-10-22); the fetcher re-checks it on every run.",
++      "terms_note": "openFDA terms (open.fda.gov/terms): data are public domain (CC0, open.fda.gov/license); do not imply FDA endorsement; results are unvalidated and not for medical decisions (shown with that note). Without a key: 240 requests a minute and 1,000 a day per address; one request a day is used. The structured twin of the fda-food-recalls feed, for OpenVibe.Food. Re-verify before enabling.",
++      "license_note": "CC0 1.0 (openFDA); US federal government work.",
++      "min_interval_ms": 60000,
++      "poll_interval_sec": 86400,
++      "max_items": 100,
++      "enabled": false,
++      "review_required": true,
++      "sensitivity": "none",
++      "default_indexability": "noindex"
++    },
++    {
++      "key": "arbeitnow-job-board",
++      "name": "Arbeitnow job board API",
++      "type": "api",
++      "category": "jobs",
++      "homepage_url": "https://www.arbeitnow.com/",
++      "endpoints": [
++        {
++          "url": "https://www.arbeitnow.com/api/job-board-api",
++          "format": "json",
++          "items_path": "data",
++          "item_kind": "job",
++          "fields": {
++            "identity": "slug",
++            "url": "url",
++            "title": "title",
++            "published_at_epoch": "created_at"
++          },
++          "extra": {
++            "company": "company_name",
++            "location": "location",
++            "remote": "remote"
++          }
++        }
++      ],
++      "robots_note": "robots.txt checked 2026-10-22: User-agent * with an empty Disallow (only ?__hstc tracking URLs excluded). The fetcher re-checks it on every run.",
++      "terms_note": "Arbeitnow publishes its job board API free of charge (the terms OpenVibe.Work records today): attribute Arbeitnow by name and link every listing to its page on arbeitnow.com. The description HTML is deliberately not mapped. The source OpenVibe.Work fetches itself today; Work moves to it here. Re-verify before enabling.",
++      "license_note": null,
++      "min_interval_ms": 60000,
++      "poll_interval_sec": 21600,
++      "max_items": 300,
++      "enabled": false,
++      "review_required": true,
++      "sensitivity": "none",
++      "default_indexability": "noindex"
      }
    ]
  }
ddiff --git a/server/registry.js b/server/registry.js
iindex 2ebb8cf..882ac0d 100644
---- a/server/registry.js
++++ b/server/registry.js
@@@ -13,7 +13,9 @@ const { checkApiMapping } = require('./adapters');
  const extract = require('./adapters/extract');
  
  const TYPES = ['rss', 'atom', 'sitemap', 'jsonld', 'api', 'manual'];
--const CATEGORIES = ['news', 'blog', 'reviews', 'deals', 'coupons', 'trade', 'hosting', 'crowdfunding', 'games'];
++// The categories are sources.source@1's (openvibe-contracts), read from the schema so the two never drift: housing, food
++// and jobs since contracts 0.210.0, for Rent and Homes, Food and Work (plan 2026-10-22, track F1).
++const CATEGORIES = Object.freeze([...require('openvibe-contracts/contracts/sources/source.v1.json').properties.category.enum]);
  const SENSITIVITY = ['none', 'financial', 'health', 'political', 'legal', 'adult'];
  const AUTH_MODES = ['none', 'header', 'bearer', 'query'];
  const KEY_RE = /^[a-z0-9][a-z0-9-]{1,63}$/;
ddiff --git a/test/categories.test.js b/test/categories.test.js
nnew file mode 100644
iindex 0000000..33c90ed
---- /dev/null
++++ b/test/categories.test.js
@@@ -0,0 +1,27 @@
++'use strict';
++/**
++ * The categories a source may have are sources.source@1's, read from openvibe-contracts: housing, food and jobs since
++ * contracts 0.210.0 (plan 2026-10-22, track F1: Rent and Homes, Food and Work move their own fetches into Sources).
++ * The registry accepts every one of them and refuses anything else, and the items API filters by them.
++ */
++const assert = require('assert');
++const { validateSource, CATEGORIES } = require('../server/registry');
++const { suite, sourceDef } = require('./helpers');
++
++const t = suite('categories');
++
++t('the categories are the contract\'s, housing, food and jobs included', () => {
++    const contract = require('openvibe-contracts/contracts/sources/source.v1.json').properties.category.enum;
++    assert.deepStrictEqual([...CATEGORIES], contract);
++    for (const c of ['news', 'hosting', 'games', 'housing', 'food', 'jobs']) assert.ok(CATEGORIES.includes(c), c);
++    assert.ok(Object.isFrozen(CATEGORIES));
++});
++
++t('the registry accepts a source in each new category and refuses one outside the contract', () => {
++    for (const category of ['housing', 'food', 'jobs']) {
++        assert.strictEqual(validateSource(sourceDef({ key: `example-${category}`, category, endpoints: ['https://feeds.example.org/feed.xml'] })).category, category);
++    }
++    assert.throws(() => validateSource(sourceDef({ key: 'example-weather', category: 'weather', endpoints: ['https://feeds.example.org/feed.xml'] })), /category must be one of .*housing\|food\|jobs/);
++});
++
++t.run();