openvibers/openvibe.sources Commits
Sources: category hosting, with Linode and Vultr plan seeds (disabled) (#34)
Changes
@@@ -45,7 +45,7 @@ the fetch queue (`sources_fetch_due`, `sources_fetch_oldest_wait_seconds`,## The registryA source records: `key`, `name`, `type` (`rss|atom|sitemap|jsonld|api|manual`), `category`--(`news|blog|reviews|deals|coupons|trade`), `endpoints` (for `api`: a field mapping, no code per++(`news|blog|reviews|deals|coupons|trade|hosting`), `endpoints` (for `api`: a field mapping, no code perprovider), `auth` (`none|header|bearer|query` with the **name** of an environment variable —names must start with `SOURCES_CRED_`, so an entry can never point at this service's ownsecrets), `robots_note`, `terms_note`, `license_note`, the rate limit (`min_interval_ms`),@@@ -9,7 +9,7 @@"properties": {"id": { "type": "string", "pattern": "^itm_[0-9A-HJKMNP-TV-Z]{26}$" },"source_key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" },-- "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade"] },++ "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting"] },"kind": { "type": "string", "pattern": "^[a-z][a-z0-9_]{1,39}$", "description": "article | url | sitemap | product | offer | review | review_signal | record | coupon …" },"identity": { "type": "string", "minLength": 1, "maxLength": 2048, "description": "The source's own id (guid, @id, API id) or the canonical URL." },"canonical_url": { "type": ["string", "null"], "format": "uri" },@@@ -10,7 +10,7 @@"key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" },"name": { "type": "string", "minLength": 1, "maxLength": 200 },"type": { "enum": ["rss", "atom", "sitemap", "jsonld", "api", "manual"] },-- "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade"] },++ "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting"] },"homepage_url": { "type": ["string", "null"], "format": "uri" },"endpoints": {"type": "array", "maxItems": 20,@@@ -12,7 +12,7 @@"dotenv": "^16.4.7","express": "^4.21.2","fast-xml-parser": "^4.5.3",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.129.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.138.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.0","pg": "^8.23.0"@@@ -769,9 +769,9 @@}},"node_modules/openvibe-contracts": {-- "version": "0.129.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.129.0",-- "integrity": "sha512-Ly0F/ap6wmAf8hYpNrOhMLsU3cZYBjsFDW2P1F/xVmHVsRDzAPAZDXtFcohP0UNeZgyxgrR0BI0pPG5guD/Efw==",++ "version": "0.138.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.138.0",++ "integrity": "sha512-YHOAU+w01TnoXKifqUOdf/JAYggzmH7R/BkZ4VBhmBzJMGNVaq1hEtG7NWl3z1Lfxcm78oqe6aJrLNfJVF2F9w==","license": "AGPL-3.0-only","dependencies": {"ajv": "^8.17.1",@@@ -27,7 +27,7 @@"dotenv": "^16.4.7","express": "^4.21.2","fast-xml-parser": "^4.5.3",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.129.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.138.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.0","pg": "^8.23.0"@@@ -36,8 +36,8 @@"node": ">=22.0.0"},"devDependencies": {++ "@electric-sql/pglite": "^0.5.8","ajv": "^8.20.0",-- "ajv-formats": "^3.0.1",-- "@electric-sql/pglite": "^0.5.8"++ "ajv-formats": "^3.0.1"}}@@@ -1,5 +1,5 @@{-- "_about": "One real source per category in the original set, all DISABLED; further news and trade sources were added later, also all DISABLED with review_required. Robots notes were checked by hand on the date each one records (2026-09-22 for the original six, 2026-10-10 for those added later); terms must be re-verified by a person before a source is enabled (PATCH enabled: true). Seeding never overwrites or enables an existing source.",++ "_about": "One real source per category in the original set, all DISABLED; further news and trade sources were added later, also all DISABLED with review_required. Robots notes were checked by hand on the date each one records (2026-09-22 for the original six, 2026-10-10 for those added later, 2026-10-11 for the two hosting seeds); terms must be re-verified by a person before a source is enabled (PATCH enabled: true). Seeding never overwrites or enables an existing source.","sources": [{"key": "nasa-news-releases",@@@ -213,6 +213,82 @@"review_required": true,"sensitivity": "financial","default_indexability": "noindex"++ },++ {++ "key": "vultr-plans",++ "name": "Vultr cloud plans",++ "type": "api",++ "category": "hosting",++ "homepage_url": "https://www.vultr.com/pricing/",++ "endpoints": [++ {++ "url": "https://api.vultr.com/v2/plans?per_page=500",++ "format": "json",++ "items_path": "plans",++ "item_kind": "plan",++ "fields": {++ "identity": "id"++ },++ "extra": {++ "vcpu_count": "vcpu_count",++ "ram": "ram",++ "disk": "disk",++ "disk_count": "disk_count",++ "bandwidth": "bandwidth",++ "monthly_cost": "monthly_cost",++ "hourly_cost": "hourly_cost",++ "type": "type"++ }++ }++ ],++ "robots_note": "robots.txt checked 2026-10-11: https://api.vultr.com/robots.txt is exactly 'User-agent: *' followed by 'Disallow: /', which disallows every path on the API host. The fetcher re-checks robots.txt on every run and would refuse /v2/plans as disallowed, so this source cannot be enabled as written — an operator must resolve the robots.txt block with Vultr first. (The endpoint itself answers without an API key.)",++ "terms_note": "Vultr's API documentation says the plans listing needs no API key for ordinary accounts ('If your account has special plans available, you will need to pass your API key ... For all other accounts, the API key is not required'), and the official govultr client documents that 'public operations like listing plans ... will still work' unauthenticated; /v2/plans answered with plan data to our keyless fetcher. www.vultr.com/api/ and www.vultr.com/legal/tos/ both answered 403 to our fetcher, so Vultr's API terms could not be read here and must be read by a person before enabling. Attribute Vultr by linking https://www.vultr.com/pricing/; show each price as an observation 'checked on <date>', never a current price, and imply no endorsement. Re-verify before enabling.",++ "license_note": "Plan specifications and prices are Vultr's data (https://www.vultr.com/pricing/); Vultr's terms pages could not be fetched (HTTP 403 to our fetcher) and no licence to republish them was found. Attribute Vultr and link the pricing page; no endorsement implied.",++ "min_interval_ms": 60000,++ "poll_interval_sec": 86400,++ "max_items": 200,++ "enabled": false,++ "review_required": true,++ "sensitivity": "none",++ "default_indexability": "noindex"++ },++ {++ "key": "linode-types",++ "name": "Akamai Cloud (Linode) plan types",++ "type": "api",++ "category": "hosting",++ "homepage_url": "https://www.linode.com/pricing/",++ "endpoints": [++ {++ "url": "https://api.linode.com/v4/linode/types?page_size=500",++ "format": "json",++ "items_path": "data",++ "item_kind": "plan",++ "fields": {++ "identity": "id",++ "title": "label"++ },++ "extra": {++ "vcpus": "vcpus",++ "memory": "memory",++ "disk": "disk",++ "transfer": "transfer",++ "class": "class",++ "price_monthly": "price.monthly",++ "price_hourly": "price.hourly"++ }++ }++ ],++ "robots_note": "robots.txt checked 2026-10-11: https://api.linode.com/robots.txt answers 301 to https://api.linode.com/v4/, which answers 404 (a JSON 'Not found'), so the API host publishes no robots.txt rules; a 404 is treated as no restrictions (never guessed as a disallow), so /v4/linode/types is allowed. The fetcher re-checks robots.txt on every run.",++ "terms_note": "Akamai's Linode API documentation says 'Some of this API's operations are publicly accessible and don't require authentication'; /v4/linode/types answered with type data to our keyless fetcher. https://www.linode.com/legal-tos/ answers 404 and https://www.linode.com/legal redirects to https://www.akamai.com/legal, where the governing documents are Akamai's Acceptable Use Policy and the Akamai Cloud (Linode) Master Services Agreement; none of those pages states a right to, or a clear ban on, republishing plan or price data (the AUP forbids using the network to build a competitive service and IP infringement), so a person must read them before enabling. Attribute Akamai/Linode by linking https://www.linode.com/pricing/; show each price as an observation 'checked on <date>', never a current price, and imply no endorsement. Re-verify before enabling.",++ "license_note": "Plan specifications and prices are Akamai/Linode's data (https://www.linode.com/pricing/); the terms hub is https://www.akamai.com/legal. No licence to republish them was found. Attribute Akamai/Linode and link the pricing page; no endorsement implied.",++ "min_interval_ms": 60000,++ "poll_interval_sec": 86400,++ "max_items": 200,++ "enabled": false,++ "review_required": true,++ "sensitivity": "none",++ "default_indexability": "noindex"}]}@@@ -12,7 +12,7 @@const { checkApiMapping } = require('./adapters');const TYPES = ['rss', 'atom', 'sitemap', 'jsonld', 'api', 'manual'];--const CATEGORIES = ['news', 'blog', 'reviews', 'deals', 'coupons', 'trade'];++const CATEGORIES = ['news', 'blog', 'reviews', 'deals', 'coupons', 'trade', 'hosting'];const SENSITIVITY = ['none', 'financial', 'health', 'political', 'legal', 'adult'];const AUTH_MODES = ['none', 'header', 'bearer', 'query'];const KEY_RE = /^[a-z0-9][a-z0-9-]{1,63}$/;nnew file mode 100644@@@ -0,0 +1,91 @@++{++ "data": [++ {++ "id": "g6-nanode-1",++ "label": "Nanode 1GB",++ "price": { "hourly": 0.0075, "monthly": 5.0 },++ "region_prices": [++ { "id": "id-cgk", "hourly": 0.009, "monthly": 6.0 },++ { "id": "br-gru", "hourly": 0.0105, "monthly": 7.0 }++ ],++ "addons": {++ "backups": {++ "price": { "hourly": 0.003, "monthly": 2.0 },++ "region_prices": [++ { "id": "id-cgk", "hourly": 0.0036, "monthly": 2.4 },++ { "id": "br-gru", "hourly": 0.004, "monthly": 2.8 }++ ]++ }++ },++ "memory": 1024,++ "disk": 25600,++ "transfer": 1000,++ "vcpus": 1,++ "gpus": 0,++ "network_out": 1000,++ "class": "nanode",++ "successor": null,++ "capabilities": [],++ "accelerated_devices": 0++ },++ {++ "id": "g6-standard-1",++ "label": "Linode 2GB",++ "price": { "hourly": 0.018, "monthly": 12.0 },++ "region_prices": [++ { "id": "id-cgk", "hourly": 0.022, "monthly": 14.4 },++ { "id": "br-gru", "hourly": 0.025, "monthly": 16.8 }++ ],++ "addons": {++ "backups": {++ "price": { "hourly": 0.004, "monthly": 2.5 },++ "region_prices": [++ { "id": "id-cgk", "hourly": 0.0045, "monthly": 3.0 },++ { "id": "br-gru", "hourly": 0.005, "monthly": 3.5 }++ ]++ }++ },++ "memory": 2048,++ "disk": 51200,++ "transfer": 2000,++ "vcpus": 1,++ "gpus": 0,++ "network_out": 2000,++ "class": "standard",++ "successor": null,++ "capabilities": [],++ "accelerated_devices": 0++ },++ {++ "id": "g6-standard-2",++ "label": "Linode 4GB",++ "price": { "hourly": 0.036, "monthly": 24.0 },++ "region_prices": [++ { "id": "id-cgk", "hourly": 0.043, "monthly": 28.8 },++ { "id": "br-gru", "hourly": 0.05, "monthly": 33.6 }++ ],++ "addons": {++ "backups": {++ "price": { "hourly": 0.008, "monthly": 5.0 },++ "region_prices": [++ { "id": "id-cgk", "hourly": 0.009, "monthly": 6.0 },++ { "id": "br-gru", "hourly": 0.01, "monthly": 7.0 }++ ]++ }++ },++ "memory": 4096,++ "disk": 81920,++ "transfer": 4000,++ "vcpus": 2,++ "gpus": 0,++ "network_out": 4000,++ "class": "standard",++ "successor": null,++ "capabilities": [],++ "accelerated_devices": 0++ }++ ],++ "page": 1,++ "pages": 1,++ "results": 75++}nnew file mode 100644@@@ -0,0 +1,77 @@++{++ "plans": [++ {++ "id": "vc2-1c-0.5gb-free",++ "vcpu_count": 1,++ "ram": 512,++ "disk": 10,++ "disk_type": "SSD",++ "disk_count": 1,++ "bandwidth": 0,++ "monthly_cost": 0,++ "hourly_cost": 0,++ "monthly_cost_preemptible": 0,++ "hourly_cost_preemptible": 0,++ "invoice_type": "monthly",++ "type": "vc2",++ "locations": ["sea", "fra", "mia"],++ "cpu_vendor": "Intel",++ "storage_type": "local_storage",++ "vcpu_type": "thread",++ "deploy_ondemand": true,++ "deploy_preemptible": false,++ "location_cost": {},++ "gpu_brand": "none"++ },++ {++ "id": "vc2-1c-0.5gb-v6",++ "vcpu_count": 1,++ "ram": 512,++ "disk": 10,++ "disk_type": "SSD",++ "disk_count": 1,++ "bandwidth": 512,++ "monthly_cost": 2.5,++ "hourly_cost": 0.003,++ "monthly_cost_preemptible": 2.5,++ "hourly_cost_preemptible": 0.003,++ "invoice_type": "monthly",++ "type": "vc2",++ "locations": ["ewr", "atl"],++ "cpu_vendor": "Intel",++ "storage_type": "local_storage",++ "vcpu_type": "thread",++ "deploy_ondemand": true,++ "deploy_preemptible": false,++ "location_cost": {},++ "gpu_brand": "none"++ },++ {++ "id": "vc2-1c-0.5gb",++ "vcpu_count": 1,++ "ram": 512,++ "disk": 10,++ "disk_type": "SSD",++ "disk_count": 1,++ "bandwidth": 512,++ "monthly_cost": 3.5,++ "hourly_cost": 0.005,++ "monthly_cost_preemptible": 3.5,++ "hourly_cost_preemptible": 0.005,++ "invoice_type": "monthly",++ "type": "vc2",++ "locations": ["ewr"],++ "cpu_vendor": "Intel",++ "storage_type": "local_storage",++ "vcpu_type": "thread",++ "deploy_ondemand": true,++ "deploy_preemptible": false,++ "location_cost": {},++ "gpu_brand": "none"++ }++ ],++ "meta": {++ "total": 151,++ "links": { "next": "", "prev": "" }++ }++}nnew file mode 100644@@@ -0,0 +1,75 @@++'use strict';++/**++ * The hosting seeds: each provider API maps through the api adapter exactly as the registry++ * declares it, and the registry accepts category `hosting` (refusing anything else). The fixtures++ * are the live responses (Vultr /v2/plans, Akamai Linode /v4/linode/types) trimmed to three records.++ */++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const api = require('../server/adapters/api');++const { validateSource } = require('../server/registry');++const { suite } = require('./helpers');++++const t = suite('hosting-sources');++const fx = (name) => fs.readFileSync(path.join(__dirname, 'fixtures', name), 'utf8');++const seeds = require('../seeds/sources.json').sources;++const seed = (key) => seeds.find((s) => s.key === key);++const endpoint = (key) => seed(key).endpoints[0];++++t('both hosting seeds declare an api mapping the adapter accepts', () => {++ for (const key of ['vultr-plans', 'linode-types']) {++ const s = seed(key);++ assert.ok(s, `${key} is seeded`);++ assert.strictEqual(s.type, 'api');++ assert.strictEqual(s.category, 'hosting');++ assert.strictEqual(api.checkMapping(endpoint(key)), null, `${key} mapping`);++ }++});++++t('Vultr /v2/plans: one item per plan, identity from id, every extra field kept', () => {++ const ep = endpoint('vultr-plans');++ const r = api.parse(fx('vultr-plans.json'), { url: ep.url, endpoint: ep });++ assert.strictEqual(r.skipped, 0);++ assert.deepStrictEqual(r.items.map((i) => i.identity), ['vc2-1c-0.5gb-free', 'vc2-1c-0.5gb-v6', 'vc2-1c-0.5gb']);++ assert.ok(r.items.every((i) => i.kind === 'plan'));++ assert.deepStrictEqual(r.items[0].fields, {++ vcpu_count: 1, ram: 512, disk: 10, disk_count: 1, bandwidth: 0, monthly_cost: 0, hourly_cost: 0, type: 'vc2',++ });++ assert.deepStrictEqual(r.items[2].fields, {++ vcpu_count: 1, ram: 512, disk: 10, disk_count: 1, bandwidth: 512, monthly_cost: 3.5, hourly_cost: 0.005, type: 'vc2',++ });++ assert.strictEqual(r.items[0].title, null, 'no title is mapped for Vultr');++ assert.strictEqual(r.items[0].canonical_url, null, 'no url is mapped for Vultr');++});++++t('Linode /v4/linode/types: identity id, title label, dotted price.monthly/hourly mapped', () => {++ const ep = endpoint('linode-types');++ const r = api.parse(fx('linode-types.json'), { url: ep.url, endpoint: ep });++ assert.strictEqual(r.skipped, 0);++ assert.deepStrictEqual(r.items.map((i) => i.identity), ['g6-nanode-1', 'g6-standard-1', 'g6-standard-2']);++ assert.deepStrictEqual(r.items.map((i) => i.title), ['Nanode 1GB', 'Linode 2GB', 'Linode 4GB']);++ assert.ok(r.items.every((i) => i.kind === 'plan'));++ assert.deepStrictEqual(r.items[0].fields, {++ vcpus: 1, memory: 1024, disk: 25600, transfer: 1000, class: 'nanode', price_monthly: 5, price_hourly: 0.0075,++ });++ assert.deepStrictEqual(r.items[2].fields, {++ vcpus: 2, memory: 4096, disk: 81920, transfer: 4000, class: 'standard', price_monthly: 24, price_hourly: 0.036,++ });++});++++t('the registry accepts category hosting and refuses an unknown category', () => {++ const rec = validateSource(seed('vultr-plans'));++ assert.strictEqual(rec.category, 'hosting');++ assert.deepStrictEqual({++ enabled: rec.enabled, review_required: rec.review_required, sensitivity: rec.sensitivity,++ default_indexability: rec.default_indexability, poll_interval_sec: rec.poll_interval_sec,++ min_interval_ms: rec.min_interval_ms, max_items: rec.max_items,++ }, {++ enabled: false, review_required: true, sensitivity: 'none', default_indexability: 'noindex',++ poll_interval_sec: 86400, min_interval_ms: 60000, max_items: 200,++ });++ assert.throws(() => validateSource({ ...seed('vultr-plans'), category: 'hostingg' }), /category must be one of/);++});++++t.run();
@@ -45,7 +45,7 @@ the fetch queue (`sources_fetch_due`, `sources_fetch_oldest_wait_seconds`,
@@ -45,7 +45,7 @@ the fetch queue (`sources_fetch_due`, `sources_fetch_oldest_wait_seconds`,
## The registry
## The registry
A source records: `key`, `name`, `type` (`rss|atom|sitemap|jsonld|api|manual`), `category`
A source records: `key`, `name`, `type` (`rss|atom|sitemap|jsonld|api|manual`), `category`
-(`news|blog|reviews|deals|coupons|trade`), `endpoints` (for `api`: a field mapping, no code per
+(`news|blog|reviews|deals|coupons|trade|hosting`), `endpoints` (for `api`: a field mapping, no code per
provider), `auth` (`none|header|bearer|query` with the **name** of an environment variable —
provider), `auth` (`none|header|bearer|query` with the **name** of an environment variable —
names must start with `SOURCES_CRED_`, so an entry can never point at this service's own
names must start with `SOURCES_CRED_`, so an entry can never point at this service's own
secrets), `robots_note`, `terms_note`, `license_note`, the rate limit (`min_interval_ms`),
secrets), `robots_note`, `terms_note`, `license_note`, the rate limit (`min_interval_ms`),
@@ -9,7 +9,7 @@
@@ -9,7 +9,7 @@
"properties": {
"properties": {
"id": { "type": "string", "pattern": "^itm_[0-9A-HJKMNP-TV-Z]{26}$" },
"id": { "type": "string", "pattern": "^itm_[0-9A-HJKMNP-TV-Z]{26}$" },
"source_key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" },
"source_key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" },
- "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade"] },
+ "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting"] },
"kind": { "type": "string", "pattern": "^[a-z][a-z0-9_]{1,39}$", "description": "article | url | sitemap | product | offer | review | review_signal | record | coupon …" },
"kind": { "type": "string", "pattern": "^[a-z][a-z0-9_]{1,39}$", "description": "article | url | sitemap | product | offer | review | review_signal | record | coupon …" },
"identity": { "type": "string", "minLength": 1, "maxLength": 2048, "description": "The source's own id (guid, @id, API id) or the canonical URL." },
"identity": { "type": "string", "minLength": 1, "maxLength": 2048, "description": "The source's own id (guid, @id, API id) or the canonical URL." },
"canonical_url": { "type": ["string", "null"], "format": "uri" },
"canonical_url": { "type": ["string", "null"], "format": "uri" },
@@ -10,7 +10,7 @@
@@ -10,7 +10,7 @@
"key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" },
"key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" },
"name": { "type": "string", "minLength": 1, "maxLength": 200 },
"name": { "type": "string", "minLength": 1, "maxLength": 200 },
"type": { "enum": ["rss", "atom", "sitemap", "jsonld", "api", "manual"] },
"type": { "enum": ["rss", "atom", "sitemap", "jsonld", "api", "manual"] },
- "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade"] },
+ "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting"] },
"homepage_url": { "type": ["string", "null"], "format": "uri" },
"homepage_url": { "type": ["string", "null"], "format": "uri" },
"endpoints": {
"endpoints": {
"type": "array", "maxItems": 20,
"type": "array", "maxItems": 20,
@@ -12,7 +12,7 @@
@@ -12,7 +12,7 @@
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"fast-xml-parser": "^4.5.3",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.129.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.138.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -769,9 +769,9 @@
@@ -769,9 +769,9 @@
}
}
},
},
"node_modules/openvibe-contracts": {
"node_modules/openvibe-contracts": {
- "version": "0.129.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.129.0",
- "integrity": "sha512-Ly0F/ap6wmAf8hYpNrOhMLsU3cZYBjsFDW2P1F/xVmHVsRDzAPAZDXtFcohP0UNeZgyxgrR0BI0pPG5guD/Efw==",
+ "version": "0.138.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.138.0",
+ "integrity": "sha512-YHOAU+w01TnoXKifqUOdf/JAYggzmH7R/BkZ4VBhmBzJMGNVaq1hEtG7NWl3z1Lfxcm78oqe6aJrLNfJVF2F9w==",
"license": "AGPL-3.0-only",
"license": "AGPL-3.0-only",
"dependencies": {
"dependencies": {
"ajv": "^8.17.1",
"ajv": "^8.17.1",
@@ -27,7 +27,7 @@
@@ -27,7 +27,7 @@
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"fast-xml-parser": "^4.5.3",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.129.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.138.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.44.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.0.0",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -36,8 +36,8 @@
@@ -36,8 +36,8 @@
"node": ">=22.0.0"
"node": ">=22.0.0"
},
},
"devDependencies": {
"devDependencies": {
+ "@electric-sql/pglite": "^0.5.8",
"ajv": "^8.20.0",
"ajv": "^8.20.0",
- "ajv-formats": "^3.0.1",
- "@electric-sql/pglite": "^0.5.8"
+ "ajv-formats": "^3.0.1"
}
}
}
}
@@ -1,5 +1,5 @@
@@ -1,5 +1,5 @@
{
{
- "_about": "One real source per category in the original set, all DISABLED; further news and trade sources were added later, also all DISABLED with review_required. Robots notes were checked by hand on the date each one records (2026-09-22 for the original six, 2026-10-10 for those added later); terms must be re-verified by a person before a source is enabled (PATCH enabled: true). Seeding never overwrites or enables an existing source.",
+ "_about": "One real source per category in the original set, all DISABLED; further news and trade sources were added later, also all DISABLED with review_required. Robots notes were checked by hand on the date each one records (2026-09-22 for the original six, 2026-10-10 for those added later, 2026-10-11 for the two hosting seeds); terms must be re-verified by a person before a source is enabled (PATCH enabled: true). Seeding never overwrites or enables an existing source.",
"sources": [
"sources": [
{
{
"key": "nasa-news-releases",
"key": "nasa-news-releases",
@@ -213,6 +213,82 @@
@@ -213,6 +213,82 @@
"review_required": true,
"review_required": true,
"sensitivity": "financial",
"sensitivity": "financial",
"default_indexability": "noindex"
"default_indexability": "noindex"
+ },
+ {
+ "key": "vultr-plans",
+ "name": "Vultr cloud plans",
+ "type": "api",
+ "category": "hosting",
+ "homepage_url": "https://www.vultr.com/pricing/",
+ "endpoints": [
+ {
+ "url": "https://api.vultr.com/v2/plans?per_page=500",
+ "format": "json",
+ "items_path": "plans",
+ "item_kind": "plan",
+ "fields": {
+ "identity": "id"
+ },
+ "extra": {
+ "vcpu_count": "vcpu_count",
+ "ram": "ram",
+ "disk": "disk",
+ "disk_count": "disk_count",
+ "bandwidth": "bandwidth",
+ "monthly_cost": "monthly_cost",
+ "hourly_cost": "hourly_cost",
+ "type": "type"
+ }
+ }
+ ],
+ "robots_note": "robots.txt checked 2026-10-11: https://api.vultr.com/robots.txt is exactly 'User-agent: *' followed by 'Disallow: /', which disallows every path on the API host. The fetcher re-checks robots.txt on every run and would refuse /v2/plans as disallowed, so this source cannot be enabled as written — an operator must resolve the robots.txt block with Vultr first. (The endpoint itself answers without an API key.)",
+ "terms_note": "Vultr's API documentation says the plans listing needs no API key for ordinary accounts ('If your account has special plans available, you will need to pass your API key ... For all other accounts, the API key is not required'), and the official govultr client documents that 'public operations like listing plans ... will still work' unauthenticated; /v2/plans answered with plan data to our keyless fetcher. www.vultr.com/api/ and www.vultr.com/legal/tos/ both answered 403 to our fetcher, so Vultr's API terms could not be read here and must be read by a person before enabling. Attribute Vultr by linking https://www.vultr.com/pricing/; show each price as an observation 'checked on <date>', never a current price, and imply no endorsement. Re-verify before enabling.",
+ "license_note": "Plan specifications and prices are Vultr's data (https://www.vultr.com/pricing/); Vultr's terms pages could not be fetched (HTTP 403 to our fetcher) and no licence to republish them was found. Attribute Vultr and link the pricing page; no endorsement implied.",
+ "min_interval_ms": 60000,
+ "poll_interval_sec": 86400,
+ "max_items": 200,
+ "enabled": false,
+ "review_required": true,
+ "sensitivity": "none",
+ "default_indexability": "noindex"
+ },
+ {
+ "key": "linode-types",
+ "name": "Akamai Cloud (Linode) plan types",
+ "type": "api",
+ "category": "hosting",
+ "homepage_url": "https://www.linode.com/pricing/",
+ "endpoints": [
+ {
+ "url": "https://api.linode.com/v4/linode/types?page_size=500",
+ "format": "json",
+ "items_path": "data",
+ "item_kind": "plan",
+ "fields": {
+ "identity": "id",
+ "title": "label"
+ },
+ "extra": {
+ "vcpus": "vcpus",
+ "memory": "memory",
+ "disk": "disk",
+ "transfer": "transfer",
+ "class": "class",
+ "price_monthly": "price.monthly",
+ "price_hourly": "price.hourly"
+ }
+ }
+ ],
+ "robots_note": "robots.txt checked 2026-10-11: https://api.linode.com/robots.txt answers 301 to https://api.linode.com/v4/, which answers 404 (a JSON 'Not found'), so the API host publishes no robots.txt rules; a 404 is treated as no restrictions (never guessed as a disallow), so /v4/linode/types is allowed. The fetcher re-checks robots.txt on every run.",
+ "terms_note": "Akamai's Linode API documentation says 'Some of this API's operations are publicly accessible and don't require authentication'; /v4/linode/types answered with type data to our keyless fetcher. https://www.linode.com/legal-tos/ answers 404 and https://www.linode.com/legal redirects to https://www.akamai.com/legal, where the governing documents are Akamai's Acceptable Use Policy and the Akamai Cloud (Linode) Master Services Agreement; none of those pages states a right to, or a clear ban on, republishing plan or price data (the AUP forbids using the network to build a competitive service and IP infringement), so a person must read them before enabling. Attribute Akamai/Linode by linking https://www.linode.com/pricing/; show each price as an observation 'checked on <date>', never a current price, and imply no endorsement. Re-verify before enabling.",
+ "license_note": "Plan specifications and prices are Akamai/Linode's data (https://www.linode.com/pricing/); the terms hub is https://www.akamai.com/legal. No licence to republish them was found. Attribute Akamai/Linode and link the pricing page; no endorsement implied.",
+ "min_interval_ms": 60000,
+ "poll_interval_sec": 86400,
+ "max_items": 200,
+ "enabled": false,
+ "review_required": true,
+ "sensitivity": "none",
+ "default_indexability": "noindex"
}
}
]
]
}
}
@@ -12,7 +12,7 @@
@@ -12,7 +12,7 @@
const { checkApiMapping } = require('./adapters');
const { checkApiMapping } = require('./adapters');
const TYPES = ['rss', 'atom', 'sitemap', 'jsonld', 'api', 'manual'];
const TYPES = ['rss', 'atom', 'sitemap', 'jsonld', 'api', 'manual'];
-const CATEGORIES = ['news', 'blog', 'reviews', 'deals', 'coupons', 'trade'];
+const CATEGORIES = ['news', 'blog', 'reviews', 'deals', 'coupons', 'trade', 'hosting'];
const SENSITIVITY = ['none', 'financial', 'health', 'political', 'legal', 'adult'];
const SENSITIVITY = ['none', 'financial', 'health', 'political', 'legal', 'adult'];
const AUTH_MODES = ['none', 'header', 'bearer', 'query'];
const AUTH_MODES = ['none', 'header', 'bearer', 'query'];
const KEY_RE = /^[a-z0-9][a-z0-9-]{1,63}$/;
const KEY_RE = /^[a-z0-9][a-z0-9-]{1,63}$/;
new file mode 100644
new file mode 100644
@@ -0,0 +1,91 @@
@@ -0,0 +1,91 @@
+{
+ "data": [
+ {
+ "id": "g6-nanode-1",
+ "label": "Nanode 1GB",
+ "price": { "hourly": 0.0075, "monthly": 5.0 },
+ "region_prices": [
+ { "id": "id-cgk", "hourly": 0.009, "monthly": 6.0 },
+ { "id": "br-gru", "hourly": 0.0105, "monthly": 7.0 }
+ ],
+ "addons": {
+ "backups": {
+ "price": { "hourly": 0.003, "monthly": 2.0 },
+ "region_prices": [
+ { "id": "id-cgk", "hourly": 0.0036, "monthly": 2.4 },
+ { "id": "br-gru", "hourly": 0.004, "monthly": 2.8 }
+ ]
+ }
+ },
+ "memory": 1024,
+ "disk": 25600,
+ "transfer": 1000,
+ "vcpus": 1,
+ "gpus": 0,
+ "network_out": 1000,
+ "class": "nanode",
+ "successor": null,
+ "capabilities": [],
+ "accelerated_devices": 0
+ },
+ {
+ "id": "g6-standard-1",
+ "label": "Linode 2GB",
+ "price": { "hourly": 0.018, "monthly": 12.0 },
+ "region_prices": [
+ { "id": "id-cgk", "hourly": 0.022, "monthly": 14.4 },
+ { "id": "br-gru", "hourly": 0.025, "monthly": 16.8 }
+ ],
+ "addons": {
+ "backups": {
+ "price": { "hourly": 0.004, "monthly": 2.5 },
+ "region_prices": [
+ { "id": "id-cgk", "hourly": 0.0045, "monthly": 3.0 },
+ { "id": "br-gru", "hourly": 0.005, "monthly": 3.5 }
+ ]
+ }
+ },
+ "memory": 2048,
+ "disk": 51200,
+ "transfer": 2000,
+ "vcpus": 1,
+ "gpus": 0,
+ "network_out": 2000,
+ "class": "standard",
+ "successor": null,
+ "capabilities": [],
+ "accelerated_devices": 0
+ },
+ {
+ "id": "g6-standard-2",
+ "label": "Linode 4GB",
+ "price": { "hourly": 0.036, "monthly": 24.0 },
+ "region_prices": [
+ { "id": "id-cgk", "hourly": 0.043, "monthly": 28.8 },
+ { "id": "br-gru", "hourly": 0.05, "monthly": 33.6 }
+ ],
+ "addons": {
+ "backups": {
+ "price": { "hourly": 0.008, "monthly": 5.0 },
+ "region_prices": [
+ { "id": "id-cgk", "hourly": 0.009, "monthly": 6.0 },
+ { "id": "br-gru", "hourly": 0.01, "monthly": 7.0 }
+ ]
+ }
+ },
+ "memory": 4096,
+ "disk": 81920,
+ "transfer": 4000,
+ "vcpus": 2,
+ "gpus": 0,
+ "network_out": 4000,
+ "class": "standard",
+ "successor": null,
+ "capabilities": [],
+ "accelerated_devices": 0
+ }
+ ],
+ "page": 1,
+ "pages": 1,
+ "results": 75
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,77 @@
@@ -0,0 +1,77 @@
+{
+ "plans": [
+ {
+ "id": "vc2-1c-0.5gb-free",
+ "vcpu_count": 1,
+ "ram": 512,
+ "disk": 10,
+ "disk_type": "SSD",
+ "disk_count": 1,
+ "bandwidth": 0,
+ "monthly_cost": 0,
+ "hourly_cost": 0,
+ "monthly_cost_preemptible": 0,
+ "hourly_cost_preemptible": 0,
+ "invoice_type": "monthly",
+ "type": "vc2",
+ "locations": ["sea", "fra", "mia"],
+ "cpu_vendor": "Intel",
+ "storage_type": "local_storage",
+ "vcpu_type": "thread",
+ "deploy_ondemand": true,
+ "deploy_preemptible": false,
+ "location_cost": {},
+ "gpu_brand": "none"
+ },
+ {
+ "id": "vc2-1c-0.5gb-v6",
+ "vcpu_count": 1,
+ "ram": 512,
+ "disk": 10,
+ "disk_type": "SSD",
+ "disk_count": 1,
+ "bandwidth": 512,
+ "monthly_cost": 2.5,
+ "hourly_cost": 0.003,
+ "monthly_cost_preemptible": 2.5,
+ "hourly_cost_preemptible": 0.003,
+ "invoice_type": "monthly",
+ "type": "vc2",
+ "locations": ["ewr", "atl"],
+ "cpu_vendor": "Intel",
+ "storage_type": "local_storage",
+ "vcpu_type": "thread",
+ "deploy_ondemand": true,
+ "deploy_preemptible": false,
+ "location_cost": {},
+ "gpu_brand": "none"
+ },
+ {
+ "id": "vc2-1c-0.5gb",
+ "vcpu_count": 1,
+ "ram": 512,
+ "disk": 10,
+ "disk_type": "SSD",
+ "disk_count": 1,
+ "bandwidth": 512,
+ "monthly_cost": 3.5,
+ "hourly_cost": 0.005,
+ "monthly_cost_preemptible": 3.5,
+ "hourly_cost_preemptible": 0.005,
+ "invoice_type": "monthly",
+ "type": "vc2",
+ "locations": ["ewr"],
+ "cpu_vendor": "Intel",
+ "storage_type": "local_storage",
+ "vcpu_type": "thread",
+ "deploy_ondemand": true,
+ "deploy_preemptible": false,
+ "location_cost": {},
+ "gpu_brand": "none"
+ }
+ ],
+ "meta": {
+ "total": 151,
+ "links": { "next": "", "prev": "" }
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,75 @@
@@ -0,0 +1,75 @@
+'use strict';
+/**
+ * The hosting seeds: each provider API maps through the api adapter exactly as the registry
+ * declares it, and the registry accepts category `hosting` (refusing anything else). The fixtures
+ * are the live responses (Vultr /v2/plans, Akamai Linode /v4/linode/types) trimmed to three records.
+ */
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const api = require('../server/adapters/api');
+const { validateSource } = require('../server/registry');
+const { suite } = require('./helpers');
+
+const t = suite('hosting-sources');
+const fx = (name) => fs.readFileSync(path.join(__dirname, 'fixtures', name), 'utf8');
+const seeds = require('../seeds/sources.json').sources;
+const seed = (key) => seeds.find((s) => s.key === key);
+const endpoint = (key) => seed(key).endpoints[0];
+
+t('both hosting seeds declare an api mapping the adapter accepts', () => {
+ for (const key of ['vultr-plans', 'linode-types']) {
+ const s = seed(key);
+ assert.ok(s, `${key} is seeded`);
+ assert.strictEqual(s.type, 'api');
+ assert.strictEqual(s.category, 'hosting');
+ assert.strictEqual(api.checkMapping(endpoint(key)), null, `${key} mapping`);
+ }
+});
+
+t('Vultr /v2/plans: one item per plan, identity from id, every extra field kept', () => {
+ const ep = endpoint('vultr-plans');
+ const r = api.parse(fx('vultr-plans.json'), { url: ep.url, endpoint: ep });
+ assert.strictEqual(r.skipped, 0);
+ assert.deepStrictEqual(r.items.map((i) => i.identity), ['vc2-1c-0.5gb-free', 'vc2-1c-0.5gb-v6', 'vc2-1c-0.5gb']);
+ assert.ok(r.items.every((i) => i.kind === 'plan'));
+ assert.deepStrictEqual(r.items[0].fields, {
+ vcpu_count: 1, ram: 512, disk: 10, disk_count: 1, bandwidth: 0, monthly_cost: 0, hourly_cost: 0, type: 'vc2',
+ });
+ assert.deepStrictEqual(r.items[2].fields, {
+ vcpu_count: 1, ram: 512, disk: 10, disk_count: 1, bandwidth: 512, monthly_cost: 3.5, hourly_cost: 0.005, type: 'vc2',
+ });
+ assert.strictEqual(r.items[0].title, null, 'no title is mapped for Vultr');
+ assert.strictEqual(r.items[0].canonical_url, null, 'no url is mapped for Vultr');
+});
+
+t('Linode /v4/linode/types: identity id, title label, dotted price.monthly/hourly mapped', () => {
+ const ep = endpoint('linode-types');
+ const r = api.parse(fx('linode-types.json'), { url: ep.url, endpoint: ep });
+ assert.strictEqual(r.skipped, 0);
+ assert.deepStrictEqual(r.items.map((i) => i.identity), ['g6-nanode-1', 'g6-standard-1', 'g6-standard-2']);
+ assert.deepStrictEqual(r.items.map((i) => i.title), ['Nanode 1GB', 'Linode 2GB', 'Linode 4GB']);
+ assert.ok(r.items.every((i) => i.kind === 'plan'));
+ assert.deepStrictEqual(r.items[0].fields, {
+ vcpus: 1, memory: 1024, disk: 25600, transfer: 1000, class: 'nanode', price_monthly: 5, price_hourly: 0.0075,
+ });
+ assert.deepStrictEqual(r.items[2].fields, {
+ vcpus: 2, memory: 4096, disk: 81920, transfer: 4000, class: 'standard', price_monthly: 24, price_hourly: 0.036,
+ });
+});
+
+t('the registry accepts category hosting and refuses an unknown category', () => {
+ const rec = validateSource(seed('vultr-plans'));
+ assert.strictEqual(rec.category, 'hosting');
+ assert.deepStrictEqual({
+ enabled: rec.enabled, review_required: rec.review_required, sensitivity: rec.sensitivity,
+ default_indexability: rec.default_indexability, poll_interval_sec: rec.poll_interval_sec,
+ min_interval_ms: rec.min_interval_ms, max_items: rec.max_items,
+ }, {
+ enabled: false, review_required: true, sensitivity: 'none', default_indexability: 'noindex',
+ poll_interval_sec: 86400, min_interval_ms: 60000, max_items: 200,
+ });
+ assert.throws(() => validateSource({ ...seed('vultr-plans'), category: 'hostingg' }), /category must be one of/);
+});
+
+t.run();