openvibers/openvibe.contracts Commits
0.224.0: a crawl source type for OpenVibe.Sources (sitemaps, links, include/exclude, recrawl; robots and terms as every source) (#232)
Changes
@@@ -4,6 +4,23 @@ All notable changes to `openvibe-contracts`. Releases are git tags (`vX.Y.Z`) thfrom `https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/<tag>`. Before v0.30.0,the notes were in the tag and commit messages (`git tag -n1`).++## 0.224.0 — 2026-10-28++++- **A `crawl` source type for OpenVibe.Sources** (the owner, 2026-10-28: "aggregate all websites and select deals on all++ sites… a full comprehensive crawler system… the open source Google"). A crawl source walks one site from its endpoints++ (the seeds):++ - **How pages are found:** the site's own sitemaps (robots.txt `Sitemap` lines, else `/sitemap.xml`, indexes++ followed), and, if asked, its same-host links.++ - **Which pages:** `include` and `exclude` patterns.++ - **Limits:** `max_pages`, `pages_per_run`, `recrawl_sec` and `max_depth`.++ - **What is kept:** `require_structured`, only pages that publish schema.org JSON-LD.++ - **The rules every source already follows still apply:** robots.txt on every request and redirect hop, per-host++ spacing and Crawl-delay, a terms note before it can be enabled, and noindex pages never kept.++ - **What is read:** only what a page publishes for machines (products and offers, articles, reviews).++ - **Price history:** a page fetched again on schedule turns a price seen twice into a history (Deals' observations).++ - **Fields:** `type` gains `crawl` and sources gain `crawl` on `sources.source@1` and its write request. There is a++ valid fixture.++## 0.223.0 — 2026-10-28- **Run sessions** (the Desks plan's D6; `OpenVibe_Agent_Desks_Plan_2026-10-27.md`). A session is a browser desk that@@@ -24,6 +24,7 @@"sitemap","jsonld","api",++ "crawl","manual"]},@@@ -144,7 +145,9 @@"pattern": "^[a-z][a-z0-9_]{1,39}$"},"request_content_type": {-- "enum": ["application/json"],++ "enum": [++ "application/json"++ ],"description": "Sent as the request's Content-Type on this endpoint's GET (no body). Open Collective's GraphQL API answers a GET only with it, as its CSRF guard (Sources#65)."},"max_bytes": {@@@ -179,6 +182,72 @@]}},++ "crawl": {++ "type": [++ "object",++ "null"++ ],++ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",++ "additionalProperties": false,++ "properties": {++ "sitemaps": {++ "type": "boolean",++ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."++ },++ "follow_links": {++ "type": "boolean",++ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."++ },++ "include": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."++ },++ "exclude": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."++ },++ "max_pages": {++ "type": "integer",++ "minimum": 1,++ "maximum": 200000,++ "description": "The most pages the frontier holds for this site (default 5000)."++ },++ "pages_per_run": {++ "type": "integer",++ "minimum": 1,++ "maximum": 1000,++ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."++ },++ "recrawl_sec": {++ "type": "integer",++ "minimum": 3600,++ "maximum": 2592000,++ "description": "How long after a fetch a page is due again (default 86400, a day)."++ },++ "max_depth": {++ "type": "integer",++ "minimum": 0,++ "maximum": 10,++ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."++ },++ "require_structured": {++ "type": "boolean",++ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."++ }++ }++ },"auth": {"type": "object","required": [@@@ -244,7 +313,10 @@"minimum": 1},"retire_after_sec": {-- "type": ["integer", "null"],++ "type": [++ "integer",++ "null"++ ],"minimum": 3600,"maximum": 31536000,"description": "Retire what the feed stopped showing (jobs, listings): once a fetch of this source succeeds, an item it has not seen for this many seconds is removed with reason \"gone_from_source\" (a change, so consumers following the change feed learn it). null (the default): items stay, as a news archive or a dataset should."@@@ -38,6 +38,7 @@"sitemap","jsonld","api",++ "crawl","manual"]},@@@ -151,7 +152,9 @@"pattern": "^[a-z][a-z0-9_]{1,39}$"},"request_content_type": {-- "enum": ["application/json"],++ "enum": [++ "application/json"++ ],"description": "Sent as the request's Content-Type on this endpoint's GET (no body). Open Collective's GraphQL API answers a GET only with it, as its CSRF guard (Sources#65)."},"max_bytes": {@@@ -249,6 +252,72 @@}}},++ "crawl": {++ "type": [++ "object",++ "null"++ ],++ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",++ "additionalProperties": false,++ "properties": {++ "sitemaps": {++ "type": "boolean",++ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."++ },++ "follow_links": {++ "type": "boolean",++ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."++ },++ "include": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."++ },++ "exclude": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."++ },++ "max_pages": {++ "type": "integer",++ "minimum": 1,++ "maximum": 200000,++ "description": "The most pages the frontier holds for this site (default 5000)."++ },++ "pages_per_run": {++ "type": "integer",++ "minimum": 1,++ "maximum": 1000,++ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."++ },++ "recrawl_sec": {++ "type": "integer",++ "minimum": 3600,++ "maximum": 2592000,++ "description": "How long after a fetch a page is due again (default 86400, a day)."++ },++ "max_depth": {++ "type": "integer",++ "minimum": 0,++ "maximum": 10,++ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."++ },++ "require_structured": {++ "type": "boolean",++ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."++ }++ }++ },"auth": {"type": "object","required": [@@@ -314,7 +383,10 @@"minimum": 1},"retire_after_sec": {-- "type": ["integer", "null"],++ "type": [++ "integer",++ "null"++ ],"minimum": 3600,"maximum": 31536000,"description": "Retire what the feed stopped showing (jobs, listings): once a fetch of this source succeeds, an item it has not seen for this many seconds is removed with reason \"gone_from_source\" (a change, so consumers following the change feed learn it). null (the default): items stay, as a news archive or a dataset should."nnew file mode 100644@@@ -0,0 +1,42 @@++{++ "key": "shop-example-products",++ "name": "Example Shop products",++ "type": "crawl",++ "category": "deals",++ "homepage_url": "https://shop.example/",++ "endpoints": [++ {++ "url": "https://shop.example/"++ }++ ],++ "crawl": {++ "sitemaps": true,++ "follow_links": false,++ "include": [++ "^/products/"++ ],++ "exclude": [++ "/cart",++ "/account"++ ],++ "max_pages": 5000,++ "pages_per_run": 100,++ "recrawl_sec": 86400,++ "max_depth": 2,++ "require_structured": true++ },++ "auth": {++ "mode": "none"++ },++ "robots_note": "robots.txt allows /products/ for every agent; Crawl-delay honoured.",++ "terms_note": "Public product pages; only the schema.org Product/Offer JSON-LD the shop publishes for search engines is read.",++ "license_note": null,++ "min_interval_ms": 60000,++ "poll_interval_sec": 3600,++ "stale_after_sec": 172800,++ "max_items": 200,++ "enabled": false,++ "review_required": true,++ "sensitivity": "none",++ "default_indexability": "noindex"++}@@@ -3747,6 +3747,7 @@"sitemap","jsonld","api",++ "crawl","manual"]},@@@ -3960,6 +3961,72 @@}}},++ "crawl": {++ "type": [++ "object",++ "null"++ ],++ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",++ "additionalProperties": false,++ "properties": {++ "sitemaps": {++ "type": "boolean",++ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."++ },++ "follow_links": {++ "type": "boolean",++ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."++ },++ "include": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."++ },++ "exclude": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."++ },++ "max_pages": {++ "type": "integer",++ "minimum": 1,++ "maximum": 200000,++ "description": "The most pages the frontier holds for this site (default 5000)."++ },++ "pages_per_run": {++ "type": "integer",++ "minimum": 1,++ "maximum": 1000,++ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."++ },++ "recrawl_sec": {++ "type": "integer",++ "minimum": 3600,++ "maximum": 2592000,++ "description": "How long after a fetch a page is due again (default 86400, a day)."++ },++ "max_depth": {++ "type": "integer",++ "minimum": 0,++ "maximum": 10,++ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."++ },++ "require_structured": {++ "type": "boolean",++ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."++ }++ }++ },"auth": {"type": "object","required": [@@@ -7339,6 +7406,7 @@"sitemap","jsonld","api",++ "crawl","manual"]},@@@ -7496,6 +7564,72 @@]}},++ "crawl": {++ "type": [++ "object",++ "null"++ ],++ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",++ "additionalProperties": false,++ "properties": {++ "sitemaps": {++ "type": "boolean",++ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."++ },++ "follow_links": {++ "type": "boolean",++ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."++ },++ "include": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."++ },++ "exclude": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."++ },++ "max_pages": {++ "type": "integer",++ "minimum": 1,++ "maximum": 200000,++ "description": "The most pages the frontier holds for this site (default 5000)."++ },++ "pages_per_run": {++ "type": "integer",++ "minimum": 1,++ "maximum": 1000,++ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."++ },++ "recrawl_sec": {++ "type": "integer",++ "minimum": 3600,++ "maximum": 2592000,++ "description": "How long after a fetch a page is due again (default 86400, a day)."++ },++ "max_depth": {++ "type": "integer",++ "minimum": 0,++ "maximum": 10,++ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."++ },++ "require_structured": {++ "type": "boolean",++ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."++ }++ }++ },"auth": {"type": "object","required": [@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Actor API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Actor as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Actor as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.AI API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.AI as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.AI as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Billing API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Billing as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Billing as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Blog API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Blog as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Blog as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Bot API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Bot as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Bot as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Chat API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Chat as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Chat as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Codes API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Codes as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Codes as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Community API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Community as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Community as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Coupons API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Coupons as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Coupons as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Deals API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Deals as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Deals as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Events API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Events as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Events as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Food API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Food as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Food as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Games API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Games as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Games as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Host API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Host as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Host as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Inventory API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Inventory as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Inventory as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Live API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Live as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Live as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.MediaHub API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.MediaHub as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.MediaHub as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Media API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Media as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Media as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Network API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Network as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Network as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.News API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.News as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.News as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenRestream API",-- "version": "0.223.0",-- "description": "Routes of OpenRestream as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenRestream as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Play API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Play as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Play as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Quest API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Quest as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Quest as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Reviews API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Reviews as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Reviews as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Run API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Run as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Run as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Search API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Search as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Search as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Services API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Services as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Services as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Sources API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Sources as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Sources as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -1460,6 +1460,7 @@"sitemap","jsonld","api",++ "crawl","manual"]},@@@ -1617,6 +1618,72 @@]}},++ "crawl": {++ "type": [++ "object",++ "null"++ ],++ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",++ "additionalProperties": false,++ "properties": {++ "sitemaps": {++ "type": "boolean",++ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."++ },++ "follow_links": {++ "type": "boolean",++ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."++ },++ "include": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."++ },++ "exclude": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."++ },++ "max_pages": {++ "type": "integer",++ "minimum": 1,++ "maximum": 200000,++ "description": "The most pages the frontier holds for this site (default 5000)."++ },++ "pages_per_run": {++ "type": "integer",++ "minimum": 1,++ "maximum": 1000,++ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."++ },++ "recrawl_sec": {++ "type": "integer",++ "minimum": 3600,++ "maximum": 2592000,++ "description": "How long after a fetch a page is due again (default 86400, a day)."++ },++ "max_depth": {++ "type": "integer",++ "minimum": 0,++ "maximum": 10,++ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."++ },++ "require_structured": {++ "type": "boolean",++ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."++ }++ }++ },"auth": {"type": "object","required": [@@@ -1859,6 +1926,7 @@"sitemap","jsonld","api",++ "crawl","manual"]},@@@ -2072,6 +2140,72 @@}}},++ "crawl": {++ "type": [++ "object",++ "null"++ ],++ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",++ "additionalProperties": false,++ "properties": {++ "sitemaps": {++ "type": "boolean",++ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."++ },++ "follow_links": {++ "type": "boolean",++ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."++ },++ "include": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."++ },++ "exclude": {++ "type": "array",++ "maxItems": 20,++ "items": {++ "type": "string",++ "minLength": 1,++ "maxLength": 200++ },++ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."++ },++ "max_pages": {++ "type": "integer",++ "minimum": 1,++ "maximum": 200000,++ "description": "The most pages the frontier holds for this site (default 5000)."++ },++ "pages_per_run": {++ "type": "integer",++ "minimum": 1,++ "maximum": 1000,++ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."++ },++ "recrawl_sec": {++ "type": "integer",++ "minimum": 3600,++ "maximum": 2592000,++ "description": "How long after a fetch a page is due again (default 86400, a day)."++ },++ "max_depth": {++ "type": "integer",++ "minimum": 0,++ "maximum": 10,++ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."++ },++ "require_structured": {++ "type": "boolean",++ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."++ }++ }++ },"auth": {"type": "object","required": [@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Space API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Space as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Space as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Tips API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Tips as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Tips as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Tools API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Tools as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Tools as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Trade API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Trade as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Trade as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.VIP API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.VIP as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.VIP as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Watch API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Watch as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Watch as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Wiki API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Wiki as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Wiki as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Work API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Work as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Work as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -3,8 +3,8 @@"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema","info": {"title": "OpenVibe.Zone API",-- "version": "0.223.0",-- "description": "Routes of OpenVibe.Zone as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",++ "version": "0.224.0",++ "description": "Routes of OpenVibe.Zone as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.","license": {"name": "MIT","identifier": "MIT"@@@ -6155,7 +6155,7 @@ export interface SearchWriteResult {export interface Source {key: string;name: string;-- type: "rss" | "atom" | "sitemap" | "jsonld" | "api" | "manual";++ type: "rss" | "atom" | "sitemap" | "jsonld" | "api" | "crawl" | "manual";category:| "news"| "blog"@@@ -75515,6 +75515,315 @@ export interface Source {};}];++ /**++ * type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.++ */++ crawl?: {++ /**++ * Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true).++ */++ sitemaps?: boolean;++ /**++ * Also follow links on crawled pages to pages of the same host that match include (default false).++ */++ follow_links?: boolean;++ /**++ * Regular expressions over a URL's path and query; a page must match one (none given: every page of the host).++ *++ * @maxItems 20++ */++ include?:++ | []++ | [string]++ | [string, string]++ | [string, string, string]++ | [string, string, string, string]++ | [string, string, string, string, string]++ | [string, string, string, string, string, string]++ | [string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string, string, string]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ];++ /**++ * Regular expressions over a URL's path and query; a page matching one is never fetched.++ *++ * @maxItems 20++ */++ exclude?:++ | []++ | [string]++ | [string, string]++ | [string, string, string]++ | [string, string, string, string]++ | [string, string, string, string, string]++ | [string, string, string, string, string, string]++ | [string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string, string, string]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ];++ /**++ * The most pages the frontier holds for this site (default 5000).++ */++ max_pages?: number;++ /**++ * Pages fetched in one run (default 100); the host spacing still applies between them.++ */++ pages_per_run?: number;++ /**++ * How long after a fetch a page is due again (default 86400, a day).++ */++ recrawl_sec?: number;++ /**++ * With follow_links, how many links away from a seed or sitemap page a page may be (default 2).++ */++ max_depth?: number;++ /**++ * Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier.++ */++ require_structured?: boolean;++ } | null;auth: {mode: "none" | "header" | "bearer" | "query";/**@@@ -78600,7 +78909,7 @@ export type SourcesSourceWriteRequest =| {key?: string;name?: string;-- type?: "rss" | "atom" | "sitemap" | "jsonld" | "api" | "manual";++ type?: "rss" | "atom" | "sitemap" | "jsonld" | "api" | "crawl" | "manual";category?:| "news"| "blog"@@@ -141028,6 +141337,345 @@ export type SourcesSourceWriteRequest =})];++ /**++ * type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.++ */++ crawl?: {++ /**++ * Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true).++ */++ sitemaps?: boolean;++ /**++ * Also follow links on crawled pages to pages of the same host that match include (default false).++ */++ follow_links?: boolean;++ /**++ * Regular expressions over a URL's path and query; a page must match one (none given: every page of the host).++ *++ * @maxItems 20++ */++ include?:++ | []++ | [string]++ | [string, string]++ | [string, string, string]++ | [string, string, string, string]++ | [string, string, string, string, string]++ | [string, string, string, string, string, string]++ | [string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string, string]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ];++ /**++ * Regular expressions over a URL's path and query; a page matching one is never fetched.++ *++ * @maxItems 20++ */++ exclude?:++ | []++ | [string]++ | [string, string]++ | [string, string, string]++ | [string, string, string, string]++ | [string, string, string, string, string]++ | [string, string, string, string, string, string]++ | [string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string]++ | [string, string, string, string, string, string, string, string, string, string, string, string, string]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ]++ | [++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string,++ string++ ];++ /**++ * The most pages the frontier holds for this site (default 5000).++ */++ max_pages?: number;++ /**++ * Pages fetched in one run (default 100); the host spacing still applies between them.++ */++ pages_per_run?: number;++ /**++ * How long after a fetch a page is due again (default 86400, a day).++ */++ recrawl_sec?: number;++ /**++ * With follow_links, how many links away from a seed or sitemap page a page may be (default 2).++ */++ max_depth?: number;++ /**++ * Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier.++ */++ require_structured?: boolean;++ } | null;auth?: {mode: "none" | "header" | "bearer" | "query";/**@@@ -1,6 +1,6 @@{"name": "openvibe-contracts",-- "version": "0.223.0",++ "version": "0.224.0","description": "Machine-readable contracts for the OpenVibe network: schemas, manifests, ids, error envelope, capability checks.","main": "index.js","types": "generated/typescript/index.d.ts",
@@ -4,6 +4,23 @@ All notable changes to `openvibe-contracts`. Releases are git tags (`vX.Y.Z`) th
@@ -4,6 +4,23 @@ All notable changes to `openvibe-contracts`. Releases are git tags (`vX.Y.Z`) th
from `https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/<tag>`. Before v0.30.0,
from `https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/<tag>`. Before v0.30.0,
the notes were in the tag and commit messages (`git tag -n1`).
the notes were in the tag and commit messages (`git tag -n1`).
+## 0.224.0 — 2026-10-28
+
+- **A `crawl` source type for OpenVibe.Sources** (the owner, 2026-10-28: "aggregate all websites and select deals on all
+ sites… a full comprehensive crawler system… the open source Google"). A crawl source walks one site from its endpoints
+ (the seeds):
+ - **How pages are found:** the site's own sitemaps (robots.txt `Sitemap` lines, else `/sitemap.xml`, indexes
+ followed), and, if asked, its same-host links.
+ - **Which pages:** `include` and `exclude` patterns.
+ - **Limits:** `max_pages`, `pages_per_run`, `recrawl_sec` and `max_depth`.
+ - **What is kept:** `require_structured`, only pages that publish schema.org JSON-LD.
+ - **The rules every source already follows still apply:** robots.txt on every request and redirect hop, per-host
+ spacing and Crawl-delay, a terms note before it can be enabled, and noindex pages never kept.
+ - **What is read:** only what a page publishes for machines (products and offers, articles, reviews).
+ - **Price history:** a page fetched again on schedule turns a price seen twice into a history (Deals' observations).
+ - **Fields:** `type` gains `crawl` and sources gain `crawl` on `sources.source@1` and its write request. There is a
+ valid fixture.
+
## 0.223.0 — 2026-10-28
## 0.223.0 — 2026-10-28
- **Run sessions** (the Desks plan's D6; `OpenVibe_Agent_Desks_Plan_2026-10-27.md`). A session is a browser desk that
- **Run sessions** (the Desks plan's D6; `OpenVibe_Agent_Desks_Plan_2026-10-27.md`). A session is a browser desk that
@@ -24,6 +24,7 @@
@@ -24,6 +24,7 @@
"sitemap",
"sitemap",
"jsonld",
"jsonld",
"api",
"api",
+ "crawl",
"manual"
"manual"
]
]
},
},
@@ -144,7 +145,9 @@
@@ -144,7 +145,9 @@
"pattern": "^[a-z][a-z0-9_]{1,39}$"
"pattern": "^[a-z][a-z0-9_]{1,39}$"
},
},
"request_content_type": {
"request_content_type": {
- "enum": ["application/json"],
+ "enum": [
+ "application/json"
+ ],
"description": "Sent as the request's Content-Type on this endpoint's GET (no body). Open Collective's GraphQL API answers a GET only with it, as its CSRF guard (Sources#65)."
"description": "Sent as the request's Content-Type on this endpoint's GET (no body). Open Collective's GraphQL API answers a GET only with it, as its CSRF guard (Sources#65)."
},
},
"max_bytes": {
"max_bytes": {
@@ -179,6 +182,72 @@
@@ -179,6 +182,72 @@
]
]
}
}
},
},
+ "crawl": {
+ "type": [
+ "object",
+ "null"
+ ],
+ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",
+ "additionalProperties": false,
+ "properties": {
+ "sitemaps": {
+ "type": "boolean",
+ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."
+ },
+ "follow_links": {
+ "type": "boolean",
+ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."
+ },
+ "include": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."
+ },
+ "exclude": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."
+ },
+ "max_pages": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 200000,
+ "description": "The most pages the frontier holds for this site (default 5000)."
+ },
+ "pages_per_run": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 1000,
+ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."
+ },
+ "recrawl_sec": {
+ "type": "integer",
+ "minimum": 3600,
+ "maximum": 2592000,
+ "description": "How long after a fetch a page is due again (default 86400, a day)."
+ },
+ "max_depth": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 10,
+ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."
+ },
+ "require_structured": {
+ "type": "boolean",
+ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."
+ }
+ }
+ },
"auth": {
"auth": {
"type": "object",
"type": "object",
"required": [
"required": [
@@ -244,7 +313,10 @@
@@ -244,7 +313,10 @@
"minimum": 1
"minimum": 1
},
},
"retire_after_sec": {
"retire_after_sec": {
- "type": ["integer", "null"],
+ "type": [
+ "integer",
+ "null"
+ ],
"minimum": 3600,
"minimum": 3600,
"maximum": 31536000,
"maximum": 31536000,
"description": "Retire what the feed stopped showing (jobs, listings): once a fetch of this source succeeds, an item it has not seen for this many seconds is removed with reason \"gone_from_source\" (a change, so consumers following the change feed learn it). null (the default): items stay, as a news archive or a dataset should."
"description": "Retire what the feed stopped showing (jobs, listings): once a fetch of this source succeeds, an item it has not seen for this many seconds is removed with reason \"gone_from_source\" (a change, so consumers following the change feed learn it). null (the default): items stay, as a news archive or a dataset should."
@@ -38,6 +38,7 @@
@@ -38,6 +38,7 @@
"sitemap",
"sitemap",
"jsonld",
"jsonld",
"api",
"api",
+ "crawl",
"manual"
"manual"
]
]
},
},
@@ -151,7 +152,9 @@
@@ -151,7 +152,9 @@
"pattern": "^[a-z][a-z0-9_]{1,39}$"
"pattern": "^[a-z][a-z0-9_]{1,39}$"
},
},
"request_content_type": {
"request_content_type": {
- "enum": ["application/json"],
+ "enum": [
+ "application/json"
+ ],
"description": "Sent as the request's Content-Type on this endpoint's GET (no body). Open Collective's GraphQL API answers a GET only with it, as its CSRF guard (Sources#65)."
"description": "Sent as the request's Content-Type on this endpoint's GET (no body). Open Collective's GraphQL API answers a GET only with it, as its CSRF guard (Sources#65)."
},
},
"max_bytes": {
"max_bytes": {
@@ -249,6 +252,72 @@
@@ -249,6 +252,72 @@
}
}
}
}
},
},
+ "crawl": {
+ "type": [
+ "object",
+ "null"
+ ],
+ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",
+ "additionalProperties": false,
+ "properties": {
+ "sitemaps": {
+ "type": "boolean",
+ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."
+ },
+ "follow_links": {
+ "type": "boolean",
+ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."
+ },
+ "include": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."
+ },
+ "exclude": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."
+ },
+ "max_pages": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 200000,
+ "description": "The most pages the frontier holds for this site (default 5000)."
+ },
+ "pages_per_run": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 1000,
+ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."
+ },
+ "recrawl_sec": {
+ "type": "integer",
+ "minimum": 3600,
+ "maximum": 2592000,
+ "description": "How long after a fetch a page is due again (default 86400, a day)."
+ },
+ "max_depth": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 10,
+ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."
+ },
+ "require_structured": {
+ "type": "boolean",
+ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."
+ }
+ }
+ },
"auth": {
"auth": {
"type": "object",
"type": "object",
"required": [
"required": [
@@ -314,7 +383,10 @@
@@ -314,7 +383,10 @@
"minimum": 1
"minimum": 1
},
},
"retire_after_sec": {
"retire_after_sec": {
- "type": ["integer", "null"],
+ "type": [
+ "integer",
+ "null"
+ ],
"minimum": 3600,
"minimum": 3600,
"maximum": 31536000,
"maximum": 31536000,
"description": "Retire what the feed stopped showing (jobs, listings): once a fetch of this source succeeds, an item it has not seen for this many seconds is removed with reason \"gone_from_source\" (a change, so consumers following the change feed learn it). null (the default): items stay, as a news archive or a dataset should."
"description": "Retire what the feed stopped showing (jobs, listings): once a fetch of this source succeeds, an item it has not seen for this many seconds is removed with reason \"gone_from_source\" (a change, so consumers following the change feed learn it). null (the default): items stay, as a news archive or a dataset should."
new file mode 100644
new file mode 100644
@@ -0,0 +1,42 @@
@@ -0,0 +1,42 @@
+{
+ "key": "shop-example-products",
+ "name": "Example Shop products",
+ "type": "crawl",
+ "category": "deals",
+ "homepage_url": "https://shop.example/",
+ "endpoints": [
+ {
+ "url": "https://shop.example/"
+ }
+ ],
+ "crawl": {
+ "sitemaps": true,
+ "follow_links": false,
+ "include": [
+ "^/products/"
+ ],
+ "exclude": [
+ "/cart",
+ "/account"
+ ],
+ "max_pages": 5000,
+ "pages_per_run": 100,
+ "recrawl_sec": 86400,
+ "max_depth": 2,
+ "require_structured": true
+ },
+ "auth": {
+ "mode": "none"
+ },
+ "robots_note": "robots.txt allows /products/ for every agent; Crawl-delay honoured.",
+ "terms_note": "Public product pages; only the schema.org Product/Offer JSON-LD the shop publishes for search engines is read.",
+ "license_note": null,
+ "min_interval_ms": 60000,
+ "poll_interval_sec": 3600,
+ "stale_after_sec": 172800,
+ "max_items": 200,
+ "enabled": false,
+ "review_required": true,
+ "sensitivity": "none",
+ "default_indexability": "noindex"
+}
@@ -3747,6 +3747,7 @@
@@ -3747,6 +3747,7 @@
"sitemap",
"sitemap",
"jsonld",
"jsonld",
"api",
"api",
+ "crawl",
"manual"
"manual"
]
]
},
},
@@ -3960,6 +3961,72 @@
@@ -3960,6 +3961,72 @@
}
}
}
}
},
},
+ "crawl": {
+ "type": [
+ "object",
+ "null"
+ ],
+ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",
+ "additionalProperties": false,
+ "properties": {
+ "sitemaps": {
+ "type": "boolean",
+ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."
+ },
+ "follow_links": {
+ "type": "boolean",
+ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."
+ },
+ "include": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."
+ },
+ "exclude": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."
+ },
+ "max_pages": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 200000,
+ "description": "The most pages the frontier holds for this site (default 5000)."
+ },
+ "pages_per_run": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 1000,
+ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."
+ },
+ "recrawl_sec": {
+ "type": "integer",
+ "minimum": 3600,
+ "maximum": 2592000,
+ "description": "How long after a fetch a page is due again (default 86400, a day)."
+ },
+ "max_depth": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 10,
+ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."
+ },
+ "require_structured": {
+ "type": "boolean",
+ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."
+ }
+ }
+ },
"auth": {
"auth": {
"type": "object",
"type": "object",
"required": [
"required": [
@@ -7339,6 +7406,7 @@
@@ -7339,6 +7406,7 @@
"sitemap",
"sitemap",
"jsonld",
"jsonld",
"api",
"api",
+ "crawl",
"manual"
"manual"
]
]
},
},
@@ -7496,6 +7564,72 @@
@@ -7496,6 +7564,72 @@
]
]
}
}
},
},
+ "crawl": {
+ "type": [
+ "object",
+ "null"
+ ],
+ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",
+ "additionalProperties": false,
+ "properties": {
+ "sitemaps": {
+ "type": "boolean",
+ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."
+ },
+ "follow_links": {
+ "type": "boolean",
+ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."
+ },
+ "include": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."
+ },
+ "exclude": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."
+ },
+ "max_pages": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 200000,
+ "description": "The most pages the frontier holds for this site (default 5000)."
+ },
+ "pages_per_run": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 1000,
+ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."
+ },
+ "recrawl_sec": {
+ "type": "integer",
+ "minimum": 3600,
+ "maximum": 2592000,
+ "description": "How long after a fetch a page is due again (default 86400, a day)."
+ },
+ "max_depth": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 10,
+ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."
+ },
+ "require_structured": {
+ "type": "boolean",
+ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."
+ }
+ }
+ },
"auth": {
"auth": {
"type": "object",
"type": "object",
"required": [
"required": [
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Actor API",
"title": "OpenVibe.Actor API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Actor as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Actor as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.AI API",
"title": "OpenVibe.AI API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.AI as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.AI as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Billing API",
"title": "OpenVibe.Billing API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Billing as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Billing as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Blog API",
"title": "OpenVibe.Blog API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Blog as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Blog as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Bot API",
"title": "OpenVibe.Bot API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Bot as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Bot as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Chat API",
"title": "OpenVibe.Chat API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Chat as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Chat as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Codes API",
"title": "OpenVibe.Codes API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Codes as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Codes as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Community API",
"title": "OpenVibe.Community API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Community as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Community as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Coupons API",
"title": "OpenVibe.Coupons API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Coupons as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Coupons as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Deals API",
"title": "OpenVibe.Deals API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Deals as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Deals as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Events API",
"title": "OpenVibe.Events API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Events as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Events as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Food API",
"title": "OpenVibe.Food API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Food as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Food as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Games API",
"title": "OpenVibe.Games API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Games as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Games as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Host API",
"title": "OpenVibe.Host API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Host as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Host as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Inventory API",
"title": "OpenVibe.Inventory API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Inventory as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Inventory as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Live API",
"title": "OpenVibe.Live API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Live as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Live as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.MediaHub API",
"title": "OpenVibe.MediaHub API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.MediaHub as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.MediaHub as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Media API",
"title": "OpenVibe.Media API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Media as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Media as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Network API",
"title": "OpenVibe.Network API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Network as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Network as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.News API",
"title": "OpenVibe.News API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.News as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.News as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenRestream API",
"title": "OpenRestream API",
- "version": "0.223.0",
- "description": "Routes of OpenRestream as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenRestream as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Play API",
"title": "OpenVibe.Play API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Play as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Play as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Quest API",
"title": "OpenVibe.Quest API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Quest as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Quest as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Reviews API",
"title": "OpenVibe.Reviews API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Reviews as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Reviews as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Run API",
"title": "OpenVibe.Run API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Run as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Run as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Search API",
"title": "OpenVibe.Search API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Search as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Search as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Services API",
"title": "OpenVibe.Services API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Services as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Services as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Sources API",
"title": "OpenVibe.Sources API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Sources as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Sources as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -1460,6 +1460,7 @@
@@ -1460,6 +1460,7 @@
"sitemap",
"sitemap",
"jsonld",
"jsonld",
"api",
"api",
+ "crawl",
"manual"
"manual"
]
]
},
},
@@ -1617,6 +1618,72 @@
@@ -1617,6 +1618,72 @@
]
]
}
}
},
},
+ "crawl": {
+ "type": [
+ "object",
+ "null"
+ ],
+ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",
+ "additionalProperties": false,
+ "properties": {
+ "sitemaps": {
+ "type": "boolean",
+ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."
+ },
+ "follow_links": {
+ "type": "boolean",
+ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."
+ },
+ "include": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."
+ },
+ "exclude": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."
+ },
+ "max_pages": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 200000,
+ "description": "The most pages the frontier holds for this site (default 5000)."
+ },
+ "pages_per_run": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 1000,
+ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."
+ },
+ "recrawl_sec": {
+ "type": "integer",
+ "minimum": 3600,
+ "maximum": 2592000,
+ "description": "How long after a fetch a page is due again (default 86400, a day)."
+ },
+ "max_depth": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 10,
+ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."
+ },
+ "require_structured": {
+ "type": "boolean",
+ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."
+ }
+ }
+ },
"auth": {
"auth": {
"type": "object",
"type": "object",
"required": [
"required": [
@@ -1859,6 +1926,7 @@
@@ -1859,6 +1926,7 @@
"sitemap",
"sitemap",
"jsonld",
"jsonld",
"api",
"api",
+ "crawl",
"manual"
"manual"
]
]
},
},
@@ -2072,6 +2140,72 @@
@@ -2072,6 +2140,72 @@
}
}
}
}
},
},
+ "crawl": {
+ "type": [
+ "object",
+ "null"
+ ],
+ "description": "type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.",
+ "additionalProperties": false,
+ "properties": {
+ "sitemaps": {
+ "type": "boolean",
+ "description": "Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true)."
+ },
+ "follow_links": {
+ "type": "boolean",
+ "description": "Also follow links on crawled pages to pages of the same host that match include (default false)."
+ },
+ "include": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page must match one (none given: every page of the host)."
+ },
+ "exclude": {
+ "type": "array",
+ "maxItems": 20,
+ "items": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 200
+ },
+ "description": "Regular expressions over a URL's path and query; a page matching one is never fetched."
+ },
+ "max_pages": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 200000,
+ "description": "The most pages the frontier holds for this site (default 5000)."
+ },
+ "pages_per_run": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 1000,
+ "description": "Pages fetched in one run (default 100); the host spacing still applies between them."
+ },
+ "recrawl_sec": {
+ "type": "integer",
+ "minimum": 3600,
+ "maximum": 2592000,
+ "description": "How long after a fetch a page is due again (default 86400, a day)."
+ },
+ "max_depth": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 10,
+ "description": "With follow_links, how many links away from a seed or sitemap page a page may be (default 2)."
+ },
+ "require_structured": {
+ "type": "boolean",
+ "description": "Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier."
+ }
+ }
+ },
"auth": {
"auth": {
"type": "object",
"type": "object",
"required": [
"required": [
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Space API",
"title": "OpenVibe.Space API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Space as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Space as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Tips API",
"title": "OpenVibe.Tips API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Tips as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Tips as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Tools API",
"title": "OpenVibe.Tools API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Tools as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Tools as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Trade API",
"title": "OpenVibe.Trade API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Trade as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Trade as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.VIP API",
"title": "OpenVibe.VIP API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.VIP as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.VIP as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Watch API",
"title": "OpenVibe.Watch API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Watch as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Watch as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Wiki API",
"title": "OpenVibe.Wiki API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Wiki as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Wiki as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Work API",
"title": "OpenVibe.Work API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Work as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Work as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -3,8 +3,8 @@
@@ -3,8 +3,8 @@
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
"info": {
"info": {
"title": "OpenVibe.Zone API",
"title": "OpenVibe.Zone API",
- "version": "0.223.0",
- "description": "Routes of OpenVibe.Zone as the OpenVibe contracts describe them (openvibe-contracts 0.223.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
+ "version": "0.224.0",
+ "description": "Routes of OpenVibe.Zone as the OpenVibe contracts describe them (openvibe-contracts 0.224.0): each operation lists the capabilities it performs; a token for the service's audience must carry one of them unless the route is open. Generated by openvibe-contracts lib/openapi.js; do not edit.",
"license": {
"license": {
"name": "MIT",
"name": "MIT",
"identifier": "MIT"
"identifier": "MIT"
@@ -6155,7 +6155,7 @@ export interface SearchWriteResult {
@@ -6155,7 +6155,7 @@ export interface SearchWriteResult {
export interface Source {
export interface Source {
key: string;
key: string;
name: string;
name: string;
- type: "rss" | "atom" | "sitemap" | "jsonld" | "api" | "manual";
+ type: "rss" | "atom" | "sitemap" | "jsonld" | "api" | "crawl" | "manual";
category:
category:
| "news"
| "news"
| "blog"
| "blog"
@@ -75515,6 +75515,315 @@ export interface Source {
@@ -75515,6 +75515,315 @@ export interface Source {
};
};
}
}
];
];
+ /**
+ * type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.
+ */
+ crawl?: {
+ /**
+ * Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true).
+ */
+ sitemaps?: boolean;
+ /**
+ * Also follow links on crawled pages to pages of the same host that match include (default false).
+ */
+ follow_links?: boolean;
+ /**
+ * Regular expressions over a URL's path and query; a page must match one (none given: every page of the host).
+ *
+ * @maxItems 20
+ */
+ include?:
+ | []
+ | [string]
+ | [string, string]
+ | [string, string, string]
+ | [string, string, string, string]
+ | [string, string, string, string, string]
+ | [string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string, string, string]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ];
+ /**
+ * Regular expressions over a URL's path and query; a page matching one is never fetched.
+ *
+ * @maxItems 20
+ */
+ exclude?:
+ | []
+ | [string]
+ | [string, string]
+ | [string, string, string]
+ | [string, string, string, string]
+ | [string, string, string, string, string]
+ | [string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string, string, string]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ];
+ /**
+ * The most pages the frontier holds for this site (default 5000).
+ */
+ max_pages?: number;
+ /**
+ * Pages fetched in one run (default 100); the host spacing still applies between them.
+ */
+ pages_per_run?: number;
+ /**
+ * How long after a fetch a page is due again (default 86400, a day).
+ */
+ recrawl_sec?: number;
+ /**
+ * With follow_links, how many links away from a seed or sitemap page a page may be (default 2).
+ */
+ max_depth?: number;
+ /**
+ * Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier.
+ */
+ require_structured?: boolean;
+ } | null;
auth: {
auth: {
mode: "none" | "header" | "bearer" | "query";
mode: "none" | "header" | "bearer" | "query";
/**
/**
@@ -78600,7 +78909,7 @@ export type SourcesSourceWriteRequest =
@@ -78600,7 +78909,7 @@ export type SourcesSourceWriteRequest =
| {
| {
key?: string;
key?: string;
name?: string;
name?: string;
- type?: "rss" | "atom" | "sitemap" | "jsonld" | "api" | "manual";
+ type?: "rss" | "atom" | "sitemap" | "jsonld" | "api" | "crawl" | "manual";
category?:
category?:
| "news"
| "news"
| "blog"
| "blog"
@@ -141028,6 +141337,345 @@ export type SourcesSourceWriteRequest =
@@ -141028,6 +141337,345 @@ export type SourcesSourceWriteRequest =
}
}
)
)
];
];
+ /**
+ * type crawl only: how the crawler walks one site from its endpoints (the seeds). Every request still passes the site's robots.txt (RFC 9309) and the per-host spacing, a page that says noindex is not kept, and only what a page publishes for machines is read (schema.org JSON-LD: products and offers for deals, articles, reviews). A crawl adds pages to its frontier from the site's own sitemaps and, if asked, its links; each run fetches the pages that are due and schedules them again after recrawl_sec, so a price seen twice becomes a history.
+ */
+ crawl?: {
+ /**
+ * Find pages from the site's sitemaps: the Sitemap lines of its robots.txt, else /sitemap.xml, following sitemap indexes (default true).
+ */
+ sitemaps?: boolean;
+ /**
+ * Also follow links on crawled pages to pages of the same host that match include (default false).
+ */
+ follow_links?: boolean;
+ /**
+ * Regular expressions over a URL's path and query; a page must match one (none given: every page of the host).
+ *
+ * @maxItems 20
+ */
+ include?:
+ | []
+ | [string]
+ | [string, string]
+ | [string, string, string]
+ | [string, string, string, string]
+ | [string, string, string, string, string]
+ | [string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string, string]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ];
+ /**
+ * Regular expressions over a URL's path and query; a page matching one is never fetched.
+ *
+ * @maxItems 20
+ */
+ exclude?:
+ | []
+ | [string]
+ | [string, string]
+ | [string, string, string]
+ | [string, string, string, string]
+ | [string, string, string, string, string]
+ | [string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string]
+ | [string, string, string, string, string, string, string, string, string, string, string, string, string]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ]
+ | [
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string,
+ string
+ ];
+ /**
+ * The most pages the frontier holds for this site (default 5000).
+ */
+ max_pages?: number;
+ /**
+ * Pages fetched in one run (default 100); the host spacing still applies between them.
+ */
+ pages_per_run?: number;
+ /**
+ * How long after a fetch a page is due again (default 86400, a day).
+ */
+ recrawl_sec?: number;
+ /**
+ * With follow_links, how many links away from a seed or sitemap page a page may be (default 2).
+ */
+ max_depth?: number;
+ /**
+ * Keep only pages that publish schema.org JSON-LD the adapter reads (default true); a page without it is fetched once and dropped from the frontier.
+ */
+ require_structured?: boolean;
+ } | null;
auth?: {
auth?: {
mode: "none" | "header" | "bearer" | "query";
mode: "none" | "header" | "bearer" | "query";
/**
/**
@@ -1,6 +1,6 @@
@@ -1,6 +1,6 @@
{
{
"name": "openvibe-contracts",
"name": "openvibe-contracts",
- "version": "0.223.0",
+ "version": "0.224.0",
"description": "Machine-readable contracts for the OpenVibe network: schemas, manifests, ids, error envelope, capability checks.",
"description": "Machine-readable contracts for the OpenVibe network: schemas, manifests, ids, error envelope, capability checks.",
"main": "index.js",
"main": "index.js",
"types": "generated/typescript/index.d.ts",
"types": "generated/typescript/index.d.ts",