openvibers/openvibe.sources Commits

Sources: category crowdfunding, request_content_type for api GETs, opencollective-accounts seed (contracts 0.191.0) (#65)

OP OpenVibers committed 29 days ago · e57602da79 · 10 files, +162 −9

Changes

ddiff --git a/README.md b/README.md
iindex f773094..6ae24ea 100644
---- a/README.md
++++ b/README.md
@@@ -53,7 +53,7 @@ start and flushes on stop.
  ## The registry
  
  A source records: `key`, `name`, `type` (`rss|atom|sitemap|jsonld|api|manual`), `category`
--(`news|blog|reviews|deals|coupons|trade|hosting`), `endpoints` (for `api`: a field mapping, no code per
++(`news|blog|reviews|deals|coupons|trade|hosting|crowdfunding`), `endpoints` (for `api`: a field mapping, no code per
  provider), `auth` (`none|header|bearer|query` with the **name** of an environment variable —
  names must start with `SOURCES_CRED_`, so an entry can never point at this service's own
  secrets), `robots_note`, `terms_note`, `license_note`, the rate limit (`min_interval_ms`),
@@@ -150,6 +150,11 @@ A value is literal text or `$rec.<path>`, read from the record itself. No match
  `max_bytes` (an integer from 1 KiB to 32 MiB) sets one endpoint's body cap instead of `SOURCES_MAX_BYTES`. OVHcloud's
  catalog is about 12 MB.
  
++`request_content_type` (`application/json` only) is sent as the Content-Type of the endpoint's GET, which has no body.
++Open Collective's public GraphQL API answers a GET only with it (its CSRF guard): `opencollective-accounts` (category
++`crowdfunding`) reads the 200 most active and the 100 newest collectives, projects and funds every six hours, each a
++`campaign` item with the amounts and counts as Open Collective states them, for OpenVibe.Deals' crowdfunding discovery.
++
  **Facts stated in prose (`extract`, rss/atom only).** A feed endpoint may carry
  `extract: { kind, require, fields }` ([server/adapters/extract.js](server/adapters/extract.js)). Each field is read from
  the item's own `title` or `summary` by a pattern with one capturing group. With `as: "date"`, "October 20" becomes
ddiff --git a/docs/contracts-proposal/contracts/sources/item.v1.json b/docs/contracts-proposal/contracts/sources/item.v1.json
iindex 5e51f96..0f9285f 100644
---- a/docs/contracts-proposal/contracts/sources/item.v1.json
++++ b/docs/contracts-proposal/contracts/sources/item.v1.json
@@@ -9,7 +9,7 @@
    "properties": {
      "id": { "type": "string", "pattern": "^itm_[0-9A-HJKMNP-TV-Z]{26}$" },
      "source_key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" },
--    "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting"] },
++    "category": { "enum": ["news", "blog", "reviews", "deals", "coupons", "trade", "hosting", "crowdfunding"] },
      "kind": { "type": "string", "pattern": "^[a-z][a-z0-9_]{1,39}$", "description": "article | url | sitemap | product | offer | review | review_signal | record | coupon …" },
      "identity": { "type": "string", "minLength": 1, "maxLength": 2048, "description": "The source's own id (guid, @id, API id) or the canonical URL." },
      "canonical_url": { "type": ["string", "null"], "format": "uri" },
ddiff --git a/docs/contracts-proposal/contracts/sources/source.v1.json b/docs/contracts-proposal/contracts/sources/source.v1.json
iindex cb9f58c..46b3e97 100644
---- a/docs/contracts-proposal/contracts/sources/source.v1.json
++++ b/docs/contracts-proposal/contracts/sources/source.v1.json
@@@ -49,7 +49,7 @@
          "deals",
          "coupons",
          "trade",
--        "hosting"
++        "hosting", "crowdfunding"
        ]
      },
      "homepage_url": {
@@@ -136,6 +136,10 @@
              "type": "string",
              "pattern": "^[a-z][a-z0-9_]{1,39}$"
            },
++          "request_content_type": {
++            "enum": ["application/json"],
++            "description": "Sent as the request's Content-Type on this endpoint's GET (no body). Open Collective's GraphQL API answers a GET only with it, as its CSRF guard (Sources#65)."
++          },
            "max_bytes": {
              "type": "integer",
              "minimum": 1024,
ddiff --git a/package-lock.json b/package-lock.json
iindex 78c88d8..8104e85 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -12,7 +12,7 @@
          "dotenv": "^16.4.7",
          "express": "^4.21.2",
          "fast-xml-parser": "^4.5.3",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.188.2",
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.191.0",
          "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.46.0",
          "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.2.1",
          "pg": "^8.23.0"
@@@ -769,9 +769,9 @@
        }
      },
      "node_modules/openvibe-contracts": {
--      "version": "0.188.2",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.188.2",
--      "integrity": "sha512-E64gQI7roKudmmJFnenyh1BjViJyI0j0MoVT4o35/jUD4rpzMkElwEh/fGbsPpnstdDFIcJR6gEcScbL0ZQhbg==",
++      "version": "0.191.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.191.0",
++      "integrity": "sha512-RA5K/K04KQce36VwTuBVVYJPeVk7FAuyHr5LU50WxyBesf/dfLiE0knnMQkDHWPLKFQ/HluWx0LUbqB+NJUo7A==",
        "license": "AGPL-3.0-only",
        "dependencies": {
          "ajv": "^8.17.1",
ddiff --git a/package.json b/package.json
iindex a076b98..007dc28 100644
---- a/package.json
++++ b/package.json
@@@ -27,7 +27,7 @@
      "dotenv": "^16.4.7",
      "express": "^4.21.2",
      "fast-xml-parser": "^4.5.3",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.188.2",
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.191.0",
      "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.46.0",
      "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.2.1",
      "pg": "^8.23.0"
ddiff --git a/seeds/sources.json b/seeds/sources.json
iindex b9412a5..080f0fd 100644
---- a/seeds/sources.json
++++ b/seeds/sources.json
@@@ -1237,6 +1237,76 @@
        "review_required": true,
        "sensitivity": "health",
        "default_indexability": "noindex"
++    },
++    {
++      "key": "opencollective-accounts",
++      "name": "Open Collective: collectives, projects and funds",
++      "type": "api",
++      "category": "crowdfunding",
++      "homepage_url": "https://opencollective.com/discover",
++      "endpoints": [
++        {
++          "url": "https://api.opencollective.com/graphql/v2?query=%7B+accounts%28type%3A+%5BCOLLECTIVE%2C+PROJECT%2C+FUND%5D%2C+limit%3A+200%2C+orderBy%3A+%7Bfield%3A+ACTIVITY%2C+direction%3A+DESC%7D%29+%7B+nodes+%7B+slug+name+type+description+currency+createdAt+imageUrl+website+tags+stats+%7B+totalAmountReceived+%7B+valueInCents+currency+%7D+yearlyBudget+%7B+valueInCents+currency+%7D+contributorsCount+%7D+%7D+%7D+%7D",
++          "format": "json",
++          "request_content_type": "application/json",
++          "items_path": "data.accounts.nodes",
++          "item_kind": "campaign",
++          "fields": {
++            "identity": "slug",
++            "title": "name",
++            "summary": "description",
++            "published_at": "createdAt"
++          },
++          "extra": {
++            "slug": "slug",
++            "account_type": "type",
++            "currency": "currency",
++            "raised_cents": "stats.totalAmountReceived.valueInCents",
++            "raised_currency": "stats.totalAmountReceived.currency",
++            "yearly_budget_cents": "stats.yearlyBudget.valueInCents",
++            "backers": "stats.contributorsCount",
++            "image_url": "imageUrl",
++            "website": "website",
++            "tag": "tags.0"
++          }
++        },
++        {
++          "url": "https://api.opencollective.com/graphql/v2?query=%7B+accounts%28type%3A+%5BCOLLECTIVE%2C+PROJECT%2C+FUND%5D%2C+limit%3A+100%2C+orderBy%3A+%7Bfield%3A+CREATED_AT%2C+direction%3A+DESC%7D%29+%7B+nodes+%7B+slug+name+type+description+currency+createdAt+imageUrl+website+tags+stats+%7B+totalAmountReceived+%7B+valueInCents+currency+%7D+yearlyBudget+%7B+valueInCents+currency+%7D+contributorsCount+%7D+%7D+%7D+%7D",
++          "format": "json",
++          "request_content_type": "application/json",
++          "items_path": "data.accounts.nodes",
++          "item_kind": "campaign",
++          "fields": {
++            "identity": "slug",
++            "title": "name",
++            "summary": "description",
++            "published_at": "createdAt"
++          },
++          "extra": {
++            "slug": "slug",
++            "account_type": "type",
++            "currency": "currency",
++            "raised_cents": "stats.totalAmountReceived.valueInCents",
++            "raised_currency": "stats.totalAmountReceived.currency",
++            "yearly_budget_cents": "stats.yearlyBudget.valueInCents",
++            "backers": "stats.contributorsCount",
++            "image_url": "imageUrl",
++            "website": "website",
++            "tag": "tags.0"
++          }
++        }
++      ],
++      "robots_note": "robots.txt checked 2026-10-15: https://api.opencollective.com/robots.txt answers 404, so the API host publishes no robots rules (a 404 is treated as no restrictions, never guessed as a disallow). The fetcher re-checks robots.txt on every run.",
++      "terms_note": "Open Collective's public GraphQL API v2 (documented at https://graphql-docs-v2.opencollective.com) answers anonymous reads of public accounts; a GET must carry Content-Type: application/json (its CSRF guard). Two requests every six hours: the 200 most active collectives, projects and funds and the 100 newest. Each becomes a campaign item: name, description, type, currency, the total received and yearly budget in cents, the number of contributors, logo URL, website and first tag, as Open Collective states them; the profile is https://opencollective.com/<slug>. Show amounts as observations with their fetch date, link each to its Open Collective page, and imply no endorsement.",
++      "license_note": "Public account data published by Open Collective; the descriptions are the collectives' own words (short summaries only, linked back).",
++      "min_interval_ms": 60000,
++      "poll_interval_sec": 21600,
++      "stale_after_sec": 86400,
++      "max_items": 300,
++      "enabled": false,
++      "review_required": true,
++      "sensitivity": "financial",
++      "default_indexability": "noindex"
      }
    ]
  }
ddiff --git a/server/adapters/api.js b/server/adapters/api.js
iindex ee99ac2..f4fa017 100644
---- a/server/adapters/api.js
++++ b/server/adapters/api.js
@@@ -48,6 +48,9 @@
   * `max_bytes` (an integer from 1 KiB to 32 MiB) raises or lowers the body cap for one endpoint
   * (SOURCES_MAX_BYTES otherwise); OVHcloud's VPS catalog is about 12 MB.
   *
++ * `request_content_type` ("application/json" only) is sent as the GET's Content-Type (the request has no body): Open
++ * Collective's public GraphQL API answers a GET only with it, as its CSRF guard.
++ *
   * Some price lists key their records by id instead of listing them: Porkbun returns
   * `{ "pricing": { "com": {…}, "net": {…} } }` and AWS Lightsail keys its price list by SKU.
   * `"items_shape": "map"` reads such an object: each own property becomes one record, in sorted
@@@ -228,6 +231,7 @@ function checkMapping(ep) {
      }
      if (ep.item_kind !== undefined && !/^[a-z][a-z0-9_]{1,39}$/.test(ep.item_kind)) return 'item_kind is malformed';
      if (ep.max_bytes !== undefined && (!Number.isInteger(ep.max_bytes) || ep.max_bytes < 1024 || ep.max_bytes > 32 * 1024 * 1024)) return 'max_bytes is an integer from 1024 to 33554432';
++    if (ep.request_content_type !== undefined && ep.request_content_type !== 'application/json') return 'request_content_type may only be application/json';
      return null;
  }
  
ddiff --git a/server/ingest.js b/server/ingest.js
iindex 16f6e3c..9d089fb 100644
---- a/server/ingest.js
++++ b/server/ingest.js
@@@ -105,6 +105,8 @@ function createIngest({ db, config, registry, items, robots, fetcher, spacer, em
  
          // credential by variable name, read now, never stored
          const headers = { Accept: adapter.accept };
++        // An api endpoint may name the Content-Type its GET carries (adapters/api.js `request_content_type`).
++        if (endpoint.request_content_type) headers['Content-Type'] = endpoint.request_content_type;
          const sensitive = [];
          let requestUrl = target;
          if (source.auth.mode !== 'none') {
ddiff --git a/server/registry.js b/server/registry.js
iindex 912618c..4d8ae21 100644
---- a/server/registry.js
++++ b/server/registry.js
@@@ -13,7 +13,7 @@ const { checkApiMapping } = require('./adapters');
  const extract = require('./adapters/extract');
  
  const TYPES = ['rss', 'atom', 'sitemap', 'jsonld', 'api', 'manual'];
--const CATEGORIES = ['news', 'blog', 'reviews', 'deals', 'coupons', 'trade', 'hosting'];
++const CATEGORIES = ['news', 'blog', 'reviews', 'deals', 'coupons', 'trade', 'hosting', 'crowdfunding'];
  const SENSITIVITY = ['none', 'financial', 'health', 'political', 'legal', 'adult'];
  const AUTH_MODES = ['none', 'header', 'bearer', 'query'];
  const KEY_RE = /^[a-z0-9][a-z0-9-]{1,63}$/;
@@@ -78,6 +78,7 @@ function validateSource(input, { maxItemsCap = 12000 } = {}) {
              // `where` is only copied when given, so records without conditions stay byte-identical.
              if (e.where) out.where = e.where;
              if (e.max_bytes !== undefined) out.max_bytes = e.max_bytes;
++            if (e.request_content_type !== undefined) out.request_content_type = e.request_content_type;
          } else {
              // A feed may read structured facts out of its items' prose (adapters/extract.js).
              for (const k of Object.keys(e)) if (k !== 'url' && !(k === 'extract' && (input.type === 'rss' || input.type === 'atom'))) throw new RegistryError(`endpoints[${i}].${k} is only for api sources`);
ddiff --git a/test/crowdfunding-source.test.js b/test/crowdfunding-source.test.js
nnew file mode 100644
iindex 0000000..ae1ca53
---- /dev/null
++++ b/test/crowdfunding-source.test.js
@@@ -0,0 +1,67 @@
++'use strict';
++/**
++ * opencollective-accounts (seeds/sources.json), category crowdfunding: Open Collective's public GraphQL API read by GET.
++ *   - the seed validates, is disabled, and both endpoints send Content-Type: application/json (the API's CSRF guard
++ *     refuses a GET without it); no other request content type is accepted;
++ *   - each account becomes one campaign item keyed by its slug, with the amounts and counts as the API states them;
++ *   - a run against a stub that insists on the header ingests, and the GET carried the header.
++ */
++const assert = require('assert');
++const api = require('../server/adapters/api');
++const { validateSource } = require('../server/registry');
++const { boot, site, sourceDef, suite } = require('./helpers');
++
++const t = suite('crowdfunding-source');
++const seed = require('../seeds/sources.json').sources.find((s) => s.key === 'opencollective-accounts');
++
++const BODY = JSON.stringify({ data: { accounts: { nodes: [
++    { slug: 'logseq', name: 'Logseq', type: 'COLLECTIVE', description: 'A privacy-first platform.', currency: 'USD', createdAt: '2020-09-22T10:40:25.537Z',
++        imageUrl: 'https://images.opencollective.com/logseq/logo.png', website: 'https://logseq.com/', tags: ['knowledge-base'],
++        stats: { totalAmountReceived: { valueInCents: 95659957, currency: 'USD' }, yearlyBudget: { valueInCents: 23811274, currency: 'USD' }, contributorsCount: 12182 } },
++    { slug: 'tiny', name: 'Tiny Project', type: 'PROJECT', description: null, currency: 'EUR', createdAt: '2026-10-14T08:00:00Z', imageUrl: null, website: null, tags: [],
++        stats: { totalAmountReceived: { valueInCents: 0, currency: 'EUR' }, yearlyBudget: { valueInCents: 0, currency: 'EUR' }, contributorsCount: 0 } },
++] } } });
++
++t('the seed is a disabled crowdfunding api source whose GETs carry Content-Type: application/json', () => {
++    assert.ok(seed, 'opencollective-accounts is seeded');
++    const rec = validateSource(seed);
++    assert.strictEqual(rec.enabled, false);
++    assert.strictEqual(rec.category, 'crowdfunding');
++    assert.strictEqual(rec.endpoints.length, 2);
++    for (const ep of rec.endpoints) assert.strictEqual(ep.request_content_type, 'application/json');
++    assert.throws(() => validateSource({ ...seed, endpoints: [{ ...seed.endpoints[0], request_content_type: 'text/plain' }] }), /request_content_type may only be application\/json/);
++    assert.throws(() => validateSource(sourceDef({ type: 'rss', endpoints: [{ url: 'https://example.org/f.xml', request_content_type: 'application/json' }] })), /only for api sources/);
++});
++
++t('each account is one campaign item keyed by slug, amounts as the API states them', () => {
++    const r = api.parse(BODY, { url: seed.endpoints[0].url, endpoint: seed.endpoints[0], maxItems: seed.max_items });
++    assert.strictEqual(r.items.length, 2);
++    const [logseq, tiny] = r.items;
++    assert.deepStrictEqual([logseq.identity, logseq.kind, logseq.title, logseq.published_at], ['logseq', 'campaign', 'Logseq', '2020-09-22T10:40:25.537Z']);
++    assert.deepStrictEqual(logseq.fields, {
++        slug: 'logseq', account_type: 'COLLECTIVE', currency: 'USD', raised_cents: 95659957, raised_currency: 'USD', yearly_budget_cents: 23811274,
++        backers: 12182, image_url: 'https://images.opencollective.com/logseq/logo.png', website: 'https://logseq.com/', tag: 'knowledge-base',
++    });
++    assert.deepStrictEqual([tiny.summary, tiny.fields.tag, tiny.fields.website], [null, null, null], 'missing values stay null');
++});
++
++t('a run sends the header and ingests what the API answered', async () => {
++    const svc = await boot();
++    const s = await site({
++        '/robots.txt': () => ({ status: 404, body: 'Not Found' }),
++        '/graphql/v2': (req) => (String(req.headers['content-type'] || '').startsWith('application/json')
++            ? { headers: { 'Content-Type': 'application/json' }, body: BODY }
++            : { status: 400, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ errors: [{ message: 'blocked as a potential CSRF' }] }) }),
++    });
++    try {
++        const ep = { ...seed.endpoints[0], url: `${s.origin}/graphql/v2?query=x` };
++        await svc.registry.create(sourceDef({ key: 'oc', type: 'api', category: 'crowdfunding', endpoints: [ep] }), 'test');
++        const r = await svc.ingest.run('oc', { trigger: 'manual' });
++        assert.deepStrictEqual([r.runs[0].state, r.runs[0].items_seen], ['ok', 2]);
++        assert.strictEqual(s.hits('/graphql/v2')[0].headers['content-type'], 'application/json');
++        const rows = await svc.db.prepare("SELECT identity, category, kind FROM items WHERE source_key = 'oc' ORDER BY identity").all();
++        assert.deepStrictEqual(rows.map((x) => [x.identity, x.category, x.kind]), [['logseq', 'crowdfunding', 'campaign'], ['tiny', 'crowdfunding', 'campaign']]);
++    } finally { await s.close(); await svc.stop(); }
++});
++
++t.run();