openvibers/openvibe.sources Commits

jsonld@2: an offer's list (strikethrough) price as stated, and a price stated only in its specification (#129)

OP OpenVibers committed 16 days ago · 991cdcaa8e · 6 files, +85 −10

Changes

ddiff --git a/README.md b/README.md
iindex f7d252f..d99cdfe 100644
---- a/README.md
++++ b/README.md
@@@ -104,9 +104,9 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled
  |---|---|---|
  | `rss`, `atom` (`feed@3`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline); its `dealnews:category` fills `fields.categories` when the item has no `<category>`, and `dealnews:staffPick` is `fields.staff_pick` (true/false, absent when not stated) |
  | `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed |
--| `jsonld` (`jsonld@1`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error` |
++| `jsonld` (`jsonld@2`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error`; `list_price` from a ListPrice/StrikethroughPrice/MSRP/SRP `priceSpecification`, as stated |
  | `api` (`api@1`) | JSON or XML official APIs via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`; epoch fields only where mapped as such |
--| `crawl` (`jsonld@1`, `page@1`) | a whole site from its seeds: its sitemaps and, if asked, its links (see below) | only pages that publish JSON-LD are kept, unless `require_structured: false` keeps a `page` item from the page's own search metadata |
++| `crawl` (`jsonld@2`, `page@1`) | a whole site from its seeds: its sitemaps and, if asked, its links (see below) | only pages that publish JSON-LD are kept, unless `require_structured: false` keeps a `page` item from the page's own search metadata |
  | `manual` (`manual@1`) | items entered through the API with an evidence URL | no fetcher |
  
  XML with DTD entity declarations is refused; the parser expands no entities (values are decoded
ddiff --git a/server/adapters/jsonld.js b/server/adapters/jsonld.js
iindex b014c24..5fe60a3 100644
---- a/server/adapters/jsonld.js
++++ b/server/adapters/jsonld.js
@@@ -5,8 +5,10 @@
   * Types read: Product, Offer, AggregateOffer, Review, NewsArticle, BlogPosting (and Article and
   * its news subtypes). Values are copied as the page states them: prices stay the strings the page
   * gave, ratings are the page's own numbers, availability is the schema.org term. Nothing is
-- * computed, averaged or defaulted; a missing price is null. Embedded reviews keep author, rating
-- * and date only (review text belongs to its author).
++ * computed, averaged or defaulted; a missing price is null. An offer's `list_price` is the price
++ * of its ListPrice/StrikethroughPrice/MSRP/SRP `priceSpecification` entry, copied exactly as the
++ * page states it. Embedded reviews keep author, rating and date only (review text belongs to its
++ * author).
   *
   * A page without any readable JSON-LD block is a parse error: the source promised structured data
   * and did not deliver it, and the run must say so instead of producing nothing silently.
@@@ -14,7 +16,7 @@
  const crypto = require('crypto');
  const { toText, toIsoDate, canonicalUrl, toId, asArray, ParseError } = require('./util');
  
--const PARSER_VERSION = 'jsonld@1';
++const PARSER_VERSION = 'jsonld@2';
  const ARTICLE_TYPES = new Set(['NewsArticle', 'BlogPosting', 'Article', 'ReportageNewsArticle', 'AnalysisNewsArticle', 'OpinionNewsArticle', 'BackgroundNewsArticle']);
  const KIND = { Product: 'product', Offer: 'offer', AggregateOffer: 'offer', Review: 'review' };
  
@@@ -61,6 +63,23 @@ const priceOf = (v) => {
      return null;
  };
  const term = (v) => (typeof v === 'string' ? v.replace(/^https?:\/\/schema\.org\//, '').slice(0, 60) : null);
++const LIST_PRICE_TYPES = new Set(['ListPrice', 'StrikethroughPrice', 'MSRP', 'SRP']);
++
++/**
++ * The price of the first priceSpecification entry whose priceType `match`es (or, when
++ * `allowUntyped`, that has no priceType), copied exactly as stated; null when there is no such
++ * entry, or when the entry's priceCurrency and the offer's are both stated and differ.
++ */
++function specPrice(o, match, allowUntyped = false) {
++    for (const spec of asArray(o.priceSpecification)) {
++        if (!spec || typeof spec !== 'object') continue;
++        const t = term(spec.priceType);
++        if (!match(t) && !(allowUntyped && t === null)) continue;
++        if (typeof spec.priceCurrency === 'string' && typeof o.priceCurrency === 'string' && spec.priceCurrency !== o.priceCurrency) return null;
++        return priceOf(spec.price);
++    }
++    return null;
++}
  const imageOf = (v, base) => {
      const first = asArray(v)[0];
      return canonicalUrl(first && typeof first === 'object' ? first.url || first.contentUrl : first, base);
@@@ -68,13 +87,24 @@ const imageOf = (v, base) => {
  
  function offerFields(o, base) {
      if (!o || typeof o !== 'object') return null;
++    let price = priceOf(o.price);
++    let currency = typeof o.priceCurrency === 'string' && /^[A-Z]{3}$/.test(o.priceCurrency) ? o.priceCurrency : null;
++    if (o.price == null) {
++        price = specPrice(o, (t) => t === 'SalePrice', true);
++        // A shop that states its price only in the specification often states the currency only there too.
++        if (price != null && !currency) {
++            const spec = asArray(o.priceSpecification).find(x => x && typeof x === 'object' && priceOf(x.price) === price);
++            if (spec && typeof spec.priceCurrency === 'string' && /^[A-Z]{3}$/.test(spec.priceCurrency)) currency = spec.priceCurrency;
++        }
++    }
      return {
          type: typesOf(o)[0] || null,
--        price: priceOf(o.price),
++        price,
          low_price: priceOf(o.lowPrice),
          high_price: priceOf(o.highPrice),
++        list_price: specPrice(o, (t) => LIST_PRICE_TYPES.has(t)),
          offer_count: numeric(o.offerCount),
--        currency: typeof o.priceCurrency === 'string' && /^[A-Z]{3}$/.test(o.priceCurrency) ? o.priceCurrency : null,
++        currency,
          availability: term(o.availability),
          condition: term(o.itemCondition),
          valid_until: toIsoDate(o.priceValidUntil),
ddiff --git a/test/adapters.test.js b/test/adapters.test.js
iindex 4bf7967..8032bf5 100644
---- a/test/adapters.test.js
++++ b/test/adapters.test.js
@@@ -132,16 +132,39 @@ t('JSON-LD Product: offers as stated, missing price stays null, rating copied no
      assert.strictEqual(p.fields.gtin, '0012345678905');
      assert.strictEqual(p.fields.brand, 'Example Co');
      assert.deepStrictEqual(p.fields.offers[0], {
--        type: 'Offer', price: '19.99', low_price: null, high_price: null, offer_count: null, currency: 'USD', availability: 'InStock',
++        type: 'Offer', price: '19.99', low_price: null, high_price: null, list_price: null, offer_count: null, currency: 'USD', availability: 'InStock',
          condition: null, valid_until: '2026-12-31T00:00:00.000Z', url: 'https://shop.example.com/p/widget-pro', seller: 'Example Shop',
      });
      assert.strictEqual(p.fields.offers[1].price, null, 'no price on the page = null, never 0 or a guess');
++    assert.strictEqual(p.fields.offers[1].list_price, null, 'no priceSpecification = no list price');
      assert.strictEqual(p.fields.offers[1].availability, 'OutOfStock');
      assert.deepStrictEqual(p.fields.aggregate_rating, { value: 4.4, best: null, worst: null, count: null, review_count: 89 });
      assert.deepStrictEqual(p.fields.reviews, [{ author: 'Sam', rating: { value: 5, best: 5, worst: null, count: null, review_count: null }, date: '2026-08-01T00:00:00.000Z' }]);
      assert.ok(!JSON.stringify(p).includes('Long text'), 'embedded review bodies are not stored');
  });
  
++t('JSON-LD priceSpecification: list price from ListPrice/StrikethroughPrice/MSRP/SRP as stated, only in the offer\'s currency; SalePrice or an untyped specification fills a missing price', () => {
++    const r = jsonld.parse(fx('list-price.html'), { url: 'https://shop.example.com/p/deals' });
++    assert.strictEqual(r.items.length, 1);
++    const o = r.items[0].fields.offers;
++    assert.strictEqual(o.length, 9);
++    const [a, b, c, d, e, f, g, h, i] = o;
++    assert.strictEqual(a.price, '99.99');
++    assert.strictEqual(a.list_price, '129.99', 'the prefixed ListPrice specification, exactly as stated');
++    assert.strictEqual(b.list_price, '79.95', 'an unprefixed StrikethroughPrice is read');
++    assert.strictEqual(c.list_price, null, 'a list price in EUR on a USD offer is not comparable');
++    assert.strictEqual(d.list_price, '99.99', 'MSRP');
++    assert.strictEqual(e.list_price, '84.99', 'SRP');
++    assert.strictEqual(f.price, '79.00', 'a SalePrice specification supplies the current price when the offer states none');
++    assert.strictEqual(f.list_price, null, 'a SalePrice is not a list price');
++    assert.strictEqual(g.price, null, 'a SalePrice in another currency does not supply the price');
++    assert.strictEqual(h.price, '44.00', 'a specification without priceType supplies the current price');
++    assert.strictEqual(i.price, '15.50');
++    assert.strictEqual(i.currency, 'GBP', 'a price stated only in the specification brings its currency');
++    assert.strictEqual(a.currency, 'USD');
++    assert.strictEqual(jsonld.PARSER_VERSION, 'jsonld@2');
++});
++
  t('JSON-LD NewsArticle, Review and BlogPosting; an invalid block is skipped, not fatal', () => {
      const r = jsonld.parse(fx('article.html'), { url: 'https://times.example.org/2026/09/budget' });
      assert.deepStrictEqual(r.items.map(i => i.kind), ['article', 'review', 'article']);
ddiff --git a/test/crawl.test.js b/test/crawl.test.js
iindex fea6cca..efa5688 100644
---- a/test/crawl.test.js
++++ b/test/crawl.test.js
@@@ -121,7 +121,7 @@ t('a crawl reads the sitemaps robots.txt names and keeps the products their page
      const lamp = items.find(i => i.title === 'Desk lamp');
      assert.strictEqual(lamp.kind, 'product');
      assert.strictEqual(JSON.parse(lamp.fields).offers[0].price, '19.99');
--    assert.strictEqual(lamp.parser_version, 'jsonld@1');
++    assert.strictEqual(lamp.parser_version, 'jsonld@2');
      assert.strictEqual(lamp.last_fetch_run_id, by['/products/lamp'].last_ok_run_id);
      const runs = await runsOf('shop');
      assert.ok(runs.every(r => r.state !== 'parse_error'));
ddiff --git a/test/fixtures/list-price.html b/test/fixtures/list-price.html
nnew file mode 100644
iindex 0000000..95a3ac7
---- /dev/null
++++ b/test/fixtures/list-price.html
@@@ -0,0 +1,22 @@
++<!doctype html>
++<html><head><meta charset="utf-8"><title>List price examples</title>
++<script type="application/ld+json">
++{
++  "@context": "https://schema.org",
++  "@type": "Product",
++  "@id": "https://shop.example.com/p/deals#product",
++  "name": "Deals example",
++  "offers": [
++    { "@type": "Offer", "price": "99.99", "priceCurrency": "USD", "priceSpecification": [{ "@type": "UnitPriceSpecification", "priceType": "https://schema.org/ListPrice", "price": "129.99", "priceCurrency": "USD" }] },
++    { "@type": "Offer", "price": "49.95", "priceCurrency": "USD", "priceSpecification": { "@type": "UnitPriceSpecification", "priceType": "StrikethroughPrice", "price": "79.95", "priceCurrency": "USD" } },
++    { "@type": "Offer", "price": "9.99", "priceCurrency": "USD", "priceSpecification": { "@type": "UnitPriceSpecification", "priceType": "http://schema.org/ListPrice", "price": "14.99", "priceCurrency": "EUR" } },
++    { "@type": "Offer", "price": "89.99", "priceCurrency": "USD", "priceSpecification": { "@type": "UnitPriceSpecification", "priceType": "https://schema.org/MSRP", "price": "99.99", "priceCurrency": "USD" } },
++    { "@type": "Offer", "price": "79.99", "priceCurrency": "USD", "priceSpecification": { "@type": "UnitPriceSpecification", "priceType": "https://schema.org/SRP", "price": "84.99", "priceCurrency": "USD" } },
++    { "@type": "Offer", "priceCurrency": "USD", "priceSpecification": { "@type": "UnitPriceSpecification", "priceType": "https://schema.org/SalePrice", "price": "79.00", "priceCurrency": "USD" } },
++    { "@type": "Offer", "priceCurrency": "USD", "priceSpecification": { "@type": "UnitPriceSpecification", "priceType": "SalePrice", "price": "59.00", "priceCurrency": "EUR" } },
++    { "@type": "Offer", "priceCurrency": "USD", "priceSpecification": { "@type": "UnitPriceSpecification", "price": "44.00", "priceCurrency": "USD" } },
++    { "@type": "Offer", "priceSpecification": { "@type": "UnitPriceSpecification", "price": "15.50", "priceCurrency": "GBP" } }
++  ]
++}
++</script>
++</head><body>Prices</body></html>
ddiff --git a/test/ingest.test.js b/test/ingest.test.js
iindex 90a6d9a..39f7db3 100644
---- a/test/ingest.test.js
++++ b/test/ingest.test.js
@@@ -323,7 +323,7 @@ t('JSON-LD and sitemap sources ingest end to end', async () => {
          const [prod] = await itemsOf(p);
          assert.strictEqual(prod.kind, 'product');
          assert.strictEqual(JSON.parse(prod.fields).offers[0].price, '19.99');
--        assert.strictEqual(prod.parser_version, 'jsonld@1');
++        assert.strictEqual(prod.parser_version, 'jsonld@2');
          const sm = await create(sourceDef({ type: 'sitemap', category: 'deals', endpoints: [`${s.origin}/sitemap.xml`] }));
          const out = await run(sm);
          assert.deepStrictEqual([out.runs[0].state, out.runs[0].items_created, out.runs[0].items_skipped], ['ok', 2, 1]);