openvibers/openvibe.deals Commits

Each offer says what it costs, where it is sold and what it looks like (#330)

OP OpenVibers committed 39 min ago · a1e36f510b · 7 files, +151 −10

The owner, 11-13: "update the listings for deals" — a run of deals in the one feed listed only titles.
feed.item@1 1.1.0 (Contracts v0.425.0) has the fields for the price, the old price, the shop and the
picture; this fills them.

- Every offer carries a "feed" block: the price now (amount and an ISO code in upper case), the price
  it was when the source states a list price, the shop that sells it and the product's picture. A
  price without a currency is no price, and a listing without a picture has none.
- The picture is the one the source stated for the listing (OpenVibe.Sources keeps it in the item's
  fields). It is stored as stated, never fetched, and only when it is https (migration 0015).

Contracts pinned to v0.425.0.

Changes

ddiff --git a/migrations/0015_offer_image.sql b/migrations/0015_offer_image.sql
nnew file mode 100644
iindex 0000000..8f1e6ef
---- /dev/null
++++ b/migrations/0015_offer_image.sql
@@@ -0,0 +1,10 @@
++-- phase: expand
++-- The picture of a product (the owner, 11-13: "update the listings for deals"): the one feed lists a
++-- deal with its price, the price it was, the shop that sells it and the product's picture
++-- (feed.item@1 1.1.0: group.items[].price / was / store, and the card's own media.thumb).
++--
++-- The image is the one the source stated for the listing (OpenVibe.Sources keeps the picture it
++-- found in the item's fields). It is stored as stated, never fetched, and only when it is https.
++--
++-- Expand only: an older release never reads this column, and nothing here changes an existing row.
++ALTER TABLE deal_offers ADD COLUMN image_url text;
ddiff --git a/package-lock.json b/package-lock.json
iindex f5fea7b..018e418 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -15,7 +15,7 @@
          "express-rate-limit": "^7.5.0",
          "helmet": "^8.1.0",
          "iovalkey": "^0.4.0",
--        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
++        "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.425.0",
          "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
          "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
          "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.241.0",
@@@ -1229,9 +1229,9 @@
        }
      },
      "node_modules/openvibe-contracts": {
--      "version": "0.419.0",
--      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
--      "integrity": "sha512-VxrTdeHchT8gvl2wWC9ASaId92en7ZKfJ8F/0hKzCAD0f2Y00rPrqgBWF9BNPoLGARoyQvsMbhBEhSkhxGNnjA==",
++      "version": "0.425.0",
++      "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.425.0",
++      "integrity": "sha512-5TGvkK5Crrnlg7g33YH3pkStkM3lJ0mHLe+BCo6KR5tQNl/cjOq7YF5QqSDL0pEVU1+zWV+pBJY3aR4OTZkYkQ==",
        "license": "AGPL-3.0-only",
        "dependencies": {
          "ajv": "^8.17.1",
ddiff --git a/package.json b/package.json
iindex 3535ae4..0c31a97 100644
---- a/package.json
++++ b/package.json
@@@ -26,7 +26,7 @@
      "express-rate-limit": "^7.5.0",
      "helmet": "^8.1.0",
      "iovalkey": "^0.4.0",
--    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.419.0",
++    "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.425.0",
      "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.7.0",
      "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.161.0",
      "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.241.0",
ddiff --git a/server/domain/publication.js b/server/domain/publication.js
iindex e873517..efb6b70 100644
---- a/server/domain/publication.js
++++ b/server/domain/publication.js
@@@ -19,7 +19,7 @@
   * drops the price, a missing availability is omitted (never an assumed InStock).
   */
  const seo = require('openvibe-publishing/seo');
--const { iso } = require('./values');
++const { iso, pictureUrl } = require('./values');
  const { attributionOf, hidesFromExtension } = require('./source-policy');
  const { priceTrend, listPriceFields } = require('./price-trend');
  
@@@ -164,6 +164,26 @@ function createPublication({ config, store, reads, catalog }) {
          };
      }
  
++    /**
++     * What the one feed needs about an offer (feed.item@1 1.1.0): what it costs now, what it cost
++     * before, the shop that sells it and its picture. A price without a currency, or a currency that is
++     * not an ISO code, is no price at all; a listing without a picture has none.
++     */
++    function feedFacts(v) {
++        const latest = v.latest;
++        const amount = latest ? Number(latest.price_num != null ? latest.price_num : latest.price) : NaN;
++        const currency = String((latest && latest.currency) || '').trim().toUpperCase();
++        const was = latest && latest.list_price_num != null ? Number(latest.list_price_num) : NaN;
++        const store = (v.store && (v.store.name || v.store.domain)) || v.root.seller || null;
++        const thumb = pictureUrl(v.root.image_url);
++        return {
++            price: Number.isFinite(amount) && amount >= 0 && /^[A-Z]{3}$/.test(currency) ? { amount, currency } : null,
++            was: Number.isFinite(was) && was >= 0 ? was : null,
++            store: store ? String(store).trim().slice(0, 60) || null : null,
++            media: thumb ? { thumb } : null,
++        };
++    }
++
      /** The JSON representation (API and /d/:slug.json): the same facts as the page. */
      function offerDto(v) {
          const r = v.root;
@@@ -179,6 +199,9 @@ function createPublication({ config, store, reads, catalog }) {
              store: v.store ? { id: v.store.id, domain: v.store.domain, name: v.store.name, url: abs(storePath(v.store)) } : null,
              // The shop a source says sells it, as stated (the store is where the link goes).
              seller: r.seller || null,
++            // What the one feed lists the offer with: the price now, the price before, the shop and the
++            // product's picture (feed.item@1 1.1.0).
++            feed: feedFacts(v),
              product: v.product ? { id: v.product.id, slug: v.product.slug, name: v.product.name, url: abs(productPath(v.product)) } : null,
              // "latest", never "current": whether it still holds is what freshness says.
              latest_observation: v.latest ? observationDto(v.latest) : null,
ddiff --git a/server/domain/source-items.js b/server/domain/source-items.js
iindex 1017d8a..391f98d 100644
---- a/server/domain/source-items.js
++++ b/server/domain/source-items.js
@@@ -47,7 +47,7 @@
   */
  const { createChangeCursor, pullChanges } = require('openvibe-publishing/ingest');
  const {
--    normalizeUrl, verbatimUrl, urlKey, parseAmount, SCHEMA_AVAILABILITY, SCHEMA_CONDITION, iso,
++    normalizeUrl, verbatimUrl, urlKey, pictureUrl, parseAmount, SCHEMA_AVAILABILITY, SCHEMA_CONDITION, iso,
  } = require('./values');
  const { keepsLinksVerbatim } = require('./source-policy');
  const { isDeal } = require('./price-trend');
@@@ -87,6 +87,18 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
      const enabled = () => Boolean(sources.enabled && config.sources.enabled);
      const cursorNow = async () => await cursor.get(CURSOR);
  
++    /**
++     * The picture the source stated for the listing, kept on the offer (the one feed lists a deal with
++     * its picture). It is stored as stated and never fetched; a listing without one keeps none.
++     */
++    async function applyImage(offerId, image) {
++        if (!image) return false;
++        const row = await db.prepare('SELECT image_url FROM deal_offers WHERE id = ?').get(offerId);
++        if (!row || row.image_url === image) return false;
++        await db.prepare('UPDATE deal_offers SET image_url = ?, updated_at = ? WHERE id = ?').run(image, store.now(), offerId);
++        return true;
++    }
++
      function amount(v) { try { return parseAmount(v, 'price'); } catch { return null; } }
      function currencyOf(v) { return typeof v === 'string' && /^[A-Z]{3}$/.test(v) ? v : null; }
  
@@@ -126,7 +138,8 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
          if (item.kind === 'offer') {
              const url = linkOf(f.url || item.canonical_url);
              if (!url || !item.title) return [];
--            return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), ...sellerOf(f.seller), ...labels }];
++            // The picture the source stated for the listing, when it stated one (the feed lists it).
++            return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), image: pictureUrl(f.image), ...sellerOf(f.seller), ...labels }];
          }
          if (item.kind === 'product') {
              if (!item.title) return [];
@@@ -139,7 +152,8 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
                  out.push({
                      // A verbatim source's title is used as stated, with no added words.
                      part: urlKey(url), url, title: !verbatim && o.seller ? `${item.title} — ${o.seller}` : item.title, description: item.summary,
--                    obs: obsFields(o), expires_at: stated(o.valid_until), ...sellerOf(o.seller), ...labels,
++                    obs: obsFields(o), expires_at: stated(o.valid_until), image: pictureUrl(f.image || o.image),
++                    ...sellerOf(o.seller), ...labels,
                      product: { name: item.title, brand: f.brand, gtin: f.gtin, mpn: f.mpn, sku: f.sku },
                  });
              }
@@@ -206,6 +220,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
                  const obs = await offers.recordObservation(existing.offer_id, existing.id, c.obs, { origin, observedAt: retrievedAt, sourceRevision: item.revision });
                  const offer = await reads.get(existing.offer_id);
                  const changed = ['observation'];
++                if (await applyImage(existing.offer_id, c.image)) changed.push('image');
                  if (origin === 'import' && c.seller && offer.origin === 'import' && c.seller !== offer.seller) {
                      await db.prepare('UPDATE deal_offers SET seller = ?, updated_at = ? WHERE id = ?').run(c.seller, store.now(), offer.id);
                      changed.push('seller');
@@@ -233,6 +248,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
                  const obs = await offers.recordObservation(dup.id, s.id, c.obs, { origin: 'import', observedAt: retrievedAt, sourceRevision: item.revision });
                  if (productId && !dup.product_id) await db.prepare('UPDATE deal_offers SET product_id = ?, updated_at = ? WHERE id = ?').run(productId, store.now(), dup.id);
                  if (c.seller && !dup.seller) await db.prepare('UPDATE deal_offers SET seller = ?, updated_at = ? WHERE id = ?').run(c.seller, store.now(), dup.id);
++                await applyImage(dup.id, c.image);
                  if (!dup.category || dup.staff_pick == null) await applyLabels(dup, { category: dup.category ? null : c.category, staff_pick: dup.staff_pick == null ? c.staff_pick : null });
                  await indexing.emitOffer('deals.offer.updated', await reads.get(dup.id), { actor: 'svc:deals', extra: { changed: ['sources'], observation_id: obs.id } });
                  await indexing.reindex(await reads.get(dup.id));
@@@ -243,6 +259,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
                  url: c.url, title: String(c.title).slice(0, 200), description: c.description, product_id: productId, category: c.category,
                  store_name: c.store_name, seller: c.seller, expires_at: c.expires_at != null && c.expires_at > store.now() ? c.expires_at : null,
              }, src, c.obs, { observedAt: retrievedAt, sourceRevision: item.revision });
++            await applyImage(made.id, c.image);
              if (c.staff_pick != null) await applyLabels(made, { staff_pick: c.staff_pick });
              outcomes.push('created');
          }
ddiff --git a/server/domain/values.js b/server/domain/values.js
iindex 50028b4..a707048 100644
---- a/server/domain/values.js
++++ b/server/domain/values.js
@@@ -86,6 +86,20 @@ function verbatimUrl(input) {
      return s.length <= 2048 ? s : null;
  }
  
++/**
++ * The picture of a listing, as the source stated it: an https address kept as it is (a picture link
++ * is not the listing's identity, so nothing is stripped from it), and null for anything else.
++ */
++function pictureUrl(input) {
++    if (input == null || input === '') return null;
++    const s = String(input).trim();
++    if (!s || s.length > 2000) return null;
++    let u;
++    try { u = new URL(s); } catch { return null; }
++    if (u.protocol !== 'https:' || u.username || u.password) return null;
++    return u.href;
++}
++
  /** The identity used to find duplicates: normalised URL without scheme and without "www.". */
  function urlKey(normalized) {
      if (!normalized) return null;
@@@ -94,7 +108,7 @@ function urlKey(normalized) {
  
  module.exports = {
      ApiError, newId, CONDITIONS, AVAILABILITY, AVAILABILITY_LABEL, SCHEMA_AVAILABILITY, SCHEMA_CONDITION,
--    normalizeUrl, urlKey, verbatimUrl,
++    normalizeUrl, urlKey, verbatimUrl, pictureUrl,
      domainOf: normalize.domainOf, slugify: normalize.slugify,
      parseAmount: asApiError(normalize.parseAmount), parseCurrency: asApiError(normalize.parseCurrency),
      parseEnum: asApiError(normalize.parseEnum), parseInstant: asApiError(normalize.parseInstant),
ddiff --git a/test/feed-export.test.js b/test/feed-export.test.js
nnew file mode 100644
iindex 0000000..afd0ae8
---- /dev/null
++++ b/test/feed-export.test.js
@@@ -0,0 +1,77 @@
++'use strict';
++/**
++ * What the one feed gets from an offer (feed.item@1 1.1.0): the price now, the price before, the shop
++ * that sells it and the product's picture. The owner, 11-13: "update the listings for deals".
++ *
++ * OpenVibe.Network's collector reads these offers (GET /api/v1/offers) and folds a run of them into
++ * one card whose items list what each deal costs.
++ */
++const assert = require('assert');
++const { boot, check, done } = require('./helpers/boot');
++const { sourceItem } = require('./helpers/mocks');
++
++(async () => {
++    const t = await boot();
++    const T0 = t.clock.now();
++
++    await check('an offer carries what it costs, the shop that sells it and its picture', async () => {
++        t.sources.put(sourceItem({
++            id: 'itm_01K5FEED00000000000000A1', title: 'Espresso machine', url: 'https://coffee.example/espresso',
++            retrievedAt: T0,
++            fields: {
++                price: '249.00', currency: 'EUR', list_price: '299.00', seller: 'Coffee Shop',
++                image: 'https://coffee.example/img/espresso.jpg',
++            },
++        }));
++        const summary = await t.ctx.importer.pull();
++        assert.strictEqual(summary.outcomes.created, 1, JSON.stringify(summary));
++        const offers = (await t.get('/api/v1/offers?sort=new')).json().offers;
++        const offer = offers.find((o) => o.title === 'Espresso machine');
++        assert.ok(offer, 'the offer is in the list the feed reads');
++        assert.deepStrictEqual(offer.feed.price, { amount: 249, currency: 'EUR' }, 'the price, with its currency in upper case');
++        assert.strictEqual(offer.feed.was, 299, 'the price it was');
++        assert.strictEqual(offer.feed.store, 'Coffee Shop', 'the shop that sells it');
++        assert.deepStrictEqual(offer.feed.media, { thumb: 'https://coffee.example/img/espresso.jpg' }, 'the picture of the product');
++    });
++
++    await check('the picture is kept as the source stated it, and only when it is https', async () => {
++        t.sources.put(sourceItem({
++            id: 'itm_01K5FEED00000000000000A2', title: 'Tea kettle', url: 'https://tea.example/kettle',
++            retrievedAt: T0, fields: { price: '39.99', currency: 'USD', image: 'http://tea.example/img/kettle.jpg' },
++        }));
++        await t.ctx.importer.pull();
++        const offers = (await t.get('/api/v1/offers?sort=new')).json().offers;
++        const offer = offers.find((o) => o.title === 'Tea kettle');
++        assert.ok(offer, 'the offer is in the list');
++        assert.strictEqual(offer.feed.media, null, 'a picture that is not https is not a picture');
++        assert.deepStrictEqual(offer.feed.price, { amount: 39.99, currency: 'USD' }, 'its price is still there');
++        assert.strictEqual(offer.feed.was, null, 'no list price stated, so no old price');
++    });
++
++    await check('an offer with no price states no price', async () => {
++        t.sources.put(sourceItem({
++            id: 'itm_01K5FEED00000000000000A3', kind: 'article', title: 'Weekend sale on garden tools',
++            url: 'https://news.example/garden', retrievedAt: T0,
++        }));
++        await t.ctx.importer.pull();
++        const offers = (await t.get('/api/v1/offers?sort=new')).json().offers;
++        const offer = offers.find((o) => o.title === 'Weekend sale on garden tools');
++        assert.ok(offer, 'the offer is in the list');
++        assert.strictEqual(offer.feed.price, null, 'no price is never a price of zero');
++        assert.strictEqual(offer.feed.was, null);
++        assert.strictEqual(offer.feed.media, null);
++    });
++
++    await check('a deal a person submitted carries its price too', async () => {
++        const alice = t.network.addUser('alice');
++        const offer = await t.submit(alice, { url: 'https://shop.example/lamp', title: 'Reading lamp', price: '24.50', currency: 'GBP' });
++        assert.ok(offer, 'the offer was submitted');
++        const json = await t.offerJson(offer.slug);
++        assert.deepStrictEqual(json.feed.price, { amount: 24.5, currency: 'GBP' }, 'the price the person stated');
++        assert.strictEqual(json.feed.was, null, 'nothing was said about an old price');
++        assert.strictEqual(json.feed.media, null, 'and there is no picture');
++    });
++
++    await t.close();
++    done();
++})().catch((err) => { console.error(err); process.exitCode = 1; });