openvibers/openvibe.deals Commits

Deals: a deal feed's stated price, retailer and end date reach the offer ("at Amazon") (#63)

OP OpenVibers committed 28 days ago · 6885baaaed · 8 files, +71 −16

Sources' feed@2 keeps DealNews' own price, currency, retailer and end date as item
fields. An article item that states them now maps like an offer: the price is an
observation, the end date the stated end, and the retailer the offer's new seller
(migration 0006), shown as "at Amazon" on cards and "Sold by" on the page. The
store stays the link's domain, and a verbatim-link source (an aggregator) never
names that store after a retailer.

Changes

ddiff --git a/README.md b/README.md
iindex 234cd39..9891097 100644
---- a/README.md
++++ b/README.md
@@@ -140,7 +140,11 @@ two subscriptions are created at boot when missing (`startSubscriptions`, needs
facet, `noindex`) — only when it actually changed.
- A price needs its currency: `10` without `USD` is refused (422), never completed. `$10` is refused
(amounts are plain decimals). Imported values are copied as stated; a malformed one is `null`. A
-- price in an imported headline is **not** parsed out of the text.
++ price in an imported headline is **not** parsed out of the text. A deal feed that states its offers in its own
++ elements (Sources `feed@2`: DealNews' price with currency, retailer and end date) is a statement: the price is an
++ observation, the end date the stated end, and the retailer the offer's `seller` ("at Amazon" on cards, "Sold by" on
++ the page, `seller` in JSON and the Offer JSON-LD). The store stays the link's domain; a source whose links are kept
++ verbatim (an aggregator's own pages) never names that store after a retailer (migration `0006_offer_seller.sql`).
- JSON-LD: `Product` with one `Offer` per offer; `price`/`priceCurrency` only from a fresh observation
that stated both, `availability`/`itemCondition` only when stated, `validThrough` only from a stated
end. Unknown fields are omitted, never 0 or `InStock`.
ddiff --git a/migrations/0006_offer_seller.sql b/migrations/0006_offer_seller.sql
nnew file mode 100644
iindex 0000000..088618f
---- /dev/null
++++ b/migrations/0006_offer_seller.sql
@@@ -0,0 +1,5 @@
++-- phase: expand
++-- The shop a source says sells the deal (server/domain/source-items.js): DealNews' feed names the retailer
++-- ("Amazon") while its links go to DealNews' own pages, so the store (the link's domain) is dealnews.com and the
++-- seller is kept here, exactly as stated. Null when no source named one.
++ALTER TABLE deal_offers ADD COLUMN IF NOT EXISTS seller text;
ddiff --git a/server/domain/offers.js b/server/domain/offers.js
iindex 8441ee9..a65a487 100644
---- a/server/domain/offers.js
++++ b/server/domain/offers.js
@@@ -33,9 +33,9 @@ function createOffers(deps) {
const q = {
byUrl: db.prepare("SELECT * FROM deal_offers WHERE url_norm = ? AND status <> 'disabled' ORDER BY created_at LIMIT 1"),
insertOffer: db.prepare(`INSERT INTO deal_offers (id, slug, title, description, url, url_norm, store_id, product_id, category, origin, submitted_by,
-- text_origin, review_state, ai_summary, status, expires_at, created_at, updated_at)
++ text_origin, review_state, ai_summary, status, expires_at, seller, created_at, updated_at)
VALUES (@id, @slug, @title, @description, @url, @url_norm, @store_id, @product_id, @category, @origin, @submitted_by,
-- @text_origin, @review_state, @ai_summary, 'active', @expires_at, @now, @now)`),
++ @text_origin, @review_state, @ai_summary, 'active', @expires_at, @seller, @now, @now)`),
insertSource: db.prepare(`INSERT INTO deal_offer_sources (id, offer_id, kind, ref_service, ref_type, ref_id, ref_part, ref_revision, source_key, url, label,
license_note, submitted_by, retrieved_at, created_at, updated_at)
VALUES (@id, @offer_id, @kind, @ref_service, @ref_type, @ref_id, @ref_part, @ref_revision, @source_key, @url, @label,
@@@ -122,7 +122,7 @@ function createOffers(deps) {
id, slug: slugify(f.title, id), title: f.title, description: f.description || null, url: f.url, url_norm: urlKey(f.url),
store_id: f.store_id || null, product_id: f.product_id || null, category: f.category || null, origin: f.origin,
submitted_by: f.submitted_by || null, text_origin: f.text_origin || 'human', review_state: f.review_state || 'not_required',
-- ai_summary: f.ai_summary || null, expires_at: f.expires_at == null ? null : f.expires_at, now: store.now(),
++ ai_summary: f.ai_summary || null, expires_at: f.expires_at == null ? null : f.expires_at, seller: f.seller || null, now: store.now(),
});
return await reads.get(id);
}
@@@ -181,7 +181,7 @@ function createOffers(deps) {
const offer = await insertOffer({
title: f.title, description: f.description, url: f.url, store_id: st ? st.id : null, product_id: f.product_id || null,
category: f.category || null, origin: 'import', submitted_by: null, text_origin: 'imported', review_state: 'pending',
-- expires_at: f.expires_at == null ? null : f.expires_at,
++ expires_at: f.expires_at == null ? null : f.expires_at, seller: f.seller || null,
});
const src = await ensureSource(offer.id, source);
await hotness.snapshot(offer.id, 'create');
ddiff --git a/server/domain/publication.js b/server/domain/publication.js
iindex 20a82c9..0536dbe 100644
---- a/server/domain/publication.js
++++ b/server/domain/publication.js
@@@ -109,6 +109,8 @@ function createPublication({ config, store, reads, catalog }) {
disabled_reason: r.status === 'disabled' ? r.disabled_reason : null,
created_at: iso(r.created_at), updated_at: iso(r.updated_at),
store: v.store ? { id: v.store.id, domain: v.store.domain, name: v.store.name, url: abs(storePath(v.store)) } : null,
++ // The shop a source says sells it, as stated (the store is where the link goes).
++ seller: r.seller || null,
product: v.product ? { id: v.product.id, slug: v.product.slug, name: v.product.name, url: abs(productPath(v.product)) } : null,
// "latest", never "current": whether it still holds is what freshness says.
latest_observation: v.latest ? observationDto(v.latest) : null,
@@@ -135,7 +137,7 @@ function createPublication({ config, store, reads, catalog }) {
const ld = {
'@type': 'Offer',
url: abs(offerPath(v.root)),
-- seller: v.store ? { '@type': 'Organization', name: v.store.name || v.store.domain } : undefined,
++ seller: v.root.seller ? { '@type': 'Organization', name: v.root.seller } : v.store ? { '@type': 'Organization', name: v.store.name || v.store.domain } : undefined,
};
if (fresh && o) {
if (o.price != null && o.currency) { ld.price = o.price; ld.priceCurrency = o.currency; }
ddiff --git a/server/domain/source-items.js b/server/domain/source-items.js
iindex d7b8fce..f21e492 100644
---- a/server/domain/source-items.js
++++ b/server/domain/source-items.js
@@@ -19,7 +19,12 @@
* seller → store name
* kind product a product (resolved by gtin / mpn / sku / name) and one offer per stated offer
* kind article (RSS/Atom) one offer with the title and link; the price is NOT parsed out of the
-- * headline — it stays "not stated" until a structured source or a person states it
++ * headline — it stays "not stated" until a structured source or a person states it. A deal
++ * feed's own offer elements (Sources feed@2: DealNews' price, currency, retailer and end date)
++ * are such a statement and map like an offer's fields.
++ * seller the shop the item names (fields.seller) is the offer's `seller`. It names the link's store
++ * too, except for a source whose links are kept verbatim (an aggregator linking its own pages:
++ * the store is the aggregator's domain, and the retailer must not become that store's name).
* other kinds skipped
*
* A source policy (server/domain/source-policy.js) may require a source's links kept verbatim and
@@@ -97,10 +102,14 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
// terms); every other source keeps today's normalisation.
const verbatim = keepsLinksVerbatim(item.source_key);
const linkOf = (raw) => (verbatim ? verbatimUrl(raw) : normalizeUrl(raw));
++ const sellerOf = (raw) => {
++ const seller = typeof raw === 'string' && raw.trim() ? raw.trim().slice(0, 120) : null;
++ return { seller, store_name: verbatim ? null : seller };
++ };
if (item.kind === 'offer') {
const url = linkOf(f.url || item.canonical_url);
if (!url || !item.title) return [];
-- return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), store_name: f.seller || null }];
++ return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), ...sellerOf(f.seller) }];
}
if (item.kind === 'product') {
if (!item.title) return [];
@@@ -113,7 +122,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
out.push({
// A verbatim source's title is used as stated, with no added words.
part: urlKey(url), url, title: !verbatim && o.seller ? `${item.title} — ${o.seller}` : item.title, description: item.summary,
-- obs: obsFields(o), expires_at: stated(o.valid_until), store_name: o.seller || null,
++ obs: obsFields(o), expires_at: stated(o.valid_until), ...sellerOf(o.seller),
product: { name: item.title, brand: f.brand, gtin: f.gtin, mpn: f.mpn, sku: f.sku },
});
}
@@@ -122,7 +131,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
if (item.kind === 'article' || item.kind === 'record') {
const url = linkOf(item.canonical_url);
if (!url || !item.title) return [];
-- return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields({}), expires_at: null, store_name: null }];
++ return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), ...sellerOf(f.seller) }];
}
return [];
}
@@@ -171,6 +180,10 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
const obs = await offers.recordObservation(existing.offer_id, existing.id, c.obs, { origin, observedAt: retrievedAt, sourceRevision: item.revision });
const offer = await reads.get(existing.offer_id);
const changed = ['observation'];
++ if (origin === 'import' && c.seller && offer.origin === 'import' && c.seller !== offer.seller) {
++ await db.prepare('UPDATE deal_offers SET seller = ?, updated_at = ? WHERE id = ?').run(c.seller, store.now(), offer.id);
++ changed.push('seller');
++ }
if (origin === 'import' && c.expires_at && offer.status === 'active' && offer.origin === 'import' && c.expires_at !== offer.expires_at) {
await db.prepare('UPDATE deal_offers SET expires_at = ?, updated_at = ? WHERE id = ?').run(c.expires_at, store.now(), offer.id);
changed.push('expires_at');
@@@ -192,6 +205,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
const s = await offers.ensureSource(dup.id, src);
const obs = await offers.recordObservation(dup.id, s.id, c.obs, { origin: 'import', observedAt: retrievedAt, sourceRevision: item.revision });
if (productId && !dup.product_id) await db.prepare('UPDATE deal_offers SET product_id = ?, updated_at = ? WHERE id = ?').run(productId, store.now(), dup.id);
++ if (c.seller && !dup.seller) await db.prepare('UPDATE deal_offers SET seller = ?, updated_at = ? WHERE id = ?').run(c.seller, store.now(), dup.id);
await indexing.emitOffer('deals.offer.updated', await reads.get(dup.id), { actor: 'svc:deals', extra: { changed: ['sources'], observation_id: obs.id } });
await indexing.reindex(await reads.get(dup.id));
outcomes.push('attached');
@@@ -199,7 +213,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
}
await offers.createImported({
url: c.url, title: String(c.title).slice(0, 200), description: c.description, product_id: productId,
-- store_name: c.store_name, expires_at: c.expires_at != null && c.expires_at > store.now() ? c.expires_at : null,
++ store_name: c.store_name, seller: c.seller, expires_at: c.expires_at != null && c.expires_at > store.now() ? c.expires_at : null,
}, src, c.obs, { observedAt: retrievedAt, sourceRevision: item.revision });
outcomes.push('created');
}
ddiff --git a/server/http/pages.js b/server/http/pages.js
iindex f2e3795..3267086 100644
---- a/server/http/pages.js
++++ b/server/http/pages.js
@@@ -181,7 +181,7 @@ function createPages(ctx) {
const canonical = publication.abs(publication.offerPath(r));
send(req, res, 200, {
title: r.title,
-- description: `${views.priceText(v.latest)}${v.latest ? ` as of ${views.when(v.latest.observed_at)}` : ''}${v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${r.description ? ` ${r.description.slice(0, 120)}` : ''}`,
++ description: `${views.priceText(v.latest)}${v.latest ? ` as of ${views.when(v.latest.observed_at)}` : ''}${r.seller ? ` at ${r.seller}` : v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${r.description ? ` ${r.description.slice(0, 120)}` : ''}`,
decision: v.decision,
canonical,
type: 'website',
@@@ -423,7 +423,7 @@ function createPages(ctx) {
const via = v.attribution ? `Deal via ${v.attribution.text}` : null;
return {
id: `deals:offer:${o.id}`, url: publication.abs(publication.offerPath(o)), title: o.title,
-- summary: `${via ? `${via}. ` : ''}${price}${v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${o.description ? ` ${o.description.slice(0, 300)}` : ''}`,
++ summary: `${via ? `${via}. ` : ''}${price}${v.root.seller ? ` at ${v.root.seller}` : v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${o.description ? ` ${o.description.slice(0, 300)}` : ''}`,
tags: via ? [via] : undefined,
published: o.created_at, updated: Math.max(o.updated_at, v.latest ? v.latest.observed_at : 0), decision: v.decision,
};
ddiff --git a/server/render/views.js b/server/render/views.js
iindex e3908df..41782e4 100644
---- a/server/render/views.js
++++ b/server/render/views.js
@@@ -84,7 +84,7 @@ function offerCard(v, urls) {
<h2 class="deal-title"><a href="${urls.offerPath(r)}">${r.title}</a> ${statusBadge(r)}</h2>
${latestLine(v)}
${attributionLine(v.attribution)}
--<p class="meta">${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a> · ` : ''}${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a> · ` : ''}posted ${timeEl(r.created_at)}${r.origin === 'import' ? ' · imported from a source' : ''}</p>
++<p class="meta">${r.seller ? html`<span class="seller">at ${r.seller}</span> · ` : ''}${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a> · ` : ''}${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a> · ` : ''}posted ${timeEl(r.created_at)}${r.origin === 'import' ? ' · imported from a source' : ''}</p>
</div></li>`;
}
@@@ -176,7 +176,8 @@ ${r.description ? html`<section class="description">${raw(renderMarkdown(r.descr
${r.ai_summary ? html`<section class="disclosure"><p><strong>AI-assisted summary</strong>${r.review_state === 'reviewed' ? ' (reviewed by a person)' : ' (not yet reviewed by a person)'}</p><p>${r.ai_summary}</p></section>` : ''}
${r.review_state === 'pending' && r.text_origin === 'imported' ? html`<p class="disclosure">This deal was imported from a source and has not been reviewed by a person yet.</p>` : ''}
<dl class="facts">
--<dt>Store</dt><dd>${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a>` : 'not stated'}</dd>
++${r.seller ? html`<dt>Sold by</dt><dd>${r.seller} <span class="meta">(as the source states it)</span></dd>` : ''}
++<dt>${r.seller ? 'Link goes to' : 'Store'}</dt><dd>${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a>` : 'not stated'}</dd>
<dt>Product</dt><dd>${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a>` : 'not linked'}</dd>
<dt>Posted</dt><dd>${timeEl(r.created_at)} ${r.origin === 'import' ? '(imported)' : '(by a member)'}</dd>
</dl>
@@@ -222,7 +223,7 @@ function productPage({ pv, urls }) {
const p = pv.product;
const rows = pv.offers.map((v) => html`<tr>
<td><a href="${urls.offerPath(v.root)}">${v.root.title}</a> ${statusBadge(v.root)}${attributionLine(v.attribution)}</td>
--<td>${v.store ? v.store.name || v.store.domain : 'not stated'}</td>
++<td>${v.root.seller || (v.store ? v.store.name || v.store.domain : 'not stated')}</td>
<td data-price="${v.latest && v.latest.price != null ? v.latest.price : ''}">${priceText(v.latest)}${v.latest && shippingText(v.latest) ? html` <span class="meta">${shippingText(v.latest)}</span>` : ''}</td>
<td>${v.latest ? timeEl(v.latest.observed_at) : html`<span class="meta">never observed</span>`}</td>
<td>${freshnessBadge(v.freshness, v.now)}</td>
ddiff --git a/test/import.test.js b/test/import.test.js
iindex ba564e9..c8a79a2 100644
---- a/test/import.test.js
++++ b/test/import.test.js
@@@ -179,6 +179,35 @@ const { signDelivery, signDeliveryHeaders } = require('openvibe-sdk/events');
assert.deepStrictEqual(normal.json().offer.attribution, { text: 'DealNews', url: 'https://www.dealnews.com/' });
});
++ await check('a deal feed\'s stated price, retailer and end date (Sources feed@2): on the offer; the retailer never names the aggregator\'s store', async () => {
++ const link = 'https://www.dealnews.com/products/Vosser/Air-Mattress/508347.html?iref=rss';
++ const ends = t.clock.now() + 30 * 86400000;
++ t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB1', sourceKey: 'dealnews-daily', kind: 'article', title: 'Vosser Queen Air Mattress for $67 + free shipping', url: link,
++ fields: { categories: [], price: '67.00', currency: 'USD', seller: 'Amazon', valid_until: new Date(ends).toISOString() }, retrievedAt: t.clock.now() }));
++ t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB2', sourceKey: 'dealnews-daily', kind: 'article', title: 'Target sale', url: 'https://www.dealnews.com/target-sale/1.html',
++ fields: { seller: 'Target' }, retrievedAt: t.clock.now() }));
++ const s = await t.ctx.importer.pull();
++ assert.deepStrictEqual(s.outcomes, { created: 2 }, JSON.stringify(s));
++ const offer = await byUrl(link);
++ assert.strictEqual(offer.seller, 'Amazon');
++ assert.strictEqual(offer.expires_at, ends, 'the stated end date');
++ const json = await t.offerJson(offer.slug);
++ assert.deepStrictEqual([json.latest_observation.price, json.latest_observation.currency, json.seller, json.store.domain, json.store.name], ['67.00', 'USD', 'Amazon', 'dealnews.com', null],
++ 'the price as stated; the store stays dealnews.com, unnamed (a retailer must not become the aggregator\'s name)');
++ const target = await byUrl('https://www.dealnews.com/target-sale/1.html');
++ assert.deepStrictEqual([target.seller, (await t.offerJson(target.slug)).latest_observation.price], ['Target', null], 'no stated price stays not stated');
++ const page = await t.get(`/d/${offer.slug}`);
++ assert.ok(page.text.includes('<dt>Sold by</dt><dd>Amazon') && page.text.includes('<dt>Link goes to</dt>'), 'the page says who sells it and where the link goes');
++ assert.ok((await t.get('/new')).text.includes('<span class="seller">at Amazon</span>'), 'the card names the seller');
++ assert.match((await t.get('/feed.xml')).text, /at Amazon\./);
++ // A later revision naming another retailer updates the offer's seller.
++ t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB2', sourceKey: 'dealnews-daily', kind: 'article', title: 'Target sale', url: 'https://www.dealnews.com/target-sale/1.html',
++ fields: { seller: 'Target.com', price: '10.00', currency: 'USD' }, revision: 2, retrievedAt: t.clock.now() + 1000 }));
++ await t.ctx.importer.pull();
++ const again = await byUrl('https://www.dealnews.com/target-sale/1.html');
++ assert.deepStrictEqual([again.seller, (await t.offerJson(again.slug)).latest_observation.price], ['Target.com', '10.00']);
++ });
++
await check('Sources down: the pull fails honestly (readiness says so), keeps its cursor, invents nothing', async () => {
const cursor = await t.ctx.importer.cursor();
const offers = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM deal_offers').get()).n;