openvibers/openvibe.deals Commits
Deals: a deal feed's stated price, retailer and end date reach the offer ("at Amazon") (#63)
Changes
@@@ -140,7 +140,11 @@ two subscriptions are created at boot when missing (`startSubscriptions`, needsfacet, `noindex`) — only when it actually changed.- A price needs its currency: `10` without `USD` is refused (422), never completed. `$10` is refused(amounts are plain decimals). Imported values are copied as stated; a malformed one is `null`. A-- price in an imported headline is **not** parsed out of the text.++ price in an imported headline is **not** parsed out of the text. A deal feed that states its offers in its own++ elements (Sources `feed@2`: DealNews' price with currency, retailer and end date) is a statement: the price is an++ observation, the end date the stated end, and the retailer the offer's `seller` ("at Amazon" on cards, "Sold by" on++ the page, `seller` in JSON and the Offer JSON-LD). The store stays the link's domain; a source whose links are kept++ verbatim (an aggregator's own pages) never names that store after a retailer (migration `0006_offer_seller.sql`).- JSON-LD: `Product` with one `Offer` per offer; `price`/`priceCurrency` only from a fresh observationthat stated both, `availability`/`itemCondition` only when stated, `validThrough` only from a statedend. Unknown fields are omitted, never 0 or `InStock`.nnew file mode 100644@@@ -0,0 +1,5 @@++-- phase: expand++-- The shop a source says sells the deal (server/domain/source-items.js): DealNews' feed names the retailer++-- ("Amazon") while its links go to DealNews' own pages, so the store (the link's domain) is dealnews.com and the++-- seller is kept here, exactly as stated. Null when no source named one.++ALTER TABLE deal_offers ADD COLUMN IF NOT EXISTS seller text;@@@ -33,9 +33,9 @@ function createOffers(deps) {const q = {byUrl: db.prepare("SELECT * FROM deal_offers WHERE url_norm = ? AND status <> 'disabled' ORDER BY created_at LIMIT 1"),insertOffer: db.prepare(`INSERT INTO deal_offers (id, slug, title, description, url, url_norm, store_id, product_id, category, origin, submitted_by,-- text_origin, review_state, ai_summary, status, expires_at, created_at, updated_at)++ text_origin, review_state, ai_summary, status, expires_at, seller, created_at, updated_at)VALUES (@id, @slug, @title, @description, @url, @url_norm, @store_id, @product_id, @category, @origin, @submitted_by,-- @text_origin, @review_state, @ai_summary, 'active', @expires_at, @now, @now)`),++ @text_origin, @review_state, @ai_summary, 'active', @expires_at, @seller, @now, @now)`),insertSource: db.prepare(`INSERT INTO deal_offer_sources (id, offer_id, kind, ref_service, ref_type, ref_id, ref_part, ref_revision, source_key, url, label,license_note, submitted_by, retrieved_at, created_at, updated_at)VALUES (@id, @offer_id, @kind, @ref_service, @ref_type, @ref_id, @ref_part, @ref_revision, @source_key, @url, @label,@@@ -122,7 +122,7 @@ function createOffers(deps) {id, slug: slugify(f.title, id), title: f.title, description: f.description || null, url: f.url, url_norm: urlKey(f.url),store_id: f.store_id || null, product_id: f.product_id || null, category: f.category || null, origin: f.origin,submitted_by: f.submitted_by || null, text_origin: f.text_origin || 'human', review_state: f.review_state || 'not_required',-- ai_summary: f.ai_summary || null, expires_at: f.expires_at == null ? null : f.expires_at, now: store.now(),++ ai_summary: f.ai_summary || null, expires_at: f.expires_at == null ? null : f.expires_at, seller: f.seller || null, now: store.now(),});return await reads.get(id);}@@@ -181,7 +181,7 @@ function createOffers(deps) {const offer = await insertOffer({title: f.title, description: f.description, url: f.url, store_id: st ? st.id : null, product_id: f.product_id || null,category: f.category || null, origin: 'import', submitted_by: null, text_origin: 'imported', review_state: 'pending',-- expires_at: f.expires_at == null ? null : f.expires_at,++ expires_at: f.expires_at == null ? null : f.expires_at, seller: f.seller || null,});const src = await ensureSource(offer.id, source);await hotness.snapshot(offer.id, 'create');@@@ -109,6 +109,8 @@ function createPublication({ config, store, reads, catalog }) {disabled_reason: r.status === 'disabled' ? r.disabled_reason : null,created_at: iso(r.created_at), updated_at: iso(r.updated_at),store: v.store ? { id: v.store.id, domain: v.store.domain, name: v.store.name, url: abs(storePath(v.store)) } : null,++ // The shop a source says sells it, as stated (the store is where the link goes).++ seller: r.seller || null,product: v.product ? { id: v.product.id, slug: v.product.slug, name: v.product.name, url: abs(productPath(v.product)) } : null,// "latest", never "current": whether it still holds is what freshness says.latest_observation: v.latest ? observationDto(v.latest) : null,@@@ -135,7 +137,7 @@ function createPublication({ config, store, reads, catalog }) {const ld = {'@type': 'Offer',url: abs(offerPath(v.root)),-- seller: v.store ? { '@type': 'Organization', name: v.store.name || v.store.domain } : undefined,++ seller: v.root.seller ? { '@type': 'Organization', name: v.root.seller } : v.store ? { '@type': 'Organization', name: v.store.name || v.store.domain } : undefined,};if (fresh && o) {if (o.price != null && o.currency) { ld.price = o.price; ld.priceCurrency = o.currency; }@@@ -19,7 +19,12 @@* seller → store name* kind product a product (resolved by gtin / mpn / sku / name) and one offer per stated offer* kind article (RSS/Atom) one offer with the title and link; the price is NOT parsed out of the-- * headline — it stays "not stated" until a structured source or a person states it++ * headline — it stays "not stated" until a structured source or a person states it. A deal++ * feed's own offer elements (Sources feed@2: DealNews' price, currency, retailer and end date)++ * are such a statement and map like an offer's fields.++ * seller the shop the item names (fields.seller) is the offer's `seller`. It names the link's store++ * too, except for a source whose links are kept verbatim (an aggregator linking its own pages:++ * the store is the aggregator's domain, and the retailer must not become that store's name).* other kinds skipped** A source policy (server/domain/source-policy.js) may require a source's links kept verbatim and@@@ -97,10 +102,14 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc// terms); every other source keeps today's normalisation.const verbatim = keepsLinksVerbatim(item.source_key);const linkOf = (raw) => (verbatim ? verbatimUrl(raw) : normalizeUrl(raw));++ const sellerOf = (raw) => {++ const seller = typeof raw === 'string' && raw.trim() ? raw.trim().slice(0, 120) : null;++ return { seller, store_name: verbatim ? null : seller };++ };if (item.kind === 'offer') {const url = linkOf(f.url || item.canonical_url);if (!url || !item.title) return [];-- return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), store_name: f.seller || null }];++ return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), ...sellerOf(f.seller) }];}if (item.kind === 'product') {if (!item.title) return [];@@@ -113,7 +122,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourcout.push({// A verbatim source's title is used as stated, with no added words.part: urlKey(url), url, title: !verbatim && o.seller ? `${item.title} — ${o.seller}` : item.title, description: item.summary,-- obs: obsFields(o), expires_at: stated(o.valid_until), store_name: o.seller || null,++ obs: obsFields(o), expires_at: stated(o.valid_until), ...sellerOf(o.seller),product: { name: item.title, brand: f.brand, gtin: f.gtin, mpn: f.mpn, sku: f.sku },});}@@@ -122,7 +131,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourcif (item.kind === 'article' || item.kind === 'record') {const url = linkOf(item.canonical_url);if (!url || !item.title) return [];-- return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields({}), expires_at: null, store_name: null }];++ return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), ...sellerOf(f.seller) }];}return [];}@@@ -171,6 +180,10 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourcconst obs = await offers.recordObservation(existing.offer_id, existing.id, c.obs, { origin, observedAt: retrievedAt, sourceRevision: item.revision });const offer = await reads.get(existing.offer_id);const changed = ['observation'];++ if (origin === 'import' && c.seller && offer.origin === 'import' && c.seller !== offer.seller) {++ await db.prepare('UPDATE deal_offers SET seller = ?, updated_at = ? WHERE id = ?').run(c.seller, store.now(), offer.id);++ changed.push('seller');++ }if (origin === 'import' && c.expires_at && offer.status === 'active' && offer.origin === 'import' && c.expires_at !== offer.expires_at) {await db.prepare('UPDATE deal_offers SET expires_at = ?, updated_at = ? WHERE id = ?').run(c.expires_at, store.now(), offer.id);changed.push('expires_at');@@@ -192,6 +205,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourcconst s = await offers.ensureSource(dup.id, src);const obs = await offers.recordObservation(dup.id, s.id, c.obs, { origin: 'import', observedAt: retrievedAt, sourceRevision: item.revision });if (productId && !dup.product_id) await db.prepare('UPDATE deal_offers SET product_id = ?, updated_at = ? WHERE id = ?').run(productId, store.now(), dup.id);++ if (c.seller && !dup.seller) await db.prepare('UPDATE deal_offers SET seller = ?, updated_at = ? WHERE id = ?').run(c.seller, store.now(), dup.id);await indexing.emitOffer('deals.offer.updated', await reads.get(dup.id), { actor: 'svc:deals', extra: { changed: ['sources'], observation_id: obs.id } });await indexing.reindex(await reads.get(dup.id));outcomes.push('attached');@@@ -199,7 +213,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc}await offers.createImported({url: c.url, title: String(c.title).slice(0, 200), description: c.description, product_id: productId,-- store_name: c.store_name, expires_at: c.expires_at != null && c.expires_at > store.now() ? c.expires_at : null,++ store_name: c.store_name, seller: c.seller, expires_at: c.expires_at != null && c.expires_at > store.now() ? c.expires_at : null,}, src, c.obs, { observedAt: retrievedAt, sourceRevision: item.revision });outcomes.push('created');}@@@ -181,7 +181,7 @@ function createPages(ctx) {const canonical = publication.abs(publication.offerPath(r));send(req, res, 200, {title: r.title,-- description: `${views.priceText(v.latest)}${v.latest ? ` as of ${views.when(v.latest.observed_at)}` : ''}${v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${r.description ? ` ${r.description.slice(0, 120)}` : ''}`,++ description: `${views.priceText(v.latest)}${v.latest ? ` as of ${views.when(v.latest.observed_at)}` : ''}${r.seller ? ` at ${r.seller}` : v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${r.description ? ` ${r.description.slice(0, 120)}` : ''}`,decision: v.decision,canonical,type: 'website',@@@ -423,7 +423,7 @@ function createPages(ctx) {const via = v.attribution ? `Deal via ${v.attribution.text}` : null;return {id: `deals:offer:${o.id}`, url: publication.abs(publication.offerPath(o)), title: o.title,-- summary: `${via ? `${via}. ` : ''}${price}${v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${o.description ? ` ${o.description.slice(0, 300)}` : ''}`,++ summary: `${via ? `${via}. ` : ''}${price}${v.root.seller ? ` at ${v.root.seller}` : v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${o.description ? ` ${o.description.slice(0, 300)}` : ''}`,tags: via ? [via] : undefined,published: o.created_at, updated: Math.max(o.updated_at, v.latest ? v.latest.observed_at : 0), decision: v.decision,};@@@ -84,7 +84,7 @@ function offerCard(v, urls) {<h2 class="deal-title"><a href="${urls.offerPath(r)}">${r.title}</a> ${statusBadge(r)}</h2>${latestLine(v)}${attributionLine(v.attribution)}--<p class="meta">${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a> · ` : ''}${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a> · ` : ''}posted ${timeEl(r.created_at)}${r.origin === 'import' ? ' · imported from a source' : ''}</p>++<p class="meta">${r.seller ? html`<span class="seller">at ${r.seller}</span> · ` : ''}${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a> · ` : ''}${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a> · ` : ''}posted ${timeEl(r.created_at)}${r.origin === 'import' ? ' · imported from a source' : ''}</p></div></li>`;}@@@ -176,7 +176,8 @@ ${r.description ? html`<section class="description">${raw(renderMarkdown(r.descr${r.ai_summary ? html`<section class="disclosure"><p><strong>AI-assisted summary</strong>${r.review_state === 'reviewed' ? ' (reviewed by a person)' : ' (not yet reviewed by a person)'}</p><p>${r.ai_summary}</p></section>` : ''}${r.review_state === 'pending' && r.text_origin === 'imported' ? html`<p class="disclosure">This deal was imported from a source and has not been reviewed by a person yet.</p>` : ''}<dl class="facts">--<dt>Store</dt><dd>${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a>` : 'not stated'}</dd>++${r.seller ? html`<dt>Sold by</dt><dd>${r.seller} <span class="meta">(as the source states it)</span></dd>` : ''}++<dt>${r.seller ? 'Link goes to' : 'Store'}</dt><dd>${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a>` : 'not stated'}</dd><dt>Product</dt><dd>${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a>` : 'not linked'}</dd><dt>Posted</dt><dd>${timeEl(r.created_at)} ${r.origin === 'import' ? '(imported)' : '(by a member)'}</dd></dl>@@@ -222,7 +223,7 @@ function productPage({ pv, urls }) {const p = pv.product;const rows = pv.offers.map((v) => html`<tr><td><a href="${urls.offerPath(v.root)}">${v.root.title}</a> ${statusBadge(v.root)}${attributionLine(v.attribution)}</td>--<td>${v.store ? v.store.name || v.store.domain : 'not stated'}</td>++<td>${v.root.seller || (v.store ? v.store.name || v.store.domain : 'not stated')}</td><td data-price="${v.latest && v.latest.price != null ? v.latest.price : ''}">${priceText(v.latest)}${v.latest && shippingText(v.latest) ? html` <span class="meta">${shippingText(v.latest)}</span>` : ''}</td><td>${v.latest ? timeEl(v.latest.observed_at) : html`<span class="meta">never observed</span>`}</td><td>${freshnessBadge(v.freshness, v.now)}</td>@@@ -179,6 +179,35 @@ const { signDelivery, signDeliveryHeaders } = require('openvibe-sdk/events');assert.deepStrictEqual(normal.json().offer.attribution, { text: 'DealNews', url: 'https://www.dealnews.com/' });});++ await check('a deal feed\'s stated price, retailer and end date (Sources feed@2): on the offer; the retailer never names the aggregator\'s store', async () => {++ const link = 'https://www.dealnews.com/products/Vosser/Air-Mattress/508347.html?iref=rss';++ const ends = t.clock.now() + 30 * 86400000;++ t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB1', sourceKey: 'dealnews-daily', kind: 'article', title: 'Vosser Queen Air Mattress for $67 + free shipping', url: link,++ fields: { categories: [], price: '67.00', currency: 'USD', seller: 'Amazon', valid_until: new Date(ends).toISOString() }, retrievedAt: t.clock.now() }));++ t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB2', sourceKey: 'dealnews-daily', kind: 'article', title: 'Target sale', url: 'https://www.dealnews.com/target-sale/1.html',++ fields: { seller: 'Target' }, retrievedAt: t.clock.now() }));++ const s = await t.ctx.importer.pull();++ assert.deepStrictEqual(s.outcomes, { created: 2 }, JSON.stringify(s));++ const offer = await byUrl(link);++ assert.strictEqual(offer.seller, 'Amazon');++ assert.strictEqual(offer.expires_at, ends, 'the stated end date');++ const json = await t.offerJson(offer.slug);++ assert.deepStrictEqual([json.latest_observation.price, json.latest_observation.currency, json.seller, json.store.domain, json.store.name], ['67.00', 'USD', 'Amazon', 'dealnews.com', null],++ 'the price as stated; the store stays dealnews.com, unnamed (a retailer must not become the aggregator\'s name)');++ const target = await byUrl('https://www.dealnews.com/target-sale/1.html');++ assert.deepStrictEqual([target.seller, (await t.offerJson(target.slug)).latest_observation.price], ['Target', null], 'no stated price stays not stated');++ const page = await t.get(`/d/${offer.slug}`);++ assert.ok(page.text.includes('<dt>Sold by</dt><dd>Amazon') && page.text.includes('<dt>Link goes to</dt>'), 'the page says who sells it and where the link goes');++ assert.ok((await t.get('/new')).text.includes('<span class="seller">at Amazon</span>'), 'the card names the seller');++ assert.match((await t.get('/feed.xml')).text, /at Amazon\./);++ // A later revision naming another retailer updates the offer's seller.++ t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB2', sourceKey: 'dealnews-daily', kind: 'article', title: 'Target sale', url: 'https://www.dealnews.com/target-sale/1.html',++ fields: { seller: 'Target.com', price: '10.00', currency: 'USD' }, revision: 2, retrievedAt: t.clock.now() + 1000 }));++ await t.ctx.importer.pull();++ const again = await byUrl('https://www.dealnews.com/target-sale/1.html');++ assert.deepStrictEqual([again.seller, (await t.offerJson(again.slug)).latest_observation.price], ['Target.com', '10.00']);++ });++await check('Sources down: the pull fails honestly (readiness says so), keeps its cursor, invents nothing', async () => {const cursor = await t.ctx.importer.cursor();const offers = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM deal_offers').get()).n;
@@ -140,7 +140,11 @@ two subscriptions are created at boot when missing (`startSubscriptions`, needs
@@ -140,7 +140,11 @@ two subscriptions are created at boot when missing (`startSubscriptions`, needs
facet, `noindex`) — only when it actually changed.
facet, `noindex`) — only when it actually changed.
- A price needs its currency: `10` without `USD` is refused (422), never completed. `$10` is refused
- A price needs its currency: `10` without `USD` is refused (422), never completed. `$10` is refused
(amounts are plain decimals). Imported values are copied as stated; a malformed one is `null`. A
(amounts are plain decimals). Imported values are copied as stated; a malformed one is `null`. A
- price in an imported headline is **not** parsed out of the text.
+ price in an imported headline is **not** parsed out of the text. A deal feed that states its offers in its own
+ elements (Sources `feed@2`: DealNews' price with currency, retailer and end date) is a statement: the price is an
+ observation, the end date the stated end, and the retailer the offer's `seller` ("at Amazon" on cards, "Sold by" on
+ the page, `seller` in JSON and the Offer JSON-LD). The store stays the link's domain; a source whose links are kept
+ verbatim (an aggregator's own pages) never names that store after a retailer (migration `0006_offer_seller.sql`).
- JSON-LD: `Product` with one `Offer` per offer; `price`/`priceCurrency` only from a fresh observation
- JSON-LD: `Product` with one `Offer` per offer; `price`/`priceCurrency` only from a fresh observation
that stated both, `availability`/`itemCondition` only when stated, `validThrough` only from a stated
that stated both, `availability`/`itemCondition` only when stated, `validThrough` only from a stated
end. Unknown fields are omitted, never 0 or `InStock`.
end. Unknown fields are omitted, never 0 or `InStock`.
new file mode 100644
new file mode 100644
@@ -0,0 +1,5 @@
@@ -0,0 +1,5 @@
+-- phase: expand
+-- The shop a source says sells the deal (server/domain/source-items.js): DealNews' feed names the retailer
+-- ("Amazon") while its links go to DealNews' own pages, so the store (the link's domain) is dealnews.com and the
+-- seller is kept here, exactly as stated. Null when no source named one.
+ALTER TABLE deal_offers ADD COLUMN IF NOT EXISTS seller text;
@@ -33,9 +33,9 @@ function createOffers(deps) {
@@ -33,9 +33,9 @@ function createOffers(deps) {
const q = {
const q = {
byUrl: db.prepare("SELECT * FROM deal_offers WHERE url_norm = ? AND status <> 'disabled' ORDER BY created_at LIMIT 1"),
byUrl: db.prepare("SELECT * FROM deal_offers WHERE url_norm = ? AND status <> 'disabled' ORDER BY created_at LIMIT 1"),
insertOffer: db.prepare(`INSERT INTO deal_offers (id, slug, title, description, url, url_norm, store_id, product_id, category, origin, submitted_by,
insertOffer: db.prepare(`INSERT INTO deal_offers (id, slug, title, description, url, url_norm, store_id, product_id, category, origin, submitted_by,
- text_origin, review_state, ai_summary, status, expires_at, created_at, updated_at)
+ text_origin, review_state, ai_summary, status, expires_at, seller, created_at, updated_at)
VALUES (@id, @slug, @title, @description, @url, @url_norm, @store_id, @product_id, @category, @origin, @submitted_by,
VALUES (@id, @slug, @title, @description, @url, @url_norm, @store_id, @product_id, @category, @origin, @submitted_by,
- @text_origin, @review_state, @ai_summary, 'active', @expires_at, @now, @now)`),
+ @text_origin, @review_state, @ai_summary, 'active', @expires_at, @seller, @now, @now)`),
insertSource: db.prepare(`INSERT INTO deal_offer_sources (id, offer_id, kind, ref_service, ref_type, ref_id, ref_part, ref_revision, source_key, url, label,
insertSource: db.prepare(`INSERT INTO deal_offer_sources (id, offer_id, kind, ref_service, ref_type, ref_id, ref_part, ref_revision, source_key, url, label,
license_note, submitted_by, retrieved_at, created_at, updated_at)
license_note, submitted_by, retrieved_at, created_at, updated_at)
VALUES (@id, @offer_id, @kind, @ref_service, @ref_type, @ref_id, @ref_part, @ref_revision, @source_key, @url, @label,
VALUES (@id, @offer_id, @kind, @ref_service, @ref_type, @ref_id, @ref_part, @ref_revision, @source_key, @url, @label,
@@ -122,7 +122,7 @@ function createOffers(deps) {
@@ -122,7 +122,7 @@ function createOffers(deps) {
id, slug: slugify(f.title, id), title: f.title, description: f.description || null, url: f.url, url_norm: urlKey(f.url),
id, slug: slugify(f.title, id), title: f.title, description: f.description || null, url: f.url, url_norm: urlKey(f.url),
store_id: f.store_id || null, product_id: f.product_id || null, category: f.category || null, origin: f.origin,
store_id: f.store_id || null, product_id: f.product_id || null, category: f.category || null, origin: f.origin,
submitted_by: f.submitted_by || null, text_origin: f.text_origin || 'human', review_state: f.review_state || 'not_required',
submitted_by: f.submitted_by || null, text_origin: f.text_origin || 'human', review_state: f.review_state || 'not_required',
- ai_summary: f.ai_summary || null, expires_at: f.expires_at == null ? null : f.expires_at, now: store.now(),
+ ai_summary: f.ai_summary || null, expires_at: f.expires_at == null ? null : f.expires_at, seller: f.seller || null, now: store.now(),
});
});
return await reads.get(id);
return await reads.get(id);
}
}
@@ -181,7 +181,7 @@ function createOffers(deps) {
@@ -181,7 +181,7 @@ function createOffers(deps) {
const offer = await insertOffer({
const offer = await insertOffer({
title: f.title, description: f.description, url: f.url, store_id: st ? st.id : null, product_id: f.product_id || null,
title: f.title, description: f.description, url: f.url, store_id: st ? st.id : null, product_id: f.product_id || null,
category: f.category || null, origin: 'import', submitted_by: null, text_origin: 'imported', review_state: 'pending',
category: f.category || null, origin: 'import', submitted_by: null, text_origin: 'imported', review_state: 'pending',
- expires_at: f.expires_at == null ? null : f.expires_at,
+ expires_at: f.expires_at == null ? null : f.expires_at, seller: f.seller || null,
});
});
const src = await ensureSource(offer.id, source);
const src = await ensureSource(offer.id, source);
await hotness.snapshot(offer.id, 'create');
await hotness.snapshot(offer.id, 'create');
@@ -109,6 +109,8 @@ function createPublication({ config, store, reads, catalog }) {
@@ -109,6 +109,8 @@ function createPublication({ config, store, reads, catalog }) {
disabled_reason: r.status === 'disabled' ? r.disabled_reason : null,
disabled_reason: r.status === 'disabled' ? r.disabled_reason : null,
created_at: iso(r.created_at), updated_at: iso(r.updated_at),
created_at: iso(r.created_at), updated_at: iso(r.updated_at),
store: v.store ? { id: v.store.id, domain: v.store.domain, name: v.store.name, url: abs(storePath(v.store)) } : null,
store: v.store ? { id: v.store.id, domain: v.store.domain, name: v.store.name, url: abs(storePath(v.store)) } : null,
+ // The shop a source says sells it, as stated (the store is where the link goes).
+ seller: r.seller || null,
product: v.product ? { id: v.product.id, slug: v.product.slug, name: v.product.name, url: abs(productPath(v.product)) } : null,
product: v.product ? { id: v.product.id, slug: v.product.slug, name: v.product.name, url: abs(productPath(v.product)) } : null,
// "latest", never "current": whether it still holds is what freshness says.
// "latest", never "current": whether it still holds is what freshness says.
latest_observation: v.latest ? observationDto(v.latest) : null,
latest_observation: v.latest ? observationDto(v.latest) : null,
@@ -135,7 +137,7 @@ function createPublication({ config, store, reads, catalog }) {
@@ -135,7 +137,7 @@ function createPublication({ config, store, reads, catalog }) {
const ld = {
const ld = {
'@type': 'Offer',
'@type': 'Offer',
url: abs(offerPath(v.root)),
url: abs(offerPath(v.root)),
- seller: v.store ? { '@type': 'Organization', name: v.store.name || v.store.domain } : undefined,
+ seller: v.root.seller ? { '@type': 'Organization', name: v.root.seller } : v.store ? { '@type': 'Organization', name: v.store.name || v.store.domain } : undefined,
};
};
if (fresh && o) {
if (fresh && o) {
if (o.price != null && o.currency) { ld.price = o.price; ld.priceCurrency = o.currency; }
if (o.price != null && o.currency) { ld.price = o.price; ld.priceCurrency = o.currency; }
@@ -19,7 +19,12 @@
@@ -19,7 +19,12 @@
* seller → store name
* seller → store name
* kind product a product (resolved by gtin / mpn / sku / name) and one offer per stated offer
* kind product a product (resolved by gtin / mpn / sku / name) and one offer per stated offer
* kind article (RSS/Atom) one offer with the title and link; the price is NOT parsed out of the
* kind article (RSS/Atom) one offer with the title and link; the price is NOT parsed out of the
- * headline — it stays "not stated" until a structured source or a person states it
+ * headline — it stays "not stated" until a structured source or a person states it. A deal
+ * feed's own offer elements (Sources feed@2: DealNews' price, currency, retailer and end date)
+ * are such a statement and map like an offer's fields.
+ * seller the shop the item names (fields.seller) is the offer's `seller`. It names the link's store
+ * too, except for a source whose links are kept verbatim (an aggregator linking its own pages:
+ * the store is the aggregator's domain, and the retailer must not become that store's name).
* other kinds skipped
* other kinds skipped
*
*
* A source policy (server/domain/source-policy.js) may require a source's links kept verbatim and
* A source policy (server/domain/source-policy.js) may require a source's links kept verbatim and
@@ -97,10 +102,14 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
@@ -97,10 +102,14 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
// terms); every other source keeps today's normalisation.
// terms); every other source keeps today's normalisation.
const verbatim = keepsLinksVerbatim(item.source_key);
const verbatim = keepsLinksVerbatim(item.source_key);
const linkOf = (raw) => (verbatim ? verbatimUrl(raw) : normalizeUrl(raw));
const linkOf = (raw) => (verbatim ? verbatimUrl(raw) : normalizeUrl(raw));
+ const sellerOf = (raw) => {
+ const seller = typeof raw === 'string' && raw.trim() ? raw.trim().slice(0, 120) : null;
+ return { seller, store_name: verbatim ? null : seller };
+ };
if (item.kind === 'offer') {
if (item.kind === 'offer') {
const url = linkOf(f.url || item.canonical_url);
const url = linkOf(f.url || item.canonical_url);
if (!url || !item.title) return [];
if (!url || !item.title) return [];
- return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), store_name: f.seller || null }];
+ return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), ...sellerOf(f.seller) }];
}
}
if (item.kind === 'product') {
if (item.kind === 'product') {
if (!item.title) return [];
if (!item.title) return [];
@@ -113,7 +122,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
@@ -113,7 +122,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
out.push({
out.push({
// A verbatim source's title is used as stated, with no added words.
// A verbatim source's title is used as stated, with no added words.
part: urlKey(url), url, title: !verbatim && o.seller ? `${item.title} — ${o.seller}` : item.title, description: item.summary,
part: urlKey(url), url, title: !verbatim && o.seller ? `${item.title} — ${o.seller}` : item.title, description: item.summary,
- obs: obsFields(o), expires_at: stated(o.valid_until), store_name: o.seller || null,
+ obs: obsFields(o), expires_at: stated(o.valid_until), ...sellerOf(o.seller),
product: { name: item.title, brand: f.brand, gtin: f.gtin, mpn: f.mpn, sku: f.sku },
product: { name: item.title, brand: f.brand, gtin: f.gtin, mpn: f.mpn, sku: f.sku },
});
});
}
}
@@ -122,7 +131,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
@@ -122,7 +131,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
if (item.kind === 'article' || item.kind === 'record') {
if (item.kind === 'article' || item.kind === 'record') {
const url = linkOf(item.canonical_url);
const url = linkOf(item.canonical_url);
if (!url || !item.title) return [];
if (!url || !item.title) return [];
- return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields({}), expires_at: null, store_name: null }];
+ return [{ part: '', url, title: item.title, description: item.summary, obs: obsFields(f), expires_at: stated(f.valid_until), ...sellerOf(f.seller) }];
}
}
return [];
return [];
}
}
@@ -171,6 +180,10 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
@@ -171,6 +180,10 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
const obs = await offers.recordObservation(existing.offer_id, existing.id, c.obs, { origin, observedAt: retrievedAt, sourceRevision: item.revision });
const obs = await offers.recordObservation(existing.offer_id, existing.id, c.obs, { origin, observedAt: retrievedAt, sourceRevision: item.revision });
const offer = await reads.get(existing.offer_id);
const offer = await reads.get(existing.offer_id);
const changed = ['observation'];
const changed = ['observation'];
+ if (origin === 'import' && c.seller && offer.origin === 'import' && c.seller !== offer.seller) {
+ await db.prepare('UPDATE deal_offers SET seller = ?, updated_at = ? WHERE id = ?').run(c.seller, store.now(), offer.id);
+ changed.push('seller');
+ }
if (origin === 'import' && c.expires_at && offer.status === 'active' && offer.origin === 'import' && c.expires_at !== offer.expires_at) {
if (origin === 'import' && c.expires_at && offer.status === 'active' && offer.origin === 'import' && c.expires_at !== offer.expires_at) {
await db.prepare('UPDATE deal_offers SET expires_at = ?, updated_at = ? WHERE id = ?').run(c.expires_at, store.now(), offer.id);
await db.prepare('UPDATE deal_offers SET expires_at = ?, updated_at = ? WHERE id = ?').run(c.expires_at, store.now(), offer.id);
changed.push('expires_at');
changed.push('expires_at');
@@ -192,6 +205,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
@@ -192,6 +205,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
const s = await offers.ensureSource(dup.id, src);
const s = await offers.ensureSource(dup.id, src);
const obs = await offers.recordObservation(dup.id, s.id, c.obs, { origin: 'import', observedAt: retrievedAt, sourceRevision: item.revision });
const obs = await offers.recordObservation(dup.id, s.id, c.obs, { origin: 'import', observedAt: retrievedAt, sourceRevision: item.revision });
if (productId && !dup.product_id) await db.prepare('UPDATE deal_offers SET product_id = ?, updated_at = ? WHERE id = ?').run(productId, store.now(), dup.id);
if (productId && !dup.product_id) await db.prepare('UPDATE deal_offers SET product_id = ?, updated_at = ? WHERE id = ?').run(productId, store.now(), dup.id);
+ if (c.seller && !dup.seller) await db.prepare('UPDATE deal_offers SET seller = ?, updated_at = ? WHERE id = ?').run(c.seller, store.now(), dup.id);
await indexing.emitOffer('deals.offer.updated', await reads.get(dup.id), { actor: 'svc:deals', extra: { changed: ['sources'], observation_id: obs.id } });
await indexing.emitOffer('deals.offer.updated', await reads.get(dup.id), { actor: 'svc:deals', extra: { changed: ['sources'], observation_id: obs.id } });
await indexing.reindex(await reads.get(dup.id));
await indexing.reindex(await reads.get(dup.id));
outcomes.push('attached');
outcomes.push('attached');
@@ -199,7 +213,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
@@ -199,7 +213,7 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
}
}
await offers.createImported({
await offers.createImported({
url: c.url, title: String(c.title).slice(0, 200), description: c.description, product_id: productId,
url: c.url, title: String(c.title).slice(0, 200), description: c.description, product_id: productId,
- store_name: c.store_name, expires_at: c.expires_at != null && c.expires_at > store.now() ? c.expires_at : null,
+ store_name: c.store_name, seller: c.seller, expires_at: c.expires_at != null && c.expires_at > store.now() ? c.expires_at : null,
}, src, c.obs, { observedAt: retrievedAt, sourceRevision: item.revision });
}, src, c.obs, { observedAt: retrievedAt, sourceRevision: item.revision });
outcomes.push('created');
outcomes.push('created');
}
}
@@ -181,7 +181,7 @@ function createPages(ctx) {
@@ -181,7 +181,7 @@ function createPages(ctx) {
const canonical = publication.abs(publication.offerPath(r));
const canonical = publication.abs(publication.offerPath(r));
send(req, res, 200, {
send(req, res, 200, {
title: r.title,
title: r.title,
- description: `${views.priceText(v.latest)}${v.latest ? ` as of ${views.when(v.latest.observed_at)}` : ''}${v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${r.description ? ` ${r.description.slice(0, 120)}` : ''}`,
+ description: `${views.priceText(v.latest)}${v.latest ? ` as of ${views.when(v.latest.observed_at)}` : ''}${r.seller ? ` at ${r.seller}` : v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${r.description ? ` ${r.description.slice(0, 120)}` : ''}`,
decision: v.decision,
decision: v.decision,
canonical,
canonical,
type: 'website',
type: 'website',
@@ -423,7 +423,7 @@ function createPages(ctx) {
@@ -423,7 +423,7 @@ function createPages(ctx) {
const via = v.attribution ? `Deal via ${v.attribution.text}` : null;
const via = v.attribution ? `Deal via ${v.attribution.text}` : null;
return {
return {
id: `deals:offer:${o.id}`, url: publication.abs(publication.offerPath(o)), title: o.title,
id: `deals:offer:${o.id}`, url: publication.abs(publication.offerPath(o)), title: o.title,
- summary: `${via ? `${via}. ` : ''}${price}${v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${o.description ? ` ${o.description.slice(0, 300)}` : ''}`,
+ summary: `${via ? `${via}. ` : ''}${price}${v.root.seller ? ` at ${v.root.seller}` : v.store ? ` at ${v.store.name || v.store.domain}` : ''}.${o.description ? ` ${o.description.slice(0, 300)}` : ''}`,
tags: via ? [via] : undefined,
tags: via ? [via] : undefined,
published: o.created_at, updated: Math.max(o.updated_at, v.latest ? v.latest.observed_at : 0), decision: v.decision,
published: o.created_at, updated: Math.max(o.updated_at, v.latest ? v.latest.observed_at : 0), decision: v.decision,
};
};
@@ -84,7 +84,7 @@ function offerCard(v, urls) {
@@ -84,7 +84,7 @@ function offerCard(v, urls) {
<h2 class="deal-title"><a href="${urls.offerPath(r)}">${r.title}</a> ${statusBadge(r)}</h2>
<h2 class="deal-title"><a href="${urls.offerPath(r)}">${r.title}</a> ${statusBadge(r)}</h2>
${latestLine(v)}
${latestLine(v)}
${attributionLine(v.attribution)}
${attributionLine(v.attribution)}
-<p class="meta">${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a> · ` : ''}${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a> · ` : ''}posted ${timeEl(r.created_at)}${r.origin === 'import' ? ' · imported from a source' : ''}</p>
+<p class="meta">${r.seller ? html`<span class="seller">at ${r.seller}</span> · ` : ''}${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a> · ` : ''}${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a> · ` : ''}posted ${timeEl(r.created_at)}${r.origin === 'import' ? ' · imported from a source' : ''}</p>
</div></li>`;
</div></li>`;
}
}
@@ -176,7 +176,8 @@ ${r.description ? html`<section class="description">${raw(renderMarkdown(r.descr
@@ -176,7 +176,8 @@ ${r.description ? html`<section class="description">${raw(renderMarkdown(r.descr
${r.ai_summary ? html`<section class="disclosure"><p><strong>AI-assisted summary</strong>${r.review_state === 'reviewed' ? ' (reviewed by a person)' : ' (not yet reviewed by a person)'}</p><p>${r.ai_summary}</p></section>` : ''}
${r.ai_summary ? html`<section class="disclosure"><p><strong>AI-assisted summary</strong>${r.review_state === 'reviewed' ? ' (reviewed by a person)' : ' (not yet reviewed by a person)'}</p><p>${r.ai_summary}</p></section>` : ''}
${r.review_state === 'pending' && r.text_origin === 'imported' ? html`<p class="disclosure">This deal was imported from a source and has not been reviewed by a person yet.</p>` : ''}
${r.review_state === 'pending' && r.text_origin === 'imported' ? html`<p class="disclosure">This deal was imported from a source and has not been reviewed by a person yet.</p>` : ''}
<dl class="facts">
<dl class="facts">
-<dt>Store</dt><dd>${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a>` : 'not stated'}</dd>
+${r.seller ? html`<dt>Sold by</dt><dd>${r.seller} <span class="meta">(as the source states it)</span></dd>` : ''}
+<dt>${r.seller ? 'Link goes to' : 'Store'}</dt><dd>${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a>` : 'not stated'}</dd>
<dt>Product</dt><dd>${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a>` : 'not linked'}</dd>
<dt>Product</dt><dd>${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a>` : 'not linked'}</dd>
<dt>Posted</dt><dd>${timeEl(r.created_at)} ${r.origin === 'import' ? '(imported)' : '(by a member)'}</dd>
<dt>Posted</dt><dd>${timeEl(r.created_at)} ${r.origin === 'import' ? '(imported)' : '(by a member)'}</dd>
</dl>
</dl>
@@ -222,7 +223,7 @@ function productPage({ pv, urls }) {
@@ -222,7 +223,7 @@ function productPage({ pv, urls }) {
const p = pv.product;
const p = pv.product;
const rows = pv.offers.map((v) => html`<tr>
const rows = pv.offers.map((v) => html`<tr>
<td><a href="${urls.offerPath(v.root)}">${v.root.title}</a> ${statusBadge(v.root)}${attributionLine(v.attribution)}</td>
<td><a href="${urls.offerPath(v.root)}">${v.root.title}</a> ${statusBadge(v.root)}${attributionLine(v.attribution)}</td>
-<td>${v.store ? v.store.name || v.store.domain : 'not stated'}</td>
+<td>${v.root.seller || (v.store ? v.store.name || v.store.domain : 'not stated')}</td>
<td data-price="${v.latest && v.latest.price != null ? v.latest.price : ''}">${priceText(v.latest)}${v.latest && shippingText(v.latest) ? html` <span class="meta">${shippingText(v.latest)}</span>` : ''}</td>
<td data-price="${v.latest && v.latest.price != null ? v.latest.price : ''}">${priceText(v.latest)}${v.latest && shippingText(v.latest) ? html` <span class="meta">${shippingText(v.latest)}</span>` : ''}</td>
<td>${v.latest ? timeEl(v.latest.observed_at) : html`<span class="meta">never observed</span>`}</td>
<td>${v.latest ? timeEl(v.latest.observed_at) : html`<span class="meta">never observed</span>`}</td>
<td>${freshnessBadge(v.freshness, v.now)}</td>
<td>${freshnessBadge(v.freshness, v.now)}</td>
@@ -179,6 +179,35 @@ const { signDelivery, signDeliveryHeaders } = require('openvibe-sdk/events');
@@ -179,6 +179,35 @@ const { signDelivery, signDeliveryHeaders } = require('openvibe-sdk/events');
assert.deepStrictEqual(normal.json().offer.attribution, { text: 'DealNews', url: 'https://www.dealnews.com/' });
assert.deepStrictEqual(normal.json().offer.attribution, { text: 'DealNews', url: 'https://www.dealnews.com/' });
});
});
+ await check('a deal feed\'s stated price, retailer and end date (Sources feed@2): on the offer; the retailer never names the aggregator\'s store', async () => {
+ const link = 'https://www.dealnews.com/products/Vosser/Air-Mattress/508347.html?iref=rss';
+ const ends = t.clock.now() + 30 * 86400000;
+ t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB1', sourceKey: 'dealnews-daily', kind: 'article', title: 'Vosser Queen Air Mattress for $67 + free shipping', url: link,
+ fields: { categories: [], price: '67.00', currency: 'USD', seller: 'Amazon', valid_until: new Date(ends).toISOString() }, retrievedAt: t.clock.now() }));
+ t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB2', sourceKey: 'dealnews-daily', kind: 'article', title: 'Target sale', url: 'https://www.dealnews.com/target-sale/1.html',
+ fields: { seller: 'Target' }, retrievedAt: t.clock.now() }));
+ const s = await t.ctx.importer.pull();
+ assert.deepStrictEqual(s.outcomes, { created: 2 }, JSON.stringify(s));
+ const offer = await byUrl(link);
+ assert.strictEqual(offer.seller, 'Amazon');
+ assert.strictEqual(offer.expires_at, ends, 'the stated end date');
+ const json = await t.offerJson(offer.slug);
+ assert.deepStrictEqual([json.latest_observation.price, json.latest_observation.currency, json.seller, json.store.domain, json.store.name], ['67.00', 'USD', 'Amazon', 'dealnews.com', null],
+ 'the price as stated; the store stays dealnews.com, unnamed (a retailer must not become the aggregator\'s name)');
+ const target = await byUrl('https://www.dealnews.com/target-sale/1.html');
+ assert.deepStrictEqual([target.seller, (await t.offerJson(target.slug)).latest_observation.price], ['Target', null], 'no stated price stays not stated');
+ const page = await t.get(`/d/${offer.slug}`);
+ assert.ok(page.text.includes('<dt>Sold by</dt><dd>Amazon') && page.text.includes('<dt>Link goes to</dt>'), 'the page says who sells it and where the link goes');
+ assert.ok((await t.get('/new')).text.includes('<span class="seller">at Amazon</span>'), 'the card names the seller');
+ assert.match((await t.get('/feed.xml')).text, /at Amazon\./);
+ // A later revision naming another retailer updates the offer's seller.
+ t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB2', sourceKey: 'dealnews-daily', kind: 'article', title: 'Target sale', url: 'https://www.dealnews.com/target-sale/1.html',
+ fields: { seller: 'Target.com', price: '10.00', currency: 'USD' }, revision: 2, retrievedAt: t.clock.now() + 1000 }));
+ await t.ctx.importer.pull();
+ const again = await byUrl('https://www.dealnews.com/target-sale/1.html');
+ assert.deepStrictEqual([again.seller, (await t.offerJson(again.slug)).latest_observation.price], ['Target.com', '10.00']);
+ });
+
await check('Sources down: the pull fails honestly (readiness says so), keeps its cursor, invents nothing', async () => {
await check('Sources down: the pull fails honestly (readiness says so), keeps its cursor, invents nothing', async () => {
const cursor = await t.ctx.importer.cursor();
const cursor = await t.ctx.importer.cursor();
const offers = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM deal_offers').get()).n;
const offers = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM deal_offers').get()).n;