openvibers/openvibe.deals Commits

Deals keeps the list price a page states (jsonld@2) and selects a deal by it, a price drop, or a watch; the offer shows it struck through with the saving (#128)

OP OpenVibers committed 16 days ago · 4820e3b760 · 10 files, +310 −25

Changes

ddiff --git a/migrations/0008_list_price.sql b/migrations/0008_list_price.sql
nnew file mode 100644
iindex 0000000..b16a4e3
---- /dev/null
++++ b/migrations/0008_list_price.sql
@@@ -0,0 +1,14 @@
++-- phase: expand
++-- The list price a source states for the same offer (server/domain/source-items.js): Sources' jsonld@2
++-- reads a ListPrice/StrikethroughPrice/MSRP/SRP priceSpecification and copies it exactly as the page
++-- stated it. It is kept only when it is in the observation's currency and greater than its price; a
++-- "list price" below the price is noise. The saving is derived (list_price_num - price_num), never
++-- stored. Existing rows stay null.
++ALTER TABLE deal_price_observations ADD COLUMN IF NOT EXISTS list_price text COLLATE "C";
++ALTER TABLE deal_price_observations ADD COLUMN IF NOT EXISTS list_price_num double precision;
++
++-- Why a crawled offer was selected as a deal (server/domain/source-items.js isDeal): 'list_price'
++-- (the page states a list price at least 5% above the price), 'price_drop' (the latest price is at
++-- its lowest / lower than the previous observation) or 'watch' (a person's price_below watch
++-- matches). Null when none of the three applies; existing rows and community offers stay null.
++ALTER TABLE deal_offers ADD COLUMN IF NOT EXISTS deal_reason text;
ddiff --git a/public/css/deals.css b/public/css/deals.css
iindex bc4baa0..00a0565 100644
---- a/public/css/deals.css
++++ b/public/css/deals.css
@@@ -63,6 +63,10 @@ form.inline button { margin: 0 0 0 8px; padding: 2px 10px; font-size: .85rem; }
.price { font-size: 1.05rem; color: var(--warning-text, var(--warning, #f59e0b)); }
.price.price-none { font-size: .8rem; font-weight: 500; color: var(--text-muted, #7386a3); }
.price-line .as-of { font-size: .8rem; }
++/* The list price the page states, struck through, and what you save against it (only ever above the price). */
++.price-line .dl-list { font-size: .82rem; color: var(--text-muted, #7386a3); text-decoration-thickness: 1px; font-variant-numeric: tabular-nums; }
++.price-line .dl-saving { padding: 1px 7px; border-radius: 999px; font-size: .74rem; font-weight: 800; font-variant-numeric: tabular-nums;
++ color: var(--success-text, var(--success, #22c55e)); background: color-mix(in srgb, var(--success, #22c55e) 14%, transparent); }
.attribution { margin: 0; font-size: .82rem; color: var(--text-muted, #7386a3); }
.deal-item .meta { margin: 2px 0 0; font-size: .82rem; }
@media (max-width: 600px) { .deal-item { grid-template-columns: 44px minmax(0, 1fr); padding: 12px; } .deal-item .score { height: 48px; } }
ddiff --git a/server/domain/account-data.js b/server/domain/account-data.js
iindex 4c5dc6d..4194ff1 100644
---- a/server/domain/account-data.js
++++ b/server/domain/account-data.js
@@@ -30,7 +30,7 @@ const TABLES = [
{ table: 'deal_offers', subject: 'disabled_by', file: null, erase: anonymize },
{ table: 'deal_offers', subject: 'merged_by', file: null, erase: anonymize },
{ table: 'deal_offer_sources', subject: 'submitted_by', file: null, erase: anonymize },
-- { table: 'deal_price_observations', subject: 'observed_by', file: 'price-reports.json', columns: ['id', 'offer_id', 'observed_at', 'price', 'currency', 'shipping', 'shipping_note', 'condition', 'availability', 'note'], order: 'observed_at', erase: anonymize },
++ { table: 'deal_price_observations', subject: 'observed_by', file: 'price-reports.json', columns: ['id', 'offer_id', 'observed_at', 'price', 'list_price', 'currency', 'shipping', 'shipping_note', 'condition', 'availability', 'note'], order: 'observed_at', erase: anonymize },
{ table: 'deal_products', subject: 'created_by', file: 'products.json', columns: ['id', 'slug', 'name', 'brand', 'category', 'description', 'created_at'], erase: anonymize },
{ table: 'deal_product_aliases', subject: 'created_by', file: null, erase: anonymize },
{ table: 'deal_flags', subject: 'reporter', file: 'reports.json', columns: ['id', 'offer_id', 'kind', 'reason', 'status', 'created_at'], erase: anonymize },
ddiff --git a/server/domain/offers.js b/server/domain/offers.js
iindex a65a487..26230a6 100644
---- a/server/domain/offers.js
++++ b/server/domain/offers.js
@@@ -18,6 +18,7 @@ const {
ApiError, newId, normalizeUrl, urlKey, domainOf, slugify, parseAmount, parseCurrency, parseEnum, parseInstant,
text, longText, CONDITIONS, AVAILABILITY,
} = require('./values');
++const { isDeal } = require('./price-trend');
const FUTURE_SKEW_MS = 5 * 60 * 1000;
@@@ -41,10 +42,10 @@ function createOffers(deps) {
VALUES (@id, @offer_id, @kind, @ref_service, @ref_type, @ref_id, @ref_part, @ref_revision, @source_key, @url, @label,
@license_note, @submitted_by, @retrieved_at, @now, @now)`),
sourceByRef: db.prepare('SELECT * FROM deal_offer_sources WHERE kind = ? AND ref_service = ? AND ref_id = ? AND ref_part = ?'),
-- insertObs: db.prepare(`INSERT INTO deal_price_observations (id, offer_id, source_id, observed_at, recorded_at, price, price_num, currency, shipping,
-- shipping_num, shipping_note, condition, availability, origin, observed_by, source_revision, note)
-- VALUES (@id, @offer_id, @source_id, @observed_at, @recorded_at, @price, @price_num, @currency, @shipping,
-- @shipping_num, @shipping_note, @condition, @availability, @origin, @observed_by, @source_revision, @note)`),
++ insertObs: db.prepare(`INSERT INTO deal_price_observations (id, offer_id, source_id, observed_at, recorded_at, price, price_num, list_price, list_price_num,
++ currency, shipping, shipping_num, shipping_note, condition, availability, origin, observed_by, source_revision, note)
++ VALUES (@id, @offer_id, @source_id, @observed_at, @recorded_at, @price, @price_num, @list_price, @list_price_num,
++ @currency, @shipping, @shipping_num, @shipping_note, @condition, @availability, @origin, @observed_by, @source_revision, @note)`),
obs: db.prepare('SELECT * FROM deal_price_observations WHERE id = ?'),
log: db.prepare('INSERT INTO moderation_log (action, offer_id, target_id, actor, reason, before, after, at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'),
logFor: db.prepare(`SELECT * FROM moderation_log WHERE offer_id IN (SELECT value FROM jsonb_array_elements_text(?::jsonb)) OR target_id IN (SELECT value FROM jsonb_array_elements_text(?::jsonb)) ORDER BY id`),
@@@ -61,6 +62,8 @@ function createOffers(deps) {
if ((price || shipping) && !currency) throw new ApiError(422, 'request.invalid', 'a price or shipping cost needs its currency (e.g. USD); Deals never assumes one');
return {
price: price ? price.text : null, price_num: price ? price.num : null,
++ // A person's report states no list price (only a crawled source does, migration 0008).
++ list_price: null, list_price_num: null,
currency,
shipping: shipping ? shipping.text : null, shipping_num: shipping ? shipping.num : null,
shipping_note: text(input.shipping_note, { field: 'shipping_note', max: 200 }),
@@@ -106,13 +109,22 @@ function createOffers(deps) {
if (observedAt > now + FUTURE_SKEW_MS) throw new ApiError(422, 'request.invalid', 'observed_at is in the future');
const row = {
id: newId('dpo'), offer_id: offerId, source_id: sourceId, observed_at: Math.min(observedAt, now), recorded_at: now,
-- price: fields.price, price_num: fields.price_num, currency: fields.currency, shipping: fields.shipping, shipping_num: fields.shipping_num,
++ price: fields.price, price_num: fields.price_num, list_price: fields.list_price == null ? null : fields.list_price, list_price_num: fields.list_price_num == null ? null : fields.list_price_num,
++ currency: fields.currency, shipping: fields.shipping, shipping_num: fields.shipping_num,
shipping_note: fields.shipping_note || null, condition: fields.condition || null, availability: fields.availability || null,
origin, observed_by: observedBy, source_revision: sourceRevision, note: fields.note || null,
};
await q.insertObs.run(row);
const obs = await q.obs.get(row.id);
-- await watches.onObservation(obs);
++ const watchEvents = await watches.onObservation(obs);
++ // A crawled offer is selected as a deal by its latest observation (price-trend.js isDeal):
++ // a stated list price, a price drop, or a person's matching price_below watch. The reason is
++ // kept on the offer where its labels are (migration 0008); a person's own observation does not
++ // select a crawled offer and leaves that decision alone.
++ if (origin === 'import' || origin === 'import_refresh') {
++ const reason = isDeal(await reads.observations([offerId]), Boolean(watchEvents && watchEvents.priceBelowMatched));
++ await db.prepare('UPDATE deal_offers SET deal_reason = ?, updated_at = ? WHERE id = ? AND deal_reason IS DISTINCT FROM ?').run(reason, now, offerId, reason);
++ }
return obs;
}
ddiff --git a/server/domain/price-trend.js b/server/domain/price-trend.js
iindex 91ad4ff..157d85d 100644
---- a/server/domain/price-trend.js
++++ b/server/domain/price-trend.js
@@@ -1,6 +1,7 @@
'use strict';
/**
-- * A deal's price over time, from its observations (reads.observations: newest first).
++ * A deal's price over time, from its observations (reads.observations: newest first), and the rule
++ * that selects a crawled offer as a deal.
*
* Only stated prices count, only in the currency of the latest stated price (a listing seen in two
* currencies is never compared across them: nothing is converted), and never from a removed source.
@@@ -8,25 +9,58 @@
*
* priceTrend(observations) → null (no stated price) or
* { currency, count, points: [{ at, num, price }] oldest first,
-- * first, latest, lowest, highest: { at, num, price },
++ * first, lowest, highest: { at, num, price },
++ * latest: { at, num, price, list_price, list_price_num, saving_num, saving_pct },
* change: { amount, percent } | null (one price only),
* atLowest: the latest price is the lowest seen (with two or more prices that differ) }
*
* The offer page draws it (render/views.js priceChart) and the JSON twin and API carry it as
-- * price_trend (publication.js trendDto).
++ * price_trend (publication.js trendDto). A list price (Sources jsonld@2) is shown only beside the
++ * price it belongs to; saving_num and saving_pct are null when the observation states none.
++ *
++ * isDeal(observations, watchHit) → 'list_price' | 'price_drop' | 'watch' | null, the one reason a
++ * crawled offer is selected as a deal (see the comment on the function).
*/
const decimals = (s) => { const m = /\.(\d+)$/.exec(String(s)); return m ? m[1].length : 0; };
++/** The saving a stated list price represents against its price: { num, pct }, both null when none. */
++function savingOf(o) {
++ // A raw observation row names its numbers `price_num` / `list_price_num`; a trend point names
++ // the price `num`. Text is accepted too, for a caller that built an observation by hand.
++ const rawList = o && (o.list_price_num != null ? o.list_price_num : o.list_price);
++ const rawPrice = o && (o.price_num != null ? o.price_num : o.num != null ? o.num : o.price);
++ const list = rawList != null ? Number(rawList) : NaN;
++ const price = rawPrice != null ? Number(rawPrice) : NaN;
++ if (!Number.isFinite(list) || !Number.isFinite(price) || list <= price) return { num: null, pct: null };
++ const places = Math.max(decimals(o.list_price != null ? o.list_price : list), decimals(o.price != null ? o.price : price));
++ return {
++ num: Number((list - price).toFixed(places)),
++ // A whole number, rounded down; the epsilon only absorbs binary floating-point noise at an exact whole percent.
++ pct: Math.floor(((list - price) / list) * 100 + 1e-9),
++ };
++}
++
++/** The four list-price fields an observation DTO carries; all null when it states no usable list price. */
++function listPriceFields(o) {
++ const saving = savingOf(o);
++ return {
++ list_price: o && o.list_price != null ? o.list_price : null,
++ list_price_num: o && o.list_price_num != null ? Number(o.list_price_num) : null,
++ saving_num: saving.num,
++ saving_pct: saving.pct,
++ };
++}
++
function priceTrend(observations) {
const stated = (observations || []).filter((o) => o.price != null && o.price_num != null && Number.isFinite(Number(o.price_num)) && !o.source_removed_at);
const ref = stated.find((o) => o.currency) || stated[0];
if (!ref) return null;
-- const points = stated.filter((o) => (o.currency || null) === (ref.currency || null))
-- .map((o) => ({ at: Number(o.observed_at), num: Number(o.price_num), price: o.price }))
-- .reverse();
++ const rows = stated.filter((o) => (o.currency || null) === (ref.currency || null));
++ const points = rows.map((o) => ({ at: Number(o.observed_at), num: Number(o.price_num), price: o.price })).reverse();
const first = points[0];
-- const latest = points[points.length - 1];
++ // rows are newest first, so its first row is the latest point (points is oldest first).
++ const latest = { ...points[points.length - 1], ...listPriceFields(rows[0]) };
// Ties go to the earliest time: "lowest since" is when it was first that low.
const lowest = points.reduce((a, p) => (p.num < a.num ? p : a));
const highest = points.reduce((a, p) => (p.num > a.num ? p : a));
@@@ -46,4 +80,23 @@ function priceTrend(observations) {
};
}
--module.exports = { priceTrend };
++/**
++ * A crawled offer is a deal for exactly one of three reasons, in this order:
++ * list_price the page states a list price at least 5% above the price (floor of the saving);
++ * price_drop the latest price is the lowest seen, or lower than the previous observation;
++ * watch a person's price_below watch matches (watches.js keeps the matching).
++ * → the reason, or null when none applies. Newest-first observations, as reads.observations returns.
++ */
++function isDeal(observations, watchHit = false) {
++ // The stored rows always carry the numbers; fill them in from the stated text for a hand-built list.
++ const stated = (observations || []).map((o) => ({ ...o, price_num: o && o.price_num != null ? o.price_num : o && o.price, list_price_num: o && o.list_price_num != null ? o.list_price_num : o && o.list_price }));
++ const t = priceTrend(stated);
++ if (t) {
++ if (t.latest.saving_pct != null && t.latest.saving_pct >= 5) return 'list_price';
++ const previous = t.points.length > 1 ? t.points[t.points.length - 2] : null;
++ if (t.atLowest || (previous && t.latest.num < previous.num)) return 'price_drop';
++ }
++ return watchHit ? 'watch' : null;
++}
++
++module.exports = { priceTrend, isDeal, savingOf, listPriceFields };
ddiff --git a/server/domain/publication.js b/server/domain/publication.js
iindex a61d378..0c5e01e 100644
---- a/server/domain/publication.js
++++ b/server/domain/publication.js
@@@ -21,7 +21,7 @@
const seo = require('openvibe-publishing/seo');
const { iso } = require('./values');
const { attributionOf, hidesFromExtension } = require('./source-policy');
--const { priceTrend } = require('./price-trend');
++const { priceTrend, listPriceFields } = require('./price-trend');
function createPublication({ config, store, reads, catalog }) {
const abs = (p) => seo.canonicalUrl(config.baseUrl, p);
@@@ -107,18 +107,26 @@ function createPublication({ config, store, reads, catalog }) {
function observationDto(o) {
return {
id: o.id, observed_at: iso(o.observed_at), recorded_at: iso(o.recorded_at),
-- price: o.price, currency: o.currency, shipping: o.shipping, shipping_note: o.shipping_note,
++ price: o.price, currency: o.currency, ...listPriceFields(o), shipping: o.shipping, shipping_note: o.shipping_note,
condition: o.condition, availability: o.availability, origin: o.origin,
observed_by: o.observed_by || null, listing: o.offer_id,
source: { id: o.source_id, kind: o.source_kind, ref: o.ref_service === 'sources' ? { service: 'sources', type: 'item', id: o.ref_id, revision: o.source_revision } : null, key: o.source_key || null, url: o.source_url || null },
};
}
-- /** The price over time as JSON: the stated prices, their times, and the change since the first. */
++ /**
++ * The price over time as JSON: the stated prices, their times, and the change since the first.
++ * The latest observation also carries its list price and the saving (all four fields null when
++ * the observation states no usable list price).
++ */
function trendDto(t) {
if (!t) return null;
const at = (p) => ({ price: p.price, observed_at: iso(p.at) });
-- return { currency: t.currency, observations: t.count, first: at(t.first), latest: at(t.latest), lowest: at(t.lowest), highest: at(t.highest), change: t.change, at_lowest: t.atLowest };
++ return {
++ currency: t.currency, observations: t.count, first: at(t.first),
++ latest: { ...at(t.latest), ...listPriceFields(t.latest) },
++ lowest: at(t.lowest), highest: at(t.highest), change: t.change, at_lowest: t.atLowest,
++ };
}
/** The JSON representation (API and /d/:slug.json): the same facts as the page. */
@@@ -127,6 +135,8 @@ function createPublication({ config, store, reads, catalog }) {
return {
id: r.id, slug: r.slug, url: abs(offerPath(r)), title: r.title, description: r.description,
link: r.url, status: r.status, category: r.category, origin: r.origin, submitted_by: r.submitted_by,
++ // Why a crawled offer was selected as a deal (migration 0008); null for community offers.
++ deal_reason: r.deal_reason || null,
text_origin: r.text_origin, review_state: r.review_state, ai_summary: r.ai_summary,
expires_at: iso(r.expires_at), expired_at: iso(r.expired_at), expired_reason: r.expired_reason,
disabled_reason: r.status === 'disabled' ? r.disabled_reason : null,
ddiff --git a/server/domain/source-items.js b/server/domain/source-items.js
iindex 43f3c74..1017d8a 100644
---- a/server/domain/source-items.js
++++ b/server/domain/source-items.js
@@@ -16,8 +16,10 @@
* Item → offer mapping (values exactly as the source stated them; missing = null):
* kind offer one offer: fields.price + fields.currency, availability / condition (schema.org
* terms mapped to Deals' enums, unknown terms → null), valid_until → stated expiry,
-- * seller → store name
-- * kind product a product (resolved by gtin / mpn / sku / name) and one offer per stated offer
++ * seller → store name, and list_price when the page states one above the price in the
++ * same currency (jsonld@2's ListPrice/StrikethroughPrice/MSRP/SRP specification)
++ * kind product a product (resolved by gtin / mpn / sku / name) and one offer per stated offer;
++ * each offer's own list_price is kept with its price (jsonld@2)
* kind article (RSS/Atom) one offer with the title and link; the price is NOT parsed out of the
* headline — it stays "not stated" until a structured source or a person states it. A deal
* feed's own offer elements (Sources feed@2: DealNews' price, currency, retailer and end date)
@@@ -48,6 +50,7 @@ const {
normalizeUrl, verbatimUrl, urlKey, parseAmount, SCHEMA_AVAILABILITY, SCHEMA_CONDITION, iso,
} = require('./values');
const { keepsLinksVerbatim } = require('./source-policy');
++const { isDeal } = require('./price-trend');
const CURSOR = 'sources.deals.after';
@@@ -90,8 +93,16 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
/** Stated fields → observation fields. A malformed value is null, never repaired or guessed. */
function obsFields(f = {}) {
const price = amount(f.price);
++ const currency = currencyOf(f.currency);
++ const list = amount(f.list_price);
++ // Sources' jsonld@2 already drops a list price in another currency; Deals keeps its own rule
++ // too, for any source that states one. list_price_currency is only present when a source makes
++ // it explicit; otherwise the list price is in the offer's own currency.
++ const listCurrency = f.list_price_currency === undefined ? currency : currencyOf(f.list_price_currency);
++ const listOk = Boolean(list && price && currency && listCurrency === currency && list.num > price.num);
return {
-- price: price ? price.text : null, price_num: price ? price.num : null, currency: currencyOf(f.currency),
++ price: price ? price.text : null, price_num: price ? price.num : null, currency,
++ list_price: listOk ? list.text : null, list_price_num: listOk ? list.num : null,
shipping: null, shipping_num: null, shipping_note: null,
condition: SCHEMA_CONDITION[f.condition] || null,
availability: SCHEMA_AVAILABILITY[f.availability] || null,
@@@ -315,4 +326,4 @@ function createImporter({ config, store, reads, catalog, offers, indexing, sourc
};
}
--module.exports = { createImporter, CURSOR };
++module.exports = { createImporter, CURSOR, isDeal };
ddiff --git a/server/domain/watches.js b/server/domain/watches.js
iindex 91bc3aa..8c6c40b 100644
---- a/server/domain/watches.js
++++ b/server/domain/watches.js
@@@ -94,7 +94,12 @@ function createWatches({ config, store, reads, catalog, publication, outbox }) {
return false;
}
-- /** Inside the observation's transaction. Returns the envelopes emitted. */
++ /**
++ * Inside the observation's transaction. Returns the envelopes emitted; the array's
++ * priceBelowMatched says whether any active price_below watch matched this observation (even
++ * when it did not emit again because it had already notified a lower price) — the deal
++ * selection reads it (server/domain/offers.js recordObservation).
++ */
async function onObservation(obs) {
const offer = await reads.get(obs.offer_id);
const v = await publication.offerView(offer);
@@@ -106,6 +111,7 @@ function createWatches({ config, store, reads, catalog, publication, outbox }) {
if (obs.observed_by && obs.observed_by === w.subject) continue;
if (v.root.submitted_by && v.root.submitted_by === w.subject && obs.origin === 'community') continue;
if (!matches(w, v, obs, words)) continue;
++ if (w.kind === 'price_below') out.priceBelowMatched = true;
const prior = await q.priorForGroup.all(w.id, JSON.stringify(v.ids));
if (w.kind !== 'price_below' && prior.length) continue;
if (w.kind === 'price_below' && prior.some((p) => p.currency === obs.currency && p.price_num != null && p.price_num <= obs.price_num)) continue;
ddiff --git a/server/render/views.js b/server/render/views.js
iindex ef04c19..d71323f 100644
---- a/server/render/views.js
++++ b/server/render/views.js
@@@ -9,6 +9,7 @@
*/
const { html, raw, renderMarkdown, paginationHtml, breadcrumbsHtml } = require('openvibe-publishing/ssr');
const { AVAILABILITY_LABEL, CONDITIONS, AVAILABILITY } = require('../domain/values');
++const { savingOf } = require('../domain/price-trend');
const pad = (n) => String(n).padStart(2, '0');
function when(ms) {
@@@ -39,6 +40,14 @@ function shippingText(o) {
return o.shipping_note || '';
}
++/** The stated list price and the saving, only beside the price they belong to (no styling: the site's CSS does that). */
++function listPriceLine(o) {
++ if (!o || o.list_price == null || o.list_price_num == null) return '';
++ const saving = savingOf(o);
++ if (saving.pct == null) return '';
++ return html` <s class="dl-list" data-list-price="${o.list_price}">${o.list_price}${o.currency ? ` ${o.currency}` : ''}</s> <span class="dl-saving">−${saving.pct}%</span>`;
++}
++
function freshnessBadge(f, now) {
if (f.state === 'fresh') return html`<span class="badge fresh" data-freshness="fresh">Fresh · observed ${ago(Date.parse(f.observed_at), now)}</span>`;
if (f.state === 'stale') return html`<span class="badge stale" data-freshness="stale">Stale · last observed ${ago(Date.parse(f.observed_at), now)} — this may no longer be available</span>`;
@@@ -68,7 +77,7 @@ function latestLine(v) {
const o = v.latest;
// A price nobody stated is said quietly: a headline-only source (DealNews' feed) states its price in the title.
const stated = o && o.price != null;
-- return html`<p class="price-line"><strong class="price${stated ? '' : ' price-none'}" data-price="${stated ? o.price : ''}">${priceText(o)}</strong>
++ return html`<p class="price-line"><strong class="price${stated ? '' : ' price-none'}" data-price="${stated ? o.price : ''}">${priceText(o)}</strong>${listPriceLine(o)}
${o && shippingText(o) ? html` <span class="meta">${shippingText(o)}</span>` : ''}
${o && o.availability ? html` · <span class="meta">${AVAILABILITY_LABEL[o.availability]}</span>` : ''}
${o ? html` <span class="meta as-of">as of ${timeEl(o.observed_at)}</span>` : ''} ${freshnessBadge(v.freshness, v.now)}</p>`;
ddiff --git a/test/list-price.test.js b/test/list-price.test.js
nnew file mode 100644
iindex 0000000..6895a01
---- /dev/null
++++ b/test/list-price.test.js
@@@ -0,0 +1,166 @@
++'use strict';
++/**
++ * A crawled source's list price (Sources jsonld@2): kept with its price only when it is in the same
++ * currency and greater, shown in the trend/JSON/page, and the three reasons a crawled offer is a
++ * deal (list_price, price_drop, watch). Migration 0008 is applied to a database that already has
++ * observations, which stay null.
++ */
++const assert = require('assert');
++const fs = require('fs');
++const os = require('os');
++const path = require('path');
++const { createDb } = require('openvibe-sdk/db');
++const { priceTrend, isDeal, savingOf } = require('../server/domain/price-trend');
++const { boot, check, done, HOUR } = require('./helpers/boot');
++const { sourceItem } = require('./helpers/mocks');
++
++const at = (n) => 1790000000000 + n * HOUR;
++const obs = (n, price, extra = {}) => ({ observed_at: at(n), price, price_num: Number(price), currency: 'USD', ...extra });
++
++(async () => {
++ await check('selection is one of list_price, price_drop or watch — and the 5% list-price threshold', async () => {
++ assert.strictEqual(isDeal([obs(1, '99.99', { list_price: '129.99', list_price_num: 129.99 })]), 'list_price');
++ assert.deepStrictEqual(savingOf({ price: '99.99', price_num: 99.99, list_price: '129.99', list_price_num: 129.99 }), { num: 30, pct: 23 }, 'a 23% saving, rounded down');
++ assert.strictEqual(isDeal([obs(1, '100.00', { list_price: '103.00', list_price_num: 103 })]), null, '3% is not enough on its own');
++ assert.strictEqual(isDeal([obs(1, '99.75', { list_price: '105.00', list_price_num: 105 })]), 'list_price', '5% is enough');
++ assert.strictEqual(isDeal([obs(2, '90.00'), obs(1, '100.00')]), 'price_drop');
++ assert.strictEqual(isDeal([obs(3, '90.00'), obs(2, '95.00'), obs(1, '100.00')]), 'price_drop', 'at its lowest');
++ assert.strictEqual(isDeal([obs(2, '100.00'), obs(1, '90.00')]), null, 'a rise is not a drop');
++ assert.strictEqual(isDeal([obs(1, '90.00')], true), 'watch');
++ assert.strictEqual(isDeal([obs(1, '90.00')]), null);
++ assert.strictEqual(priceTrend([obs(1, '99.99', { list_price: '129.99', list_price_num: 129.99 })]).latest.saving_pct, 23);
++ });
++
++ await check('migration 0008 applies over a database that already has observations; they stay null', async () => {
++ const migrations = path.join(__dirname, '..', 'migrations');
++ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'deals-lp-migrate-'));
++ const db = createDb({ pglite: true, service: 'deals-lp-migration-test' });
++ try {
++ for (const f of fs.readdirSync(migrations).filter((f) => f.endsWith('.sql')).sort()) {
++ if (f < '0008') fs.copyFileSync(path.join(migrations, f), path.join(dir, f));
++ }
++ await db.migrate({ dir, log: { log() {} } });
++ await db.query(`INSERT INTO deal_offers (id, slug, title, url, url_norm, origin, created_at, updated_at)
++ VALUES ('dof_old', 'old-deal', 'Old deal', 'https://old.example/deal', 'old.example/deal', 'import', 1, 1)`);
++ await db.query(`INSERT INTO deal_offer_sources (id, offer_id, kind, ref_service, ref_type, ref_id, created_at, updated_at)
++ VALUES ('dos_old', 'dof_old', 'sources_item', 'sources', 'item', 'itm_old', 1, 1)`);
++ await db.query(`INSERT INTO deal_price_observations (id, offer_id, source_id, observed_at, recorded_at, price, price_num, currency, origin)
++ VALUES ('dpo_old', 'dof_old', 'dos_old', 1, 1, '10.00', 10, 'USD', 'import')`);
++
++ fs.copyFileSync(path.join(migrations, '0008_list_price.sql'), path.join(dir, '0008_list_price.sql'));
++ const applied = await db.migrate({ dir, log: { log() {} } });
++ assert.deepStrictEqual(applied.applied.map((m) => m.id), ['0008']);
++
++ const old = await db.prepare('SELECT list_price, list_price_num FROM deal_price_observations WHERE id = ?').get('dpo_old');
++ assert.deepStrictEqual([old.list_price, old.list_price_num], [null, null], 'an existing observation stays null');
++ const offer = await db.prepare('SELECT deal_reason FROM deal_offers WHERE id = ?').get('dof_old');
++ assert.strictEqual(offer.deal_reason, null, 'an existing offer has no selection reason');
++
++ await db.query(`INSERT INTO deal_price_observations (id, offer_id, source_id, observed_at, recorded_at, price, price_num, list_price, list_price_num, currency, origin)
++ VALUES ('dpo_new', 'dof_old', 'dos_old', 2, 2, '99.99', 99.99, '129.99', 129.99, 'USD', 'import')`);
++ const fresh = await db.prepare('SELECT list_price, list_price_num FROM deal_price_observations WHERE id = ?').get('dpo_new');
++ assert.deepStrictEqual([fresh.list_price, fresh.list_price_num], ['129.99', 129.99]);
++ } finally {
++ await db.close();
++ fs.rmSync(dir, { recursive: true, force: true });
++ }
++ });
++
++ const t = await boot();
++ const alice = t.network.addUser('alice');
++ const byUrl = async (u) => await t.ctx.store.db.prepare('SELECT * FROM deal_offers WHERE url = ?').get(u);
++ const obsOf = async (id) => await t.ctx.store.db.prepare('SELECT * FROM deal_price_observations WHERE offer_id = ? ORDER BY observed_at, seq').all(id);
++
++ await check('a list price 129.99 against 99.99 USD is kept and shown as a 23% saving; other/equal/below list prices are null', async () => {
++ t.sources.put(sourceItem({ id: 'itm_LP000000000000000000001', title: 'List price keeper', url: 'https://lp.example/keep', retrievedAt: t.clock.now(),
++ fields: { price: '99.99', currency: 'USD', list_price: '129.99' } }));
++ await t.ctx.importer.pull();
++ const kept = await byUrl('https://lp.example/keep');
++ const o = (await obsOf(kept.id))[0];
++ assert.deepStrictEqual([o.price, o.list_price, o.list_price_num, o.currency], ['99.99', '129.99', 129.99, 'USD']);
++ assert.strictEqual(kept.deal_reason, 'list_price');
++
++ const json = await t.offerJson(kept.slug);
++ assert.deepStrictEqual(
++ [json.latest_observation.list_price, json.latest_observation.list_price_num, json.latest_observation.saving_num, json.latest_observation.saving_pct],
++ ['129.99', 129.99, 30, 23],
++ );
++ assert.deepStrictEqual(
++ [json.price_trend.latest.list_price, json.price_trend.latest.list_price_num, json.price_trend.latest.saving_num, json.price_trend.latest.saving_pct],
++ ['129.99', 129.99, 30, 23],
++ );
++ assert.strictEqual(json.price_trend.at_lowest, false, 'one price is not a drop');
++ const page = await t.get(`/d/${kept.slug}`);
++ assert.match(page.text, /<s class="dl-list" data-list-price="129\.99">129\.99 USD<\/s> <span class="dl-saving">−23%<\/span>/);
++
++ for (const [suffix, list] of [['equal', '99.99'], ['below', '89.99']]) {
++ t.sources.put(sourceItem({ id: `itm_LP0000000000000000000${suffix === 'equal' ? '02' : '03'}`, title: `List price ${suffix}`,
++ url: `https://lp.example/${suffix}`, retrievedAt: t.clock.now(), fields: { price: '99.99', currency: 'USD', list_price: list } }));
++ }
++ t.sources.put(sourceItem({ id: 'itm_LP000000000000000000004', title: 'List price another currency', url: 'https://lp.example/eur', retrievedAt: t.clock.now(),
++ fields: { price: '99.99', currency: 'USD', list_price: '129.99', list_price_currency: 'EUR' } }));
++ await t.ctx.importer.pull();
++ for (const suffix of ['equal', 'below', 'eur']) {
++ const row = await byUrl(`https://lp.example/${suffix}`);
++ const x = (await obsOf(row.id))[0];
++ assert.deepStrictEqual([x.list_price, x.list_price_num], [null, null], `${suffix}: dropped`);
++ assert.strictEqual(row.deal_reason, null, `${suffix}: not selected by a list price`);
++ }
++ });
++
++ await check('each offer of a product keeps its own list price; the JSON and the page carry only that offer\'s', async () => {
++ t.sources.put(sourceItem({ id: 'itm_LP000000000000000000005', kind: 'product', title: 'Widget', url: 'https://lp.example/widget', retrievedAt: t.clock.now(),
++ fields: { brand: 'W', offers: [
++ { price: '50.00', currency: 'USD', list_price: '100.00', url: 'https://lp.example/widget-a', seller: 'A' },
++ { price: '20.00', currency: 'USD', list_price: '19.00', url: 'https://lp.example/widget-b', seller: 'B' },
++ { price: '10.00', currency: 'EUR', list_price: '15.00', url: 'https://lp.example/widget-c', seller: 'C' },
++ ] } }));
++ await t.ctx.importer.pull();
++ const a = await byUrl('https://lp.example/widget-a');
++ const b = await byUrl('https://lp.example/widget-b');
++ const c = await byUrl('https://lp.example/widget-c');
++ assert.deepStrictEqual((await obsOf(a.id))[0].list_price, '100.00');
++ assert.deepStrictEqual((await obsOf(b.id))[0].list_price, null);
++ assert.deepStrictEqual((await obsOf(c.id))[0].list_price, '15.00');
++ assert.deepStrictEqual([a.deal_reason, b.deal_reason, c.deal_reason], ['list_price', null, 'list_price']);
++ assert.strictEqual((await t.offerJson(a.slug)).latest_observation.saving_pct, 50);
++ assert.strictEqual((await t.offerJson(b.slug)).latest_observation.list_price, null);
++ assert.strictEqual((await t.offerJson(c.slug)).latest_observation.saving_pct, 33);
++ });
++
++ await check('a lower price than the previous observation selects price_drop; a first price does not', async () => {
++ t.sources.put(sourceItem({ id: 'itm_LP000000000000000000006', title: 'Drop widget', url: 'https://lp.example/drop', retrievedAt: t.clock.now(),
++ fields: { price: '100.00', currency: 'USD' } }));
++ await t.ctx.importer.pull();
++ const drop = await byUrl('https://lp.example/drop');
++ assert.strictEqual(drop.deal_reason, null, 'one price is not a drop');
++
++ t.clock.advance(2 * HOUR);
++ t.sources.put(sourceItem({ id: 'itm_LP000000000000000000006', title: 'Drop widget', url: 'https://lp.example/drop', retrievedAt: t.clock.now(), revision: 2,
++ fields: { price: '90.00', currency: 'USD' } }));
++ await t.ctx.importer.pull();
++ assert.strictEqual((await byUrl('https://lp.example/drop')).deal_reason, 'price_drop');
++ assert.strictEqual((await t.offerJson(drop.slug)).price_trend.at_lowest, true);
++ });
++
++ await check('a matching price_below watch selects watch; a normal import with none of the three is not a deal', async () => {
++ const w = await t.ctx.watches.create(alice.subject, { kind: 'price_below', query: 'watchwidget', max_price: '50', currency: 'USD' });
++ t.sources.put(sourceItem({ id: 'itm_LP000000000000000000007', title: 'WatchWidget 3000', url: 'https://lp.example/watch', retrievedAt: t.clock.now(),
++ fields: { price: '40.00', currency: 'USD' } }));
++ await t.ctx.importer.pull();
++ const watched = await byUrl('https://lp.example/watch');
++ assert.strictEqual(watched.deal_reason, 'watch');
++ assert.ok((await t.events('deals.watch.matched')).some((e) => e.payload.watch_id === w.id && e.payload.offer_id === watched.id));
++
++ t.sources.put(sourceItem({ id: 'itm_LP000000000000000000008', title: 'Ordinary thing', url: 'https://lp.example/ordinary', retrievedAt: t.clock.now(),
++ fields: { price: '25.00', currency: 'USD' } }));
++ await t.ctx.importer.pull();
++ const ordinary = await byUrl('https://lp.example/ordinary');
++ assert.ok(ordinary, 'a crawled offer is still imported (existing selection behaviour)');
++ assert.strictEqual(ordinary.deal_reason, null);
++ assert.strictEqual((await t.offerJson(ordinary.slug)).latest_observation.list_price, null);
++ });
++
++ await t.close();
++ done();
++})();