openvibers/openvibe.deals Commits

Deals: shops — /at lists every seller, /at/<seller> its deals; "at Amazon" links there (#65)

OP OpenVibers committed 28 days ago · df67036705 · 8 files, +80 −8

Changes

ddiff --git a/README.md b/README.md
iindex 9891097..3372895 100644
---- a/README.md
++++ b/README.md
@@@ -145,6 +145,8 @@ two subscriptions are created at boot when missing (`startSubscriptions`, needs
observation, the end date the stated end, and the retailer the offer's `seller` ("at Amazon" on cards, "Sold by" on
the page, `seller` in JSON and the Offer JSON-LD). The store stays the link's domain; a source whose links are kept
verbatim (an aggregator's own pages) never names that store after a retailer (migration `0006_offer_seller.sql`).
++ **Shops:** `/at` lists every seller with its count and `/at/<seller>` its deals (newest first, paginated); the
++ "at Amazon" on a card and "Sold by" on a page link there. Like store pages, they are not in the sitemap.
- JSON-LD: `Product` with one `Offer` per offer; `price`/`priceCurrency` only from a fresh observation
that stated both, `availability`/`itemCondition` only when stated, `validThrough` only from a stated
end. Unknown fields are omitted, never 0 or `InStock`.
ddiff --git a/public/css/deals.css b/public/css/deals.css
iindex 30c340f..fda765c 100644
---- a/public/css/deals.css
++++ b/public/css/deals.css
@@@ -97,3 +97,5 @@ code { overflow-wrap: anywhere; }
.cf-group h3, .cf-campaign h3 { margin: 0 0 4px; font-size: 1.05rem; }
.cf-links { list-style: none; padding: 0; margin: 8px 0 0; display: flex; flex-wrap: wrap; gap: 8px; }
.cf-links .button { margin: 0; padding: 6px 12px; }
++.seller-list { columns: 3 220px; padding-left: 1.1rem; }
++.seller-list li { margin: 4px 0; break-inside: avoid; }
ddiff --git a/server/domain/listings.js b/server/domain/listings.js
iindex 8c3cba8..af86f48 100644
---- a/server/domain/listings.js
++++ b/server/domain/listings.js
@@@ -25,6 +25,10 @@ function createListings({ store }) {
countActive: db.prepare("SELECT COUNT(*) AS n FROM deal_offers WHERE status = 'active' AND merged_into IS NULL"),
byStore: db.prepare(`SELECT * FROM deal_offers WHERE store_id = ? AND merged_into IS NULL AND status <> 'disabled' ORDER BY status = 'active' DESC, created_at DESC LIMIT ? OFFSET ?`),
countStore: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE store_id = ? AND merged_into IS NULL AND status <> 'disabled'`),
++ // Sellers: the shop a source names (migration 0006), as stated; a seller page lists by that exact name.
++ sellers: db.prepare(`SELECT seller, COUNT(*) AS n, MAX(created_at) AS latest FROM deal_offers WHERE seller IS NOT NULL AND merged_into IS NULL AND status <> 'disabled' GROUP BY seller ORDER BY n DESC, seller LIMIT 2000`),
++ bySeller: db.prepare(`SELECT * FROM deal_offers WHERE seller = ? AND merged_into IS NULL AND status <> 'disabled' ORDER BY status = 'active' DESC, created_at DESC LIMIT ? OFFSET ?`),
++ countSeller: db.prepare(`SELECT COUNT(*) AS n FROM deal_offers WHERE seller = ? AND merged_into IS NULL AND status <> 'disabled'`),
indexable: db.prepare(`SELECT * FROM deal_offers WHERE merged_into IS NULL AND status <> 'disabled' ORDER BY created_at DESC LIMIT 50000`),
products: db.prepare(`SELECT DISTINCT p.* FROM deal_products p JOIN deal_offers o ON o.product_id = p.id WHERE o.merged_into IS NULL AND o.status <> 'disabled' LIMIT 50000`),
duplicates: db.prepare(`SELECT a.id AS a, b.id AS b FROM deal_offers a JOIN deal_offers b
@@@ -55,6 +59,9 @@ function createListings({ store }) {
countActive: async () => (await q.countActive.get()).n,
byStore: async (storeId, limit, offset) => await q.byStore.all(storeId, limit, offset),
countStore: async (storeId) => (await q.countStore.get(storeId)).n,
++ sellers: async () => (await q.sellers.all()).map((r) => ({ seller: r.seller, slug: sellerSlug(r.seller), count: Number(r.n), latest: r.latest })),
++ bySeller: async (seller, limit, offset) => await q.bySeller.all(seller, limit, offset),
++ countSeller: async (seller) => (await q.countSeller.get(seller)).n,
indexable: async () => await q.indexable.all(),
products: async () => await q.products.all(),
possibleDuplicates: async (limit = 50) => await q.duplicates.all(limit),
@@@ -63,4 +70,7 @@ function createListings({ store }) {
};
}
--module.exports = { createListings };
++/** "Best Buy" → "best-buy", "Target.com" → "target-com": a seller's address under /at/. */
++const sellerSlug = (name) => String(name || '').normalize('NFKD').replace(/[\u0300-\u036f]/g, '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80);
++
++module.exports = { createListings, sellerSlug };
ddiff --git a/server/domain/publication.js b/server/domain/publication.js
iindex 0536dbe..634f12f 100644
---- a/server/domain/publication.js
++++ b/server/domain/publication.js
@@@ -27,6 +27,7 @@ function createPublication({ config, store, reads, catalog }) {
const offerPath = (o) => `/d/${encodeURIComponent(o.slug)}`;
const productPath = (p) => `/p/${encodeURIComponent(p.slug)}`;
const storePath = (s) => `/s/${encodeURIComponent(s.domain)}`;
++ const sellerPath = (name) => `/at/${require('./listings').sellerSlug(name)}`;
function freshness(obs, now = store.now()) {
if (!obs) return { state: 'unobserved', observed_at: null, age_ms: null, max_age_ms: config.freshnessMs };
@@@ -189,7 +190,7 @@ function createPublication({ config, store, reads, catalog }) {
});
}
-- return { abs, offerPath, productPath, storePath, freshness, decideOffer, decideProduct, offerView, offerDto, observationDto, offerLd, offerJsonLd, productView, productJsonLd };
++ return { abs, offerPath, productPath, storePath, sellerPath, freshness, decideOffer, decideProduct, offerView, offerDto, observationDto, offerLd, offerJsonLd, productView, productJsonLd };
}
module.exports = { createPublication };
ddiff --git a/server/http/pages.js b/server/http/pages.js
iindex 3267086..aff8677 100644
---- a/server/http/pages.js
++++ b/server/http/pages.js
@@@ -41,7 +41,7 @@ function createPages(ctx) {
// Per-actor limits on the forms (http/actor-limits.js), before the form is read: each shares its budget
// with the API route that does the same thing. Reading pages is left to the per-address limits.
const B = (name) => actorLimits.budget(name);
-- const urls = { offerPath: publication.offerPath, productPath: publication.productPath, storePath: publication.storePath };
++ const urls = { offerPath: publication.offerPath, productPath: publication.productPath, storePath: publication.storePath, sellerPath: publication.sellerPath };
const form = express.urlencoded({ extended: false, limit: '32kb' });
const wrap = (fn) => (req, res, next) => Promise.resolve(fn(req, res, next)).catch(next);
@@@ -296,6 +296,32 @@ function createPages(ctx) {
}, { cacheable: true });
}));
++ // ── shops (the seller a source names, migration 0006) ──
++ router.get('/at', wrap(async (req, res) => {
++ const sellers = await listings.sellers();
++ send(req, res, 200, {
++ title: 'Deals by shop', decision: pageDecision('/at', { empty: sellers.length === 0 }), canonical: seo.canonicalUrl(config.baseUrl, '/at'),
++ body: views.sellersPage({ sellers }),
++ }, { cacheable: true });
++ }));
++ router.get('/at/:seller', wrap(async (req, res) => {
++ const want = String(req.params.seller || '').toLowerCase();
++ const hit = /^[a-z0-9-]{1,80}$/.test(want) ? (await listings.sellers()).find((x) => x.slug === want) : null;
++ if (!hit) return notFound(req, res);
++ const path = `/at/${hit.slug}`;
++ const total = await listings.countSeller(hit.seller);
++ const pager = ssr.paginate({ page: pageNumber(req), perPage: PER_PAGE, total, href: (p) => (p === 1 ? path : `${path}?page=${p}`) });
++ if (pager.outOfRange && total) return notFound(req, res);
++ const list = await Promise.all((await listings.bySeller(hit.seller, PER_PAGE, pager.offset)).map(async (o) => await publication.offerView(o)));
++ const canonical = seo.canonicalUrl(config.baseUrl, pager.page === 1 ? path : `${path}?page=${pager.page}`, { query: ['page'] });
++ send(req, res, 200, {
++ title: `Deals at ${hit.seller}`, decision: pageDecision(canonical, { empty: total === 0, query: ['page'] }), canonical,
++ description: `${total} deal${total === 1 ? '' : 's'} at ${hit.seller}, each with its source, price and when it was seen.`,
++ prev: pager.prev ? pager.prev.href : null, next: pager.next ? pager.next.href : null,
++ body: views.sellerPage({ seller: hit.seller, views: list, pager, urls, total }),
++ }, { cacheable: true });
++ }));
++
// ── search ──────────────────────────────────────────────
router.get('/search', actorLimits.reads('deals.search'), wrap(async (req, res) => {
ddiff --git a/server/render/layout.js b/server/render/layout.js
iindex 0c793e5..345c357 100644
---- a/server/render/layout.js
++++ b/server/render/layout.js
@@@ -23,6 +23,7 @@ const PUBLIC_DIR = path.join(__dirname, '..', '..', 'public');
const NAV_LINKS = [
{ label: 'Hot', href: '/' },
{ label: 'New', href: '/new' },
++ { label: 'Shops', href: '/at' },
{ label: 'Crowdfunding', href: '/crowdfunding' },
{ label: 'Submit', href: '/submit', icon: 'fa-plus' },
{ label: 'Watches', href: '/watches', icon: 'fa-bell' },
ddiff --git a/server/render/views.js b/server/render/views.js
iindex 41782e4..d8a8121 100644
---- a/server/render/views.js
++++ b/server/render/views.js
@@@ -84,7 +84,7 @@ function offerCard(v, urls) {
<h2 class="deal-title"><a href="${urls.offerPath(r)}">${r.title}</a> ${statusBadge(r)}</h2>
${latestLine(v)}
${attributionLine(v.attribution)}
--<p class="meta">${r.seller ? html`<span class="seller">at ${r.seller}</span> · ` : ''}${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a> · ` : ''}${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a> · ` : ''}posted ${timeEl(r.created_at)}${r.origin === 'import' ? ' · imported from a source' : ''}</p>
++<p class="meta">${r.seller ? html`<a class="seller" href="${urls.sellerPath ? urls.sellerPath(r.seller) : '#'}">at ${r.seller}</a> · ` : ''}${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a> · ` : ''}${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a> · ` : ''}posted ${timeEl(r.created_at)}${r.origin === 'import' ? ' · imported from a source' : ''}</p>
</div></li>`;
}
@@@ -176,7 +176,7 @@ ${r.description ? html`<section class="description">${raw(renderMarkdown(r.descr
${r.ai_summary ? html`<section class="disclosure"><p><strong>AI-assisted summary</strong>${r.review_state === 'reviewed' ? ' (reviewed by a person)' : ' (not yet reviewed by a person)'}</p><p>${r.ai_summary}</p></section>` : ''}
${r.review_state === 'pending' && r.text_origin === 'imported' ? html`<p class="disclosure">This deal was imported from a source and has not been reviewed by a person yet.</p>` : ''}
<dl class="facts">
--${r.seller ? html`<dt>Sold by</dt><dd>${r.seller} <span class="meta">(as the source states it)</span></dd>` : ''}
++${r.seller ? html`<dt>Sold by</dt><dd>${urls.sellerPath ? html`<a href="${urls.sellerPath(r.seller)}">${r.seller}</a>` : r.seller} <span class="meta">(as the source states it)</span></dd>` : ''}
<dt>${r.seller ? 'Link goes to' : 'Store'}</dt><dd>${v.store ? html`<a href="${urls.storePath(v.store)}">${v.store.name || v.store.domain}</a>` : 'not stated'}</dd>
<dt>Product</dt><dd>${v.product ? html`<a href="${urls.productPath(v.product)}">${v.product.name}</a>` : 'not linked'}</dd>
<dt>Posted</dt><dd>${timeEl(r.created_at)} ${r.origin === 'import' ? '(imported)' : '(by a member)'}</dd>
@@@ -313,8 +313,25 @@ ${views.length ? html`<ol class="deal-list">${views.map((v) => offerCard(v, urls
${raw(pager ? paginationHtml(pager) : '')}`;
}
++/** /at/:seller: deals at one shop, as their sources name it. */
++function sellerPage({ seller, views: list, pager, urls, total }) {
++ return html`${raw(breadcrumbsHtml([{ name: 'Deals', url: '/' }, { name: 'Shops', url: '/at' }, { name: seller }]))}
++<h1>Deals at ${seller}</h1>
++<p class="lede">${total} deal${total === 1 ? '' : 's'} at ${seller}, as the deal sources name the shop, newest first.</p>
++${list.length ? html`<ol class="deal-list">${list.map((v) => offerCard(v, urls))}</ol>` : html`<p class="empty">No deals at this shop right now.</p>`}
++${raw(pager ? paginationHtml(pager) : '')}`;
++}
++
++/** /at: every shop with deals here, most deals first. */
++function sellersPage({ sellers }) {
++ return html`${raw(breadcrumbsHtml([{ name: 'Deals', url: '/' }, { name: 'Shops' }]))}
++<h1>Deals by shop</h1>
++<p class="lede">Every shop with a deal here, as the deal sources name it, the most deals first.</p>
++${sellers.length ? html`<ul class="seller-list">${sellers.map((x) => html`<li><a href="/at/${x.slug}">${x.seller}</a> <span class="meta">${x.count} deal${x.count === 1 ? '' : 's'}</span></li>`)}</ul>` : html`<p class="empty">No shops yet.</p>`}`;
++}
++
function message({ heading, text, action }) {
return html`<h1>${heading}</h1><p>${text}</p>${action ? html`<p><a href="${action.href}">${action.label}</a></p>` : ''}`;
}
--module.exports = { dealsShowcase, offerList, tabs, offerPage, productPage, submitForm, watchesPage, searchPage, modPage, storePage, message, priceText, when };
++module.exports = { dealsShowcase, offerList, tabs, offerPage, productPage, submitForm, watchesPage, searchPage, modPage, storePage, sellerPage, sellersPage, message, priceText, when };
ddiff --git a/test/import.test.js b/test/import.test.js
iindex c8a79a2..8580e60 100644
---- a/test/import.test.js
++++ b/test/import.test.js
@@@ -197,8 +197,8 @@ const { signDelivery, signDeliveryHeaders } = require('openvibe-sdk/events');
const target = await byUrl('https://www.dealnews.com/target-sale/1.html');
assert.deepStrictEqual([target.seller, (await t.offerJson(target.slug)).latest_observation.price], ['Target', null], 'no stated price stays not stated');
const page = await t.get(`/d/${offer.slug}`);
-- assert.ok(page.text.includes('<dt>Sold by</dt><dd>Amazon') && page.text.includes('<dt>Link goes to</dt>'), 'the page says who sells it and where the link goes');
-- assert.ok((await t.get('/new')).text.includes('<span class="seller">at Amazon</span>'), 'the card names the seller');
++ assert.ok(page.text.includes('<dt>Sold by</dt><dd><a href="/at/amazon">Amazon</a>') && page.text.includes('<dt>Link goes to</dt>'), 'the page says who sells it and where the link goes');
++ assert.ok((await t.get('/new')).text.includes('>at Amazon</a>'), 'the card names the seller');
assert.match((await t.get('/feed.xml')).text, /at Amazon\./);
// A later revision naming another retailer updates the offer's seller.
t.sources.put(sourceItem({ id: 'itm_01K5AAAAAAAAAAAAAAAAAAAAB2', sourceKey: 'dealnews-daily', kind: 'article', title: 'Target sale', url: 'https://www.dealnews.com/target-sale/1.html',
@@@ -208,6 +208,19 @@ const { signDelivery, signDeliveryHeaders } = require('openvibe-sdk/events');
assert.deepStrictEqual([again.seller, (await t.offerJson(again.slug)).latest_observation.price], ['Target.com', '10.00']);
});
++ await check('shops: /at lists every seller with its count, /at/<seller> its deals; cards and pages link there', async () => {
++ const all = await t.get('/at');
++ assert.strictEqual(all.status, 200);
++ assert.ok(all.text.includes('<a href="/at/amazon">Amazon</a>') && all.text.includes('<a href="/at/target-com">Target.com</a>'));
++ const amazon = await t.get('/at/amazon');
++ assert.strictEqual(amazon.status, 200);
++ assert.ok(amazon.text.includes('<h1>Deals at Amazon</h1>') && amazon.text.includes('Vosser Queen Air Mattress'));
++ assert.ok(!amazon.text.includes('Target sale'), 'only that seller');
++ assert.ok((await t.get('/new')).text.includes('<a class="seller" href="/at/amazon">at Amazon</a>'), 'the card links the shop');
++ assert.strictEqual((await t.get('/at/nobody')).status, 404);
++ assert.strictEqual((await t.get('/at/..%2Fadmin')).status, 404);
++ });
++
await check('Sources down: the pull fails honestly (readiness says so), keeps its cursor, invents nothing', async () => {
const cursor = await t.ctx.importer.cursor();
const offers = (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM deal_offers').get()).n;