openvibers/openvibe.sources Commits

DealNews deals keep their category and their editors' pick (feed@3), so OpenVibe.Deals can sort deals by category and show staff picks (#82)

OP OpenVibers committed 22 days ago · 6e1c0f9487 · 5 files, +26 −9

Changes

ddiff --git a/README.md b/README.md
iindex 5e72953..194a97b 100644
---- a/README.md
++++ b/README.md
@@@ -101,7 +101,7 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled
| Type | Reads | Notes |
|---|---|---|
--| `rss`, `atom` (`feed@2`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline) |
++| `rss`, `atom` (`feed@3`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline); its `dealnews:category` fills `fields.categories` when the item has no `<category>`, and `dealnews:staffPick` is `fields.staff_pick` (true/false, absent when not stated) |
| `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed |
| `jsonld` (`jsonld@1`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error` |
| `api` (`api@1`) | JSON or XML official APIs via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`; epoch fields only where mapped as such |
ddiff --git a/server/adapters/feed.js b/server/adapters/feed.js
iindex 1b8c53e..6caf532 100644
---- a/server/adapters/feed.js
++++ b/server/adapters/feed.js
@@@ -8,13 +8,13 @@
*/
const { parseXml, toText, toIsoDate, canonicalUrl, toId, asArray, pick, ParseError } = require('./util');
--const PARSER_VERSION = 'feed@2';
++const PARSER_VERSION = 'feed@3';
/**
* Offer facts a deal feed states in its own elements (not read out of the headline): DealNews' namespace gives the
* price (with its currency), the retailer and when the deal ends. They become fields.price / currency / seller /
* valid_until, the names an `offer` item uses, exactly as stated; the item stays an article. Anything malformed is
-- * left out, never guessed.
++ * left out, never guessed. Its category and editors' pick are kept too (statedPick, below).
*/
const STATED_OFFER = [{ price: 'dealnews:price', seller: 'dealnews:retailer', valid_until: 'dealnews:expires' }];
function statedOffer(it) {
@@@ -36,6 +36,18 @@ function statedOffer(it) {
return null;
}
++/**
++ * A deal feed's own labels (feed@3): DealNews states each deal's category (<dealnews:category>, kept in
++ * fields.categories when the item has no RSS <category>) and whether its editors picked it (<dealnews:staffPick>,
++ * fields.staff_pick true or false; absent when not stated).
++ */
++function statedPick(it) {
++ const raw = toText(it['dealnews:staffPick'], 10);
++ if (raw === 'true') return { staff_pick: true };
++ if (raw === 'false') return { staff_pick: false };
++ return null;
++}
++
function atomLink(entry, base) {
const links = asArray(entry.link);
const alt = links.find(l => typeof l === 'object' && (!l['@_rel'] || l['@_rel'] === 'alternate') && l['@_href'])
@@@ -81,10 +93,11 @@ function rssItem(it, base, summaryMax) {
published_at: toIsoDate(pick(it, 'pubDate', 'dc:date')),
updated_at: toIsoDate(pick(it, 'atom:updated', 'dc:modified')),
fields: {
-- categories: categoriesOf(it.category ?? it['dc:subject']),
++ categories: categoriesOf(it.category ?? it['dc:subject'] ?? it['dealnews:category']),
enclosure: enclosure ? { url: canonicalUrl(enclosure['@_url'], base), type: toId(enclosure['@_type'], 100), length: /^\d{1,15}$/.test(String(enclosure['@_length'] || '')) ? Number(enclosure['@_length']) : null } : null,
comments_url: canonicalUrl(pick(it, 'comments'), base),
...(statedOffer(it) || {}),
++ ...(statedPick(it) || {}),
},
};
}
ddiff --git a/test/adapters.test.js b/test/adapters.test.js
iindex 1d34c3f..06adb82 100644
---- a/test/adapters.test.js
++++ b/test/adapters.test.js
@@@ -36,9 +36,10 @@ t('RSS 2.0: guid identity, cleaned link, text-only summary, dates, skips', () =>
t('RSS 2.0 deal feed (DealNews namespace): price, currency, retailer and end date as stated; malformed ones left out', () => {
const xml = `<?xml version="1.0"?><rss version="2.0" xmlns:dealnews="https://www.dealnews.com/ns/rss/1.0.htm"><channel><title>D</title>
<item><title>Air Mattress for $67 + free shipping</title><link>https://www.dealnews.com/p/1.html?iref=rss</link><guid>https://www.dealnews.com/1.html</guid>
--<dealnews:retailer>Amazon</dealnews:retailer><dealnews:expires>2027-01-13T19:54:28-05:00</dealnews:expires><dealnews:price currency="USD">67.00</dealnews:price></item>
++<dealnews:retailer>Amazon</dealnews:retailer><dealnews:expires>2027-01-13T19:54:28-05:00</dealnews:expires><dealnews:price currency="USD">67.00</dealnews:price>
++<dealnews:category>Bed &amp; Bath</dealnews:category><dealnews:staffPick>true</dealnews:staffPick></item>
<item><title>Sale at Target</title><link>https://www.dealnews.com/p/2.html</link><guid>https://www.dealnews.com/2.html</guid>
--<dealnews:retailer>Target</dealnews:retailer><dealnews:price currency="dollars">about 10</dealnews:price></item>
++<dealnews:retailer>Target</dealnews:retailer><dealnews:price currency="dollars">about 10</dealnews:price><dealnews:category></dealnews:category><dealnews:staffPick>false</dealnews:staffPick></item>
<item><title>Plain news</title><link>https://www.dealnews.com/p/3.html</link><guid>https://www.dealnews.com/3.html</guid></item>
</channel></rss>`;
const r = feed.parse(xml, { url: 'https://www.dealnews.com/?rss=1' });
@@@ -47,7 +48,10 @@ t('RSS 2.0 deal feed (DealNews namespace): price, currency, retailer and end dat
assert.deepStrictEqual([a.fields.price, a.fields.currency, a.fields.seller, a.fields.valid_until], ['67.00', 'USD', 'Amazon', '2027-01-14T00:54:28.000Z']);
assert.deepStrictEqual([b.fields.price, b.fields.currency, b.fields.seller], [undefined, undefined, 'Target'], 'a price that is not a number is not kept');
assert.ok(!('price' in c.fields) && !('seller' in c.fields), 'an item that states none has none');
-- assert.strictEqual(feed.PARSER_VERSION, 'feed@2');
++ assert.deepStrictEqual([a.fields.categories, a.fields.staff_pick], [['Bed & Bath'], true], 'its category and editors\' pick as stated');
++ assert.deepStrictEqual([b.fields.categories, b.fields.staff_pick], [[], false], 'an empty category is no category');
++ assert.ok(!('staff_pick' in c.fields), 'a pick that is not stated is not there');
++ assert.strictEqual(feed.PARSER_VERSION, 'feed@3');
});
t('Atom: id identity, relative alternate link, html title, bad dates are null', () => {
ddiff --git a/test/api.test.js b/test/api.test.js
iindex d62a21c..20d49a7 100644
---- a/test/api.test.js
++++ b/test/api.test.js
@@@ -88,7 +88,7 @@ t('items page in change order and carry provenance and source staleness', async
assert.strictEqual(p2.body.items.length, 3);
assert.strictEqual(p2.body.more, false);
const it = p1.body.items[0];
-- assert.ok(it.provenance.retrieved_at && it.provenance.content_hash && it.provenance.raw_body_hash && it.provenance.parser_version === 'feed@2');
++ assert.ok(it.provenance.retrieved_at && it.provenance.content_hash && it.provenance.raw_body_hash && it.provenance.parser_version === 'feed@3');
assert.strictEqual(it.provenance.terms_note, 'test fixture');
assert.deepStrictEqual(Object.keys(p1.body.sources), ['feed-a']);
assert.strictEqual(p1.body.sources['feed-a'].stale, false);
ddiff --git a/test/ingest.test.js b/test/ingest.test.js
iindex 08cb168..90a6d9a 100644
---- a/test/ingest.test.js
++++ b/test/ingest.test.js
@@@ -56,7 +56,7 @@ t('a good fetch creates items with full provenance and an ok run', async () => {
assert.strictEqual(a.revision, 1);
assert.match(a.content_hash, /^[0-9a-f]{64}$/);
assert.match(a.raw_body_hash, /^[0-9a-f]{64}$/);
-- assert.strictEqual(a.parser_version, 'feed@2');
++ assert.strictEqual(a.parser_version, 'feed@3');
assert.strictEqual(a.license_note, 'headlines only');
assert.strictEqual(a.terms_note, 'fixture terms');
assert.ok(a.retrieved_at > 0 && a.first_seen_at === a.retrieved_at);