openvibers/openvibe.sources Commits
DealNews deals keep their category and their editors' pick (feed@3), so OpenVibe.Deals can sort deals by category and show staff picks (#82)
Changes
@@@ -101,7 +101,7 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled| Type | Reads | Notes ||---|---|---|--| `rss`, `atom` (`feed@2`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline) |++| `rss`, `atom` (`feed@3`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline); its `dealnews:category` fills `fields.categories` when the item has no `<category>`, and `dealnews:staffPick` is `fields.staff_pick` (true/false, absent when not stated) || `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed || `jsonld` (`jsonld@1`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error` || `api` (`api@1`) | JSON or XML official APIs via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`; epoch fields only where mapped as such |@@@ -8,13 +8,13 @@*/const { parseXml, toText, toIsoDate, canonicalUrl, toId, asArray, pick, ParseError } = require('./util');--const PARSER_VERSION = 'feed@2';++const PARSER_VERSION = 'feed@3';/*** Offer facts a deal feed states in its own elements (not read out of the headline): DealNews' namespace gives the* price (with its currency), the retailer and when the deal ends. They become fields.price / currency / seller /* valid_until, the names an `offer` item uses, exactly as stated; the item stays an article. Anything malformed is-- * left out, never guessed.++ * left out, never guessed. Its category and editors' pick are kept too (statedPick, below).*/const STATED_OFFER = [{ price: 'dealnews:price', seller: 'dealnews:retailer', valid_until: 'dealnews:expires' }];function statedOffer(it) {@@@ -36,6 +36,18 @@ function statedOffer(it) {return null;}++/**++ * A deal feed's own labels (feed@3): DealNews states each deal's category (<dealnews:category>, kept in++ * fields.categories when the item has no RSS <category>) and whether its editors picked it (<dealnews:staffPick>,++ * fields.staff_pick true or false; absent when not stated).++ */++function statedPick(it) {++ const raw = toText(it['dealnews:staffPick'], 10);++ if (raw === 'true') return { staff_pick: true };++ if (raw === 'false') return { staff_pick: false };++ return null;++}++function atomLink(entry, base) {const links = asArray(entry.link);const alt = links.find(l => typeof l === 'object' && (!l['@_rel'] || l['@_rel'] === 'alternate') && l['@_href'])@@@ -81,10 +93,11 @@ function rssItem(it, base, summaryMax) {published_at: toIsoDate(pick(it, 'pubDate', 'dc:date')),updated_at: toIsoDate(pick(it, 'atom:updated', 'dc:modified')),fields: {-- categories: categoriesOf(it.category ?? it['dc:subject']),++ categories: categoriesOf(it.category ?? it['dc:subject'] ?? it['dealnews:category']),enclosure: enclosure ? { url: canonicalUrl(enclosure['@_url'], base), type: toId(enclosure['@_type'], 100), length: /^\d{1,15}$/.test(String(enclosure['@_length'] || '')) ? Number(enclosure['@_length']) : null } : null,comments_url: canonicalUrl(pick(it, 'comments'), base),...(statedOffer(it) || {}),++ ...(statedPick(it) || {}),},};}@@@ -36,9 +36,10 @@ t('RSS 2.0: guid identity, cleaned link, text-only summary, dates, skips', () =>t('RSS 2.0 deal feed (DealNews namespace): price, currency, retailer and end date as stated; malformed ones left out', () => {const xml = `<?xml version="1.0"?><rss version="2.0" xmlns:dealnews="https://www.dealnews.com/ns/rss/1.0.htm"><channel><title>D</title><item><title>Air Mattress for $67 + free shipping</title><link>https://www.dealnews.com/p/1.html?iref=rss</link><guid>https://www.dealnews.com/1.html</guid>--<dealnews:retailer>Amazon</dealnews:retailer><dealnews:expires>2027-01-13T19:54:28-05:00</dealnews:expires><dealnews:price currency="USD">67.00</dealnews:price></item>++<dealnews:retailer>Amazon</dealnews:retailer><dealnews:expires>2027-01-13T19:54:28-05:00</dealnews:expires><dealnews:price currency="USD">67.00</dealnews:price>++<dealnews:category>Bed & Bath</dealnews:category><dealnews:staffPick>true</dealnews:staffPick></item><item><title>Sale at Target</title><link>https://www.dealnews.com/p/2.html</link><guid>https://www.dealnews.com/2.html</guid>--<dealnews:retailer>Target</dealnews:retailer><dealnews:price currency="dollars">about 10</dealnews:price></item>++<dealnews:retailer>Target</dealnews:retailer><dealnews:price currency="dollars">about 10</dealnews:price><dealnews:category></dealnews:category><dealnews:staffPick>false</dealnews:staffPick></item><item><title>Plain news</title><link>https://www.dealnews.com/p/3.html</link><guid>https://www.dealnews.com/3.html</guid></item></channel></rss>`;const r = feed.parse(xml, { url: 'https://www.dealnews.com/?rss=1' });@@@ -47,7 +48,10 @@ t('RSS 2.0 deal feed (DealNews namespace): price, currency, retailer and end datassert.deepStrictEqual([a.fields.price, a.fields.currency, a.fields.seller, a.fields.valid_until], ['67.00', 'USD', 'Amazon', '2027-01-14T00:54:28.000Z']);assert.deepStrictEqual([b.fields.price, b.fields.currency, b.fields.seller], [undefined, undefined, 'Target'], 'a price that is not a number is not kept');assert.ok(!('price' in c.fields) && !('seller' in c.fields), 'an item that states none has none');-- assert.strictEqual(feed.PARSER_VERSION, 'feed@2');++ assert.deepStrictEqual([a.fields.categories, a.fields.staff_pick], [['Bed & Bath'], true], 'its category and editors\' pick as stated');++ assert.deepStrictEqual([b.fields.categories, b.fields.staff_pick], [[], false], 'an empty category is no category');++ assert.ok(!('staff_pick' in c.fields), 'a pick that is not stated is not there');++ assert.strictEqual(feed.PARSER_VERSION, 'feed@3');});t('Atom: id identity, relative alternate link, html title, bad dates are null', () => {@@@ -88,7 +88,7 @@ t('items page in change order and carry provenance and source staleness', asyncassert.strictEqual(p2.body.items.length, 3);assert.strictEqual(p2.body.more, false);const it = p1.body.items[0];-- assert.ok(it.provenance.retrieved_at && it.provenance.content_hash && it.provenance.raw_body_hash && it.provenance.parser_version === 'feed@2');++ assert.ok(it.provenance.retrieved_at && it.provenance.content_hash && it.provenance.raw_body_hash && it.provenance.parser_version === 'feed@3');assert.strictEqual(it.provenance.terms_note, 'test fixture');assert.deepStrictEqual(Object.keys(p1.body.sources), ['feed-a']);assert.strictEqual(p1.body.sources['feed-a'].stale, false);@@@ -56,7 +56,7 @@ t('a good fetch creates items with full provenance and an ok run', async () => {assert.strictEqual(a.revision, 1);assert.match(a.content_hash, /^[0-9a-f]{64}$/);assert.match(a.raw_body_hash, /^[0-9a-f]{64}$/);-- assert.strictEqual(a.parser_version, 'feed@2');++ assert.strictEqual(a.parser_version, 'feed@3');assert.strictEqual(a.license_note, 'headlines only');assert.strictEqual(a.terms_note, 'fixture terms');assert.ok(a.retrieved_at > 0 && a.first_seen_at === a.retrieved_at);
@@ -101,7 +101,7 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled
@@ -101,7 +101,7 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled
| Type | Reads | Notes |
| Type | Reads | Notes |
|---|---|---|
|---|---|---|
-| `rss`, `atom` (`feed@2`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline) |
+| `rss`, `atom` (`feed@3`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline); its `dealnews:category` fills `fields.categories` when the item has no `<category>`, and `dealnews:staffPick` is `fields.staff_pick` (true/false, absent when not stated) |
| `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed |
| `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed |
| `jsonld` (`jsonld@1`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error` |
| `jsonld` (`jsonld@1`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error` |
| `api` (`api@1`) | JSON or XML official APIs via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`; epoch fields only where mapped as such |
| `api` (`api@1`) | JSON or XML official APIs via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`; epoch fields only where mapped as such |
@@ -8,13 +8,13 @@
@@ -8,13 +8,13 @@
*/
*/
const { parseXml, toText, toIsoDate, canonicalUrl, toId, asArray, pick, ParseError } = require('./util');
const { parseXml, toText, toIsoDate, canonicalUrl, toId, asArray, pick, ParseError } = require('./util');
-const PARSER_VERSION = 'feed@2';
+const PARSER_VERSION = 'feed@3';
/**
/**
* Offer facts a deal feed states in its own elements (not read out of the headline): DealNews' namespace gives the
* Offer facts a deal feed states in its own elements (not read out of the headline): DealNews' namespace gives the
* price (with its currency), the retailer and when the deal ends. They become fields.price / currency / seller /
* price (with its currency), the retailer and when the deal ends. They become fields.price / currency / seller /
* valid_until, the names an `offer` item uses, exactly as stated; the item stays an article. Anything malformed is
* valid_until, the names an `offer` item uses, exactly as stated; the item stays an article. Anything malformed is
- * left out, never guessed.
+ * left out, never guessed. Its category and editors' pick are kept too (statedPick, below).
*/
*/
const STATED_OFFER = [{ price: 'dealnews:price', seller: 'dealnews:retailer', valid_until: 'dealnews:expires' }];
const STATED_OFFER = [{ price: 'dealnews:price', seller: 'dealnews:retailer', valid_until: 'dealnews:expires' }];
function statedOffer(it) {
function statedOffer(it) {
@@ -36,6 +36,18 @@ function statedOffer(it) {
@@ -36,6 +36,18 @@ function statedOffer(it) {
return null;
return null;
}
}
+/**
+ * A deal feed's own labels (feed@3): DealNews states each deal's category (<dealnews:category>, kept in
+ * fields.categories when the item has no RSS <category>) and whether its editors picked it (<dealnews:staffPick>,
+ * fields.staff_pick true or false; absent when not stated).
+ */
+function statedPick(it) {
+ const raw = toText(it['dealnews:staffPick'], 10);
+ if (raw === 'true') return { staff_pick: true };
+ if (raw === 'false') return { staff_pick: false };
+ return null;
+}
+
function atomLink(entry, base) {
function atomLink(entry, base) {
const links = asArray(entry.link);
const links = asArray(entry.link);
const alt = links.find(l => typeof l === 'object' && (!l['@_rel'] || l['@_rel'] === 'alternate') && l['@_href'])
const alt = links.find(l => typeof l === 'object' && (!l['@_rel'] || l['@_rel'] === 'alternate') && l['@_href'])
@@ -81,10 +93,11 @@ function rssItem(it, base, summaryMax) {
@@ -81,10 +93,11 @@ function rssItem(it, base, summaryMax) {
published_at: toIsoDate(pick(it, 'pubDate', 'dc:date')),
published_at: toIsoDate(pick(it, 'pubDate', 'dc:date')),
updated_at: toIsoDate(pick(it, 'atom:updated', 'dc:modified')),
updated_at: toIsoDate(pick(it, 'atom:updated', 'dc:modified')),
fields: {
fields: {
- categories: categoriesOf(it.category ?? it['dc:subject']),
+ categories: categoriesOf(it.category ?? it['dc:subject'] ?? it['dealnews:category']),
enclosure: enclosure ? { url: canonicalUrl(enclosure['@_url'], base), type: toId(enclosure['@_type'], 100), length: /^\d{1,15}$/.test(String(enclosure['@_length'] || '')) ? Number(enclosure['@_length']) : null } : null,
enclosure: enclosure ? { url: canonicalUrl(enclosure['@_url'], base), type: toId(enclosure['@_type'], 100), length: /^\d{1,15}$/.test(String(enclosure['@_length'] || '')) ? Number(enclosure['@_length']) : null } : null,
comments_url: canonicalUrl(pick(it, 'comments'), base),
comments_url: canonicalUrl(pick(it, 'comments'), base),
...(statedOffer(it) || {}),
...(statedOffer(it) || {}),
+ ...(statedPick(it) || {}),
},
},
};
};
}
}
@@ -36,9 +36,10 @@ t('RSS 2.0: guid identity, cleaned link, text-only summary, dates, skips', () =>
@@ -36,9 +36,10 @@ t('RSS 2.0: guid identity, cleaned link, text-only summary, dates, skips', () =>
t('RSS 2.0 deal feed (DealNews namespace): price, currency, retailer and end date as stated; malformed ones left out', () => {
t('RSS 2.0 deal feed (DealNews namespace): price, currency, retailer and end date as stated; malformed ones left out', () => {
const xml = `<?xml version="1.0"?><rss version="2.0" xmlns:dealnews="https://www.dealnews.com/ns/rss/1.0.htm"><channel><title>D</title>
const xml = `<?xml version="1.0"?><rss version="2.0" xmlns:dealnews="https://www.dealnews.com/ns/rss/1.0.htm"><channel><title>D</title>
<item><title>Air Mattress for $67 + free shipping</title><link>https://www.dealnews.com/p/1.html?iref=rss</link><guid>https://www.dealnews.com/1.html</guid>
<item><title>Air Mattress for $67 + free shipping</title><link>https://www.dealnews.com/p/1.html?iref=rss</link><guid>https://www.dealnews.com/1.html</guid>
-<dealnews:retailer>Amazon</dealnews:retailer><dealnews:expires>2027-01-13T19:54:28-05:00</dealnews:expires><dealnews:price currency="USD">67.00</dealnews:price></item>
+<dealnews:retailer>Amazon</dealnews:retailer><dealnews:expires>2027-01-13T19:54:28-05:00</dealnews:expires><dealnews:price currency="USD">67.00</dealnews:price>
+<dealnews:category>Bed & Bath</dealnews:category><dealnews:staffPick>true</dealnews:staffPick></item>
<item><title>Sale at Target</title><link>https://www.dealnews.com/p/2.html</link><guid>https://www.dealnews.com/2.html</guid>
<item><title>Sale at Target</title><link>https://www.dealnews.com/p/2.html</link><guid>https://www.dealnews.com/2.html</guid>
-<dealnews:retailer>Target</dealnews:retailer><dealnews:price currency="dollars">about 10</dealnews:price></item>
+<dealnews:retailer>Target</dealnews:retailer><dealnews:price currency="dollars">about 10</dealnews:price><dealnews:category></dealnews:category><dealnews:staffPick>false</dealnews:staffPick></item>
<item><title>Plain news</title><link>https://www.dealnews.com/p/3.html</link><guid>https://www.dealnews.com/3.html</guid></item>
<item><title>Plain news</title><link>https://www.dealnews.com/p/3.html</link><guid>https://www.dealnews.com/3.html</guid></item>
</channel></rss>`;
</channel></rss>`;
const r = feed.parse(xml, { url: 'https://www.dealnews.com/?rss=1' });
const r = feed.parse(xml, { url: 'https://www.dealnews.com/?rss=1' });
@@ -47,7 +48,10 @@ t('RSS 2.0 deal feed (DealNews namespace): price, currency, retailer and end dat
@@ -47,7 +48,10 @@ t('RSS 2.0 deal feed (DealNews namespace): price, currency, retailer and end dat
assert.deepStrictEqual([a.fields.price, a.fields.currency, a.fields.seller, a.fields.valid_until], ['67.00', 'USD', 'Amazon', '2027-01-14T00:54:28.000Z']);
assert.deepStrictEqual([a.fields.price, a.fields.currency, a.fields.seller, a.fields.valid_until], ['67.00', 'USD', 'Amazon', '2027-01-14T00:54:28.000Z']);
assert.deepStrictEqual([b.fields.price, b.fields.currency, b.fields.seller], [undefined, undefined, 'Target'], 'a price that is not a number is not kept');
assert.deepStrictEqual([b.fields.price, b.fields.currency, b.fields.seller], [undefined, undefined, 'Target'], 'a price that is not a number is not kept');
assert.ok(!('price' in c.fields) && !('seller' in c.fields), 'an item that states none has none');
assert.ok(!('price' in c.fields) && !('seller' in c.fields), 'an item that states none has none');
- assert.strictEqual(feed.PARSER_VERSION, 'feed@2');
+ assert.deepStrictEqual([a.fields.categories, a.fields.staff_pick], [['Bed & Bath'], true], 'its category and editors\' pick as stated');
+ assert.deepStrictEqual([b.fields.categories, b.fields.staff_pick], [[], false], 'an empty category is no category');
+ assert.ok(!('staff_pick' in c.fields), 'a pick that is not stated is not there');
+ assert.strictEqual(feed.PARSER_VERSION, 'feed@3');
});
});
t('Atom: id identity, relative alternate link, html title, bad dates are null', () => {
t('Atom: id identity, relative alternate link, html title, bad dates are null', () => {
@@ -88,7 +88,7 @@ t('items page in change order and carry provenance and source staleness', async
@@ -88,7 +88,7 @@ t('items page in change order and carry provenance and source staleness', async
assert.strictEqual(p2.body.items.length, 3);
assert.strictEqual(p2.body.items.length, 3);
assert.strictEqual(p2.body.more, false);
assert.strictEqual(p2.body.more, false);
const it = p1.body.items[0];
const it = p1.body.items[0];
- assert.ok(it.provenance.retrieved_at && it.provenance.content_hash && it.provenance.raw_body_hash && it.provenance.parser_version === 'feed@2');
+ assert.ok(it.provenance.retrieved_at && it.provenance.content_hash && it.provenance.raw_body_hash && it.provenance.parser_version === 'feed@3');
assert.strictEqual(it.provenance.terms_note, 'test fixture');
assert.strictEqual(it.provenance.terms_note, 'test fixture');
assert.deepStrictEqual(Object.keys(p1.body.sources), ['feed-a']);
assert.deepStrictEqual(Object.keys(p1.body.sources), ['feed-a']);
assert.strictEqual(p1.body.sources['feed-a'].stale, false);
assert.strictEqual(p1.body.sources['feed-a'].stale, false);
@@ -56,7 +56,7 @@ t('a good fetch creates items with full provenance and an ok run', async () => {
@@ -56,7 +56,7 @@ t('a good fetch creates items with full provenance and an ok run', async () => {
assert.strictEqual(a.revision, 1);
assert.strictEqual(a.revision, 1);
assert.match(a.content_hash, /^[0-9a-f]{64}$/);
assert.match(a.content_hash, /^[0-9a-f]{64}$/);
assert.match(a.raw_body_hash, /^[0-9a-f]{64}$/);
assert.match(a.raw_body_hash, /^[0-9a-f]{64}$/);
- assert.strictEqual(a.parser_version, 'feed@2');
+ assert.strictEqual(a.parser_version, 'feed@3');
assert.strictEqual(a.license_note, 'headlines only');
assert.strictEqual(a.license_note, 'headlines only');
assert.strictEqual(a.terms_note, 'fixture terms');
assert.strictEqual(a.terms_note, 'fixture terms');
assert.ok(a.retrieved_at > 0 && a.first_seen_at === a.retrieved_at);
assert.ok(a.retrieved_at > 0 && a.first_seen_at === a.retrieved_at);