openvibers/openvibe.sources Commits

Sources: a deal feed's own price, retailer and end date are kept (DealNews' namespace) (#68)

OP OpenVibers committed 28 days ago · ac236f971d · 5 files, +48 −4

DealNews states each deal's price (with currency), retailer and expiry in its own
RSS elements; the feed parser dropped them, so every OpenVibe.Deals offer from it
had no price and was filed under dealnews.com. They now become fields.price,
currency, seller and valid_until exactly as stated (feed@2); the item stays an
article and nothing is parsed out of the headline.

Changes

ddiff --git a/README.md b/README.md
iindex 9bd609a..46168de 100644
---- a/README.md
++++ b/README.md
@@@ -101,7 +101,7 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled
| Type | Reads | Notes |
|---|---|---|
--| `rss`, `atom` (`feed@1`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars |
++| `rss`, `atom` (`feed@2`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline) |
| `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed |
| `jsonld` (`jsonld@1`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error` |
| `api` (`api@1`) | JSON or XML official APIs via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`; epoch fields only where mapped as such |
ddiff --git a/server/adapters/feed.js b/server/adapters/feed.js
iindex b581031..1b8c53e 100644
---- a/server/adapters/feed.js
++++ b/server/adapters/feed.js
@@@ -8,7 +8,33 @@
*/
const { parseXml, toText, toIsoDate, canonicalUrl, toId, asArray, pick, ParseError } = require('./util');
--const PARSER_VERSION = 'feed@1';
++const PARSER_VERSION = 'feed@2';
++
++/**
++ * Offer facts a deal feed states in its own elements (not read out of the headline): DealNews' namespace gives the
++ * price (with its currency), the retailer and when the deal ends. They become fields.price / currency / seller /
++ * valid_until, the names an `offer` item uses, exactly as stated; the item stays an article. Anything malformed is
++ * left out, never guessed.
++ */
++const STATED_OFFER = [{ price: 'dealnews:price', seller: 'dealnews:retailer', valid_until: 'dealnews:expires' }];
++function statedOffer(it) {
++ for (const map of STATED_OFFER) {
++ if (it[map.price] == null && it[map.seller] == null) continue;
++ const node = it[map.price];
++ const raw = node != null && typeof node === 'object' ? node['#text'] : node;
++ const price = /^\d{1,9}(?:\.\d{1,2})?$/.test(String(raw ?? '').trim()) ? String(raw).trim() : null;
++ const cur = node != null && typeof node === 'object' ? String(node['@_currency'] || '').trim().toUpperCase() : '';
++ const seller = toText(it[map.seller], 120);
++ const until = toIsoDate(it[map.valid_until]);
++ const out = {
++ ...(price ? { price, ...(/^[A-Z]{3}$/.test(cur) ? { currency: cur } : {}) } : {}),
++ ...(seller ? { seller } : {}),
++ ...(until ? { valid_until: until } : {}),
++ };
++ return Object.keys(out).length ? out : null;
++ }
++ return null;
++}
function atomLink(entry, base) {
const links = asArray(entry.link);
@@@ -58,6 +84,7 @@ function rssItem(it, base, summaryMax) {
categories: categoriesOf(it.category ?? it['dc:subject']),
enclosure: enclosure ? { url: canonicalUrl(enclosure['@_url'], base), type: toId(enclosure['@_type'], 100), length: /^\d{1,15}$/.test(String(enclosure['@_length'] || '')) ? Number(enclosure['@_length']) : null } : null,
comments_url: canonicalUrl(pick(it, 'comments'), base),
++ ...(statedOffer(it) || {}),
},
};
}
ddiff --git a/test/adapters.test.js b/test/adapters.test.js
iindex 210d7bd..1d34c3f 100644
---- a/test/adapters.test.js
++++ b/test/adapters.test.js
@@@ -33,6 +33,23 @@ t('RSS 2.0: guid identity, cleaned link, text-only summary, dates, skips', () =>
assert.deepStrictEqual(b.authors, []);
});
++t('RSS 2.0 deal feed (DealNews namespace): price, currency, retailer and end date as stated; malformed ones left out', () => {
++ const xml = `<?xml version="1.0"?><rss version="2.0" xmlns:dealnews="https://www.dealnews.com/ns/rss/1.0.htm"><channel><title>D</title>
++<item><title>Air Mattress for $67 + free shipping</title><link>https://www.dealnews.com/p/1.html?iref=rss</link><guid>https://www.dealnews.com/1.html</guid>
++<dealnews:retailer>Amazon</dealnews:retailer><dealnews:expires>2027-01-13T19:54:28-05:00</dealnews:expires><dealnews:price currency="USD">67.00</dealnews:price></item>
++<item><title>Sale at Target</title><link>https://www.dealnews.com/p/2.html</link><guid>https://www.dealnews.com/2.html</guid>
++<dealnews:retailer>Target</dealnews:retailer><dealnews:price currency="dollars">about 10</dealnews:price></item>
++<item><title>Plain news</title><link>https://www.dealnews.com/p/3.html</link><guid>https://www.dealnews.com/3.html</guid></item>
++</channel></rss>`;
++ const r = feed.parse(xml, { url: 'https://www.dealnews.com/?rss=1' });
++ const [a, b, c] = r.items;
++ assert.strictEqual(a.kind, 'article', 'still an article');
++ assert.deepStrictEqual([a.fields.price, a.fields.currency, a.fields.seller, a.fields.valid_until], ['67.00', 'USD', 'Amazon', '2027-01-14T00:54:28.000Z']);
++ assert.deepStrictEqual([b.fields.price, b.fields.currency, b.fields.seller], [undefined, undefined, 'Target'], 'a price that is not a number is not kept');
++ assert.ok(!('price' in c.fields) && !('seller' in c.fields), 'an item that states none has none');
++ assert.strictEqual(feed.PARSER_VERSION, 'feed@2');
++});
++
t('Atom: id identity, relative alternate link, html title, bad dates are null', () => {
const r = feed.parse(fx('atom.xml'), { url: 'https://blog.example.com/feed.atom' });
assert.strictEqual(r.meta.format, 'atom');
ddiff --git a/test/api.test.js b/test/api.test.js
iindex 0d7ef65..d62a21c 100644
---- a/test/api.test.js
++++ b/test/api.test.js
@@@ -88,7 +88,7 @@ t('items page in change order and carry provenance and source staleness', async
assert.strictEqual(p2.body.items.length, 3);
assert.strictEqual(p2.body.more, false);
const it = p1.body.items[0];
-- assert.ok(it.provenance.retrieved_at && it.provenance.content_hash && it.provenance.raw_body_hash && it.provenance.parser_version === 'feed@1');
++ assert.ok(it.provenance.retrieved_at && it.provenance.content_hash && it.provenance.raw_body_hash && it.provenance.parser_version === 'feed@2');
assert.strictEqual(it.provenance.terms_note, 'test fixture');
assert.deepStrictEqual(Object.keys(p1.body.sources), ['feed-a']);
assert.strictEqual(p1.body.sources['feed-a'].stale, false);
ddiff --git a/test/ingest.test.js b/test/ingest.test.js
iindex c686937..c86e229 100644
---- a/test/ingest.test.js
++++ b/test/ingest.test.js
@@@ -56,7 +56,7 @@ t('a good fetch creates items with full provenance and an ok run', async () => {
assert.strictEqual(a.revision, 1);
assert.match(a.content_hash, /^[0-9a-f]{64}$/);
assert.match(a.raw_body_hash, /^[0-9a-f]{64}$/);
-- assert.strictEqual(a.parser_version, 'feed@1');
++ assert.strictEqual(a.parser_version, 'feed@2');
assert.strictEqual(a.license_note, 'headlines only');
assert.strictEqual(a.terms_note, 'fixture terms');
assert.ok(a.retrieved_at > 0 && a.first_seen_at === a.retrieved_at);