openvibers/openvibe.sources Commits

Sources: an Atom entry with only <updated> is dated by it (GOV.UK's reports had no date) (#48)

OP OpenVibers committed 29 days ago · 3ed5fa18dd · 2 files, +19 −3

**GOV.UK's reports get a date.**

**Found:** while checking openvibe.news after the clustering fixes. All 133 GOV.UK reports had no `published_at`, so the wire showed no date for them, and News placed them by when it first saw them.

**Why:** GOV.UK's Atom feed states only `<updated>`. Atom requires `<updated>` and makes `<published>` optional, but the adapter read only `<published>`/`<issued>`.

**Change (`server/adapters/feed.js`):** an Atom entry with no `<published>` is dated by its `<updated>`, as feed readers do. `fields.dated_by: 'updated'` records that the date came from `<updated>`. A stated `<published>` still wins, and an entry with neither valid date keeps `published_at: null`.

Entries still in the feed get a new revision on their next fetch, which carries the date to News. Older entries keep no date.

**Test:** `adapters.test.js` covers an entry with only `<updated>` (dated by it, with `dated_by`) and an entry with both (`<published>` wins, no `dated_by`). The existing Atom fixture's checks are unchanged.

Changes

ddiff --git a/server/adapters/feed.js b/server/adapters/feed.js
iindex 0bc6837..b581031 100644
---- a/server/adapters/feed.js
++++ b/server/adapters/feed.js
@@@ -67,6 +67,10 @@ function atomEntry(e, base, summaryMax) {
      const url = atomLink(e, base);
      const identity = id || url;
      if (!identity) return null;
++    // Atom requires <updated> and makes <published> optional. A feed that states only <updated> (GOV.UK's) is dated by
++    // it, as feed readers do; fields.dated_by says so, and published_at stays null when neither is a date.
++    const published = toIsoDate(pick(e, 'published', 'issued'));
++    const updated = toIsoDate(pick(e, 'updated', 'modified'));
      return {
          identity,
          kind: 'article',
@@@ -74,9 +78,9 @@ function atomEntry(e, base, summaryMax) {
          title: toText(pick(e, 'title'), 500),
          summary: toText(pick(e, 'summary', 'content'), summaryMax),
          authors: authorsOf(e.author),
--        published_at: toIsoDate(pick(e, 'published', 'issued')),
--        updated_at: toIsoDate(pick(e, 'updated', 'modified')),
--        fields: { categories: categoriesOf(e.category) },
++        published_at: published || updated,
++        updated_at: updated,
++        fields: { categories: categoriesOf(e.category), ...(!published && updated ? { dated_by: 'updated' } : {}) },
      };
  }
  
ddiff --git a/test/adapters.test.js b/test/adapters.test.js
iindex 7709f53..210d7bd 100644
---- a/test/adapters.test.js
++++ b/test/adapters.test.js
@@@ -49,6 +49,18 @@ t('Atom: id identity, relative alternate link, html title, bad dates are null',
      assert.strictEqual(b.summary, 'Body');
  });
  
++t('Atom: an entry with only <updated> (GOV.UK) is dated by it, and says so', () => {
++    const xml = `<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom"><id>https://www.gov.uk/news</id><title>GOV.UK</title><updated>2026-10-14T12:00:00+01:00</updated>
++<entry><id>https://www.gov.uk/government/news/a</id><link rel="alternate" href="https://www.gov.uk/government/news/a"/><title>Rail strategy</title><updated>2026-10-14T11:15:00+01:00</updated><summary>s</summary></entry>
++<entry><id>https://www.gov.uk/government/news/b</id><title>Both dates</title><published>2026-10-13T09:00:00Z</published><updated>2026-10-14T09:00:00Z</updated></entry></feed>`;
++    const [only, both] = feed.parse(xml, { url: 'https://www.gov.uk/news.atom' }).items;
++    assert.strictEqual(only.published_at, '2026-10-14T10:15:00.000Z');
++    assert.strictEqual(only.updated_at, '2026-10-14T10:15:00.000Z');
++    assert.strictEqual(only.fields.dated_by, 'updated');
++    assert.strictEqual(both.published_at, '2026-10-13T09:00:00.000Z', 'a stated <published> wins');
++    assert.strictEqual(both.fields.dated_by, undefined);
++});
++
  t('a large real-world-sized feed with thousands of entities parses (no expansion limit tripped)', () => {
      const items = Array.from({ length: 1500 }, (_, i) => `<item><title>Deal ${i} &amp; more &#8212; $${i}</title><link>https://deals.example.com/d/${i}?a=1&amp;b=2</link><guid>d-${i}</guid></item>`).join('');
      const r = feed.parse(`<?xml version="1.0"?><rss version="2.0"><channel><title>Big &amp; busy</title>${items}</channel></rss>`, { url: 'https://deals.example.com/rss', maxItems: 2000 });