openvibers/openvibe.sources Commits

Sources: trending is the last 24 hours, not GitHub's empty same-day bucket (#286)

OP OpenVibers committed 1 days ago · bcacc8d513 · 4 files, +8 −6

Changes

ddiff --git a/README.md b/README.md
iindex 3af261d..d6ddf4e 100644
---- a/README.md
++++ b/README.md
@@@ -110,7 +110,7 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled
| `rss`, `atom` (`feed@3`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline); its `dealnews:category` fills `fields.categories` when the item has no `<category>`, and `dealnews:staffPick` is `fields.staff_pick` (true/false, absent when not stated) |
| `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed |
| `jsonld` (`jsonld@2`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error`; `list_price` from a ListPrice/StrikethroughPrice/MSRP/SRP `priceSpecification`, as stated |
--| `api` (`api@1`) | JSON, XML, TSV or CSV official files via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`, `where`/`filter`; epoch fields only where mapped as such; `resolver: "hn-whoishiring"` walks the Hacker News thread's JSON documents instead; `resolver: "github"` walks GitHub's repository search plan (server/github.js) |
++| `api` (`api@1`) | JSON, XML, TSV or CSV official files via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`, `where`/`filter`; epoch fields only where mapped as such; `resolver: "hn-whoishiring"` walks the Hacker News thread's JSON documents instead; `resolver: "github"` walks GitHub's repository search plan (server/github.js; trending is the last 24 hours because GitHub's index has no same-day bucket) |
| `bulk` (`bulk@1`) | one dataset file, streamed (CSV, JSONL or TSV) | `max_bytes` is the size cap, `max_rows` the per-run row limit; the stream stops at the limit, so a multi-GB file is never held in memory. `resolver: "off-delta"` reads a text index and only the gzipped JSONL delta parts it lists, keeping the last finished part and the current part's row offset in `source_state` (Open Food Facts) |
| `crawl` (`jsonld@2`, `page@1`) | a whole site from its seeds: its sitemaps and, if asked, its links (see below) | only pages that publish JSON-LD are kept, unless `require_structured: false` keeps a `page` item from the page's own search metadata |
| `manual` (`manual@1`) | items entered through the API with an evidence URL | no fetcher |
ddiff --git a/server/adapters/github.js b/server/adapters/github.js
iindex 70c2b43..abd3573 100644
---- a/server/adapters/github.js
++++ b/server/adapters/github.js
@@@ -35,12 +35,14 @@ const daysAgo = (ms, n) => isoDay(ms - n * 86400000);
/** The queries, in a fixed order: base lists first, then one recent list per language. */
function plan(now = Date.now()) {
-- const today = isoDay(now);
++ const yesterday = daysAgo(now, 1);
const week = daysAgo(now, 7);
const month = daysAgo(now, 30);
const pushed = daysAgo(now, 30);
const base = [
-- { key: 'trending', q: `created:>${today}`, sort: 'stars', order: 'desc' },
++ // GitHub's search index has no 'today' bucket until the day closes, so the last 24 hours is the window;
++ // the API applies the same 24-hour cutoff to the list.
++ { key: 'trending', q: `created:>${yesterday}`, sort: 'stars', order: 'desc' },
{ key: 'week', q: `created:>${week}`, sort: 'stars', order: 'desc' },
{ key: 'month', q: `created:>${month}`, sort: 'stars', order: 'desc' },
{ key: 'pushed', q: `pushed:>${pushed} stars:>100`, sort: 'stars', order: 'desc' },
ddiff --git a/server/api/code.js b/server/api/code.js
iindex 80e72c9..b2aa0b6 100644
---- a/server/api/code.js
++++ b/server/api/code.js
@@@ -86,7 +86,7 @@ function createCodeRouter({ db, registry, auth, limits, github: client }) {
const topic = text(req.query.topic, 50);
const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 30, 1), 100);
const t = Date.now();
-- const dayStart = Date.parse(github.isoDay(t));
++ const lastDay = t - DAY;
// Ordering by stars happens in JS: the tracked set is bounded by the source's max_items (3,000) and
// the gains need the snapshots anyway. A filter narrows it before the sort.
const rows = (await db.query("SELECT * FROM items WHERE category = 'code' AND removed_at IS NULL ORDER BY retrieved_at DESC LIMIT 5000")).rows;
@@@ -96,7 +96,7 @@ function createCodeRouter({ db, registry, auth, limits, github: client }) {
if (lang && String(f.language || '').toLowerCase() !== lang.toLowerCase()) return false;
if (topic && !(Array.isArray(f.topics) ? f.topics : []).some((x) => String(x).toLowerCase() === topic.toLowerCase())) return false;
const created = Date.parse(f.created || '');
-- if (list === 'trending' && (!Number.isFinite(created) || created < dayStart)) return false;
++ if (list === 'trending' && (!Number.isFinite(created) || created < lastDay)) return false;
if (list === 'week' && (!Number.isFinite(created) || created < t - 7 * DAY)) return false;
if (list === 'month' && (!Number.isFinite(created) || created < t - 30 * DAY)) return false;
if (list === 'new' && (!Number.isFinite(created) || created < t - 7 * DAY)) return false;
ddiff --git a/test/github.test.js b/test/github.test.js
iindex ec5d1db..1f7a844 100644
---- a/test/github.test.js
++++ b/test/github.test.js
@@@ -29,7 +29,7 @@ function fakeFetcher(handler) {
t('the plan is today, this week, this month, recently pushed, all-time top, new, and twelve languages', () => {
const plan = github.plan(Date.parse('2026-11-12T12:00:00Z'));
assert.deepStrictEqual(plan.slice(0, 6).map((q) => q.key), ['trending', 'week', 'month', 'pushed', 'top', 'new']);
-- assert.strictEqual(plan[0].q, 'created:>2026-11-12');
++ assert.strictEqual(plan[0].q, 'created:>2026-11-11');
assert.strictEqual(plan[1].q, 'created:>2026-11-05');
assert.strictEqual(plan[2].q, 'created:>2026-10-13');
assert.strictEqual(plan[3].q, 'pushed:>2026-10-13 stars:>100');