openvibers/openvibe.sources Commits
Sources: trending is the last 24 hours, not GitHub's empty same-day bucket (#286)
Changes
@@@ -110,7 +110,7 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled| `rss`, `atom` (`feed@3`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline); its `dealnews:category` fills `fields.categories` when the item has no `<category>`, and `dealnews:staffPick` is `fields.staff_pick` (true/false, absent when not stated) || `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed || `jsonld` (`jsonld@2`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error`; `list_price` from a ListPrice/StrikethroughPrice/MSRP/SRP `priceSpecification`, as stated |--| `api` (`api@1`) | JSON, XML, TSV or CSV official files via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`, `where`/`filter`; epoch fields only where mapped as such; `resolver: "hn-whoishiring"` walks the Hacker News thread's JSON documents instead; `resolver: "github"` walks GitHub's repository search plan (server/github.js) |++| `api` (`api@1`) | JSON, XML, TSV or CSV official files via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`, `where`/`filter`; epoch fields only where mapped as such; `resolver: "hn-whoishiring"` walks the Hacker News thread's JSON documents instead; `resolver: "github"` walks GitHub's repository search plan (server/github.js; trending is the last 24 hours because GitHub's index has no same-day bucket) || `bulk` (`bulk@1`) | one dataset file, streamed (CSV, JSONL or TSV) | `max_bytes` is the size cap, `max_rows` the per-run row limit; the stream stops at the limit, so a multi-GB file is never held in memory. `resolver: "off-delta"` reads a text index and only the gzipped JSONL delta parts it lists, keeping the last finished part and the current part's row offset in `source_state` (Open Food Facts) || `crawl` (`jsonld@2`, `page@1`) | a whole site from its seeds: its sitemaps and, if asked, its links (see below) | only pages that publish JSON-LD are kept, unless `require_structured: false` keeps a `page` item from the page's own search metadata || `manual` (`manual@1`) | items entered through the API with an evidence URL | no fetcher |@@@ -35,12 +35,14 @@ const daysAgo = (ms, n) => isoDay(ms - n * 86400000);/** The queries, in a fixed order: base lists first, then one recent list per language. */function plan(now = Date.now()) {-- const today = isoDay(now);++ const yesterday = daysAgo(now, 1);const week = daysAgo(now, 7);const month = daysAgo(now, 30);const pushed = daysAgo(now, 30);const base = [-- { key: 'trending', q: `created:>${today}`, sort: 'stars', order: 'desc' },++ // GitHub's search index has no 'today' bucket until the day closes, so the last 24 hours is the window;++ // the API applies the same 24-hour cutoff to the list.++ { key: 'trending', q: `created:>${yesterday}`, sort: 'stars', order: 'desc' },{ key: 'week', q: `created:>${week}`, sort: 'stars', order: 'desc' },{ key: 'month', q: `created:>${month}`, sort: 'stars', order: 'desc' },{ key: 'pushed', q: `pushed:>${pushed} stars:>100`, sort: 'stars', order: 'desc' },@@@ -86,7 +86,7 @@ function createCodeRouter({ db, registry, auth, limits, github: client }) {const topic = text(req.query.topic, 50);const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 30, 1), 100);const t = Date.now();-- const dayStart = Date.parse(github.isoDay(t));++ const lastDay = t - DAY;// Ordering by stars happens in JS: the tracked set is bounded by the source's max_items (3,000) and// the gains need the snapshots anyway. A filter narrows it before the sort.const rows = (await db.query("SELECT * FROM items WHERE category = 'code' AND removed_at IS NULL ORDER BY retrieved_at DESC LIMIT 5000")).rows;@@@ -96,7 +96,7 @@ function createCodeRouter({ db, registry, auth, limits, github: client }) {if (lang && String(f.language || '').toLowerCase() !== lang.toLowerCase()) return false;if (topic && !(Array.isArray(f.topics) ? f.topics : []).some((x) => String(x).toLowerCase() === topic.toLowerCase())) return false;const created = Date.parse(f.created || '');-- if (list === 'trending' && (!Number.isFinite(created) || created < dayStart)) return false;++ if (list === 'trending' && (!Number.isFinite(created) || created < lastDay)) return false;if (list === 'week' && (!Number.isFinite(created) || created < t - 7 * DAY)) return false;if (list === 'month' && (!Number.isFinite(created) || created < t - 30 * DAY)) return false;if (list === 'new' && (!Number.isFinite(created) || created < t - 7 * DAY)) return false;@@@ -29,7 +29,7 @@ function fakeFetcher(handler) {t('the plan is today, this week, this month, recently pushed, all-time top, new, and twelve languages', () => {const plan = github.plan(Date.parse('2026-11-12T12:00:00Z'));assert.deepStrictEqual(plan.slice(0, 6).map((q) => q.key), ['trending', 'week', 'month', 'pushed', 'top', 'new']);-- assert.strictEqual(plan[0].q, 'created:>2026-11-12');++ assert.strictEqual(plan[0].q, 'created:>2026-11-11');assert.strictEqual(plan[1].q, 'created:>2026-11-05');assert.strictEqual(plan[2].q, 'created:>2026-10-13');assert.strictEqual(plan[3].q, 'pushed:>2026-10-13 stars:>100');
@@ -110,7 +110,7 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled
@@ -110,7 +110,7 @@ succeeded. Health status: `healthy | stale | failing | never_fetched | disabled
| `rss`, `atom` (`feed@3`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline); its `dealnews:category` fills `fields.categories` when the item has no `<category>`, and `dealnews:staffPick` is `fields.staff_pick` (true/false, absent when not stated) |
| `rss`, `atom` (`feed@3`) | RSS 2.0, RSS 1.0/RDF, Atom 1.0 | guid/id identity, else link; text-only summary capped at 1000 chars; a deal feed's own offer elements (DealNews' `dealnews:price` with its currency, `dealnews:retailer`, `dealnews:expires`) become `fields.price`, `currency`, `seller` and `valid_until` exactly as stated (the item stays an article; nothing is read out of the headline); its `dealnews:category` fills `fields.categories` when the item has no `<category>`, and `dealnews:staffPick` is `fields.staff_pick` (true/false, absent when not stated) |
| `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed |
| `sitemap` (`sitemap@1`) | `<urlset>` (+ Google News extension), `<sitemapindex>` | index children are listed, not followed |
| `jsonld` (`jsonld@2`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error`; `list_price` from a ListPrice/StrikethroughPrice/MSRP/SRP `priceSpecification`, as stated |
| `jsonld` (`jsonld@2`) | schema.org Product, Offer/AggregateOffer, Review, NewsArticle, BlogPosting (+ Article) | prices, ratings and availability exactly as the page states; missing = null; embedded review text not kept; a page without JSON-LD is a `parse_error`; `list_price` from a ListPrice/StrikethroughPrice/MSRP/SRP `priceSpecification`, as stated |
-| `api` (`api@1`) | JSON, XML, TSV or CSV official files via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`, `where`/`filter`; epoch fields only where mapped as such; `resolver: "hn-whoishiring"` walks the Hacker News thread's JSON documents instead; `resolver: "github"` walks GitHub's repository search plan (server/github.js) |
+| `api` (`api@1`) | JSON, XML, TSV or CSV official files via a registry mapping | `items_path`, `items_shape` (`array` default, `map` for records keyed by id), `fields`, `extra`, `where`/`filter`; epoch fields only where mapped as such; `resolver: "hn-whoishiring"` walks the Hacker News thread's JSON documents instead; `resolver: "github"` walks GitHub's repository search plan (server/github.js; trending is the last 24 hours because GitHub's index has no same-day bucket) |
| `bulk` (`bulk@1`) | one dataset file, streamed (CSV, JSONL or TSV) | `max_bytes` is the size cap, `max_rows` the per-run row limit; the stream stops at the limit, so a multi-GB file is never held in memory. `resolver: "off-delta"` reads a text index and only the gzipped JSONL delta parts it lists, keeping the last finished part and the current part's row offset in `source_state` (Open Food Facts) |
| `bulk` (`bulk@1`) | one dataset file, streamed (CSV, JSONL or TSV) | `max_bytes` is the size cap, `max_rows` the per-run row limit; the stream stops at the limit, so a multi-GB file is never held in memory. `resolver: "off-delta"` reads a text index and only the gzipped JSONL delta parts it lists, keeping the last finished part and the current part's row offset in `source_state` (Open Food Facts) |
| `crawl` (`jsonld@2`, `page@1`) | a whole site from its seeds: its sitemaps and, if asked, its links (see below) | only pages that publish JSON-LD are kept, unless `require_structured: false` keeps a `page` item from the page's own search metadata |
| `crawl` (`jsonld@2`, `page@1`) | a whole site from its seeds: its sitemaps and, if asked, its links (see below) | only pages that publish JSON-LD are kept, unless `require_structured: false` keeps a `page` item from the page's own search metadata |
| `manual` (`manual@1`) | items entered through the API with an evidence URL | no fetcher |
| `manual` (`manual@1`) | items entered through the API with an evidence URL | no fetcher |
@@ -35,12 +35,14 @@ const daysAgo = (ms, n) => isoDay(ms - n * 86400000);
@@ -35,12 +35,14 @@ const daysAgo = (ms, n) => isoDay(ms - n * 86400000);
/** The queries, in a fixed order: base lists first, then one recent list per language. */
/** The queries, in a fixed order: base lists first, then one recent list per language. */
function plan(now = Date.now()) {
function plan(now = Date.now()) {
- const today = isoDay(now);
+ const yesterday = daysAgo(now, 1);
const week = daysAgo(now, 7);
const week = daysAgo(now, 7);
const month = daysAgo(now, 30);
const month = daysAgo(now, 30);
const pushed = daysAgo(now, 30);
const pushed = daysAgo(now, 30);
const base = [
const base = [
- { key: 'trending', q: `created:>${today}`, sort: 'stars', order: 'desc' },
+ // GitHub's search index has no 'today' bucket until the day closes, so the last 24 hours is the window;
+ // the API applies the same 24-hour cutoff to the list.
+ { key: 'trending', q: `created:>${yesterday}`, sort: 'stars', order: 'desc' },
{ key: 'week', q: `created:>${week}`, sort: 'stars', order: 'desc' },
{ key: 'week', q: `created:>${week}`, sort: 'stars', order: 'desc' },
{ key: 'month', q: `created:>${month}`, sort: 'stars', order: 'desc' },
{ key: 'month', q: `created:>${month}`, sort: 'stars', order: 'desc' },
{ key: 'pushed', q: `pushed:>${pushed} stars:>100`, sort: 'stars', order: 'desc' },
{ key: 'pushed', q: `pushed:>${pushed} stars:>100`, sort: 'stars', order: 'desc' },
@@ -86,7 +86,7 @@ function createCodeRouter({ db, registry, auth, limits, github: client }) {
@@ -86,7 +86,7 @@ function createCodeRouter({ db, registry, auth, limits, github: client }) {
const topic = text(req.query.topic, 50);
const topic = text(req.query.topic, 50);
const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 30, 1), 100);
const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 30, 1), 100);
const t = Date.now();
const t = Date.now();
- const dayStart = Date.parse(github.isoDay(t));
+ const lastDay = t - DAY;
// Ordering by stars happens in JS: the tracked set is bounded by the source's max_items (3,000) and
// Ordering by stars happens in JS: the tracked set is bounded by the source's max_items (3,000) and
// the gains need the snapshots anyway. A filter narrows it before the sort.
// the gains need the snapshots anyway. A filter narrows it before the sort.
const rows = (await db.query("SELECT * FROM items WHERE category = 'code' AND removed_at IS NULL ORDER BY retrieved_at DESC LIMIT 5000")).rows;
const rows = (await db.query("SELECT * FROM items WHERE category = 'code' AND removed_at IS NULL ORDER BY retrieved_at DESC LIMIT 5000")).rows;
@@ -96,7 +96,7 @@ function createCodeRouter({ db, registry, auth, limits, github: client }) {
@@ -96,7 +96,7 @@ function createCodeRouter({ db, registry, auth, limits, github: client }) {
if (lang && String(f.language || '').toLowerCase() !== lang.toLowerCase()) return false;
if (lang && String(f.language || '').toLowerCase() !== lang.toLowerCase()) return false;
if (topic && !(Array.isArray(f.topics) ? f.topics : []).some((x) => String(x).toLowerCase() === topic.toLowerCase())) return false;
if (topic && !(Array.isArray(f.topics) ? f.topics : []).some((x) => String(x).toLowerCase() === topic.toLowerCase())) return false;
const created = Date.parse(f.created || '');
const created = Date.parse(f.created || '');
- if (list === 'trending' && (!Number.isFinite(created) || created < dayStart)) return false;
+ if (list === 'trending' && (!Number.isFinite(created) || created < lastDay)) return false;
if (list === 'week' && (!Number.isFinite(created) || created < t - 7 * DAY)) return false;
if (list === 'week' && (!Number.isFinite(created) || created < t - 7 * DAY)) return false;
if (list === 'month' && (!Number.isFinite(created) || created < t - 30 * DAY)) return false;
if (list === 'month' && (!Number.isFinite(created) || created < t - 30 * DAY)) return false;
if (list === 'new' && (!Number.isFinite(created) || created < t - 7 * DAY)) return false;
if (list === 'new' && (!Number.isFinite(created) || created < t - 7 * DAY)) return false;
@@ -29,7 +29,7 @@ function fakeFetcher(handler) {
@@ -29,7 +29,7 @@ function fakeFetcher(handler) {
t('the plan is today, this week, this month, recently pushed, all-time top, new, and twelve languages', () => {
t('the plan is today, this week, this month, recently pushed, all-time top, new, and twelve languages', () => {
const plan = github.plan(Date.parse('2026-11-12T12:00:00Z'));
const plan = github.plan(Date.parse('2026-11-12T12:00:00Z'));
assert.deepStrictEqual(plan.slice(0, 6).map((q) => q.key), ['trending', 'week', 'month', 'pushed', 'top', 'new']);
assert.deepStrictEqual(plan.slice(0, 6).map((q) => q.key), ['trending', 'week', 'month', 'pushed', 'top', 'new']);
- assert.strictEqual(plan[0].q, 'created:>2026-11-12');
+ assert.strictEqual(plan[0].q, 'created:>2026-11-11');
assert.strictEqual(plan[1].q, 'created:>2026-11-05');
assert.strictEqual(plan[1].q, 'created:>2026-11-05');
assert.strictEqual(plan[2].q, 'created:>2026-10-13');
assert.strictEqual(plan[2].q, 'created:>2026-10-13');
assert.strictEqual(plan[3].q, 'pushed:>2026-10-13 stars:>100');
assert.strictEqual(plan[3].q, 'pushed:>2026-10-13 stars:>100');