openvibers/openvibe.network Commits
Feed: one card per burst, words not Markdown, pictures that load, no lost cards (#887)
Changes
@@@ -16,7 +16,7 @@*/const express = require('express');const store = require('./store');--const { cardOf, foldCards, rankCards, searchMatches, searchExamples, wallItem } = require('./card');++const { cardOf, foldCards, rankCards, searchMatches, searchExamples, wallItem, bulkId, BULK_WORD } = require('./card');const { createFeedSources } = require('./sources');const { createEventsPull } = require('./events');const session = require('../auth/session');@@@ -99,10 +99,42 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publ} catch { /* a card without a count */ }if (q) cards = cards.filter((c) => searchMatches(c, q));cards = foldCards(cards, now());++ // A burst's card counts the whole hour, not just the rows this page read (11-12: three "112 new deals").++ for (const card of cards.filter((c) => c.bulk)) {++ const b = card.bulk;++ delete card.bulk;++ let n = card.group ? card.group.count : 1;++ try { n = Math.max(n, await store.countBulk(db, b)); } catch { /* the page's own count */ }++ if (n < 2) continue;++ if (!card.group) card.group = { count: n, items: [{ id: card.id, url: card.url, title: card.title, media: card.media }] };++ card.id = bulkId(b.kind, b.site, b.hour);++ card.title = `${n.toLocaleString('en-US')} new ${b.many}`;++ card.group.count = n;++ card.counts = { ...card.counts, people: n };++ }cards = rankCards(cards, { followed: new Set(followed), affinity, now: now() });const items = cards.slice(0, want);-- const last = read.rows[read.rows.length - 1];-- const next = read.hasMore && last ? encodeCursor([last.occurred_at, last.id]) : null;++ // The next page starts at the newest row whose card this page did not show (it used to start after every row++ // read, so the cards past the first `want` were never seen); a row of a shown burst or group counts as shown.++ const shown = new Set();++ for (const c of items) { shown.add(c.id); for (const it of (c.group && c.group.items) || []) shown.add(it.id); }++ const rowShown = (r) => {++ const hour = r.occurred_at ? new Date(r.occurred_at).toISOString().slice(0, 13) : '';++ return shown.has(`f:${r.source_service}:${r.source_type}:${r.source_id}`)++ || (!r.actor_subject && BULK_WORD[r.kind] && shown.has(bulkId(r.kind, r.site, hour)))++ || (r.kind === 'work' && shown.has(`g:work:${hour}`));++ };++ const firstOut = read.rows.findIndex((r) => !rowShown(r));++ const edge = firstOut > 0 ? read.rows[firstOut - 1] : read.rows[read.rows.length - 1];++ let next = (firstOut > 0 || read.hasMore) && edge ? encodeCursor([edge.occurred_at, edge.id]) : null;++ // A page that ends inside a burst (a Deals poll) moves past the rest of it when nothing else happened in that++ // span, so the next page does not open with the same card again.++ if (next && edge && !edge.actor_subject && BULK_WORD[edge.kind]) {++ try {++ const start = await store.burstStart(db, { kind: edge.kind, site: edge.site, before: [edge.occurred_at, edge.id] });++ if (start) next = encodeCursor([start, 0]);++ } catch { /* the plain cursor */ }++ }if (wall) return { items: items.map(wallItem), next_cursor: next };return { items, next_cursor: next };}@@@ -134,7 +166,7 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publconst followed = viewer && viewer.subject ? await store.followedSubjects(db, viewer.subject) : [];const read = await store.listItems(db, { q, followed, viewerSubject: viewer && viewer.subject, limit: 400, since: new Date(now() - 365 * 86400000).toISOString() });const actors = await actorsFor(db, read.rows);-- const cards = foldCards(read.rows.map((r) => cardOf(r, actors.get(r.actor_subject) || null)).filter((c) => searchMatches(c, q)), now());++ const cards = foldCards(read.rows.map((r) => cardOf(r, actors.get(r.actor_subject) || null)).filter((c) => searchMatches(c, q)), now()).map(({ bulk, ...c }) => c);res.json({ items: rankCards(cards, { followed: new Set(followed), now: now() }).slice(0, want), next_cursor: null });} catch (err) { next(err); }};@@@ -22,6 +22,26 @@ const https = (u) => {try { const x = new URL(String(u)); return x.protocol === 'https:' ? x.href : (x.protocol === 'http:' ? x.href : null); } catch { return null; }};++/** The site a row comes from, as an origin: a relative address in its text or picture is on that site, not on the++ * page that shows the card (the menu on openvibe.link showed Community's "/p/x" as a broken link and picture). */++const ORIGINS = { community: 'https://openvibe.community', 'media-hub': 'https://openvibe.video', rundown: 'https://openvibe.live', stream: 'https://openvibe.live' };++const originOf = (site) => ORIGINS[site] || (/^[a-z][a-z0-9-]{0,30}$/.test(String(site || '')) ? `https://openvibe.${site}` : null);++const absolute = (u, site) => {++ const s = String(u || '').trim();++ if (!s) return null;++ try { const x = new URL(s, originOf(site) || undefined); return x.protocol === 'https:' || x.protocol === 'http:' ? x.href : null; } catch { return null; }++};++const MD_IMAGE = /!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;++const MD_LINK = /\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;++/** Markdown a source left in a card's text, as plain words, and the first picture it named (11-12: the menu printed it). */++function plainText(text, site) {++ const raw = String(text == null ? '' : text);++ let image = null;++ const noImages = raw.replace(MD_IMAGE, (m, alt, src) => { if (!image) image = absolute(src, site); return ' '; });++ const words = noImages.replace(MD_LINK, '$1').replace(/(^|\n)[ \t]*(?:#{1,6}|>)[ \t]*/g, '$1').replace(/\*{1,3}|`+|~~/g, '');++ return { text: words, image };++}++/** What a source row is, as a feed kind. An unknown type is a post, never dropped silently. */function kindOf(service, type, row = {}) {const s = String(service || '');@@@ -79,7 +99,7 @@ function cardOf(row, actor = null) {kind: row.kind,url: https(row.url) || String(row.url || ''),title: clean(row.title, 300),-- text: clean(row.text, 280),++ text: '',at: row.occurred_at ? new Date(row.occurred_at).toISOString() : null,site: String(row.site || row.source_service || ''),site_name: String(row.site_name || SITE_NAMES[row.site] || SITE_NAMES[row.source_service] || row.site || ''),@@@ -90,11 +110,24 @@ function cardOf(row, actor = null) {: null,counts: safe,};++ // The words as words: Markdown out, relative addresses on the source's own site, and no text that only repeats++ // the title (a shared post's body is "[its title](its link)"). A shared post's picture becomes the card's picture.++ const plain = plainText(row.text, card.site);++ card.text = clean(plain.text, 280);++ if (card.text && card.text.toLowerCase() === card.title.toLowerCase()) card.text = '';++ if (!/^https?:/.test(card.url) && card.url) card.url = absolute(card.url, card.site) || card.url;++ if (!card.media && plain.image) card.media = { thumb: plain.image, ratio: Number.isFinite(ratio) && ratio > 0 ? ratio : 16 / 9 };++ // A Community paste's picture is served by Community (Media keeps it private to the paste): the small copy, never++ // Media's original address, which answers 404 to everyone else (the menu's broken picture, 11-12).++ if (row.source_service === 'community' && row.source_type === 'paste' && card.media && /^https:\/\/openvibe\.media\/o\//.test(card.media.thumb)) {++ card.media.thumb = `https://openvibe.community/p/${encodeURIComponent(String(row.source_id))}/screenshot?w=640`;++ }++ if (card.media && card.media.thumb && /^https:\/\/openvibe\.community\/p\/[^/]+\/screenshot$/.test(card.media.thumb)) card.media.thumb += '?w=640';if (row.starts_at) card.starts_at = new Date(row.starts_at).toISOString();if (row.label) card.label = row.label;// The menu labels AI output and system/aggregate rows; keep the old Pulse's promise that AI is never a person.if (!card.label && row.origin === 'ai') card.label = 'AI';-- else if (!card.label && row.origin === 'system' && !card.actor) card.label = 'System';++ else if (!card.label && row.origin === 'system' && !card.actor && !card.site_name) card.label = 'System';if (row.visibility === 'followers') card.visibility = 'followers';if (card.kind === 'live' || card.kind === 'stream') {card.live_now = row.kind === 'live' && row.live_now === true;@@@ -117,6 +150,9 @@ const BURST_WORD = { pic: ['photo', 'photos'], clip: ['clip', 'clips'], video: [// A poll can deliver forty of the same kind at once with no person behind them (deals, news): one card.const BULK_WORD = { deal: ['deal', 'deals'], news: ['story', 'stories'], event: ['event', 'events'] };++/** The id of one site's run of one bulk kind in one UTC hour ('2026-11-12T17'): the same on every page that reads it. */++const bulkId = (kind, site, hour) => `g:${kind}:${site}:${hour}`;++function titleKey(card) {return String(card.title || '').toLowerCase().replace(/\s+/g, ' ').trim();}@@@ -142,11 +178,11 @@ function foldCards(cards, now = Date.now()) {const prior = BURST_WORD[card.kind] && who && Number.isFinite(at) && groups.find((g) => g.subject != null&& g.subject === who && g.site === card.site && g.card.kind === card.kind && Math.abs(g.at - at) <= BURST_MS);if (prior) { prior.cards.push(card); continue; }-- // No person behind a run (a Deals poll, a News import): the same kind and site in the window is one card.-- const bulk = !who && BULK_WORD[card.kind] && Number.isFinite(at) && groups.find((g) => g.bulk-- && g.card.kind === card.kind && g.site === card.site && Math.abs(g.at - at) <= BURST_MS);++ // No person behind a run (a Deals poll, a News import): the same kind and site in the same hour is one card.++ const hour = String(card.at || '').slice(0, 13);++ const bulk = !who && BULK_WORD[card.kind] && hour && groups.find((g) => g.bulk && g.card.kind === card.kind && g.site === card.site && g.hour === hour);if (bulk) { bulk.cards.push(card); continue; }-- groups.push({ subject: who, at, site: card.site, card, cards: [card], bulk: !who && BULK_WORD[card.kind] ? true : undefined });++ groups.push({ subject: who, at, site: card.site, card, cards: [card], bulk: !who && BULK_WORD[card.kind] ? true : undefined, hour });}return groups.map((g) => {if (g.workHour !== undefined) {@@@ -163,16 +199,21 @@ function foldCards(cards, now = Date.now()) {site_name: sites.size === 1 ? [...sites][0] : g.card.site_name,};}-- if (g.cards.length < 2) return g.card;-- if (BULK_WORD[g.card.kind]) {++ if (g.bulk) {++ // One row stays itself (its own title) unless the API finds more of its hour; a run is one card per hour.const [one, many] = BULK_WORD[g.card.kind];++ const bulk = { kind: g.card.kind, site: g.site, hour: g.hour, one, many };++ if (g.cards.length < 2) return { ...g.card, bulk };return {...g.card,-- title: `${g.cards.length} new ${g.cards.length === 1 ? one : many}`,++ id: bulkId(g.card.kind, g.site, g.hour),++ bulk,++ title: `${g.cards.length} new ${many}`,group: { count: g.cards.length, items: g.cards.slice(0, 20).map((c) => ({ id: c.id, url: c.url, title: c.title, media: c.media })) },counts: { ...g.card.counts, people: g.cards.length },};}++ if (g.cards.length < 2) return g.card;const [one, many] = BURST_WORD[g.card.kind];const name = (g.card.actor && g.card.actor.name) || 'Someone';return {@@@ -272,4 +313,4 @@ function wallItem(card) {};}--module.exports = { SITE_NAMES, kindOf, cardOf, foldCards, compareRankedCards, rankCards, searchMatches, searchExamples, wallItem, titleKey, clean, https };++module.exports = { bulkId, BULK_WORD, SITE_NAMES, kindOf, cardOf, foldCards, compareRankedCards, rankCards, searchMatches, searchExamples, wallItem, titleKey, clean, https };@@@ -171,4 +171,31 @@ async function listItems(db, { kind = '', site = '', q = '', followed = [], follreturn { rows, hasMore };}--module.exports = { upsertItem, upsertLiveItem, removeItem, removeMissing, followedSubjects, affinityServices, listItems, iso, ms };++/** How many public rows a burst holds: one site's rows of one kind with no person, in one UTC hour ('2026-11-12T17'). */++async function countBulk(db, { kind, site, hour }) {++ if (!/^\d{4}-\d\d-\d\dT\d\d$/.test(String(hour || ''))) return 0;++ const from = new Date(`${hour}:00:00.000Z`).toISOString();++ const to = new Date(Date.parse(from) + 3600_000).toISOString();++ const row = await db.prepare(`SELECT COUNT(*) AS n FROM feed_items WHERE kind = ? AND site = ? AND actor_subject IS NULL++ AND visibility = 'public' AND occurred_at >= ? AND occurred_at < ?`).get(kind, site, from, to);++ return Number(row && row.n) || 0;++}++++/**++ * Where the burst a page ended in begins, when the rest of it is only that burst: the oldest row of one site's run of++ * one kind with no person in the same UTC hour, older than `before`, with no other public row between them. Null when++ * something else sits in that span (a person's post is never skipped).++ */++async function burstStart(db, { kind, site, before }) {++ const at = String(before && before[0] || '');++ const hour = new Date(at).toISOString().slice(0, 13);++ const from = new Date(`${hour}:00:00.000Z`).toISOString();++ const first = await db.prepare(`SELECT MIN(occurred_at) AS at FROM feed_items WHERE kind = ? AND site = ? AND actor_subject IS NULL++ AND visibility = 'public' AND occurred_at >= ? AND occurred_at < ?`).get(kind, site, from, at);++ if (!first || !first.at) return null;++ const other = await db.prepare(`SELECT 1 FROM feed_items WHERE visibility = 'public' AND occurred_at >= ? AND occurred_at < ?++ AND NOT (kind = ? AND site = ? AND actor_subject IS NULL) LIMIT 1`).get(first.at, at, kind, site);++ return other ? null : iso(first.at);++}++++module.exports = { upsertItem, upsertLiveItem, removeItem, removeMissing, followedSubjects, affinityServices, listItems, countBulk, burstStart, iso, ms };@@@ -63,6 +63,23 @@ const envelope = (type, payload, over = {}) => ({assert.strictEqual(deals.length, 1, 'a poll of deals is one card');assert.strictEqual(deals[0].title, '3 new deals');assert.strictEqual(deals[0].group.count, 3);++ assert.match(deals[0].id, /^g:deal:deals:\d{4}-\d\d-\d\dT\d\d$/, 'a burst has one id per site and hour, the same on every page (11-12: three "112 new deals")');++ assert.strictEqual(deals[0].label, undefined, 'named by its site, never "System"');++++ // ── Markdown and relative addresses (11-12: the menu printed "[title](/p/x) ") ──++ const shared = cardOf({ source_service: 'community', source_type: 'thread', source_id: '95', kind: 'post', title: '1000012166',++ text: '[1000012166](/p/lone-path-83) ', url: '/forum/status/x', site: 'community', counts: {}, occurred_at: '2026-11-12T16:00:00Z', origin: 'user' });++ assert.strictEqual(shared.text, '', 'a body that only repeats the title is dropped');++ assert.strictEqual(shared.media.thumb, 'https://openvibe.community/p/lone-path-83/screenshot?w=640', 'its picture, on Community, the small copy');++ assert.strictEqual(shared.url, 'https://openvibe.community/forum/status/x', 'a relative address is on its own site');++ const words = cardOf({ source_service: 'community', source_type: 'thread', source_id: '93', kind: 'post', title: 'UR',++ text: '[UR](https://openvibe.video/c/46)  Shared by **Maticus**', url: 'https://openvibe.community/t/93', site: 'community', counts: {}, occurred_at: '2026-11-12T16:00:00Z' });++ assert.strictEqual(words.text, 'UR Shared by Maticus', 'links keep their words, pictures and marks go');++ assert.strictEqual(words.media.thumb, 'https://openvibe.media/t/clip-46.jpg');++ const paste = cardOf({ source_service: 'community', source_type: 'paste', source_id: 'lone-path-83', kind: 'pic', title: 'A read', text: '# Current read',++ picture_url: 'https://openvibe.media/o/med_01M78B3X3FSAA9V8DRERBV4DTT', url: 'https://openvibe.community/p/lone-path-83', site: 'community', counts: {}, occurred_at: '2026-11-12T16:00:00Z' });++ assert.strictEqual(paste.media.thumb, 'https://openvibe.community/p/lone-path-83/screenshot?w=640', 'a paste\'s picture comes from Community (Media keeps it private), never a 404');++ assert.strictEqual(paste.text, 'Current read', 'a heading mark goes');// ── ranking: live, followed, fresh, then older than a week ─const live = cardOf({ source_service: 'live', source_type: 'stream', source_id: 'l', kind: 'live', title: 'Live', url: 'https://openvibe.live/@alex', site: 'live', site_name: 'Live', actor_subject: B, live_now: true, occurred_at: iso(now - 86400000) });@@@ -107,6 +124,23 @@ const envelope = (type, payload, over = {}) => ({assert.strictEqual(r.status, 401, 'Following needs a session');({ d } = await get('/api/v1/feed/search?q=public'));assert.strictEqual(d.items[0].title, 'Public post');++ // A Deals poll (11-12: the menu showed "112 new deals" three times, one per page): the burst is one card counted++ // whole, and the next page goes past it to the burst before.++ {++ const mid = new Date(Date.now() - 2 * 86400000); mid.setUTCMinutes(30, 0, 0);++ for (let i = 0; i < 30; i++) {++ await store.upsertItem(db, { source_service: 'deals', source_type: 'offer', source_id: `burst${i}`, kind: 'deal', title: `Deal ${i}`, text: '', url: 'https://openvibe.deals/d/x', site: 'deals', site_name: 'Deals', counts: {}, occurred_at: new Date(mid.getTime() - i * 1000).toISOString(), origin: 'system' });++ }++ await store.upsertItem(db, { source_service: 'deals', source_type: 'offer', source_id: 'older', kind: 'deal', title: 'Older deal', text: '', url: 'https://openvibe.deals/d/y', site: 'deals', site_name: 'Deals', counts: {}, occurred_at: new Date(mid.getTime() - 3 * 3600000).toISOString(), origin: 'system' });++ const p1 = await get('/api/v1/feed?site=deals&limit=1');++ assert.strictEqual(p1.d.items.length, 1);++ assert.strictEqual(p1.d.items[0].title, '30 new deals', 'the whole burst, counted once');++ assert.strictEqual(p1.d.items[0].label, undefined, 'named by its site, not "System"');++ assert.ok(p1.d.next_cursor, 'more after it');++ const p2 = await get(`/api/v1/feed?site=deals&limit=1&cursor=${encodeURIComponent(p1.d.next_cursor)}`);++ assert.ok(p2.d.items.length >= 1 && p2.d.items[0].id !== p1.d.items[0].id, 'the next page is not the same burst again');++ assert.strictEqual(p2.d.items[0].title, 'Older deal', 'the deal before it, its own title (one deal is not "1 new deal")');++ }server.close();console.log('feed: mappings, grouping, privacy, ranking and the menu/feed ids pass');})().catch((err) => { console.error(err); process.exit(1); });
@@ -16,7 +16,7 @@
@@ -16,7 +16,7 @@
*/
*/
const express = require('express');
const express = require('express');
const store = require('./store');
const store = require('./store');
-const { cardOf, foldCards, rankCards, searchMatches, searchExamples, wallItem } = require('./card');
+const { cardOf, foldCards, rankCards, searchMatches, searchExamples, wallItem, bulkId, BULK_WORD } = require('./card');
const { createFeedSources } = require('./sources');
const { createFeedSources } = require('./sources');
const { createEventsPull } = require('./events');
const { createEventsPull } = require('./events');
const session = require('../auth/session');
const session = require('../auth/session');
@@ -99,10 +99,42 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publ
@@ -99,10 +99,42 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publ
} catch { /* a card without a count */ }
} catch { /* a card without a count */ }
if (q) cards = cards.filter((c) => searchMatches(c, q));
if (q) cards = cards.filter((c) => searchMatches(c, q));
cards = foldCards(cards, now());
cards = foldCards(cards, now());
+ // A burst's card counts the whole hour, not just the rows this page read (11-12: three "112 new deals").
+ for (const card of cards.filter((c) => c.bulk)) {
+ const b = card.bulk;
+ delete card.bulk;
+ let n = card.group ? card.group.count : 1;
+ try { n = Math.max(n, await store.countBulk(db, b)); } catch { /* the page's own count */ }
+ if (n < 2) continue;
+ if (!card.group) card.group = { count: n, items: [{ id: card.id, url: card.url, title: card.title, media: card.media }] };
+ card.id = bulkId(b.kind, b.site, b.hour);
+ card.title = `${n.toLocaleString('en-US')} new ${b.many}`;
+ card.group.count = n;
+ card.counts = { ...card.counts, people: n };
+ }
cards = rankCards(cards, { followed: new Set(followed), affinity, now: now() });
cards = rankCards(cards, { followed: new Set(followed), affinity, now: now() });
const items = cards.slice(0, want);
const items = cards.slice(0, want);
- const last = read.rows[read.rows.length - 1];
- const next = read.hasMore && last ? encodeCursor([last.occurred_at, last.id]) : null;
+ // The next page starts at the newest row whose card this page did not show (it used to start after every row
+ // read, so the cards past the first `want` were never seen); a row of a shown burst or group counts as shown.
+ const shown = new Set();
+ for (const c of items) { shown.add(c.id); for (const it of (c.group && c.group.items) || []) shown.add(it.id); }
+ const rowShown = (r) => {
+ const hour = r.occurred_at ? new Date(r.occurred_at).toISOString().slice(0, 13) : '';
+ return shown.has(`f:${r.source_service}:${r.source_type}:${r.source_id}`)
+ || (!r.actor_subject && BULK_WORD[r.kind] && shown.has(bulkId(r.kind, r.site, hour)))
+ || (r.kind === 'work' && shown.has(`g:work:${hour}`));
+ };
+ const firstOut = read.rows.findIndex((r) => !rowShown(r));
+ const edge = firstOut > 0 ? read.rows[firstOut - 1] : read.rows[read.rows.length - 1];
+ let next = (firstOut > 0 || read.hasMore) && edge ? encodeCursor([edge.occurred_at, edge.id]) : null;
+ // A page that ends inside a burst (a Deals poll) moves past the rest of it when nothing else happened in that
+ // span, so the next page does not open with the same card again.
+ if (next && edge && !edge.actor_subject && BULK_WORD[edge.kind]) {
+ try {
+ const start = await store.burstStart(db, { kind: edge.kind, site: edge.site, before: [edge.occurred_at, edge.id] });
+ if (start) next = encodeCursor([start, 0]);
+ } catch { /* the plain cursor */ }
+ }
if (wall) return { items: items.map(wallItem), next_cursor: next };
if (wall) return { items: items.map(wallItem), next_cursor: next };
return { items, next_cursor: next };
return { items, next_cursor: next };
}
}
@@ -134,7 +166,7 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publ
@@ -134,7 +166,7 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publ
const followed = viewer && viewer.subject ? await store.followedSubjects(db, viewer.subject) : [];
const followed = viewer && viewer.subject ? await store.followedSubjects(db, viewer.subject) : [];
const read = await store.listItems(db, { q, followed, viewerSubject: viewer && viewer.subject, limit: 400, since: new Date(now() - 365 * 86400000).toISOString() });
const read = await store.listItems(db, { q, followed, viewerSubject: viewer && viewer.subject, limit: 400, since: new Date(now() - 365 * 86400000).toISOString() });
const actors = await actorsFor(db, read.rows);
const actors = await actorsFor(db, read.rows);
- const cards = foldCards(read.rows.map((r) => cardOf(r, actors.get(r.actor_subject) || null)).filter((c) => searchMatches(c, q)), now());
+ const cards = foldCards(read.rows.map((r) => cardOf(r, actors.get(r.actor_subject) || null)).filter((c) => searchMatches(c, q)), now()).map(({ bulk, ...c }) => c);
res.json({ items: rankCards(cards, { followed: new Set(followed), now: now() }).slice(0, want), next_cursor: null });
res.json({ items: rankCards(cards, { followed: new Set(followed), now: now() }).slice(0, want), next_cursor: null });
} catch (err) { next(err); }
} catch (err) { next(err); }
};
};
@@ -22,6 +22,26 @@ const https = (u) => {
@@ -22,6 +22,26 @@ const https = (u) => {
try { const x = new URL(String(u)); return x.protocol === 'https:' ? x.href : (x.protocol === 'http:' ? x.href : null); } catch { return null; }
try { const x = new URL(String(u)); return x.protocol === 'https:' ? x.href : (x.protocol === 'http:' ? x.href : null); } catch { return null; }
};
};
+/** The site a row comes from, as an origin: a relative address in its text or picture is on that site, not on the
+ * page that shows the card (the menu on openvibe.link showed Community's "/p/x" as a broken link and picture). */
+const ORIGINS = { community: 'https://openvibe.community', 'media-hub': 'https://openvibe.video', rundown: 'https://openvibe.live', stream: 'https://openvibe.live' };
+const originOf = (site) => ORIGINS[site] || (/^[a-z][a-z0-9-]{0,30}$/.test(String(site || '')) ? `https://openvibe.${site}` : null);
+const absolute = (u, site) => {
+ const s = String(u || '').trim();
+ if (!s) return null;
+ try { const x = new URL(s, originOf(site) || undefined); return x.protocol === 'https:' || x.protocol === 'http:' ? x.href : null; } catch { return null; }
+};
+const MD_IMAGE = /!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
+const MD_LINK = /\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
+/** Markdown a source left in a card's text, as plain words, and the first picture it named (11-12: the menu printed it). */
+function plainText(text, site) {
+ const raw = String(text == null ? '' : text);
+ let image = null;
+ const noImages = raw.replace(MD_IMAGE, (m, alt, src) => { if (!image) image = absolute(src, site); return ' '; });
+ const words = noImages.replace(MD_LINK, '$1').replace(/(^|\n)[ \t]*(?:#{1,6}|>)[ \t]*/g, '$1').replace(/\*{1,3}|`+|~~/g, '');
+ return { text: words, image };
+}
+
/** What a source row is, as a feed kind. An unknown type is a post, never dropped silently. */
/** What a source row is, as a feed kind. An unknown type is a post, never dropped silently. */
function kindOf(service, type, row = {}) {
function kindOf(service, type, row = {}) {
const s = String(service || '');
const s = String(service || '');
@@ -79,7 +99,7 @@ function cardOf(row, actor = null) {
@@ -79,7 +99,7 @@ function cardOf(row, actor = null) {
kind: row.kind,
kind: row.kind,
url: https(row.url) || String(row.url || ''),
url: https(row.url) || String(row.url || ''),
title: clean(row.title, 300),
title: clean(row.title, 300),
- text: clean(row.text, 280),
+ text: '',
at: row.occurred_at ? new Date(row.occurred_at).toISOString() : null,
at: row.occurred_at ? new Date(row.occurred_at).toISOString() : null,
site: String(row.site || row.source_service || ''),
site: String(row.site || row.source_service || ''),
site_name: String(row.site_name || SITE_NAMES[row.site] || SITE_NAMES[row.source_service] || row.site || ''),
site_name: String(row.site_name || SITE_NAMES[row.site] || SITE_NAMES[row.source_service] || row.site || ''),
@@ -90,11 +110,24 @@ function cardOf(row, actor = null) {
@@ -90,11 +110,24 @@ function cardOf(row, actor = null) {
: null,
: null,
counts: safe,
counts: safe,
};
};
+ // The words as words: Markdown out, relative addresses on the source's own site, and no text that only repeats
+ // the title (a shared post's body is "[its title](its link)"). A shared post's picture becomes the card's picture.
+ const plain = plainText(row.text, card.site);
+ card.text = clean(plain.text, 280);
+ if (card.text && card.text.toLowerCase() === card.title.toLowerCase()) card.text = '';
+ if (!/^https?:/.test(card.url) && card.url) card.url = absolute(card.url, card.site) || card.url;
+ if (!card.media && plain.image) card.media = { thumb: plain.image, ratio: Number.isFinite(ratio) && ratio > 0 ? ratio : 16 / 9 };
+ // A Community paste's picture is served by Community (Media keeps it private to the paste): the small copy, never
+ // Media's original address, which answers 404 to everyone else (the menu's broken picture, 11-12).
+ if (row.source_service === 'community' && row.source_type === 'paste' && card.media && /^https:\/\/openvibe\.media\/o\//.test(card.media.thumb)) {
+ card.media.thumb = `https://openvibe.community/p/${encodeURIComponent(String(row.source_id))}/screenshot?w=640`;
+ }
+ if (card.media && card.media.thumb && /^https:\/\/openvibe\.community\/p\/[^/]+\/screenshot$/.test(card.media.thumb)) card.media.thumb += '?w=640';
if (row.starts_at) card.starts_at = new Date(row.starts_at).toISOString();
if (row.starts_at) card.starts_at = new Date(row.starts_at).toISOString();
if (row.label) card.label = row.label;
if (row.label) card.label = row.label;
// The menu labels AI output and system/aggregate rows; keep the old Pulse's promise that AI is never a person.
// The menu labels AI output and system/aggregate rows; keep the old Pulse's promise that AI is never a person.
if (!card.label && row.origin === 'ai') card.label = 'AI';
if (!card.label && row.origin === 'ai') card.label = 'AI';
- else if (!card.label && row.origin === 'system' && !card.actor) card.label = 'System';
+ else if (!card.label && row.origin === 'system' && !card.actor && !card.site_name) card.label = 'System';
if (row.visibility === 'followers') card.visibility = 'followers';
if (row.visibility === 'followers') card.visibility = 'followers';
if (card.kind === 'live' || card.kind === 'stream') {
if (card.kind === 'live' || card.kind === 'stream') {
card.live_now = row.kind === 'live' && row.live_now === true;
card.live_now = row.kind === 'live' && row.live_now === true;
@@ -117,6 +150,9 @@ const BURST_WORD = { pic: ['photo', 'photos'], clip: ['clip', 'clips'], video: [
@@ -117,6 +150,9 @@ const BURST_WORD = { pic: ['photo', 'photos'], clip: ['clip', 'clips'], video: [
// A poll can deliver forty of the same kind at once with no person behind them (deals, news): one card.
// A poll can deliver forty of the same kind at once with no person behind them (deals, news): one card.
const BULK_WORD = { deal: ['deal', 'deals'], news: ['story', 'stories'], event: ['event', 'events'] };
const BULK_WORD = { deal: ['deal', 'deals'], news: ['story', 'stories'], event: ['event', 'events'] };
+/** The id of one site's run of one bulk kind in one UTC hour ('2026-11-12T17'): the same on every page that reads it. */
+const bulkId = (kind, site, hour) => `g:${kind}:${site}:${hour}`;
+
function titleKey(card) {
function titleKey(card) {
return String(card.title || '').toLowerCase().replace(/\s+/g, ' ').trim();
return String(card.title || '').toLowerCase().replace(/\s+/g, ' ').trim();
}
}
@@ -142,11 +178,11 @@ function foldCards(cards, now = Date.now()) {
@@ -142,11 +178,11 @@ function foldCards(cards, now = Date.now()) {
const prior = BURST_WORD[card.kind] && who && Number.isFinite(at) && groups.find((g) => g.subject != null
const prior = BURST_WORD[card.kind] && who && Number.isFinite(at) && groups.find((g) => g.subject != null
&& g.subject === who && g.site === card.site && g.card.kind === card.kind && Math.abs(g.at - at) <= BURST_MS);
&& g.subject === who && g.site === card.site && g.card.kind === card.kind && Math.abs(g.at - at) <= BURST_MS);
if (prior) { prior.cards.push(card); continue; }
if (prior) { prior.cards.push(card); continue; }
- // No person behind a run (a Deals poll, a News import): the same kind and site in the window is one card.
- const bulk = !who && BULK_WORD[card.kind] && Number.isFinite(at) && groups.find((g) => g.bulk
- && g.card.kind === card.kind && g.site === card.site && Math.abs(g.at - at) <= BURST_MS);
+ // No person behind a run (a Deals poll, a News import): the same kind and site in the same hour is one card.
+ const hour = String(card.at || '').slice(0, 13);
+ const bulk = !who && BULK_WORD[card.kind] && hour && groups.find((g) => g.bulk && g.card.kind === card.kind && g.site === card.site && g.hour === hour);
if (bulk) { bulk.cards.push(card); continue; }
if (bulk) { bulk.cards.push(card); continue; }
- groups.push({ subject: who, at, site: card.site, card, cards: [card], bulk: !who && BULK_WORD[card.kind] ? true : undefined });
+ groups.push({ subject: who, at, site: card.site, card, cards: [card], bulk: !who && BULK_WORD[card.kind] ? true : undefined, hour });
}
}
return groups.map((g) => {
return groups.map((g) => {
if (g.workHour !== undefined) {
if (g.workHour !== undefined) {
@@ -163,16 +199,21 @@ function foldCards(cards, now = Date.now()) {
@@ -163,16 +199,21 @@ function foldCards(cards, now = Date.now()) {
site_name: sites.size === 1 ? [...sites][0] : g.card.site_name,
site_name: sites.size === 1 ? [...sites][0] : g.card.site_name,
};
};
}
}
- if (g.cards.length < 2) return g.card;
- if (BULK_WORD[g.card.kind]) {
+ if (g.bulk) {
+ // One row stays itself (its own title) unless the API finds more of its hour; a run is one card per hour.
const [one, many] = BULK_WORD[g.card.kind];
const [one, many] = BULK_WORD[g.card.kind];
+ const bulk = { kind: g.card.kind, site: g.site, hour: g.hour, one, many };
+ if (g.cards.length < 2) return { ...g.card, bulk };
return {
return {
...g.card,
...g.card,
- title: `${g.cards.length} new ${g.cards.length === 1 ? one : many}`,
+ id: bulkId(g.card.kind, g.site, g.hour),
+ bulk,
+ title: `${g.cards.length} new ${many}`,
group: { count: g.cards.length, items: g.cards.slice(0, 20).map((c) => ({ id: c.id, url: c.url, title: c.title, media: c.media })) },
group: { count: g.cards.length, items: g.cards.slice(0, 20).map((c) => ({ id: c.id, url: c.url, title: c.title, media: c.media })) },
counts: { ...g.card.counts, people: g.cards.length },
counts: { ...g.card.counts, people: g.cards.length },
};
};
}
}
+ if (g.cards.length < 2) return g.card;
const [one, many] = BURST_WORD[g.card.kind];
const [one, many] = BURST_WORD[g.card.kind];
const name = (g.card.actor && g.card.actor.name) || 'Someone';
const name = (g.card.actor && g.card.actor.name) || 'Someone';
return {
return {
@@ -272,4 +313,4 @@ function wallItem(card) {
@@ -272,4 +313,4 @@ function wallItem(card) {
};
};
}
}
-module.exports = { SITE_NAMES, kindOf, cardOf, foldCards, compareRankedCards, rankCards, searchMatches, searchExamples, wallItem, titleKey, clean, https };
+module.exports = { bulkId, BULK_WORD, SITE_NAMES, kindOf, cardOf, foldCards, compareRankedCards, rankCards, searchMatches, searchExamples, wallItem, titleKey, clean, https };
@@ -171,4 +171,31 @@ async function listItems(db, { kind = '', site = '', q = '', followed = [], foll
@@ -171,4 +171,31 @@ async function listItems(db, { kind = '', site = '', q = '', followed = [], foll
return { rows, hasMore };
return { rows, hasMore };
}
}
-module.exports = { upsertItem, upsertLiveItem, removeItem, removeMissing, followedSubjects, affinityServices, listItems, iso, ms };
+/** How many public rows a burst holds: one site's rows of one kind with no person, in one UTC hour ('2026-11-12T17'). */
+async function countBulk(db, { kind, site, hour }) {
+ if (!/^\d{4}-\d\d-\d\dT\d\d$/.test(String(hour || ''))) return 0;
+ const from = new Date(`${hour}:00:00.000Z`).toISOString();
+ const to = new Date(Date.parse(from) + 3600_000).toISOString();
+ const row = await db.prepare(`SELECT COUNT(*) AS n FROM feed_items WHERE kind = ? AND site = ? AND actor_subject IS NULL
+ AND visibility = 'public' AND occurred_at >= ? AND occurred_at < ?`).get(kind, site, from, to);
+ return Number(row && row.n) || 0;
+}
+
+/**
+ * Where the burst a page ended in begins, when the rest of it is only that burst: the oldest row of one site's run of
+ * one kind with no person in the same UTC hour, older than `before`, with no other public row between them. Null when
+ * something else sits in that span (a person's post is never skipped).
+ */
+async function burstStart(db, { kind, site, before }) {
+ const at = String(before && before[0] || '');
+ const hour = new Date(at).toISOString().slice(0, 13);
+ const from = new Date(`${hour}:00:00.000Z`).toISOString();
+ const first = await db.prepare(`SELECT MIN(occurred_at) AS at FROM feed_items WHERE kind = ? AND site = ? AND actor_subject IS NULL
+ AND visibility = 'public' AND occurred_at >= ? AND occurred_at < ?`).get(kind, site, from, at);
+ if (!first || !first.at) return null;
+ const other = await db.prepare(`SELECT 1 FROM feed_items WHERE visibility = 'public' AND occurred_at >= ? AND occurred_at < ?
+ AND NOT (kind = ? AND site = ? AND actor_subject IS NULL) LIMIT 1`).get(first.at, at, kind, site);
+ return other ? null : iso(first.at);
+}
+
+module.exports = { upsertItem, upsertLiveItem, removeItem, removeMissing, followedSubjects, affinityServices, listItems, countBulk, burstStart, iso, ms };
@@ -63,6 +63,23 @@ const envelope = (type, payload, over = {}) => ({
@@ -63,6 +63,23 @@ const envelope = (type, payload, over = {}) => ({
assert.strictEqual(deals.length, 1, 'a poll of deals is one card');
assert.strictEqual(deals.length, 1, 'a poll of deals is one card');
assert.strictEqual(deals[0].title, '3 new deals');
assert.strictEqual(deals[0].title, '3 new deals');
assert.strictEqual(deals[0].group.count, 3);
assert.strictEqual(deals[0].group.count, 3);
+ assert.match(deals[0].id, /^g:deal:deals:\d{4}-\d\d-\d\dT\d\d$/, 'a burst has one id per site and hour, the same on every page (11-12: three "112 new deals")');
+ assert.strictEqual(deals[0].label, undefined, 'named by its site, never "System"');
+
+ // ── Markdown and relative addresses (11-12: the menu printed "[title](/p/x) ") ──
+ const shared = cardOf({ source_service: 'community', source_type: 'thread', source_id: '95', kind: 'post', title: '1000012166',
+ text: '[1000012166](/p/lone-path-83) ', url: '/forum/status/x', site: 'community', counts: {}, occurred_at: '2026-11-12T16:00:00Z', origin: 'user' });
+ assert.strictEqual(shared.text, '', 'a body that only repeats the title is dropped');
+ assert.strictEqual(shared.media.thumb, 'https://openvibe.community/p/lone-path-83/screenshot?w=640', 'its picture, on Community, the small copy');
+ assert.strictEqual(shared.url, 'https://openvibe.community/forum/status/x', 'a relative address is on its own site');
+ const words = cardOf({ source_service: 'community', source_type: 'thread', source_id: '93', kind: 'post', title: 'UR',
+ text: '[UR](https://openvibe.video/c/46)  Shared by **Maticus**', url: 'https://openvibe.community/t/93', site: 'community', counts: {}, occurred_at: '2026-11-12T16:00:00Z' });
+ assert.strictEqual(words.text, 'UR Shared by Maticus', 'links keep their words, pictures and marks go');
+ assert.strictEqual(words.media.thumb, 'https://openvibe.media/t/clip-46.jpg');
+ const paste = cardOf({ source_service: 'community', source_type: 'paste', source_id: 'lone-path-83', kind: 'pic', title: 'A read', text: '# Current read',
+ picture_url: 'https://openvibe.media/o/med_01M78B3X3FSAA9V8DRERBV4DTT', url: 'https://openvibe.community/p/lone-path-83', site: 'community', counts: {}, occurred_at: '2026-11-12T16:00:00Z' });
+ assert.strictEqual(paste.media.thumb, 'https://openvibe.community/p/lone-path-83/screenshot?w=640', 'a paste\'s picture comes from Community (Media keeps it private), never a 404');
+ assert.strictEqual(paste.text, 'Current read', 'a heading mark goes');
// ── ranking: live, followed, fresh, then older than a week ─
// ── ranking: live, followed, fresh, then older than a week ─
const live = cardOf({ source_service: 'live', source_type: 'stream', source_id: 'l', kind: 'live', title: 'Live', url: 'https://openvibe.live/@alex', site: 'live', site_name: 'Live', actor_subject: B, live_now: true, occurred_at: iso(now - 86400000) });
const live = cardOf({ source_service: 'live', source_type: 'stream', source_id: 'l', kind: 'live', title: 'Live', url: 'https://openvibe.live/@alex', site: 'live', site_name: 'Live', actor_subject: B, live_now: true, occurred_at: iso(now - 86400000) });
@@ -107,6 +124,23 @@ const envelope = (type, payload, over = {}) => ({
@@ -107,6 +124,23 @@ const envelope = (type, payload, over = {}) => ({
assert.strictEqual(r.status, 401, 'Following needs a session');
assert.strictEqual(r.status, 401, 'Following needs a session');
({ d } = await get('/api/v1/feed/search?q=public'));
({ d } = await get('/api/v1/feed/search?q=public'));
assert.strictEqual(d.items[0].title, 'Public post');
assert.strictEqual(d.items[0].title, 'Public post');
+ // A Deals poll (11-12: the menu showed "112 new deals" three times, one per page): the burst is one card counted
+ // whole, and the next page goes past it to the burst before.
+ {
+ const mid = new Date(Date.now() - 2 * 86400000); mid.setUTCMinutes(30, 0, 0);
+ for (let i = 0; i < 30; i++) {
+ await store.upsertItem(db, { source_service: 'deals', source_type: 'offer', source_id: `burst${i}`, kind: 'deal', title: `Deal ${i}`, text: '', url: 'https://openvibe.deals/d/x', site: 'deals', site_name: 'Deals', counts: {}, occurred_at: new Date(mid.getTime() - i * 1000).toISOString(), origin: 'system' });
+ }
+ await store.upsertItem(db, { source_service: 'deals', source_type: 'offer', source_id: 'older', kind: 'deal', title: 'Older deal', text: '', url: 'https://openvibe.deals/d/y', site: 'deals', site_name: 'Deals', counts: {}, occurred_at: new Date(mid.getTime() - 3 * 3600000).toISOString(), origin: 'system' });
+ const p1 = await get('/api/v1/feed?site=deals&limit=1');
+ assert.strictEqual(p1.d.items.length, 1);
+ assert.strictEqual(p1.d.items[0].title, '30 new deals', 'the whole burst, counted once');
+ assert.strictEqual(p1.d.items[0].label, undefined, 'named by its site, not "System"');
+ assert.ok(p1.d.next_cursor, 'more after it');
+ const p2 = await get(`/api/v1/feed?site=deals&limit=1&cursor=${encodeURIComponent(p1.d.next_cursor)}`);
+ assert.ok(p2.d.items.length >= 1 && p2.d.items[0].id !== p1.d.items[0].id, 'the next page is not the same burst again');
+ assert.strictEqual(p2.d.items[0].title, 'Older deal', 'the deal before it, its own title (one deal is not "1 new deal")');
+ }
server.close();
server.close();
console.log('feed: mappings, grouping, privacy, ranking and the menu/feed ids pass');
console.log('feed: mappings, grouping, privacy, ranking and the menu/feed ids pass');
})().catch((err) => { console.error(err); process.exit(1); });
})().catch((err) => { console.error(err); process.exit(1); });