openvibers/openvibe.network Commits

Feed: one card per burst, words not Markdown, pictures that load, no lost cards (#887)

OP OpenVibers committed 15 h ago · 52cad23ee2 · 4 files, +149 −15

From the owner's phone (11-12):
- "112 new deals" three times: each page read 112 rows of one Deals poll and folded them into a card of its own. A burst (deals, news, events with no person) is now one card per site and hour with one id on every page, counted whole from the store, and a page that ends inside a burst moves past the rest of it when nothing else happened in that span.
- A page kept its first N cards after ranking but moved its cursor past every row it read, so the rest were never shown; the next page now starts at the newest row whose card was not shown.
- Community's shared posts printed their Markdown ("[title](/p/x) ![title](/p/x/screenshot)"): text is plain words, the first picture is the card's picture, relative addresses are on the source's own site, and a body that only repeats the title is dropped.
- A paste's picture pointed at Media's private original (404): it is Community's small copy (/p/<slug>/screenshot?w=640).
- A site's system row is named by its site (Deals), not "System".

Changes

ddiff --git a/server/feed/api.js b/server/feed/api.js
iindex 4dde0021..0ee8b85d 100644
---- a/server/feed/api.js
++++ b/server/feed/api.js
@@@ -16,7 +16,7 @@
*/
const express = require('express');
const store = require('./store');
--const { cardOf, foldCards, rankCards, searchMatches, searchExamples, wallItem } = require('./card');
++const { cardOf, foldCards, rankCards, searchMatches, searchExamples, wallItem, bulkId, BULK_WORD } = require('./card');
const { createFeedSources } = require('./sources');
const { createEventsPull } = require('./events');
const session = require('../auth/session');
@@@ -99,10 +99,42 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publ
} catch { /* a card without a count */ }
if (q) cards = cards.filter((c) => searchMatches(c, q));
cards = foldCards(cards, now());
++ // A burst's card counts the whole hour, not just the rows this page read (11-12: three "112 new deals").
++ for (const card of cards.filter((c) => c.bulk)) {
++ const b = card.bulk;
++ delete card.bulk;
++ let n = card.group ? card.group.count : 1;
++ try { n = Math.max(n, await store.countBulk(db, b)); } catch { /* the page's own count */ }
++ if (n < 2) continue;
++ if (!card.group) card.group = { count: n, items: [{ id: card.id, url: card.url, title: card.title, media: card.media }] };
++ card.id = bulkId(b.kind, b.site, b.hour);
++ card.title = `${n.toLocaleString('en-US')} new ${b.many}`;
++ card.group.count = n;
++ card.counts = { ...card.counts, people: n };
++ }
cards = rankCards(cards, { followed: new Set(followed), affinity, now: now() });
const items = cards.slice(0, want);
-- const last = read.rows[read.rows.length - 1];
-- const next = read.hasMore && last ? encodeCursor([last.occurred_at, last.id]) : null;
++ // The next page starts at the newest row whose card this page did not show (it used to start after every row
++ // read, so the cards past the first `want` were never seen); a row of a shown burst or group counts as shown.
++ const shown = new Set();
++ for (const c of items) { shown.add(c.id); for (const it of (c.group && c.group.items) || []) shown.add(it.id); }
++ const rowShown = (r) => {
++ const hour = r.occurred_at ? new Date(r.occurred_at).toISOString().slice(0, 13) : '';
++ return shown.has(`f:${r.source_service}:${r.source_type}:${r.source_id}`)
++ || (!r.actor_subject && BULK_WORD[r.kind] && shown.has(bulkId(r.kind, r.site, hour)))
++ || (r.kind === 'work' && shown.has(`g:work:${hour}`));
++ };
++ const firstOut = read.rows.findIndex((r) => !rowShown(r));
++ const edge = firstOut > 0 ? read.rows[firstOut - 1] : read.rows[read.rows.length - 1];
++ let next = (firstOut > 0 || read.hasMore) && edge ? encodeCursor([edge.occurred_at, edge.id]) : null;
++ // A page that ends inside a burst (a Deals poll) moves past the rest of it when nothing else happened in that
++ // span, so the next page does not open with the same card again.
++ if (next && edge && !edge.actor_subject && BULK_WORD[edge.kind]) {
++ try {
++ const start = await store.burstStart(db, { kind: edge.kind, site: edge.site, before: [edge.occurred_at, edge.id] });
++ if (start) next = encodeCursor([start, 0]);
++ } catch { /* the plain cursor */ }
++ }
if (wall) return { items: items.map(wallItem), next_cursor: next };
return { items, next_cursor: next };
}
@@@ -134,7 +166,7 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publ
const followed = viewer && viewer.subject ? await store.followedSubjects(db, viewer.subject) : [];
const read = await store.listItems(db, { q, followed, viewerSubject: viewer && viewer.subject, limit: 400, since: new Date(now() - 365 * 86400000).toISOString() });
const actors = await actorsFor(db, read.rows);
-- const cards = foldCards(read.rows.map((r) => cardOf(r, actors.get(r.actor_subject) || null)).filter((c) => searchMatches(c, q)), now());
++ const cards = foldCards(read.rows.map((r) => cardOf(r, actors.get(r.actor_subject) || null)).filter((c) => searchMatches(c, q)), now()).map(({ bulk, ...c }) => c);
res.json({ items: rankCards(cards, { followed: new Set(followed), now: now() }).slice(0, want), next_cursor: null });
} catch (err) { next(err); }
};
ddiff --git a/server/feed/card.js b/server/feed/card.js
iindex b6941e3a..81865e1b 100644
---- a/server/feed/card.js
++++ b/server/feed/card.js
@@@ -22,6 +22,26 @@ const https = (u) => {
try { const x = new URL(String(u)); return x.protocol === 'https:' ? x.href : (x.protocol === 'http:' ? x.href : null); } catch { return null; }
};
++/** The site a row comes from, as an origin: a relative address in its text or picture is on that site, not on the
++ * page that shows the card (the menu on openvibe.link showed Community's "/p/x" as a broken link and picture). */
++const ORIGINS = { community: 'https://openvibe.community', 'media-hub': 'https://openvibe.video', rundown: 'https://openvibe.live', stream: 'https://openvibe.live' };
++const originOf = (site) => ORIGINS[site] || (/^[a-z][a-z0-9-]{0,30}$/.test(String(site || '')) ? `https://openvibe.${site}` : null);
++const absolute = (u, site) => {
++ const s = String(u || '').trim();
++ if (!s) return null;
++ try { const x = new URL(s, originOf(site) || undefined); return x.protocol === 'https:' || x.protocol === 'http:' ? x.href : null; } catch { return null; }
++};
++const MD_IMAGE = /!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
++const MD_LINK = /\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
++/** Markdown a source left in a card's text, as plain words, and the first picture it named (11-12: the menu printed it). */
++function plainText(text, site) {
++ const raw = String(text == null ? '' : text);
++ let image = null;
++ const noImages = raw.replace(MD_IMAGE, (m, alt, src) => { if (!image) image = absolute(src, site); return ' '; });
++ const words = noImages.replace(MD_LINK, '$1').replace(/(^|\n)[ \t]*(?:#{1,6}|>)[ \t]*/g, '$1').replace(/\*{1,3}|`+|~~/g, '');
++ return { text: words, image };
++}
++
/** What a source row is, as a feed kind. An unknown type is a post, never dropped silently. */
function kindOf(service, type, row = {}) {
const s = String(service || '');
@@@ -79,7 +99,7 @@ function cardOf(row, actor = null) {
kind: row.kind,
url: https(row.url) || String(row.url || ''),
title: clean(row.title, 300),
-- text: clean(row.text, 280),
++ text: '',
at: row.occurred_at ? new Date(row.occurred_at).toISOString() : null,
site: String(row.site || row.source_service || ''),
site_name: String(row.site_name || SITE_NAMES[row.site] || SITE_NAMES[row.source_service] || row.site || ''),
@@@ -90,11 +110,24 @@ function cardOf(row, actor = null) {
: null,
counts: safe,
};
++ // The words as words: Markdown out, relative addresses on the source's own site, and no text that only repeats
++ // the title (a shared post's body is "[its title](its link)"). A shared post's picture becomes the card's picture.
++ const plain = plainText(row.text, card.site);
++ card.text = clean(plain.text, 280);
++ if (card.text && card.text.toLowerCase() === card.title.toLowerCase()) card.text = '';
++ if (!/^https?:/.test(card.url) && card.url) card.url = absolute(card.url, card.site) || card.url;
++ if (!card.media && plain.image) card.media = { thumb: plain.image, ratio: Number.isFinite(ratio) && ratio > 0 ? ratio : 16 / 9 };
++ // A Community paste's picture is served by Community (Media keeps it private to the paste): the small copy, never
++ // Media's original address, which answers 404 to everyone else (the menu's broken picture, 11-12).
++ if (row.source_service === 'community' && row.source_type === 'paste' && card.media && /^https:\/\/openvibe\.media\/o\//.test(card.media.thumb)) {
++ card.media.thumb = `https://openvibe.community/p/${encodeURIComponent(String(row.source_id))}/screenshot?w=640`;
++ }
++ if (card.media && card.media.thumb && /^https:\/\/openvibe\.community\/p\/[^/]+\/screenshot$/.test(card.media.thumb)) card.media.thumb += '?w=640';
if (row.starts_at) card.starts_at = new Date(row.starts_at).toISOString();
if (row.label) card.label = row.label;
// The menu labels AI output and system/aggregate rows; keep the old Pulse's promise that AI is never a person.
if (!card.label && row.origin === 'ai') card.label = 'AI';
-- else if (!card.label && row.origin === 'system' && !card.actor) card.label = 'System';
++ else if (!card.label && row.origin === 'system' && !card.actor && !card.site_name) card.label = 'System';
if (row.visibility === 'followers') card.visibility = 'followers';
if (card.kind === 'live' || card.kind === 'stream') {
card.live_now = row.kind === 'live' && row.live_now === true;
@@@ -117,6 +150,9 @@ const BURST_WORD = { pic: ['photo', 'photos'], clip: ['clip', 'clips'], video: [
// A poll can deliver forty of the same kind at once with no person behind them (deals, news): one card.
const BULK_WORD = { deal: ['deal', 'deals'], news: ['story', 'stories'], event: ['event', 'events'] };
++/** The id of one site's run of one bulk kind in one UTC hour ('2026-11-12T17'): the same on every page that reads it. */
++const bulkId = (kind, site, hour) => `g:${kind}:${site}:${hour}`;
++
function titleKey(card) {
return String(card.title || '').toLowerCase().replace(/\s+/g, ' ').trim();
}
@@@ -142,11 +178,11 @@ function foldCards(cards, now = Date.now()) {
const prior = BURST_WORD[card.kind] && who && Number.isFinite(at) && groups.find((g) => g.subject != null
&& g.subject === who && g.site === card.site && g.card.kind === card.kind && Math.abs(g.at - at) <= BURST_MS);
if (prior) { prior.cards.push(card); continue; }
-- // No person behind a run (a Deals poll, a News import): the same kind and site in the window is one card.
-- const bulk = !who && BULK_WORD[card.kind] && Number.isFinite(at) && groups.find((g) => g.bulk
-- && g.card.kind === card.kind && g.site === card.site && Math.abs(g.at - at) <= BURST_MS);
++ // No person behind a run (a Deals poll, a News import): the same kind and site in the same hour is one card.
++ const hour = String(card.at || '').slice(0, 13);
++ const bulk = !who && BULK_WORD[card.kind] && hour && groups.find((g) => g.bulk && g.card.kind === card.kind && g.site === card.site && g.hour === hour);
if (bulk) { bulk.cards.push(card); continue; }
-- groups.push({ subject: who, at, site: card.site, card, cards: [card], bulk: !who && BULK_WORD[card.kind] ? true : undefined });
++ groups.push({ subject: who, at, site: card.site, card, cards: [card], bulk: !who && BULK_WORD[card.kind] ? true : undefined, hour });
}
return groups.map((g) => {
if (g.workHour !== undefined) {
@@@ -163,16 +199,21 @@ function foldCards(cards, now = Date.now()) {
site_name: sites.size === 1 ? [...sites][0] : g.card.site_name,
};
}
-- if (g.cards.length < 2) return g.card;
-- if (BULK_WORD[g.card.kind]) {
++ if (g.bulk) {
++ // One row stays itself (its own title) unless the API finds more of its hour; a run is one card per hour.
const [one, many] = BULK_WORD[g.card.kind];
++ const bulk = { kind: g.card.kind, site: g.site, hour: g.hour, one, many };
++ if (g.cards.length < 2) return { ...g.card, bulk };
return {
...g.card,
-- title: `${g.cards.length} new ${g.cards.length === 1 ? one : many}`,
++ id: bulkId(g.card.kind, g.site, g.hour),
++ bulk,
++ title: `${g.cards.length} new ${many}`,
group: { count: g.cards.length, items: g.cards.slice(0, 20).map((c) => ({ id: c.id, url: c.url, title: c.title, media: c.media })) },
counts: { ...g.card.counts, people: g.cards.length },
};
}
++ if (g.cards.length < 2) return g.card;
const [one, many] = BURST_WORD[g.card.kind];
const name = (g.card.actor && g.card.actor.name) || 'Someone';
return {
@@@ -272,4 +313,4 @@ function wallItem(card) {
};
}
--module.exports = { SITE_NAMES, kindOf, cardOf, foldCards, compareRankedCards, rankCards, searchMatches, searchExamples, wallItem, titleKey, clean, https };
++module.exports = { bulkId, BULK_WORD, SITE_NAMES, kindOf, cardOf, foldCards, compareRankedCards, rankCards, searchMatches, searchExamples, wallItem, titleKey, clean, https };
ddiff --git a/server/feed/store.js b/server/feed/store.js
iindex 0d33688b..a8b4ba87 100644
---- a/server/feed/store.js
++++ b/server/feed/store.js
@@@ -171,4 +171,31 @@ async function listItems(db, { kind = '', site = '', q = '', followed = [], foll
return { rows, hasMore };
}
--module.exports = { upsertItem, upsertLiveItem, removeItem, removeMissing, followedSubjects, affinityServices, listItems, iso, ms };
++/** How many public rows a burst holds: one site's rows of one kind with no person, in one UTC hour ('2026-11-12T17'). */
++async function countBulk(db, { kind, site, hour }) {
++ if (!/^\d{4}-\d\d-\d\dT\d\d$/.test(String(hour || ''))) return 0;
++ const from = new Date(`${hour}:00:00.000Z`).toISOString();
++ const to = new Date(Date.parse(from) + 3600_000).toISOString();
++ const row = await db.prepare(`SELECT COUNT(*) AS n FROM feed_items WHERE kind = ? AND site = ? AND actor_subject IS NULL
++ AND visibility = 'public' AND occurred_at >= ? AND occurred_at < ?`).get(kind, site, from, to);
++ return Number(row && row.n) || 0;
++}
++
++/**
++ * Where the burst a page ended in begins, when the rest of it is only that burst: the oldest row of one site's run of
++ * one kind with no person in the same UTC hour, older than `before`, with no other public row between them. Null when
++ * something else sits in that span (a person's post is never skipped).
++ */
++async function burstStart(db, { kind, site, before }) {
++ const at = String(before && before[0] || '');
++ const hour = new Date(at).toISOString().slice(0, 13);
++ const from = new Date(`${hour}:00:00.000Z`).toISOString();
++ const first = await db.prepare(`SELECT MIN(occurred_at) AS at FROM feed_items WHERE kind = ? AND site = ? AND actor_subject IS NULL
++ AND visibility = 'public' AND occurred_at >= ? AND occurred_at < ?`).get(kind, site, from, at);
++ if (!first || !first.at) return null;
++ const other = await db.prepare(`SELECT 1 FROM feed_items WHERE visibility = 'public' AND occurred_at >= ? AND occurred_at < ?
++ AND NOT (kind = ? AND site = ? AND actor_subject IS NULL) LIMIT 1`).get(first.at, at, kind, site);
++ return other ? null : iso(first.at);
++}
++
++module.exports = { upsertItem, upsertLiveItem, removeItem, removeMissing, followedSubjects, affinityServices, listItems, countBulk, burstStart, iso, ms };
ddiff --git a/test/feed.test.js b/test/feed.test.js
iindex e41d4420..cdc1ab24 100644
---- a/test/feed.test.js
++++ b/test/feed.test.js
@@@ -63,6 +63,23 @@ const envelope = (type, payload, over = {}) => ({
assert.strictEqual(deals.length, 1, 'a poll of deals is one card');
assert.strictEqual(deals[0].title, '3 new deals');
assert.strictEqual(deals[0].group.count, 3);
++ assert.match(deals[0].id, /^g:deal:deals:\d{4}-\d\d-\d\dT\d\d$/, 'a burst has one id per site and hour, the same on every page (11-12: three "112 new deals")');
++ assert.strictEqual(deals[0].label, undefined, 'named by its site, never "System"');
++
++ // ── Markdown and relative addresses (11-12: the menu printed "[title](/p/x) ![title](/p/x/screenshot)") ──
++ const shared = cardOf({ source_service: 'community', source_type: 'thread', source_id: '95', kind: 'post', title: '1000012166',
++ text: '[1000012166](/p/lone-path-83) ![1000012166](/p/lone-path-83/screenshot)', url: '/forum/status/x', site: 'community', counts: {}, occurred_at: '2026-11-12T16:00:00Z', origin: 'user' });
++ assert.strictEqual(shared.text, '', 'a body that only repeats the title is dropped');
++ assert.strictEqual(shared.media.thumb, 'https://openvibe.community/p/lone-path-83/screenshot?w=640', 'its picture, on Community, the small copy');
++ assert.strictEqual(shared.url, 'https://openvibe.community/forum/status/x', 'a relative address is on its own site');
++ const words = cardOf({ source_service: 'community', source_type: 'thread', source_id: '93', kind: 'post', title: 'UR',
++ text: '[UR](https://openvibe.video/c/46) ![UR](https://openvibe.media/t/clip-46.jpg) Shared by **Maticus**', url: 'https://openvibe.community/t/93', site: 'community', counts: {}, occurred_at: '2026-11-12T16:00:00Z' });
++ assert.strictEqual(words.text, 'UR Shared by Maticus', 'links keep their words, pictures and marks go');
++ assert.strictEqual(words.media.thumb, 'https://openvibe.media/t/clip-46.jpg');
++ const paste = cardOf({ source_service: 'community', source_type: 'paste', source_id: 'lone-path-83', kind: 'pic', title: 'A read', text: '# Current read',
++ picture_url: 'https://openvibe.media/o/med_01M78B3X3FSAA9V8DRERBV4DTT', url: 'https://openvibe.community/p/lone-path-83', site: 'community', counts: {}, occurred_at: '2026-11-12T16:00:00Z' });
++ assert.strictEqual(paste.media.thumb, 'https://openvibe.community/p/lone-path-83/screenshot?w=640', 'a paste\'s picture comes from Community (Media keeps it private), never a 404');
++ assert.strictEqual(paste.text, 'Current read', 'a heading mark goes');
// ── ranking: live, followed, fresh, then older than a week ─
const live = cardOf({ source_service: 'live', source_type: 'stream', source_id: 'l', kind: 'live', title: 'Live', url: 'https://openvibe.live/@alex', site: 'live', site_name: 'Live', actor_subject: B, live_now: true, occurred_at: iso(now - 86400000) });
@@@ -107,6 +124,23 @@ const envelope = (type, payload, over = {}) => ({
assert.strictEqual(r.status, 401, 'Following needs a session');
({ d } = await get('/api/v1/feed/search?q=public'));
assert.strictEqual(d.items[0].title, 'Public post');
++ // A Deals poll (11-12: the menu showed "112 new deals" three times, one per page): the burst is one card counted
++ // whole, and the next page goes past it to the burst before.
++ {
++ const mid = new Date(Date.now() - 2 * 86400000); mid.setUTCMinutes(30, 0, 0);
++ for (let i = 0; i < 30; i++) {
++ await store.upsertItem(db, { source_service: 'deals', source_type: 'offer', source_id: `burst${i}`, kind: 'deal', title: `Deal ${i}`, text: '', url: 'https://openvibe.deals/d/x', site: 'deals', site_name: 'Deals', counts: {}, occurred_at: new Date(mid.getTime() - i * 1000).toISOString(), origin: 'system' });
++ }
++ await store.upsertItem(db, { source_service: 'deals', source_type: 'offer', source_id: 'older', kind: 'deal', title: 'Older deal', text: '', url: 'https://openvibe.deals/d/y', site: 'deals', site_name: 'Deals', counts: {}, occurred_at: new Date(mid.getTime() - 3 * 3600000).toISOString(), origin: 'system' });
++ const p1 = await get('/api/v1/feed?site=deals&limit=1');
++ assert.strictEqual(p1.d.items.length, 1);
++ assert.strictEqual(p1.d.items[0].title, '30 new deals', 'the whole burst, counted once');
++ assert.strictEqual(p1.d.items[0].label, undefined, 'named by its site, not "System"');
++ assert.ok(p1.d.next_cursor, 'more after it');
++ const p2 = await get(`/api/v1/feed?site=deals&limit=1&cursor=${encodeURIComponent(p1.d.next_cursor)}`);
++ assert.ok(p2.d.items.length >= 1 && p2.d.items[0].id !== p1.d.items[0].id, 'the next page is not the same burst again');
++ assert.strictEqual(p2.d.items[0].title, 'Older deal', 'the deal before it, its own title (one deal is not "1 new deal")');
++ }
server.close();
console.log('feed: mappings, grouping, privacy, ranking and the menu/feed ids pass');
})().catch((err) => { console.error(err); process.exit(1); });