openvibers/openvibe.network Commits

Feed: a burst bigger than a page is still one card (#888)

OP OpenVibers committed 14 h ago · a7b991ee94 · 3 files, +46 −15

After #887 production still showed "358 new deals" on every page: a page that ended inside a Deals poll moved past it only when nothing else happened in that hour (something always does), and that cursor named id 0, which the decoder refused, so the next page started over. The cursor now carries the bursts the pages before showed (kind, site, hour; at most eight, dropped once the cursor is past their hour) and the next read leaves their rows out. A test walks 150 deals with a person's post in the middle: one card, and the post is met.

Changes

ddiff --git a/server/feed/api.js b/server/feed/api.js
iindex 0ee8b85d..6bb7ef90 100644
---- a/server/feed/api.js
++++ b/server/feed/api.js
@@@ -24,14 +24,20 @@ const session = require('../auth/session');
  const clean = (v, n) => String(v == null ? '' : v).replace(/[\u0000-\u001f\u007f]+/g, ' ').replace(/\s+/g, ' ').trim().slice(0, n);
  const KINDS = ['post', 'pic', 'clip', 'video', 'live', 'stream', 'rundown', 'quest', 'member', 'chat', 'event', 'wiki', 'update', 'work', 'deal', 'news', 'report', 'poll', 'game', 'meme', 'paste', 'episode', 'space', 'avatar', 'banner', 'follow', 'site'];
  const SITE_RE = /^[a-z0-9][a-z0-9:-]{0,39}$/;
--const encodeCursor = ([at, id]) => Buffer.from(JSON.stringify([new Date(at).toISOString(), Number(id)])).toString('base64url');
++// A cursor is [occurred_at, id] and, from 11-12, the bursts the pages before already showed ("deal|deals|2026-11-12T18"):
++// the next page leaves their rows out, so a Deals poll is one card however many pages it spans.
++const BURST_KEY = /^(deal|news|event)\|[a-z][a-z0-9-]{0,30}\|\d{4}-\d\d-\d\dT\d\d$/;
++const encodeCursor = ([at, id], done = []) => Buffer.from(JSON.stringify([new Date(at).toISOString(), Number(id), ...(done.length ? [done.slice(0, 8)] : [])])).toString('base64url');
  const decodeCursor = (v) => {
--    if (!v || typeof v !== 'string' || v.length > 400) return null;
++    if (!v || typeof v !== 'string' || v.length > 1200) return null;
      try {
--        const [at, id] = JSON.parse(Buffer.from(v, 'base64url').toString('utf8'));
--        return typeof at === 'string' && Number.isSafeInteger(id) && id > 0 ? [at, id] : null;
++        const [at, id, done] = JSON.parse(Buffer.from(v, 'base64url').toString('utf8'));
++        if (typeof at !== 'string' || !Number.isSafeInteger(id) || id <= 0) return null;
++        const kept = Array.isArray(done) ? done.filter((k) => typeof k === 'string' && BURST_KEY.test(k)).slice(0, 8) : [];
++        return Object.assign([at, id], { done: kept });
      } catch { return null; }
  };
++const burstsOf = (keys) => keys.map((k) => { const [kind, site, hour] = k.split('|'); return { kind, site, hour }; });
  const problem = (res, status, code, detail) => res.status(status).json({ type: `https://openvibe.network/problems/${code}`, title: status === 401 ? 'Unauthorized' : 'Bad request', status, code, detail });
  
  /** The person asking, when they carry a live session (Bearer or page cookie); never a service token. */
@@@ -80,7 +86,7 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publ
          const before = decodeCursor(String(req.query.cursor || ''));
  
          const read = await store.listItems(db, {
--            kind, site, q, followed, following, before, viewerSubject: viewer && viewer.subject,
++            kind, site, q, followed, following, before, viewerSubject: viewer && viewer.subject, skipBursts: before ? burstsOf(before.done || []) : [],
              limit: Math.min(400, want * 6 + 40), since: new Date(now() - 60 * 86400000).toISOString(),
          });
          const actors = await actorsFor(db, read.rows);
@@@ -126,15 +132,13 @@ function createFeedApi({ db, config = {}, privateKey = null, issuer = null, publ
          };
          const firstOut = read.rows.findIndex((r) => !rowShown(r));
          const edge = firstOut > 0 ? read.rows[firstOut - 1] : read.rows[read.rows.length - 1];
--        let next = (firstOut > 0 || read.hasMore) && edge ? encodeCursor([edge.occurred_at, edge.id]) : null;
--        // A page that ends inside a burst (a Deals poll) moves past the rest of it when nothing else happened in that
--        // span, so the next page does not open with the same card again.
--        if (next && edge && !edge.actor_subject && BULK_WORD[edge.kind]) {
--            try {
--                const start = await store.burstStart(db, { kind: edge.kind, site: edge.site, before: [edge.occurred_at, edge.id] });
--                if (start) next = encodeCursor([start, 0]);
--            } catch { /* the plain cursor */ }
--        }
++        // The bursts shown so far ride in the cursor while the next page can still meet their rows (an hour that has not
++        // ended before the cursor's time).
++        const edgeAt = edge ? new Date(edge.occurred_at).toISOString() : '';
++        const done = [...new Set([...(before ? before.done || [] : []),
++            ...items.filter((c) => /^g:(deal|news|event):/.test(c.id)).map((c) => c.id.slice(2).replace(/:/, '|').replace(/:(?=\d{4}-)/, '|'))])]
++            .filter((k) => BURST_KEY.test(k) && `${k.split('|')[2]}:00:00.000Z` < edgeAt).slice(-8);
++        const next = (firstOut > 0 || read.hasMore) && edge ? encodeCursor([edge.occurred_at, edge.id], done) : null;
          if (wall) return { items: items.map(wallItem), next_cursor: next };
          return { items, next_cursor: next };
      }
ddiff --git a/server/feed/store.js b/server/feed/store.js
iindex a8b4ba87..44c17a48 100644
---- a/server/feed/store.js
++++ b/server/feed/store.js
@@@ -139,9 +139,16 @@ async function affinityServices(db, userId) {
   *   opts.since                          only this recent (the collectors read the last 7 days)
   * → { rows, hasMore }
   */
--async function listItems(db, { kind = '', site = '', q = '', followed = [], following = false, before = null, viewerSubject = null, limit = 40, since = null } = {}) {
++async function listItems(db, { kind = '', site = '', q = '', followed = [], following = false, before = null, viewerSubject = null, limit = 40, since = null, skipBursts = [] } = {}) {
      const where = [];
      const args = [];
++    // A burst an earlier page showed as one card (its kind, site and UTC hour, no person): its rows are not read again.
++    for (const b of (skipBursts || []).slice(0, 8)) {
++        const from = new Date(`${b.hour}:00:00.000Z`);
++        if (Number.isNaN(from.getTime())) continue;
++        where.push('NOT (i.kind = ? AND i.site = ? AND i.actor_subject IS NULL AND i.occurred_at >= ? AND i.occurred_at < ?)');
++        args.push(b.kind, b.site, from.toISOString(), new Date(from.getTime() + 3600_000).toISOString());
++    }
      if (kind) { where.push('i.kind = ?'); args.push(kind); }
      if (site) { where.push('i.site = ?'); args.push(site); }
      if (q) { where.push('(i.title ILIKE ? OR i.text ILIKE ?)'); args.push(`%${q}%`, `%${q}%`); }
ddiff --git a/test/feed.test.js b/test/feed.test.js
iindex cdc1ab24..1a7e86b8 100644
---- a/test/feed.test.js
++++ b/test/feed.test.js
@@@ -141,6 +141,26 @@ const envelope = (type, payload, over = {}) => ({
          assert.ok(p2.d.items.length >= 1 && p2.d.items[0].id !== p1.d.items[0].id, 'the next page is not the same burst again');
          assert.strictEqual(p2.d.items[0].title, 'Older deal', 'the deal before it, its own title (one deal is not "1 new deal")');
      }
++    // A burst bigger than a page, with a person's post in the middle of it (production, 11-12: "358 new deals" on every
++    // page): the burst is one card across all pages, and the post is still met.
++    {
++        const hourAt = new Date(Date.now() - 4 * 86400000); hourAt.setUTCMinutes(59, 0, 0);
++        for (let i = 0; i < 150; i++) {
++            await store.upsertItem(db, { source_service: 'deals', source_type: 'offer', source_id: `big${i}`, kind: 'deal', title: `Big ${i}`, text: '', url: 'https://openvibe.deals/d/z', site: 'deals', site_name: 'Deals', counts: {}, occurred_at: new Date(hourAt.getTime() - i * 20000).toISOString(), origin: 'system' });
++        }
++        await store.upsertItem(db, { source_service: 'community', source_type: 'thread', source_id: 'mid', kind: 'post', title: 'In the middle', text: 'hello', url: 'https://openvibe.community/forum/status/mid', site: 'community', site_name: 'Community', actor_subject: A, counts: {}, occurred_at: new Date(hourAt.getTime() - 75 * 20000 - 5000).toISOString() });
++        const seen = [];
++        let cursor = '';
++        for (let page = 0; page < 8; page++) {
++            const r = await get(`/api/v1/feed?site=&limit=2${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`);
++            seen.push(...r.d.items.map((x) => x.id));
++            if (!r.d.next_cursor) break;
++            cursor = r.d.next_cursor;
++        }
++        const burst = `g:deal:deals:${hourAt.toISOString().slice(0, 13)}`;
++        assert.strictEqual(seen.filter((id) => id === burst).length, 1, `the burst is one card across the pages (${seen.join(', ')})`);
++        assert.ok(seen.includes('f:community:thread:mid'), 'the post in the middle of the burst is still met');
++    }
      server.close();
      console.log('feed: mappings, grouping, privacy, ranking and the menu/feed ids pass');
  })().catch((err) => { console.error(err); process.exit(1); });