openvibers/openvibe.search Commits

Search breaks same-type ties on Live activity, then recency (#177)

OP OpenVibers committed 12 days ago · 140bcc2979 · 6 files, +116 −27

A rank tie is now broken by the importance signal the index already holds (a live channel, then followers; a VOD's duration), then by documents.sort_at (updated_at, else published_at). The keyword cursor carries the whole sort key. Channels are no longer thin and a match only in Live's boilerplate summary ties inside its field tier, because that sentence matches every channel and the length-normalized ts_rank made the busiest channel score lowest. Tested with seeded channels (live before followed before followerless, title still first) and equal VODs (longer first).

Changes

ddiff --git a/README.md b/README.md
iindex 8f0e015..ba49a9d 100644
---- a/README.md
++++ b/README.md
@@@ -116,6 +116,17 @@ a summary-only match outranks a body-only one. Within a tier, a thin document (a
title with no real summary: the recorded-stream VODs) ranks below one with a real title and summary, and
one page keeps one result per title and owner.
++A tie in `score` is broken by Live's activity, then by recency (`documents.sort_at`, i.e. `updated_at`,
++else `published_at`): a channel that is live now comes before one that is not, a channel with more
++followers before one with fewer, and a VOD with a longer duration before a shorter one (Search is not
++sent views). A channel is never "thin": its handle title and service summary are normal, so the thin
++sink stays for the VODs it was written for. A Live channel whose only match is the service sentence
++"<name> streams on OpenVibe.Live." is tied with every other such channel inside its field tier: the
++sentence matches all of them, and the length-normalized `ts_rank` would otherwise rank a busier channel
++lower because its body holds more past stream titles. A channel whose *title* matches still comes first,
++like any other title match. Only Live publishes an activity signal; for other owners the tie-break is
++just recency. `test/ranking.test.js` seeds tied channels for both rules.
++
`age_days` counts from `published_at`, else `updated_at` (an edit does not make an old document
new); a future date counts as today and a missing one gets no boost. With the defaults a document
published today counts ×2.0, 30 days old ×1.5, 60 days ×1.25, 90 days ×1.125 and a year old ×1.0002:
ddiff --git a/docs/adr-engine.md b/docs/adr-engine.md
iindex ad0b822..4c7c45f 100644
---- a/docs/adr-engine.md
++++ b/docs/adr-engine.md
@@@ -47,7 +47,11 @@ There is no corpus to measure: no product indexes anything today.
title/summary/body weights 10/4/1 times a freshness boost ([README "Ranking"](../README.md#ranking)),
returned as minus the score so ascending keyset cursors work. The owner and type are not text
fields, thin username-titled VODs sink inside a tier, and one page keeps one result per title and
-- owner.
++ owner. A score tie is broken by Live's activity (live now, then followers; a VOD's duration), then
++ by `documents.sort_at`, and the cursor carries that whole key. Channels are excluded from the thin
++ sink and a boilerplate-only channel match is tied inside its field tier, because the service
++ sentence matches every channel and a length-normalized `ts_rank` would otherwise reward the channel
++ with the least stream history.
- Text is folded (diacritics removed) and split into runs of letters and digits before the
`simple` configuration sees it, so tokens are predictable across languages (Japanese/Chinese get
no word segmentation; a later engine or ICU tokenizer fixes that) and a query and the index
ddiff --git a/server/api/query.js b/server/api/query.js
iindex 92bb5a1..bab1259 100644
---- a/server/api/query.js
++++ b/server/api/query.js
@@@ -81,7 +81,10 @@ function decodeCursor(s, hash) {
// f: a hybrid page is an offset into the fused list (the keyword and vector ranks are recomputed for
// each page, so the fused order is stable; `i` carries the last rid for the shared cursor shape).
if (c.k === 'f' && Number.isInteger(c.o) && c.o >= 0) return { offset: c.o, rid: c.i, now: Number.isFinite(c.n) ? c.n : null };
-- if (c.k === 'r' && typeof c.r === 'number') return { rank: c.r, rid: c.i, now: Number.isFinite(c.n) ? c.n : null };
++ // A keyword page cursor carries the whole sort key: rank, importance, recency, rid.
++ if (c.k === 'r' && typeof c.r === 'number' && typeof c.m === 'number' && typeof c.s === 'string') {
++ return { rank: c.r, importance: c.m, recency: c.s, rid: c.i, now: Number.isFinite(c.n) ? c.n : null };
++ }
if (c.k === 't' && typeof c.t === 'string') return { sort_at: c.t, rid: c.i };
throw new QueryError('search.bad_cursor', 'cursor is malformed');
}
@@@ -198,7 +201,7 @@ async function rankOrder(fused, { match, store }) {
* reciprocal rank fusion (k = 60). The keyword list still supplies snippets, filters, facet counts and
* the ACL: the vector list is public-only, and every fused hit is re-checked by canView. When vectors
* are not available, AI fails or the query is too short, the answer is exactly today's keyword page and
-- * `explain` says which reason applied. Keyword paging keeps its rank/rid cursor; a hybrid page carries
++ * `explain` says which reason applied. Keyword paging keeps its rank/importance/recency/rid cursor; a hybrid page carries
* an offset into the fused list. `ranks` on each hit are 1-based positions in the two lists (null when
* the hit is absent from one).
*/
@@@ -309,7 +312,7 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
seen.add(key);
results.push(projection(found.doc, found.row, h.snip, { keyword: position, vector: null }));
}
-- next = page.next ? encodeCursor({ k: 'r', r: page.next.rank, i: page.next.rid, n: refNow, h: hash }) : null;
++ next = page.next ? encodeCursor({ k: 'r', r: page.next.rank, m: page.next.importance, s: page.next.recency, i: page.next.rid, n: refNow, h: hash }) : null;
explain = { ranking: 'keyword', reason: state.reason };
}
} else {
ddiff --git a/server/engine/pg.js b/server/engine/pg.js
iindex 0ec27b0..d101925 100644
---- a/server/engine/pg.js
++++ b/server/engine/pg.js
@@@ -119,12 +119,20 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
}
/**
-- * The visible, matching set as (rid, rank) rows. match = toMatch()'s query. rank is minus the
-- * tier (site 10 + title 3 / summary 2 / body 1) and ts_rank times the freshness boost at
-- * reference time `now` (ms), so lower ranks first and pages of one query (which carry their
-- * first page's `now` in the cursor) rank against the same clock. The owner and type are not in
-- * the tsvector: a word that only equals an owner never matches. Rows whose owner and title are
++ * The visible, matching set as (rid, rank, importance, recency) rows. match = toMatch()'s query.
++ * rank is minus the tier (site 10 + title 3 / summary 2 / body 1) and ts_rank times the freshness
++ * boost at reference time `now` (ms), so lower ranks first and pages of one query (which carry
++ * their first page's `now` in the cursor) rank against the same clock. The owner and type are not
++ * in the tsvector: a word that only equals an owner never matches. Rows whose owner and title are
* the same keep only the best-ranked one.
++ *
++ * A rank tie between records is broken by `importance`, then by `recency` (documents.sort_at =
++ * updated_at, else published_at). Only Live publishes a signal, in facets the index already holds:
++ * a live channel, then one with more followers (`followers`); for a VOD, its duration (Search is
++ * not sent views). Every channel in the index has streamed at least once, because Live only
++ * publishes a channel after its first stream. Other documents tie at importance 0, so their only
++ * new tie-break is recency. `recency` is empty when freshness (weight 0) is off, which keeps that
++ * mode's old index-order tie-break.
*/
function matchedSet({ match, filters, viewer, now = Date.now() }, params) {
params.match = tsquery(match);
@@@ -143,11 +151,32 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
const site = `(@site_re::text <> '' AND d.canonical_url ~* @site_re)`;
const field = `CASE WHEN ${title} > 0 THEN 3 WHEN ${summary} > 0 THEN 2 WHEN ${body} > 0 THEN 1 ELSE 0 END`;
const tier = `(CASE WHEN ${site} THEN 10 ELSE 0 END + ${field})`;
-- const thin = `(char_length(btrim(f.title)) <= 24 AND char_length(btrim(f.summary)) <= 80)`;
++ // A channel's title is its handle and its summary the service sentence; "thin" is normal for a
++ // channel, not a quality signal, so the thin sink stays for the documents it was written for.
++ const thin = `(d.type <> 'channel' AND char_length(btrim(f.title)) <= 24 AND char_length(btrim(f.summary)) <= 80)`;
++ // Every channel's summary is Live's one sentence, "<name> streams on OpenVibe.Live."; matching
++ // it says nothing about this channel. The body is the last stream titles, so the length-
++ // normalized ts_rank makes a busier channel score *lower*. Inside the field tier those channels
++ // are tied on relevance, and importance/recency decide. A channel whose title matches keeps
++ // its score: that is a real match.
++ const boilerplate = `(d.owner = 'live' AND d.type = 'channel' AND ${title} = 0 AND f.summary ~ ' streams on OpenVibe[.]Live[.]$')`;
++ const relevance = `CASE WHEN ${boilerplate} THEN 0 ELSE ${base} END`;
// Thin VODs sink below documents with a real title and summary inside the same tier; -2 is
// larger than any base score (ts_rank × freshness < 2), so it decides before relevance does.
-- const rank = `-(${tier} * 100 + (${base}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;
-- const select = `SELECT d.rid AS rid, d.owner AS owner, f.title AS title, ${rank} AS rank`;
++ const rank = `-(${tier} * 100 + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;
++ const facetNumber = (key) => `(SELECT x.value::float8 FROM doc_facets x
++ WHERE x.rid = d.rid AND x.key = '${key}' AND x.value ~ '^[0-9]+([.][0-9]+)?$'
++ ORDER BY x.value LIMIT 1)`;
++ const importance = `CASE
++ WHEN d.owner = 'live' AND d.type = 'channel' THEN
++ (CASE WHEN EXISTS (SELECT 1 FROM doc_facets x WHERE x.rid = d.rid AND x.key = 'live' AND x.value = 'true')
++ THEN 1000000 ELSE 0 END)
++ + LEAST(COALESCE(${facetNumber('followers')}, 0), 999999)
++ WHEN d.owner = 'live' AND d.type = 'vod' THEN LEAST(COALESCE(${facetNumber('duration_seconds')}, 0), 999999)
++ ELSE 0 END`;
++ const recency = `CASE WHEN @r_weight::float8 = 0 THEN '' ELSE d.sort_at END`;
++ const select = `SELECT d.rid AS rid, d.owner AS owner, f.title AS title, ${rank} AS rank,
++ ${importance} AS importance, ${recency} AS recency`;
const branches = [`${select}
FROM search_fts f JOIN documents d ON d.rid = f.rid
WHERE f.audience = ${EXPOSURE.public_listed} AND f.tsv @@ ${q}
@@@ -174,9 +203,10 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
AND d.exposure = ${EXPOSURE.restricted} AND d.deleted = 0${where} AND ${acl}`);
}
}
-- return `SELECT rid, rank FROM (
-- SELECT m.rid, m.rank,
-- ROW_NUMBER() OVER (PARTITION BY m.owner, lower(btrim(m.title)) ORDER BY m.rank, m.rid) AS rn
++ return `SELECT rid, rank, importance, recency FROM (
++ SELECT m.rid, m.rank, m.importance, m.recency,
++ ROW_NUMBER() OVER (PARTITION BY m.owner, lower(btrim(m.title))
++ ORDER BY m.rank, m.importance DESC, m.recency DESC, m.rid) AS rn
FROM (${branches.join('\nUNION ALL\n')}) m
) x WHERE x.rn = 1`;
}
@@@ -194,29 +224,35 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
// ── Queries ──────────────────────────────────────────────
/**
-- * Ranked full-text search. after = { rank, rid } keyset cursor. The snippet is computed for the page's rows only:
-- * the summary and body when they match, else the title.
++ * Ranked full-text search. after = { rank, importance, recency, rid } keyset cursor. The snippet is
++ * computed for the page's rows only: the summary and body when they match, else the title.
*/
async function search({ match, filters = {}, viewer, limit, after, now = Date.now() }) {
const params = { limit: limit + 1 };
-- let sql = `SELECT rid, rank FROM (${matchedSet({ match, filters, viewer, now }, params)}) m`;
++ let sql = `SELECT rid, rank, importance, recency FROM (${matchedSet({ match, filters, viewer, now }, params)}) m`;
if (after) {
-- sql += ' WHERE (rank > @c_rank::float8 OR (rank = @c_rank::float8 AND rid > @c_rid::bigint))';
++ sql += ` WHERE (rank > @c_rank::float8
++ OR (rank = @c_rank::float8 AND importance < @c_importance::float8)
++ OR (rank = @c_rank::float8 AND importance = @c_importance::float8 AND recency < @c_recency::text)
++ OR (rank = @c_rank::float8 AND importance = @c_importance::float8 AND recency = @c_recency::text AND rid > @c_rid::bigint))`;
params.c_rank = after.rank;
++ params.c_importance = after.importance;
++ params.c_recency = after.recency;
params.c_rid = after.rid;
}
-- sql += ' ORDER BY rank, rid LIMIT @limit';
-- const page = `SELECT p.rid, p.rank,
++ sql += ' ORDER BY rank, importance DESC, recency DESC, rid LIMIT @limit';
++ const page = `SELECT p.rid, p.rank, p.importance, p.recency,
CASE WHEN to_tsvector('${CONFIG}', f.summary || ' ' || f.body) @@ to_tsquery('${CONFIG}', @match)
THEN ts_headline('${CONFIG}', f.summary || ' ' || f.body, to_tsquery('${CONFIG}', @match), @hl)
ELSE ts_headline('${CONFIG}', f.title, to_tsquery('${CONFIG}', @match), @hl) END AS snip
-- FROM (${sql}) p JOIN search_fts f ON f.rid = p.rid ORDER BY p.rank, p.rid`;
++ FROM (${sql}) p JOIN search_fts f ON f.rid = p.rid
++ ORDER BY p.rank, p.importance DESC, p.recency DESC, p.rid`;
params.hl = HEADLINE;
const rows = await db.prepare(page).all(params);
const more = rows.length > limit;
const hits = rows.slice(0, limit);
const last = hits[hits.length - 1];
-- return { hits, next: more && last ? { rank: last.rank, rid: last.rid } : null };
++ return { hits, next: more && last ? { rank: last.rank, importance: last.importance, recency: last.recency, rid: last.rid } : null };
}
/** Newest first, no text query. after = { sort_at, rid }. */
@@@ -255,7 +291,8 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
/** Title prefix suggestions over the visible set. */
async function suggest({ match, filters = {}, viewer, limit }) {
const params = { limit };
-- return await db.prepare(`SELECT rid, rank FROM (${matchedSet({ match, filters, viewer }, params)}) m ORDER BY rank, rid LIMIT @limit`).all(params);
++ return await db.prepare(`SELECT rid, rank FROM (${matchedSet({ match, filters, viewer }, params)}) m
++ ORDER BY rank, importance DESC, recency DESC, rid LIMIT @limit`).all(params);
}
async function counts() {
ddiff --git a/server/ranking.js b/server/ranking.js
iindex d4efca4..c01fd6d 100644
---- a/server/ranking.js
++++ b/server/ranking.js
@@@ -139,8 +139,11 @@ function docTier(doc, match, hosts = siteHosts(match)) {
* A thin document: a username or a very short title with no real summary ("aaooo.ee", "Omar",
* "Seattle Maybe 🤔" — a recorded stream whose summary is the site's sentence). It ranks below a
* document with a real title and summary inside the same tier; the field and site tiers are unchanged.
++ * A Live channel is never thin: its title is the handle and its summary the service sentence, and
++ * Search orders channels by their activity signal instead (server/engine/pg.js).
*/
function isThin(doc) {
++ if (doc && doc.type === 'channel') return false;
const title = String((doc && doc.title) || '').trim();
const summary = String((doc && doc.summary) || '').trim();
if (!title) return true;
ddiff --git a/test/ranking.test.js b/test/ranking.test.js
iindex e48c698..7d40952 100644
---- a/test/ranking.test.js
++++ b/test/ranking.test.js
@@@ -39,6 +39,21 @@ t('boot and seed', async () => {
await put({ owner: 'help', type: 'article', id: 'help-go-live', title: 'Can I go live without OBS?', summary: 'Yes. Go live straight from your browser, camera or screen on OpenVibe.Live, with no software to install.', canonical_url: 'https://openvibe.help/a/openvibe.live/can-i-go-live-without-obs' });
await put({ owner: 'live', type: 'channel', id: 'channel-arman', title: 'Arman (@Arman)', summary: 'Arman streams on OpenVibe.Live.', canonical_url: 'https://openvibe.live/@Arman' });
++ // Live channels carry only the activity fields the index has: live now, followers. Their text is
++ // the same service sentence, so they tie inside the tier; the signal, then updated_at, decides.
++ const channel = (id, title, facets, updated_at) => put({
++ owner: 'live', type: 'channel', id, title, summary: `${title.split(' ')[0]} streams on OpenVibe.Live.`,
++ canonical_url: `https://openvibe.live/@${id}`, facets, updated_at,
++ });
++ await channel('channel-fans', 'Bea (@bea)', { category: 'irl', live: false, followers: 12 }, '2026-10-29T00:00:00Z');
++ await channel('channel-live', 'Cy (@cy)', { category: 'irl', live: true, followers: 0 }, '2026-10-28T00:00:00Z');
++ await channel('channel-idle', 'Di (@di)', { category: 'irl', live: false, followers: 0 }, '2026-10-27T00:00:00Z');
++ // A channel whose title says what was asked for outranks all the boilerplate-only ones.
++ await put({ owner: 'live', type: 'channel', id: 'channel-title', title: 'OpenVibe Live Now', summary: 'A real description of this channel.', canonical_url: 'https://openvibe.live/@now', facets: { category: 'irl', live: false, followers: 0 }, updated_at: '2026-10-26T00:00:00Z' });
++
++ // Equal VODs (same title, summary and body) carry their only difference in duration_seconds.
++ await put({ owner: 'live', type: 'vod', id: 'vod-meteor-short', title: 'Meteor run', summary: 'A recorded stream.', body: 'meteor', canonical_url: 'https://openvibe.video/v/8011', facets: { duration_seconds: 60 } });
++ await put({ owner: 'live', type: 'vod', id: 'vod-meteor-long', title: 'Meteor pass', summary: 'A recorded stream.', body: 'meteor', canonical_url: 'https://openvibe.video/v/8012', facets: { duration_seconds: 7200 } });
// Thin VODs, two of each title: the page keeps one.
await put({ owner: 'live', type: 'vod', id: 'vod-omar-a', title: 'Omar', summary: 'Recorded live stream by omar on OpenVibe.Live.', canonical_url: 'https://openvibe.video/v/9003' });
await put({ owner: 'live', type: 'vod', id: 'vod-omar-b', title: 'Omar', summary: 'Recorded live stream by omar on OpenVibe.Live.', canonical_url: 'https://openvibe.video/v/9004' });
@@@ -71,8 +86,8 @@ t('a site query puts that site\'s own records first', async () => {
const r = await request(svc.base, 'GET', '/api/v1/search?q=openvibe%20live&limit=10');
assert.strictEqual(r.status, 200, r.text);
const first = r.body.results[0];
-- assert.ok(['help-live-default', 'help-stream-obs', 'help-go-live', 'channel-arman'].includes(first.id), `first is a Live record: ${first.id}`);
-- const site = ['help-live-default', 'help-stream-obs', 'help-go-live', 'channel-arman'];
++ assert.ok(['help-live-default', 'help-stream-obs', 'help-go-live', 'channel-arman', 'channel-title'].includes(first.id), `first is a Live record: ${first.id}`);
++ const site = ['help-live-default', 'help-stream-obs', 'help-go-live', 'channel-arman', 'channel-title'];
const lastSite = Math.max(...site.map((id) => indexOf(r, id)).filter((i) => i >= 0));
const firstVod = Math.min(...['vod-omar-a', 'vod-seattle-a', 'vod-real'].map((id) => indexOf(r, id)).filter((i) => i >= 0));
assert.ok(lastSite < firstVod, 'every Live record precedes the first VOD');
@@@ -108,8 +123,24 @@ t('a thin username title loses the score tie to a real title and summary', async
assert.strictEqual(isThin(real.doc), false);
});
++t('equal VODs order by duration, the importance field the index holds', async () => {
++ const r = await request(svc.base, 'GET', '/api/v1/search?q=meteor&limit=10');
++ assert.deepStrictEqual(ids(r), ['vod-meteor-long', 'vod-meteor-short'], 'the longer recording first');
++});
++
++t('tied channels order by activity (live, then followers), then by updated_at', async () => {
++ const r = await request(svc.base, 'GET', '/api/v1/search?q=openvibe%20live&type=channel&limit=10');
++ assert.strictEqual(r.status, 200, r.text);
++ const order = ids(r);
++ assert.strictEqual(order[0], 'channel-title', 'a title match is still text first');
++ assert.ok(indexOf(r, 'channel-live') < indexOf(r, 'channel-fans'), 'a live channel before a followed one');
++ assert.ok(indexOf(r, 'channel-fans') < indexOf(r, 'channel-idle'), 'a followed channel before a followerless one');
++ // Equal signal (both 0): updated_at decides, not the published_at they share.
++ assert.ok(indexOf(r, 'channel-arman') < indexOf(r, 'channel-idle'), 'the newer channel first when the signal ties');
++});
++
t('one page keeps one result per title and owner', async () => {
-- const r = await request(svc.base, 'GET', '/api/v1/search?q=openvibe%20live&limit=10');
++ const r = await request(svc.base, 'GET', '/api/v1/search?q=openvibe%20live&limit=20');
const keys = r.body.results.map(dedupeKey);
assert.strictEqual(new Set(keys).size, keys.length, 'no repeated title+owner');
assert.strictEqual(r.body.results.filter((x) => x.title === 'Omar').length, 1);