openvibers/openvibe.search Commits
A question's answer reaches the candidates (engine lifts articles half a tier) (#192)
Changes
@@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');const { fuseRrf } = require('../fabric/rrf');const { createQueryEmbeddingCache } = require('../fabric/embed');--const { docTier, isThin, dedupeKey, siteHosts } = require('../ranking');++const { docTier, isThin, dedupeKey, siteHosts, isQuestion } = require('../ranking');const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;@@@ -184,11 +184,7 @@ async function dealsOrder(fused, { match, store }) {// - a title match (3) outranks a summary-only match (2), which outranks a body-only one (1);// - a thin username-titled VOD ranks below a real title+summary inside the same tier.// The fused order is the final tie-break, so relevance still decides among equals.--// A question ("how do I stream", "can I go live without OBS") wants an answer: inside its tier, an article (Help's--// answers, a site's docs) comes before a recording or a post that only shares a word (2026-11-02: "how do I stream"--// ranked four VODs titled "… Stream" above Help's "Can I stream without OBS?").--const QUESTION_WORDS = new Set(['how', 'what', 'why', 'when', 'where', 'which', 'who', 'can', 'do', 'does', 'is', 'are', 'should', 'will']);--const isQuestion = (match) => Boolean(match && match.words.length >= 2 && QUESTION_WORDS.has(match.words[0]));++// A question wants an answer first inside its tier (ranking.js isQuestion; the engine lifts articles the same way).const isAnswer = (doc) => Boolean(doc && doc.type === 'article');async function rankOrder(fused, { match, store }) {@@@ -22,7 +22,7 @@* Nothing outside this file writes SQL against search_fts.*/const { EXPOSURE } = require('../document');--const { fold, words, siteOnlyQuery, siteUrlPattern } = require('../ranking');++const { fold, words, siteOnlyQuery, siteUrlPattern, isQuestion } = require('../ranking');const CONFIG = 'simple';const WEIGHTS = '{0.1, 0.1, 0.4, 1.0}'; // D, C body, B summary, A title: ts_rank weights 1 / 4 / 10@@@ -163,7 +163,10 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})const relevance = `CASE WHEN ${boilerplate} THEN 0 ELSE ${base} END`;// Thin VODs sink below documents with a real title and summary inside the same tier; -2 is// larger than any base score (ts_rank × freshness < 2), so it decides before relevance does.-- const rank = `-(${tier} * 100 + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;++ // A question lifts an article half a tier: above the recordings that share its tier, below the next tier up,++ // so it is among the candidates the page ranks (ranking.js isQuestion).++ const answer = isQuestion(match) ? `CASE WHEN d.type = 'article' THEN 50 ELSE 0 END` : '0';++ const rank = `-(${tier} * 100 + ${answer} + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;const facetNumber = (key) => `(SELECT x.value::float8 FROM doc_facets xWHERE x.rid = d.rid AND x.key = '${key}' AND x.value ~ '^[0-9]+([.][0-9]+)?$'ORDER BY x.value LIMIT 1)`;@@@ -158,7 +158,16 @@ function dedupeKey(doc) {return `${String((doc && doc.owner) || '').toLowerCase()}\u0000${title}`;}++// A question ("how do I stream", "can I go live without OBS") wants an answer: an article (Help's answers, a site's++// docs) comes before a recording or a post that only shares a word, inside its tier (2026-11-02: "how do I stream"++// ranked hundreds of VODs titled "… Stream" above Help's "Can I stream without OBS?", which never reached the top 100).++const QUESTION_WORDS = new Set(['how', 'what', 'why', 'when', 'where', 'which', 'who', 'can', 'do', 'does', 'is', 'are', 'should', 'will']);++function isQuestion(match) {++ return Boolean(match && !match.column && Array.isArray(match.words) && match.words.length >= 2 && QUESTION_WORDS.has(match.words[0]));++}++module.exports = {++ isQuestion,SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,textMatches, fieldTier, docTier, isThin, dedupeKey,};nnew file mode 100644@@@ -0,0 +1,41 @@++'use strict';++/**++ * A question's answer must reach the candidates: on production "how do I stream" matched hundreds of recent VODs titled++ * "… Stream" in the title tier, and Help's older "Can I stream without OBS?" never made the top 100 the page ranks.++ * The engine lifts an article half a tier on a question: above the recordings that share its tier, below the next++ * tier up. A plain query is unchanged.++ */++const assert = require('assert');++const { boot, request, doc, suite } = require('./helpers');++++const t = suite('question candidates');++let svc;++const put = (overrides) => svc.store.apply(doc({ summary: '', body: '', ...overrides }));++const ids = (r) => r.body.results.map((x) => x.id);++const q = (text) => request(svc.base, 'GET', `/api/v1/search?q=${encodeURIComponent(text)}&limit=10`);++++t('boot and seed', async () => {++ svc = await boot();++ const recent = new Date(Date.now() - 3600e3).toISOString();++ const old = '2026-06-01T00:00:00Z';++ for (let i = 1; i <= 8; i++) {++ await put({ owner: 'live', type: 'vod', id: `vod-${i}`, title: `Late night stream number ${i}`, summary: 'A recorded stream with chat, music and talk about the week and the plans ahead.', canonical_url: `https://openvibe.video/v/${8100 + i}`, published_at: recent, updated_at: recent });++ }++ await put({ owner: 'help', type: 'article', id: 'help-obs', title: 'Can I stream without OBS?', summary: 'Yes. Go live from your browser on OpenVibe.Live: press Go Live and you are live in seconds, no software.', canonical_url: 'https://openvibe.help/a/openvibe.live/can-i-stream-without-obs', published_at: old, updated_at: old });++ await put({ owner: 'wiki', type: 'article', id: 'wiki-body', title: 'Camera notes', summary: 'Lenses and light.', body: 'how to stream with a phone camera', canonical_url: 'https://openvibe.wiki/p/camera-notes', published_at: old, updated_at: old });++});++++t('a question puts the answer first, even older than the recordings in its tier', async () => {++ const got = ids(await q('how do I stream'));++ assert.strictEqual(got[0], 'help-obs', `the answer first (${got})`);++ assert.ok(got.indexOf('wiki-body') > got.indexOf('vod-1'), 'an article that only mentions it in its body stays below the title tier');++});++++t('a plain query is unchanged: the recordings first, by freshness', async () => {++ const got = ids(await q('stream'));++ assert.ok(got.indexOf('help-obs') > 0, `not lifted without a question (${got})`);++});++++t('close', async () => { await svc.stop(); });++++t.run();
@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');
@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');
const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');
const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');
const { fuseRrf } = require('../fabric/rrf');
const { fuseRrf } = require('../fabric/rrf');
const { createQueryEmbeddingCache } = require('../fabric/embed');
const { createQueryEmbeddingCache } = require('../fabric/embed');
-const { docTier, isThin, dedupeKey, siteHosts } = require('../ranking');
+const { docTier, isThin, dedupeKey, siteHosts, isQuestion } = require('../ranking');
const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;
const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;
const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;
const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;
@@ -184,11 +184,7 @@ async function dealsOrder(fused, { match, store }) {
@@ -184,11 +184,7 @@ async function dealsOrder(fused, { match, store }) {
// - a title match (3) outranks a summary-only match (2), which outranks a body-only one (1);
// - a title match (3) outranks a summary-only match (2), which outranks a body-only one (1);
// - a thin username-titled VOD ranks below a real title+summary inside the same tier.
// - a thin username-titled VOD ranks below a real title+summary inside the same tier.
// The fused order is the final tie-break, so relevance still decides among equals.
// The fused order is the final tie-break, so relevance still decides among equals.
-// A question ("how do I stream", "can I go live without OBS") wants an answer: inside its tier, an article (Help's
-// answers, a site's docs) comes before a recording or a post that only shares a word (2026-11-02: "how do I stream"
-// ranked four VODs titled "… Stream" above Help's "Can I stream without OBS?").
-const QUESTION_WORDS = new Set(['how', 'what', 'why', 'when', 'where', 'which', 'who', 'can', 'do', 'does', 'is', 'are', 'should', 'will']);
-const isQuestion = (match) => Boolean(match && match.words.length >= 2 && QUESTION_WORDS.has(match.words[0]));
+// A question wants an answer first inside its tier (ranking.js isQuestion; the engine lifts articles the same way).
const isAnswer = (doc) => Boolean(doc && doc.type === 'article');
const isAnswer = (doc) => Boolean(doc && doc.type === 'article');
async function rankOrder(fused, { match, store }) {
async function rankOrder(fused, { match, store }) {
@@ -22,7 +22,7 @@
@@ -22,7 +22,7 @@
* Nothing outside this file writes SQL against search_fts.
* Nothing outside this file writes SQL against search_fts.
*/
*/
const { EXPOSURE } = require('../document');
const { EXPOSURE } = require('../document');
-const { fold, words, siteOnlyQuery, siteUrlPattern } = require('../ranking');
+const { fold, words, siteOnlyQuery, siteUrlPattern, isQuestion } = require('../ranking');
const CONFIG = 'simple';
const CONFIG = 'simple';
const WEIGHTS = '{0.1, 0.1, 0.4, 1.0}'; // D, C body, B summary, A title: ts_rank weights 1 / 4 / 10
const WEIGHTS = '{0.1, 0.1, 0.4, 1.0}'; // D, C body, B summary, A title: ts_rank weights 1 / 4 / 10
@@ -163,7 +163,10 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
@@ -163,7 +163,10 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
const relevance = `CASE WHEN ${boilerplate} THEN 0 ELSE ${base} END`;
const relevance = `CASE WHEN ${boilerplate} THEN 0 ELSE ${base} END`;
// Thin VODs sink below documents with a real title and summary inside the same tier; -2 is
// Thin VODs sink below documents with a real title and summary inside the same tier; -2 is
// larger than any base score (ts_rank × freshness < 2), so it decides before relevance does.
// larger than any base score (ts_rank × freshness < 2), so it decides before relevance does.
- const rank = `-(${tier} * 100 + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;
+ // A question lifts an article half a tier: above the recordings that share its tier, below the next tier up,
+ // so it is among the candidates the page ranks (ranking.js isQuestion).
+ const answer = isQuestion(match) ? `CASE WHEN d.type = 'article' THEN 50 ELSE 0 END` : '0';
+ const rank = `-(${tier} * 100 + ${answer} + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;
const facetNumber = (key) => `(SELECT x.value::float8 FROM doc_facets x
const facetNumber = (key) => `(SELECT x.value::float8 FROM doc_facets x
WHERE x.rid = d.rid AND x.key = '${key}' AND x.value ~ '^[0-9]+([.][0-9]+)?$'
WHERE x.rid = d.rid AND x.key = '${key}' AND x.value ~ '^[0-9]+([.][0-9]+)?$'
ORDER BY x.value LIMIT 1)`;
ORDER BY x.value LIMIT 1)`;
@@ -158,7 +158,16 @@ function dedupeKey(doc) {
@@ -158,7 +158,16 @@ function dedupeKey(doc) {
return `${String((doc && doc.owner) || '').toLowerCase()}\u0000${title}`;
return `${String((doc && doc.owner) || '').toLowerCase()}\u0000${title}`;
}
}
+// A question ("how do I stream", "can I go live without OBS") wants an answer: an article (Help's answers, a site's
+// docs) comes before a recording or a post that only shares a word, inside its tier (2026-11-02: "how do I stream"
+// ranked hundreds of VODs titled "… Stream" above Help's "Can I stream without OBS?", which never reached the top 100).
+const QUESTION_WORDS = new Set(['how', 'what', 'why', 'when', 'where', 'which', 'who', 'can', 'do', 'does', 'is', 'are', 'should', 'will']);
+function isQuestion(match) {
+ return Boolean(match && !match.column && Array.isArray(match.words) && match.words.length >= 2 && QUESTION_WORDS.has(match.words[0]));
+}
+
module.exports = {
module.exports = {
+ isQuestion,
SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,
SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,
textMatches, fieldTier, docTier, isThin, dedupeKey,
textMatches, fieldTier, docTier, isThin, dedupeKey,
};
};
new file mode 100644
new file mode 100644
@@ -0,0 +1,41 @@
@@ -0,0 +1,41 @@
+'use strict';
+/**
+ * A question's answer must reach the candidates: on production "how do I stream" matched hundreds of recent VODs titled
+ * "… Stream" in the title tier, and Help's older "Can I stream without OBS?" never made the top 100 the page ranks.
+ * The engine lifts an article half a tier on a question: above the recordings that share its tier, below the next
+ * tier up. A plain query is unchanged.
+ */
+const assert = require('assert');
+const { boot, request, doc, suite } = require('./helpers');
+
+const t = suite('question candidates');
+let svc;
+const put = (overrides) => svc.store.apply(doc({ summary: '', body: '', ...overrides }));
+const ids = (r) => r.body.results.map((x) => x.id);
+const q = (text) => request(svc.base, 'GET', `/api/v1/search?q=${encodeURIComponent(text)}&limit=10`);
+
+t('boot and seed', async () => {
+ svc = await boot();
+ const recent = new Date(Date.now() - 3600e3).toISOString();
+ const old = '2026-06-01T00:00:00Z';
+ for (let i = 1; i <= 8; i++) {
+ await put({ owner: 'live', type: 'vod', id: `vod-${i}`, title: `Late night stream number ${i}`, summary: 'A recorded stream with chat, music and talk about the week and the plans ahead.', canonical_url: `https://openvibe.video/v/${8100 + i}`, published_at: recent, updated_at: recent });
+ }
+ await put({ owner: 'help', type: 'article', id: 'help-obs', title: 'Can I stream without OBS?', summary: 'Yes. Go live from your browser on OpenVibe.Live: press Go Live and you are live in seconds, no software.', canonical_url: 'https://openvibe.help/a/openvibe.live/can-i-stream-without-obs', published_at: old, updated_at: old });
+ await put({ owner: 'wiki', type: 'article', id: 'wiki-body', title: 'Camera notes', summary: 'Lenses and light.', body: 'how to stream with a phone camera', canonical_url: 'https://openvibe.wiki/p/camera-notes', published_at: old, updated_at: old });
+});
+
+t('a question puts the answer first, even older than the recordings in its tier', async () => {
+ const got = ids(await q('how do I stream'));
+ assert.strictEqual(got[0], 'help-obs', `the answer first (${got})`);
+ assert.ok(got.indexOf('wiki-body') > got.indexOf('vod-1'), 'an article that only mentions it in its body stays below the title tier');
+});
+
+t('a plain query is unchanged: the recordings first, by freshness', async () => {
+ const got = ids(await q('stream'));
+ assert.ok(got.indexOf('help-obs') > 0, `not lifted without a question (${got})`);
+});
+
+t('close', async () => { await svc.stop(); });
+
+t.run();