openvibers/openvibe.search Commits

A question matches on its content words (the index keeps stop words) (#193)

OP OpenVibers committed 11 days ago · f1ea6f2954 · 3 files, +25 −6

Changes

ddiff --git a/server/api/query.js b/server/api/query.js
iindex abf2d26..f969dac 100644
---- a/server/api/query.js
++++ b/server/api/query.js
@@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');
const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');
const { fuseRrf } = require('../fabric/rrf');
const { createQueryEmbeddingCache } = require('../fabric/embed');
--const { docTier, isThin, dedupeKey, siteHosts, isQuestion } = require('../ranking');
++const { docTier, isThin, dedupeKey, siteHosts, isQuestion, questionMatch } = require('../ranking');
const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;
const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;
@@@ -321,7 +321,8 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
if (facetKeys.length > 5 || facetKeys.some(k => !FACET_KEY_RE.test(k))) throw new QueryError('search.bad_query', 'facets: at most 5 well-formed keys');
const trimmed = text.trim();
-- const exact = trimmed ? toMatch(text, { maxTerms: config.query.maxTerms }) : null;
++ // A question matches on its content words (ranking.js questionMatch) and keeps its answer-first order.
++ const exact = trimmed ? questionMatch(toMatch(text, { maxTerms: config.query.maxTerms })) : null;
const relaxedMatch = exact && exact.words.length > 1 ? { ...exact, any: true } : null;
// A later page of a relaxed answer says so in its cursor (x), and continues that query.
const relaxedCursor = Boolean(relaxedMatch && cursor && cursorFlag(cursor, 'x'));
ddiff --git a/server/ranking.js b/server/ranking.js
iindex f261035..0687622 100644
---- a/server/ranking.js
++++ b/server/ranking.js
@@@ -163,11 +163,25 @@ function dedupeKey(doc) {
// ranked hundreds of VODs titled "… Stream" above Help's "Can I stream without OBS?", which never reached the top 100).
const QUESTION_WORDS = new Set(['how', 'what', 'why', 'when', 'where', 'which', 'who', 'can', 'do', 'does', 'is', 'are', 'should', 'will']);
function isQuestion(match) {
++ if (match && match.question) return true;
return Boolean(match && !match.column && Array.isArray(match.words) && match.words.length >= 2 && QUESTION_WORDS.has(match.words[0]));
}
++// The words a question is made of that say nothing about what it asks. The index uses the 'simple' configuration
++// (no stop words), so "how do I stream" had to find how, do, i and stream all together: long VOD transcripts did,
++// Help's "Can I stream without OBS?" did not (production, 2026-11-02).
++const FUNCTION_WORDS = new Set([...QUESTION_WORDS, 'could', 'would', 'did', 'was', 'were', 'be', 'i', 'me', 'my', 'we', 'our', 'you',
++ 'your', 'it', 'its', 'a', 'an', 'the', 'to', 'of', 'in', 'on', 'for', 'with', 'from', 'and', 'or', 'there', 'this', 'that', 'get', 'make']);
++
++/** A question's match on its content words ("how do I stream" → stream), marked as a question; else the match as it is. */
++function questionMatch(match) {
++ if (!isQuestion(match)) return match;
++ const content = match.words.filter((w) => !FUNCTION_WORDS.has(w));
++ return content.length ? { ...match, words: content, prefix: match.prefix && content[content.length - 1] === match.words[match.words.length - 1], question: true } : match;
++}
++
module.exports = {
-- isQuestion,
++ isQuestion, questionMatch,
SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,
textMatches, fieldTier, docTier, isThin, dedupeKey,
};
ddiff --git a/test/question-candidates.test.js b/test/question-candidates.test.js
iindex 2fdff3e..c6e2939 100644
---- a/test/question-candidates.test.js
++++ b/test/question-candidates.test.js
@@@ -2,8 +2,10 @@
/**
* A question's answer must reach the candidates: on production "how do I stream" matched hundreds of recent VODs titled
* "… Stream" in the title tier, and Help's older "Can I stream without OBS?" never made the top 100 the page ranks.
-- * The engine lifts an article half a tier on a question: above the recordings that share its tier, below the next
-- * tier up. A plain query is unchanged.
++ * The index has no stop words, so a question matches on its content words ("how do I stream" → stream), and the engine
++ * lifts an article half a tier on a question: above the recordings that share its tier, below the next tier up. The
++ * recordings here hold every word of the question in their bodies, as production's transcripts do. A plain query is
++ * unchanged.
*/
const assert = require('assert');
const { boot, request, doc, suite } = require('./helpers');
@@@ -19,7 +21,9 @@ t('boot and seed', async () => {
const recent = new Date(Date.now() - 3600e3).toISOString();
const old = '2026-06-01T00:00:00Z';
for (let i = 1; i <= 8; i++) {
-- await put({ owner: 'live', type: 'vod', id: `vod-${i}`, title: `Late night stream number ${i}`, summary: 'A recorded stream with chat, music and talk about the week and the plans ahead.', canonical_url: `https://openvibe.video/v/${8100 + i}`, published_at: recent, updated_at: recent });
++ // Like production's transcripts: every word of the question is somewhere in the body, so all four words
++ // together find these and the relaxed answer never runs.
++ await put({ owner: 'live', type: 'vod', id: `vod-${i}`, title: `Late night stream number ${i}`, summary: 'A recorded stream with chat, music and talk about the week and the plans ahead.', body: 'so how do i stream this, chat asked, and i said do it like this', canonical_url: `https://openvibe.video/v/${8100 + i}`, published_at: recent, updated_at: recent });
}
await put({ owner: 'help', type: 'article', id: 'help-obs', title: 'Can I stream without OBS?', summary: 'Yes. Go live from your browser on OpenVibe.Live: press Go Live and you are live in seconds, no software.', canonical_url: 'https://openvibe.help/a/openvibe.live/can-i-stream-without-obs', published_at: old, updated_at: old });
await put({ owner: 'wiki', type: 'article', id: 'wiki-body', title: 'Camera notes', summary: 'Lenses and light.', body: 'how to stream with a phone camera', canonical_url: 'https://openvibe.wiki/p/camera-notes', published_at: old, updated_at: old });