openvibers/openvibe.search Commits
A question matches on its content words (the index keeps stop words) (#193)
Changes
@@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');const { fuseRrf } = require('../fabric/rrf');const { createQueryEmbeddingCache } = require('../fabric/embed');--const { docTier, isThin, dedupeKey, siteHosts, isQuestion } = require('../ranking');++const { docTier, isThin, dedupeKey, siteHosts, isQuestion, questionMatch } = require('../ranking');const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;@@@ -321,7 +321,8 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddingsif (facetKeys.length > 5 || facetKeys.some(k => !FACET_KEY_RE.test(k))) throw new QueryError('search.bad_query', 'facets: at most 5 well-formed keys');const trimmed = text.trim();-- const exact = trimmed ? toMatch(text, { maxTerms: config.query.maxTerms }) : null;++ // A question matches on its content words (ranking.js questionMatch) and keeps its answer-first order.++ const exact = trimmed ? questionMatch(toMatch(text, { maxTerms: config.query.maxTerms })) : null;const relaxedMatch = exact && exact.words.length > 1 ? { ...exact, any: true } : null;// A later page of a relaxed answer says so in its cursor (x), and continues that query.const relaxedCursor = Boolean(relaxedMatch && cursor && cursorFlag(cursor, 'x'));@@@ -163,11 +163,25 @@ function dedupeKey(doc) {// ranked hundreds of VODs titled "… Stream" above Help's "Can I stream without OBS?", which never reached the top 100).const QUESTION_WORDS = new Set(['how', 'what', 'why', 'when', 'where', 'which', 'who', 'can', 'do', 'does', 'is', 'are', 'should', 'will']);function isQuestion(match) {++ if (match && match.question) return true;return Boolean(match && !match.column && Array.isArray(match.words) && match.words.length >= 2 && QUESTION_WORDS.has(match.words[0]));}++// The words a question is made of that say nothing about what it asks. The index uses the 'simple' configuration++// (no stop words), so "how do I stream" had to find how, do, i and stream all together: long VOD transcripts did,++// Help's "Can I stream without OBS?" did not (production, 2026-11-02).++const FUNCTION_WORDS = new Set([...QUESTION_WORDS, 'could', 'would', 'did', 'was', 'were', 'be', 'i', 'me', 'my', 'we', 'our', 'you',++ 'your', 'it', 'its', 'a', 'an', 'the', 'to', 'of', 'in', 'on', 'for', 'with', 'from', 'and', 'or', 'there', 'this', 'that', 'get', 'make']);++++/** A question's match on its content words ("how do I stream" → stream), marked as a question; else the match as it is. */++function questionMatch(match) {++ if (!isQuestion(match)) return match;++ const content = match.words.filter((w) => !FUNCTION_WORDS.has(w));++ return content.length ? { ...match, words: content, prefix: match.prefix && content[content.length - 1] === match.words[match.words.length - 1], question: true } : match;++}++module.exports = {-- isQuestion,++ isQuestion, questionMatch,SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,textMatches, fieldTier, docTier, isThin, dedupeKey,};@@@ -2,8 +2,10 @@/*** A question's answer must reach the candidates: on production "how do I stream" matched hundreds of recent VODs titled* "… Stream" in the title tier, and Help's older "Can I stream without OBS?" never made the top 100 the page ranks.-- * The engine lifts an article half a tier on a question: above the recordings that share its tier, below the next-- * tier up. A plain query is unchanged.++ * The index has no stop words, so a question matches on its content words ("how do I stream" → stream), and the engine++ * lifts an article half a tier on a question: above the recordings that share its tier, below the next tier up. The++ * recordings here hold every word of the question in their bodies, as production's transcripts do. A plain query is++ * unchanged.*/const assert = require('assert');const { boot, request, doc, suite } = require('./helpers');@@@ -19,7 +21,9 @@ t('boot and seed', async () => {const recent = new Date(Date.now() - 3600e3).toISOString();const old = '2026-06-01T00:00:00Z';for (let i = 1; i <= 8; i++) {-- await put({ owner: 'live', type: 'vod', id: `vod-${i}`, title: `Late night stream number ${i}`, summary: 'A recorded stream with chat, music and talk about the week and the plans ahead.', canonical_url: `https://openvibe.video/v/${8100 + i}`, published_at: recent, updated_at: recent });++ // Like production's transcripts: every word of the question is somewhere in the body, so all four words++ // together find these and the relaxed answer never runs.++ await put({ owner: 'live', type: 'vod', id: `vod-${i}`, title: `Late night stream number ${i}`, summary: 'A recorded stream with chat, music and talk about the week and the plans ahead.', body: 'so how do i stream this, chat asked, and i said do it like this', canonical_url: `https://openvibe.video/v/${8100 + i}`, published_at: recent, updated_at: recent });}await put({ owner: 'help', type: 'article', id: 'help-obs', title: 'Can I stream without OBS?', summary: 'Yes. Go live from your browser on OpenVibe.Live: press Go Live and you are live in seconds, no software.', canonical_url: 'https://openvibe.help/a/openvibe.live/can-i-stream-without-obs', published_at: old, updated_at: old });await put({ owner: 'wiki', type: 'article', id: 'wiki-body', title: 'Camera notes', summary: 'Lenses and light.', body: 'how to stream with a phone camera', canonical_url: 'https://openvibe.wiki/p/camera-notes', published_at: old, updated_at: old });
@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');
@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');
const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');
const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');
const { fuseRrf } = require('../fabric/rrf');
const { fuseRrf } = require('../fabric/rrf');
const { createQueryEmbeddingCache } = require('../fabric/embed');
const { createQueryEmbeddingCache } = require('../fabric/embed');
-const { docTier, isThin, dedupeKey, siteHosts, isQuestion } = require('../ranking');
+const { docTier, isThin, dedupeKey, siteHosts, isQuestion, questionMatch } = require('../ranking');
const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;
const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;
const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;
const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;
@@ -321,7 +321,8 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
@@ -321,7 +321,8 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
if (facetKeys.length > 5 || facetKeys.some(k => !FACET_KEY_RE.test(k))) throw new QueryError('search.bad_query', 'facets: at most 5 well-formed keys');
if (facetKeys.length > 5 || facetKeys.some(k => !FACET_KEY_RE.test(k))) throw new QueryError('search.bad_query', 'facets: at most 5 well-formed keys');
const trimmed = text.trim();
const trimmed = text.trim();
- const exact = trimmed ? toMatch(text, { maxTerms: config.query.maxTerms }) : null;
+ // A question matches on its content words (ranking.js questionMatch) and keeps its answer-first order.
+ const exact = trimmed ? questionMatch(toMatch(text, { maxTerms: config.query.maxTerms })) : null;
const relaxedMatch = exact && exact.words.length > 1 ? { ...exact, any: true } : null;
const relaxedMatch = exact && exact.words.length > 1 ? { ...exact, any: true } : null;
// A later page of a relaxed answer says so in its cursor (x), and continues that query.
// A later page of a relaxed answer says so in its cursor (x), and continues that query.
const relaxedCursor = Boolean(relaxedMatch && cursor && cursorFlag(cursor, 'x'));
const relaxedCursor = Boolean(relaxedMatch && cursor && cursorFlag(cursor, 'x'));
@@ -163,11 +163,25 @@ function dedupeKey(doc) {
@@ -163,11 +163,25 @@ function dedupeKey(doc) {
// ranked hundreds of VODs titled "… Stream" above Help's "Can I stream without OBS?", which never reached the top 100).
// ranked hundreds of VODs titled "… Stream" above Help's "Can I stream without OBS?", which never reached the top 100).
const QUESTION_WORDS = new Set(['how', 'what', 'why', 'when', 'where', 'which', 'who', 'can', 'do', 'does', 'is', 'are', 'should', 'will']);
const QUESTION_WORDS = new Set(['how', 'what', 'why', 'when', 'where', 'which', 'who', 'can', 'do', 'does', 'is', 'are', 'should', 'will']);
function isQuestion(match) {
function isQuestion(match) {
+ if (match && match.question) return true;
return Boolean(match && !match.column && Array.isArray(match.words) && match.words.length >= 2 && QUESTION_WORDS.has(match.words[0]));
return Boolean(match && !match.column && Array.isArray(match.words) && match.words.length >= 2 && QUESTION_WORDS.has(match.words[0]));
}
}
+// The words a question is made of that say nothing about what it asks. The index uses the 'simple' configuration
+// (no stop words), so "how do I stream" had to find how, do, i and stream all together: long VOD transcripts did,
+// Help's "Can I stream without OBS?" did not (production, 2026-11-02).
+const FUNCTION_WORDS = new Set([...QUESTION_WORDS, 'could', 'would', 'did', 'was', 'were', 'be', 'i', 'me', 'my', 'we', 'our', 'you',
+ 'your', 'it', 'its', 'a', 'an', 'the', 'to', 'of', 'in', 'on', 'for', 'with', 'from', 'and', 'or', 'there', 'this', 'that', 'get', 'make']);
+
+/** A question's match on its content words ("how do I stream" → stream), marked as a question; else the match as it is. */
+function questionMatch(match) {
+ if (!isQuestion(match)) return match;
+ const content = match.words.filter((w) => !FUNCTION_WORDS.has(w));
+ return content.length ? { ...match, words: content, prefix: match.prefix && content[content.length - 1] === match.words[match.words.length - 1], question: true } : match;
+}
+
module.exports = {
module.exports = {
- isQuestion,
+ isQuestion, questionMatch,
SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,
SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,
textMatches, fieldTier, docTier, isThin, dedupeKey,
textMatches, fieldTier, docTier, isThin, dedupeKey,
};
};
@@ -2,8 +2,10 @@
@@ -2,8 +2,10 @@
/**
/**
* A question's answer must reach the candidates: on production "how do I stream" matched hundreds of recent VODs titled
* A question's answer must reach the candidates: on production "how do I stream" matched hundreds of recent VODs titled
* "… Stream" in the title tier, and Help's older "Can I stream without OBS?" never made the top 100 the page ranks.
* "… Stream" in the title tier, and Help's older "Can I stream without OBS?" never made the top 100 the page ranks.
- * The engine lifts an article half a tier on a question: above the recordings that share its tier, below the next
- * tier up. A plain query is unchanged.
+ * The index has no stop words, so a question matches on its content words ("how do I stream" → stream), and the engine
+ * lifts an article half a tier on a question: above the recordings that share its tier, below the next tier up. The
+ * recordings here hold every word of the question in their bodies, as production's transcripts do. A plain query is
+ * unchanged.
*/
*/
const assert = require('assert');
const assert = require('assert');
const { boot, request, doc, suite } = require('./helpers');
const { boot, request, doc, suite } = require('./helpers');
@@ -19,7 +21,9 @@ t('boot and seed', async () => {
@@ -19,7 +21,9 @@ t('boot and seed', async () => {
const recent = new Date(Date.now() - 3600e3).toISOString();
const recent = new Date(Date.now() - 3600e3).toISOString();
const old = '2026-06-01T00:00:00Z';
const old = '2026-06-01T00:00:00Z';
for (let i = 1; i <= 8; i++) {
for (let i = 1; i <= 8; i++) {
- await put({ owner: 'live', type: 'vod', id: `vod-${i}`, title: `Late night stream number ${i}`, summary: 'A recorded stream with chat, music and talk about the week and the plans ahead.', canonical_url: `https://openvibe.video/v/${8100 + i}`, published_at: recent, updated_at: recent });
+ // Like production's transcripts: every word of the question is somewhere in the body, so all four words
+ // together find these and the relaxed answer never runs.
+ await put({ owner: 'live', type: 'vod', id: `vod-${i}`, title: `Late night stream number ${i}`, summary: 'A recorded stream with chat, music and talk about the week and the plans ahead.', body: 'so how do i stream this, chat asked, and i said do it like this', canonical_url: `https://openvibe.video/v/${8100 + i}`, published_at: recent, updated_at: recent });
}
}
await put({ owner: 'help', type: 'article', id: 'help-obs', title: 'Can I stream without OBS?', summary: 'Yes. Go live from your browser on OpenVibe.Live: press Go Live and you are live in seconds, no software.', canonical_url: 'https://openvibe.help/a/openvibe.live/can-i-stream-without-obs', published_at: old, updated_at: old });
await put({ owner: 'help', type: 'article', id: 'help-obs', title: 'Can I stream without OBS?', summary: 'Yes. Go live from your browser on OpenVibe.Live: press Go Live and you are live in seconds, no software.', canonical_url: 'https://openvibe.help/a/openvibe.live/can-i-stream-without-obs', published_at: old, updated_at: old });
await put({ owner: 'wiki', type: 'article', id: 'wiki-body', title: 'Camera notes', summary: 'Lenses and light.', body: 'how to stream with a phone camera', canonical_url: 'https://openvibe.wiki/p/camera-notes', published_at: old, updated_at: old });
await put({ owner: 'wiki', type: 'article', id: 'wiki-body', title: 'Camera notes', summary: 'Lenses and light.', body: 'how to stream with a phone camera', canonical_url: 'https://openvibe.wiki/p/camera-notes', published_at: old, updated_at: old });