openvibers/openvibe.search Commits
Inside a tier, a title holding more of the query's words ranks first (#199)
Changes
@@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');const { fuseRrf } = require('../fabric/rrf');const { createQueryEmbeddingCache } = require('../fabric/embed');--const { docTier, isThin, dedupeKey, siteHosts, isQuestion, questionMatch, contentWords, coverage } = require('../ranking');++const { docTier, isThin, dedupeKey, siteHosts, isQuestion, questionMatch, contentWords, coverage, titleWords } = require('../ranking');const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;@@@ -197,10 +197,10 @@ async function rankOrder(fused, { match, store }) {.filter((hit) => rows.get(hit.rid)).map((hit, i) => {const found = rows.get(hit.rid);-- return { hit, found, i, cover: coverage(found.doc, match), tier: docTier(found.doc, match, hosts), thin: isThin(found.doc), answer: ask && isAnswer(found.doc) ? 1 : 0 };++ return { hit, found, i, cover: coverage(found.doc, match), tier: docTier(found.doc, match, hosts), words: match.words.length > 1 ? titleWords(found.doc, match) : 0, thin: isThin(found.doc), answer: ask && isAnswer(found.doc) ? 1 : 0 };})// A relaxed query: how much of it a document holds comes first (rare words weigh more), as in the engine.-- .sort((a, b) => b.cover - a.cover || b.tier - a.tier || b.answer - a.answer || (a.thin === b.thin ? 0 : a.thin ? 1 : -1) || a.i - b.i)++ .sort((a, b) => b.cover - a.cover || b.tier - a.tier || b.answer - a.answer || b.words - a.words || (a.thin === b.thin ? 0 : a.thin ? 1 : -1) || a.i - b.i).map(({ hit, found }) => ({ ...hit, found }));}@@@ -22,7 +22,7 @@* Nothing outside this file writes SQL against search_fts.*/const { EXPOSURE } = require('../document');--const { fold, words, siteOnlyQuery, siteUrlPattern, isQuestion, FIELD_FACTORS } = require('../ranking');++const { fold, words, siteOnlyQuery, siteUrlPattern, isQuestion, FIELD_FACTORS, TITLE_WORDS_MAX } = require('../ranking');const FIELD = Object.fromEntries(FIELD_FACTORS);const CONFIG = 'simple';@@@ -179,7 +179,13 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})WHEN f.tsv @@ to_tsquery('${CONFIG}', @cwb_${i}) THEN ${wt * FIELD.summary}WHEN f.tsv @@ to_tsquery('${CONFIG}', @cw_${i}) THEN ${wt * FIELD.body} ELSE 0 END`;}).join(' + ')})::numeric, 1)::float8` : '0';-- const rank = `-(${cover} * 10000 + ${tier} * 100 + ${answer} + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;++ // Inside a tier, a title holding more of the query's words first (ranking.js titleWords): 10 a word, at most++ // TITLE_WORDS_MAX, under a question's 50 and over relevance (< 2).++ const titleCount = match.words.length > 1 ? `LEAST(${TITLE_WORDS_MAX}, ${match.words.map((_, i) => {++ params[`tw_${i}`] = tsTerm(match, i, 'A');++ return `(CASE WHEN f.tsv @@ to_tsquery('${CONFIG}', @tw_${i}) THEN 1 ELSE 0 END)`;++ }).join(' + ')})` : '0';++ const rank = `-(${cover} * 10000 + ${tier} * 100 + ${answer} + ${titleCount} * 10 + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;const facetNumber = (key) => `(SELECT x.value::float8 FROM doc_facets xWHERE x.rid = d.rid AND x.key = '${key}' AND x.value ~ '^[0-9]+([.][0-9]+)?$'ORDER BY x.value LIMIT 1)`;@@@ -120,6 +120,20 @@ function textMatches(text, match) {return match.words.some((word, i) => ts.some((token) => wordMatches(token, word, Boolean(match.prefix && i === match.words.length - 1))));}++// Inside a field tier, a title that holds more of the query's words comes first, up to TITLE_WORDS_MAX of them:++// "central street harbin" put "Harbin Confucius Temple" (one word) above "Central Street (中央大街)" (two), because++// both titles matched and relevance decided (production, 2026-11-02). Capped so it never outweighs the half tier a++// question's answer gets (the engine's 50 against 10 a word).++const TITLE_WORDS_MAX = 4;++++/** How many of the query's words the title holds (the last a prefix when the query says so), at most TITLE_WORDS_MAX. */++function titleWords(doc, match) {++ if (!doc || !match || !Array.isArray(match.words) || !match.words.length) return 0;++ const ts = tokens(doc.title);++ const n = match.words.filter((word, i) => ts.some((t) => wordMatches(t, word, Boolean(match.prefix && i === match.words.length - 1)))).length;++ return Math.min(TITLE_WORDS_MAX, n);++}++/** 3 when the title matches, 2 the summary, 1 the body, 0 no text field (a vector-only hit). */function fieldTier(doc, match) {if (!doc) return 0;@@@ -214,7 +228,7 @@ function coverage(doc, match) {}module.exports = {-- isQuestion, questionMatch, contentWords, coverage, FIELD_FACTORS,++ isQuestion, questionMatch, contentWords, coverage, FIELD_FACTORS, titleWords, TITLE_WORDS_MAX,SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,textMatches, fieldTier, docTier, isThin, dedupeKey,};@@@ -156,6 +156,17 @@ t('helper tiers and dedupe agree with the SQL order', () => {assert.strictEqual(dedupeKey({ owner: 'live', title: ' Omar ' }), dedupeKey({ owner: 'LIVE', title: 'omar' }));});++t('inside a tier, a title holding more of the query\'s words comes first ("central street harbin")', async () => {++ await put({ owner: 'tours', type: 'place', id: 'plc-temple', title: 'Harbin Confucius Temple', summary: 'Temple in Harbin, China. A short walk from Central Street.', body: 'Harbin Confucius Temple\nCentral Street is near.', canonical_url: 'https://openvibe.tours/place/plc-temple' });++ await put({ owner: 'tours', type: 'place', id: 'plc-central', title: 'Central Street (中央大街)', summary: 'Attraction in Harbin, China.', body: 'Central Street (中央大街)\nAttraction in Harbin, China.', canonical_url: 'https://openvibe.tours/place/plc-central' });++ const r = await request(svc.base, 'GET', '/api/v1/search?q=central%20street%20harbin&limit=10');++ assert.strictEqual(r.status, 200, r.text);++ assert.ok(indexOf(r, 'plc-central') >= 0 && indexOf(r, 'plc-central') < indexOf(r, 'plc-temple'), `two title words before one: ${ids(r).join(', ')}`);++ const { titleWords } = require('../server/ranking');++ assert.strictEqual(titleWords({ title: 'Central Street (中央大街)' }, { words: ['central', 'street', 'harbin'] }), 2);++ assert.strictEqual(titleWords({ title: 'a b c d e f' }, { words: ['a', 'b', 'c', 'd', 'e', 'f'] }), 4, 'capped');++});++t('close', async () => { await svc.stop(); });t.run();
@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');
@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');
const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');
const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');
const { fuseRrf } = require('../fabric/rrf');
const { fuseRrf } = require('../fabric/rrf');
const { createQueryEmbeddingCache } = require('../fabric/embed');
const { createQueryEmbeddingCache } = require('../fabric/embed');
-const { docTier, isThin, dedupeKey, siteHosts, isQuestion, questionMatch, contentWords, coverage } = require('../ranking');
+const { docTier, isThin, dedupeKey, siteHosts, isQuestion, questionMatch, contentWords, coverage, titleWords } = require('../ranking');
const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;
const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;
const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;
const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;
@@ -197,10 +197,10 @@ async function rankOrder(fused, { match, store }) {
@@ -197,10 +197,10 @@ async function rankOrder(fused, { match, store }) {
.filter((hit) => rows.get(hit.rid))
.filter((hit) => rows.get(hit.rid))
.map((hit, i) => {
.map((hit, i) => {
const found = rows.get(hit.rid);
const found = rows.get(hit.rid);
- return { hit, found, i, cover: coverage(found.doc, match), tier: docTier(found.doc, match, hosts), thin: isThin(found.doc), answer: ask && isAnswer(found.doc) ? 1 : 0 };
+ return { hit, found, i, cover: coverage(found.doc, match), tier: docTier(found.doc, match, hosts), words: match.words.length > 1 ? titleWords(found.doc, match) : 0, thin: isThin(found.doc), answer: ask && isAnswer(found.doc) ? 1 : 0 };
})
})
// A relaxed query: how much of it a document holds comes first (rare words weigh more), as in the engine.
// A relaxed query: how much of it a document holds comes first (rare words weigh more), as in the engine.
- .sort((a, b) => b.cover - a.cover || b.tier - a.tier || b.answer - a.answer || (a.thin === b.thin ? 0 : a.thin ? 1 : -1) || a.i - b.i)
+ .sort((a, b) => b.cover - a.cover || b.tier - a.tier || b.answer - a.answer || b.words - a.words || (a.thin === b.thin ? 0 : a.thin ? 1 : -1) || a.i - b.i)
.map(({ hit, found }) => ({ ...hit, found }));
.map(({ hit, found }) => ({ ...hit, found }));
}
}
@@ -22,7 +22,7 @@
@@ -22,7 +22,7 @@
* Nothing outside this file writes SQL against search_fts.
* Nothing outside this file writes SQL against search_fts.
*/
*/
const { EXPOSURE } = require('../document');
const { EXPOSURE } = require('../document');
-const { fold, words, siteOnlyQuery, siteUrlPattern, isQuestion, FIELD_FACTORS } = require('../ranking');
+const { fold, words, siteOnlyQuery, siteUrlPattern, isQuestion, FIELD_FACTORS, TITLE_WORDS_MAX } = require('../ranking');
const FIELD = Object.fromEntries(FIELD_FACTORS);
const FIELD = Object.fromEntries(FIELD_FACTORS);
const CONFIG = 'simple';
const CONFIG = 'simple';
@@ -179,7 +179,13 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
@@ -179,7 +179,13 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
WHEN f.tsv @@ to_tsquery('${CONFIG}', @cwb_${i}) THEN ${wt * FIELD.summary}
WHEN f.tsv @@ to_tsquery('${CONFIG}', @cwb_${i}) THEN ${wt * FIELD.summary}
WHEN f.tsv @@ to_tsquery('${CONFIG}', @cw_${i}) THEN ${wt * FIELD.body} ELSE 0 END`;
WHEN f.tsv @@ to_tsquery('${CONFIG}', @cw_${i}) THEN ${wt * FIELD.body} ELSE 0 END`;
}).join(' + ')})::numeric, 1)::float8` : '0';
}).join(' + ')})::numeric, 1)::float8` : '0';
- const rank = `-(${cover} * 10000 + ${tier} * 100 + ${answer} + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;
+ // Inside a tier, a title holding more of the query's words first (ranking.js titleWords): 10 a word, at most
+ // TITLE_WORDS_MAX, under a question's 50 and over relevance (< 2).
+ const titleCount = match.words.length > 1 ? `LEAST(${TITLE_WORDS_MAX}, ${match.words.map((_, i) => {
+ params[`tw_${i}`] = tsTerm(match, i, 'A');
+ return `(CASE WHEN f.tsv @@ to_tsquery('${CONFIG}', @tw_${i}) THEN 1 ELSE 0 END)`;
+ }).join(' + ')})` : '0';
+ const rank = `-(${cover} * 10000 + ${tier} * 100 + ${answer} + ${titleCount} * 10 + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;
const facetNumber = (key) => `(SELECT x.value::float8 FROM doc_facets x
const facetNumber = (key) => `(SELECT x.value::float8 FROM doc_facets x
WHERE x.rid = d.rid AND x.key = '${key}' AND x.value ~ '^[0-9]+([.][0-9]+)?$'
WHERE x.rid = d.rid AND x.key = '${key}' AND x.value ~ '^[0-9]+([.][0-9]+)?$'
ORDER BY x.value LIMIT 1)`;
ORDER BY x.value LIMIT 1)`;
@@ -120,6 +120,20 @@ function textMatches(text, match) {
@@ -120,6 +120,20 @@ function textMatches(text, match) {
return match.words.some((word, i) => ts.some((token) => wordMatches(token, word, Boolean(match.prefix && i === match.words.length - 1))));
return match.words.some((word, i) => ts.some((token) => wordMatches(token, word, Boolean(match.prefix && i === match.words.length - 1))));
}
}
+// Inside a field tier, a title that holds more of the query's words comes first, up to TITLE_WORDS_MAX of them:
+// "central street harbin" put "Harbin Confucius Temple" (one word) above "Central Street (中央大街)" (two), because
+// both titles matched and relevance decided (production, 2026-11-02). Capped so it never outweighs the half tier a
+// question's answer gets (the engine's 50 against 10 a word).
+const TITLE_WORDS_MAX = 4;
+
+/** How many of the query's words the title holds (the last a prefix when the query says so), at most TITLE_WORDS_MAX. */
+function titleWords(doc, match) {
+ if (!doc || !match || !Array.isArray(match.words) || !match.words.length) return 0;
+ const ts = tokens(doc.title);
+ const n = match.words.filter((word, i) => ts.some((t) => wordMatches(t, word, Boolean(match.prefix && i === match.words.length - 1)))).length;
+ return Math.min(TITLE_WORDS_MAX, n);
+}
+
/** 3 when the title matches, 2 the summary, 1 the body, 0 no text field (a vector-only hit). */
/** 3 when the title matches, 2 the summary, 1 the body, 0 no text field (a vector-only hit). */
function fieldTier(doc, match) {
function fieldTier(doc, match) {
if (!doc) return 0;
if (!doc) return 0;
@@ -214,7 +228,7 @@ function coverage(doc, match) {
@@ -214,7 +228,7 @@ function coverage(doc, match) {
}
}
module.exports = {
module.exports = {
- isQuestion, questionMatch, contentWords, coverage, FIELD_FACTORS,
+ isQuestion, questionMatch, contentWords, coverage, FIELD_FACTORS, titleWords, TITLE_WORDS_MAX,
SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,
SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,
textMatches, fieldTier, docTier, isThin, dedupeKey,
textMatches, fieldTier, docTier, isThin, dedupeKey,
};
};
@@ -156,6 +156,17 @@ t('helper tiers and dedupe agree with the SQL order', () => {
@@ -156,6 +156,17 @@ t('helper tiers and dedupe agree with the SQL order', () => {
assert.strictEqual(dedupeKey({ owner: 'live', title: ' Omar ' }), dedupeKey({ owner: 'LIVE', title: 'omar' }));
assert.strictEqual(dedupeKey({ owner: 'live', title: ' Omar ' }), dedupeKey({ owner: 'LIVE', title: 'omar' }));
});
});
+t('inside a tier, a title holding more of the query\'s words comes first ("central street harbin")', async () => {
+ await put({ owner: 'tours', type: 'place', id: 'plc-temple', title: 'Harbin Confucius Temple', summary: 'Temple in Harbin, China. A short walk from Central Street.', body: 'Harbin Confucius Temple\nCentral Street is near.', canonical_url: 'https://openvibe.tours/place/plc-temple' });
+ await put({ owner: 'tours', type: 'place', id: 'plc-central', title: 'Central Street (中央大街)', summary: 'Attraction in Harbin, China.', body: 'Central Street (中央大街)\nAttraction in Harbin, China.', canonical_url: 'https://openvibe.tours/place/plc-central' });
+ const r = await request(svc.base, 'GET', '/api/v1/search?q=central%20street%20harbin&limit=10');
+ assert.strictEqual(r.status, 200, r.text);
+ assert.ok(indexOf(r, 'plc-central') >= 0 && indexOf(r, 'plc-central') < indexOf(r, 'plc-temple'), `two title words before one: ${ids(r).join(', ')}`);
+ const { titleWords } = require('../server/ranking');
+ assert.strictEqual(titleWords({ title: 'Central Street (中央大街)' }, { words: ['central', 'street', 'harbin'] }), 2);
+ assert.strictEqual(titleWords({ title: 'a b c d e f' }, { words: ['a', 'b', 'c', 'd', 'e', 'f'] }), 4, 'capped');
+});
+
t('close', async () => { await svc.stop(); });
t('close', async () => { await svc.stop(); });
t.run();
t.run();