openvibers/openvibe.search Commits

Inside a tier, a title holding more of the query's words ranks first (#199)

OP OpenVibers committed 11 days ago · 5246a5404d · 4 files, +37 −6

"central street harbin" put "Harbin Confucius Temple" (one title word) above "Central Street
(中央大街)" (two): both titles matched, so both were tier 3, and relevance decided. The engine's
rank and rankOrder now count the query words a title holds (10 each, at most 4, under a
question's half-tier lift of 50 and over relevance) for queries of two or more words.

Changes

ddiff --git a/server/api/query.js b/server/api/query.js
iindex b4cea16..709fdf3 100644
---- a/server/api/query.js
++++ b/server/api/query.js
@@@ -29,7 +29,7 @@ const { AuthError } = require('../auth');
const { OWNER_RE, TYPE_RE, ID_RE } = require('./documents');
const { fuseRrf } = require('../fabric/rrf');
const { createQueryEmbeddingCache } = require('../fabric/embed');
--const { docTier, isThin, dedupeKey, siteHosts, isQuestion, questionMatch, contentWords, coverage } = require('../ranking');
++const { docTier, isThin, dedupeKey, siteHosts, isQuestion, questionMatch, contentWords, coverage, titleWords } = require('../ranking');
const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;
const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;
@@@ -197,10 +197,10 @@ async function rankOrder(fused, { match, store }) {
.filter((hit) => rows.get(hit.rid))
.map((hit, i) => {
const found = rows.get(hit.rid);
-- return { hit, found, i, cover: coverage(found.doc, match), tier: docTier(found.doc, match, hosts), thin: isThin(found.doc), answer: ask && isAnswer(found.doc) ? 1 : 0 };
++ return { hit, found, i, cover: coverage(found.doc, match), tier: docTier(found.doc, match, hosts), words: match.words.length > 1 ? titleWords(found.doc, match) : 0, thin: isThin(found.doc), answer: ask && isAnswer(found.doc) ? 1 : 0 };
})
// A relaxed query: how much of it a document holds comes first (rare words weigh more), as in the engine.
-- .sort((a, b) => b.cover - a.cover || b.tier - a.tier || b.answer - a.answer || (a.thin === b.thin ? 0 : a.thin ? 1 : -1) || a.i - b.i)
++ .sort((a, b) => b.cover - a.cover || b.tier - a.tier || b.answer - a.answer || b.words - a.words || (a.thin === b.thin ? 0 : a.thin ? 1 : -1) || a.i - b.i)
.map(({ hit, found }) => ({ ...hit, found }));
}
ddiff --git a/server/engine/pg.js b/server/engine/pg.js
iindex 395a187..23ac01c 100644
---- a/server/engine/pg.js
++++ b/server/engine/pg.js
@@@ -22,7 +22,7 @@
* Nothing outside this file writes SQL against search_fts.
*/
const { EXPOSURE } = require('../document');
--const { fold, words, siteOnlyQuery, siteUrlPattern, isQuestion, FIELD_FACTORS } = require('../ranking');
++const { fold, words, siteOnlyQuery, siteUrlPattern, isQuestion, FIELD_FACTORS, TITLE_WORDS_MAX } = require('../ranking');
const FIELD = Object.fromEntries(FIELD_FACTORS);
const CONFIG = 'simple';
@@@ -179,7 +179,13 @@ function createEngine(db, { freshness = { weight: 1, halfLifeDays: 30 } } = {})
WHEN f.tsv @@ to_tsquery('${CONFIG}', @cwb_${i}) THEN ${wt * FIELD.summary}
WHEN f.tsv @@ to_tsquery('${CONFIG}', @cw_${i}) THEN ${wt * FIELD.body} ELSE 0 END`;
}).join(' + ')})::numeric, 1)::float8` : '0';
-- const rank = `-(${cover} * 10000 + ${tier} * 100 + ${answer} + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;
++ // Inside a tier, a title holding more of the query's words first (ranking.js titleWords): 10 a word, at most
++ // TITLE_WORDS_MAX, under a question's 50 and over relevance (< 2).
++ const titleCount = match.words.length > 1 ? `LEAST(${TITLE_WORDS_MAX}, ${match.words.map((_, i) => {
++ params[`tw_${i}`] = tsTerm(match, i, 'A');
++ return `(CASE WHEN f.tsv @@ to_tsquery('${CONFIG}', @tw_${i}) THEN 1 ELSE 0 END)`;
++ }).join(' + ')})` : '0';
++ const rank = `-(${cover} * 10000 + ${tier} * 100 + ${answer} + ${titleCount} * 10 + (${relevance}) + CASE WHEN ${thin} THEN -2 ELSE 0 END)`;
const facetNumber = (key) => `(SELECT x.value::float8 FROM doc_facets x
WHERE x.rid = d.rid AND x.key = '${key}' AND x.value ~ '^[0-9]+([.][0-9]+)?$'
ORDER BY x.value LIMIT 1)`;
ddiff --git a/server/ranking.js b/server/ranking.js
iindex e183027..64bad81 100644
---- a/server/ranking.js
++++ b/server/ranking.js
@@@ -120,6 +120,20 @@ function textMatches(text, match) {
return match.words.some((word, i) => ts.some((token) => wordMatches(token, word, Boolean(match.prefix && i === match.words.length - 1))));
}
++// Inside a field tier, a title that holds more of the query's words comes first, up to TITLE_WORDS_MAX of them:
++// "central street harbin" put "Harbin Confucius Temple" (one word) above "Central Street (中央大街)" (two), because
++// both titles matched and relevance decided (production, 2026-11-02). Capped so it never outweighs the half tier a
++// question's answer gets (the engine's 50 against 10 a word).
++const TITLE_WORDS_MAX = 4;
++
++/** How many of the query's words the title holds (the last a prefix when the query says so), at most TITLE_WORDS_MAX. */
++function titleWords(doc, match) {
++ if (!doc || !match || !Array.isArray(match.words) || !match.words.length) return 0;
++ const ts = tokens(doc.title);
++ const n = match.words.filter((word, i) => ts.some((t) => wordMatches(t, word, Boolean(match.prefix && i === match.words.length - 1)))).length;
++ return Math.min(TITLE_WORDS_MAX, n);
++}
++
/** 3 when the title matches, 2 the summary, 1 the body, 0 no text field (a vector-only hit). */
function fieldTier(doc, match) {
if (!doc) return 0;
@@@ -214,7 +228,7 @@ function coverage(doc, match) {
}
module.exports = {
-- isQuestion, questionMatch, contentWords, coverage, FIELD_FACTORS,
++ isQuestion, questionMatch, contentWords, coverage, FIELD_FACTORS, titleWords, TITLE_WORDS_MAX,
SITE_ALIASES, fold, words, tokens, siteHosts, siteOnlyQuery, siteUrlPattern, urlBelongsToSites,
textMatches, fieldTier, docTier, isThin, dedupeKey,
};
ddiff --git a/test/ranking.test.js b/test/ranking.test.js
iindex 7d40952..a0d838d 100644
---- a/test/ranking.test.js
++++ b/test/ranking.test.js
@@@ -156,6 +156,17 @@ t('helper tiers and dedupe agree with the SQL order', () => {
assert.strictEqual(dedupeKey({ owner: 'live', title: ' Omar ' }), dedupeKey({ owner: 'LIVE', title: 'omar' }));
});
++t('inside a tier, a title holding more of the query\'s words comes first ("central street harbin")', async () => {
++ await put({ owner: 'tours', type: 'place', id: 'plc-temple', title: 'Harbin Confucius Temple', summary: 'Temple in Harbin, China. A short walk from Central Street.', body: 'Harbin Confucius Temple\nCentral Street is near.', canonical_url: 'https://openvibe.tours/place/plc-temple' });
++ await put({ owner: 'tours', type: 'place', id: 'plc-central', title: 'Central Street (中央大街)', summary: 'Attraction in Harbin, China.', body: 'Central Street (中央大街)\nAttraction in Harbin, China.', canonical_url: 'https://openvibe.tours/place/plc-central' });
++ const r = await request(svc.base, 'GET', '/api/v1/search?q=central%20street%20harbin&limit=10');
++ assert.strictEqual(r.status, 200, r.text);
++ assert.ok(indexOf(r, 'plc-central') >= 0 && indexOf(r, 'plc-central') < indexOf(r, 'plc-temple'), `two title words before one: ${ids(r).join(', ')}`);
++ const { titleWords } = require('../server/ranking');
++ assert.strictEqual(titleWords({ title: 'Central Street (中央大街)' }, { words: ['central', 'street', 'harbin'] }), 2);
++ assert.strictEqual(titleWords({ title: 'a b c d e f' }, { words: ['a', 'b', 'c', 'd', 'e', 'f'] }), 4, 'capped');
++});
++
t('close', async () => { await svc.stop(); });
t.run();