openvibers/openvibe.search Commits
Deals: a title or category match outranks a description-only match (#144)
Changes
@@@ -123,6 +123,45 @@ function canView(viewer, row, doc) {return false;}++// ── Deals: a title or category match outranks a description-only one (11-01) ───────────────────++// "mechanical keyboard" ranked Bit Blaster XL as the only deal: its description says "Keyboard + Gamepad Support",++// and a near vector hit (under the 0.30 floor) plus a description keyword hit beat the real product. For the deals++// owner, every result whose title or category names the thing asked for (the query's head noun: a keyboard, not a++// game) comes before one that only matched in its summary or body, and a vector-only hit (no keyword match at all)++// comes last. The order inside each group stays the fused order, so relevance still decides among equals.++const WORD_RE = /[\p{L}\p{N}]+/gu;++function foldedWords(text) {++ return String(text || '').normalize('NFKD').replace(/\p{M}+/gu, '').toLowerCase().match(WORD_RE) || [];++}++/** "keyboard" carries "keyboards" and the other way round once the shorter word is at least four letters. */++function wordMatches(token, word) {++ return token === word || token.startsWith(word) || (token.length >= 4 && word.startsWith(token));++}++function titleOrCategoryMatch(doc, match) {++ if (!doc || !match || !match.words.length) return false;++ const category = doc.facets && doc.facets.category;++ const text = `${doc.title || ''} ${Array.isArray(category) ? category.join(' ') : (category || '')}`;++ const tokens = foldedWords(text);++ // The head noun is what the person wants ("mechanical keyboard" -> keyboard); a modifier may only be in the++ // description, and that is still a title/category match for the product.++ const head = match.words[match.words.length - 1];++ return tokens.some((t) => wordMatches(t, head));++}++async function dealsOrder(fused, { match, store }) {++ if (!match || !match.words.length) return fused;++ const rows = new Map();++ for (const hit of fused) rows.set(hit.rid, await store.byRid(hit.rid));++ const tierOf = (hit) => {++ const found = rows.get(hit.rid);++ if (found && titleOrCategoryMatch(found.doc, match)) return 0;++ return hit.ranks && hit.ranks.keyword != null ? 1 : 2;++ };++ return fused++ .map((hit, i) => ({ hit, i, tier: tierOf(hit) }))++ .sort((a, b) => a.tier - b.tier || a.i - b.i)++ .map(({ hit }) => ({ ...hit, found: rows.get(hit.rid) }));++}++/*** One search as a viewer: the query API, saved-search runs and the HTML page all go through here.* run({ text, query, viewer, limit, cursor, facetKeys, now, since }) → { results, next_cursor, explain, facets? }@@@ -209,15 +248,17 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddingsif (state.ranking === 'hybrid') {if (after && after.offset === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');if (after && after.now != null) refNow = after.now;-- const fused = await hybridCandidates({ match, filters, viewer, now: refNow, vector: state.vector });++ let fused = await hybridCandidates({ match, filters, viewer, now: refNow, vector: state.vector });++ // Deals: the product the person asked for (its title or category) beats a description-only match.++ if (filters.owner === 'deals') fused = await dealsOrder(fused, { match, store });let consumed = after ? after.offset : 0;let lastRid = after ? after.rid : null;for (let i = consumed; i < fused.length && results.length < limit; i++) {const f = fused[i];consumed = i + 1;lastRid = f.rid;-- const found = await store.byRid(f.rid);// Defence in depth: the vector store is public-only, the keyword SQL is ACL-aware.++ const found = f.found !== undefined ? f.found : await store.byRid(f.rid);if (!found || !canView(viewer, found.row, found.doc)) continue;results.push(projection(found.doc, found.row, f.snip, f.ranks));}@@@ -284,4 +284,43 @@ t('hybrid: a vector hit farther than the floor is dropped, so an unrelated query}});++t('deals: a title or category match outranks a description-only match (11-01)', async () => {++ const vectors = new FakeVectors();++ const embeddings = fakeEmbeddings();++ const svc = await boot({ vectorStore: vectors, embeddingClient: embeddings });++ try {++ // The two offers production ranked wrong for "mechanical keyboard": the real product (title and category)++ // and Bit Blaster XL, a game whose description says "Keyboard + Gamepad Support".++ await svc.store.apply(doc({++ owner: 'deals', type: 'offer', id: 'k2', title: 'Keychron K2 Mechanical Keyboard',++ summary: 'A compact mechanical keyboard.', body: 'Keychron K2 Mechanical Keyboard\nA compact mechanical keyboard.',++ facets: { category: 'Keyboards' }, canonical_url: 'https://openvibe.deals/d/k2',++ }));++ await svc.store.apply(doc({++ owner: 'deals', type: 'offer', id: 'bit-blaster', title: 'Bit Blaster XL',++ summary: 'Mechanical keyboard support and a gamepad. An arcade shoot \'em up.',++ body: 'Bit Blaster XL\nMechanical keyboard support and a gamepad.',++ facets: { category: 'Games' }, canonical_url: 'https://openvibe.deals/d/bit-blaster',++ }));++ const rid = await ridMap(svc.db);++ // The vector side admitted the game under the 0.30 floor (config.query.vectorMaxDistance) and ranked it++ // first; the keyword side also matched "mechanical keyboard" in its description. Without the deals++ // order, RRF puts the game above the one result whose title and category are the product asked for.++ vectors.hits = [{ rid: rid['bit-blaster'], distance: 0.12 }];++++ const r = await request(svc.base, 'GET', '/api/v1/search?q=mechanical%20keyboard&owner=deals&limit=10');++ assert.strictEqual(r.status, 200, r.text);++ assert.deepStrictEqual(r.body.explain, { ranking: 'hybrid' });++ assert.deepStrictEqual(r.body.results.map((x) => x.id), ['k2', 'bit-blaster'], 'the keyboard is the deal, the game is not');++ assert.deepStrictEqual(r.body.results.find((x) => x.id === 'k2').ranks, { keyword: 1, vector: null });++ assert.deepStrictEqual(r.body.results.find((x) => x.id === 'bit-blaster').ranks, { keyword: 2, vector: 1 });++++ // The order is only the deals owner's: the same two documents unowned keep the fused order.++ const all = await request(svc.base, 'GET', '/api/v1/search?q=mechanical%20keyboard&limit=10');++ assert.deepStrictEqual(all.body.results.map((x) => x.id), ['bit-blaster', 'k2'], 'other owners stay on RRF');++ } finally {++ await svc.stop();++ }++});++t.run();
@@ -123,6 +123,45 @@ function canView(viewer, row, doc) {
@@ -123,6 +123,45 @@ function canView(viewer, row, doc) {
return false;
return false;
}
}
+// ── Deals: a title or category match outranks a description-only one (11-01) ───────────────────
+// "mechanical keyboard" ranked Bit Blaster XL as the only deal: its description says "Keyboard + Gamepad Support",
+// and a near vector hit (under the 0.30 floor) plus a description keyword hit beat the real product. For the deals
+// owner, every result whose title or category names the thing asked for (the query's head noun: a keyboard, not a
+// game) comes before one that only matched in its summary or body, and a vector-only hit (no keyword match at all)
+// comes last. The order inside each group stays the fused order, so relevance still decides among equals.
+const WORD_RE = /[\p{L}\p{N}]+/gu;
+function foldedWords(text) {
+ return String(text || '').normalize('NFKD').replace(/\p{M}+/gu, '').toLowerCase().match(WORD_RE) || [];
+}
+/** "keyboard" carries "keyboards" and the other way round once the shorter word is at least four letters. */
+function wordMatches(token, word) {
+ return token === word || token.startsWith(word) || (token.length >= 4 && word.startsWith(token));
+}
+function titleOrCategoryMatch(doc, match) {
+ if (!doc || !match || !match.words.length) return false;
+ const category = doc.facets && doc.facets.category;
+ const text = `${doc.title || ''} ${Array.isArray(category) ? category.join(' ') : (category || '')}`;
+ const tokens = foldedWords(text);
+ // The head noun is what the person wants ("mechanical keyboard" -> keyboard); a modifier may only be in the
+ // description, and that is still a title/category match for the product.
+ const head = match.words[match.words.length - 1];
+ return tokens.some((t) => wordMatches(t, head));
+}
+async function dealsOrder(fused, { match, store }) {
+ if (!match || !match.words.length) return fused;
+ const rows = new Map();
+ for (const hit of fused) rows.set(hit.rid, await store.byRid(hit.rid));
+ const tierOf = (hit) => {
+ const found = rows.get(hit.rid);
+ if (found && titleOrCategoryMatch(found.doc, match)) return 0;
+ return hit.ranks && hit.ranks.keyword != null ? 1 : 2;
+ };
+ return fused
+ .map((hit, i) => ({ hit, i, tier: tierOf(hit) }))
+ .sort((a, b) => a.tier - b.tier || a.i - b.i)
+ .map(({ hit }) => ({ ...hit, found: rows.get(hit.rid) }));
+}
+
/**
/**
* One search as a viewer: the query API, saved-search runs and the HTML page all go through here.
* One search as a viewer: the query API, saved-search runs and the HTML page all go through here.
* run({ text, query, viewer, limit, cursor, facetKeys, now, since }) → { results, next_cursor, explain, facets? }
* run({ text, query, viewer, limit, cursor, facetKeys, now, since }) → { results, next_cursor, explain, facets? }
@@ -209,15 +248,17 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
@@ -209,15 +248,17 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
if (state.ranking === 'hybrid') {
if (state.ranking === 'hybrid') {
if (after && after.offset === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');
if (after && after.offset === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');
if (after && after.now != null) refNow = after.now;
if (after && after.now != null) refNow = after.now;
- const fused = await hybridCandidates({ match, filters, viewer, now: refNow, vector: state.vector });
+ let fused = await hybridCandidates({ match, filters, viewer, now: refNow, vector: state.vector });
+ // Deals: the product the person asked for (its title or category) beats a description-only match.
+ if (filters.owner === 'deals') fused = await dealsOrder(fused, { match, store });
let consumed = after ? after.offset : 0;
let consumed = after ? after.offset : 0;
let lastRid = after ? after.rid : null;
let lastRid = after ? after.rid : null;
for (let i = consumed; i < fused.length && results.length < limit; i++) {
for (let i = consumed; i < fused.length && results.length < limit; i++) {
const f = fused[i];
const f = fused[i];
consumed = i + 1;
consumed = i + 1;
lastRid = f.rid;
lastRid = f.rid;
- const found = await store.byRid(f.rid);
// Defence in depth: the vector store is public-only, the keyword SQL is ACL-aware.
// Defence in depth: the vector store is public-only, the keyword SQL is ACL-aware.
+ const found = f.found !== undefined ? f.found : await store.byRid(f.rid);
if (!found || !canView(viewer, found.row, found.doc)) continue;
if (!found || !canView(viewer, found.row, found.doc)) continue;
results.push(projection(found.doc, found.row, f.snip, f.ranks));
results.push(projection(found.doc, found.row, f.snip, f.ranks));
}
}
@@ -284,4 +284,43 @@ t('hybrid: a vector hit farther than the floor is dropped, so an unrelated query
@@ -284,4 +284,43 @@ t('hybrid: a vector hit farther than the floor is dropped, so an unrelated query
}
}
});
});
+t('deals: a title or category match outranks a description-only match (11-01)', async () => {
+ const vectors = new FakeVectors();
+ const embeddings = fakeEmbeddings();
+ const svc = await boot({ vectorStore: vectors, embeddingClient: embeddings });
+ try {
+ // The two offers production ranked wrong for "mechanical keyboard": the real product (title and category)
+ // and Bit Blaster XL, a game whose description says "Keyboard + Gamepad Support".
+ await svc.store.apply(doc({
+ owner: 'deals', type: 'offer', id: 'k2', title: 'Keychron K2 Mechanical Keyboard',
+ summary: 'A compact mechanical keyboard.', body: 'Keychron K2 Mechanical Keyboard\nA compact mechanical keyboard.',
+ facets: { category: 'Keyboards' }, canonical_url: 'https://openvibe.deals/d/k2',
+ }));
+ await svc.store.apply(doc({
+ owner: 'deals', type: 'offer', id: 'bit-blaster', title: 'Bit Blaster XL',
+ summary: 'Mechanical keyboard support and a gamepad. An arcade shoot \'em up.',
+ body: 'Bit Blaster XL\nMechanical keyboard support and a gamepad.',
+ facets: { category: 'Games' }, canonical_url: 'https://openvibe.deals/d/bit-blaster',
+ }));
+ const rid = await ridMap(svc.db);
+ // The vector side admitted the game under the 0.30 floor (config.query.vectorMaxDistance) and ranked it
+ // first; the keyword side also matched "mechanical keyboard" in its description. Without the deals
+ // order, RRF puts the game above the one result whose title and category are the product asked for.
+ vectors.hits = [{ rid: rid['bit-blaster'], distance: 0.12 }];
+
+ const r = await request(svc.base, 'GET', '/api/v1/search?q=mechanical%20keyboard&owner=deals&limit=10');
+ assert.strictEqual(r.status, 200, r.text);
+ assert.deepStrictEqual(r.body.explain, { ranking: 'hybrid' });
+ assert.deepStrictEqual(r.body.results.map((x) => x.id), ['k2', 'bit-blaster'], 'the keyboard is the deal, the game is not');
+ assert.deepStrictEqual(r.body.results.find((x) => x.id === 'k2').ranks, { keyword: 1, vector: null });
+ assert.deepStrictEqual(r.body.results.find((x) => x.id === 'bit-blaster').ranks, { keyword: 2, vector: 1 });
+
+ // The order is only the deals owner's: the same two documents unowned keep the fused order.
+ const all = await request(svc.base, 'GET', '/api/v1/search?q=mechanical%20keyboard&limit=10');
+ assert.deepStrictEqual(all.body.results.map((x) => x.id), ['bit-blaster', 'k2'], 'other owners stay on RRF');
+ } finally {
+ await svc.stop();
+ }
+});
+
t.run();
t.run();