openvibers/openvibe.search Commits

Several words that find little: answer with any of the words (#190)

OP OpenVibers committed 11 days ago · f4f6490c1d · 3 files, +152 −48

Changes

ddiff --git a/server/api/query.js b/server/api/query.js
iindex bab1259..78e48e7 100644
---- a/server/api/query.js
++++ b/server/api/query.js
@@@ -34,6 +34,11 @@ const { docTier, isThin, dedupeKey, siteHosts } = require('../ranking');
const FACET_KEY_RE = /^[a-z][a-z0-9_]{0,39}$/;
const LANG_RE = /^[a-zA-Z]{2,3}(-[A-Za-z0-9]{2,8})*$/;
const HYBRID_CANDIDATES = 100;
++// A query of several words is answered with every word first. When that finds fewer than this, the answer is the
++// documents with any of the words instead ("sony headphones" found nothing while "headphones" found deals): ranked by
++// the same tiers, so a title holding a word comes before a body that mentions one, and the ones with every word rank
++// first by relevance. Its cursor carries x, so the next page continues the same relaxed query.
++const RELAX_BELOW = 5;
class QueryError extends Error {
constructor(code, detail) { super(detail); this.code = code; }
@@@ -72,6 +77,11 @@ function encodeCursor(obj) {
return Buffer.from(JSON.stringify(obj)).toString('base64url');
}
++/** Whether a cursor carries a flag (x: a relaxed answer's page), before it is checked against the query. */
++function cursorFlag(s, key) {
++ try { return Boolean(JSON.parse(Buffer.from(String(s), 'base64url').toString('utf8'))[key]); } catch { return false; }
++}
++
function decodeCursor(s, hash) {
if (s === undefined || s === '') return null;
let c;
@@@ -246,6 +256,59 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
return fuseRrf(keyword.hits, near).map((h) => ({ ...h, snip: snips.get(h.rid) }));
}
++ /** One page for a word query (every word, or any with relaxed): hybrid when vectors are there, keyword else. */
++ async function matchedPage({ match, hash, after, state, filters, viewer, limit, now, relaxed }) {
++ const results = [];
++ const seen = new Set();
++ let next = null;
++ let explain;
++ let refNow = now;
++ const x = relaxed ? { x: 1 } : {};
++ if (state.ranking === 'hybrid') {
++ if (after && after.offset === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');
++ if (after && after.now != null) refNow = after.now;
++ let fused = await hybridCandidates({ match, filters, viewer, now: refNow, vector: state.vector });
++ // A plain query's tiers: site first, then title > summary > body, thin VODs last within a tier.
++ fused = await rankOrder(fused, { match, store });
++ // Deals: the product the person asked for (its title or category) beats a description-only match.
++ if (filters.owner === 'deals') fused = await dealsOrder(fused, { match, store });
++ let consumed = after ? after.offset : 0;
++ let lastRid = after ? after.rid : null;
++ for (let i = consumed; i < fused.length && results.length < limit; i++) {
++ const f = fused[i];
++ consumed = i + 1;
++ lastRid = f.rid;
++ // Defence in depth: the vector store is public-only, the keyword SQL is ACL-aware.
++ const found = f.found !== undefined ? f.found : await store.byRid(f.rid);
++ if (!found || !canView(viewer, found.row, found.doc)) continue;
++ const key = dedupeKey(found.doc);
++ if (seen.has(key)) continue;
++ seen.add(key);
++ results.push(projection(found.doc, found.row, f.snip, f.ranks));
++ }
++ if (consumed < fused.length) next = encodeCursor({ k: 'f', o: consumed, i: lastRid, n: refNow, h: hash, ...x });
++ explain = { ranking: 'hybrid' };
++ } else {
++ if (after && after.rank === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');
++ if (after && after.now != null) refNow = after.now;
++ const page = await engine.search({ match, filters, viewer, limit, after, now: refNow });
++ let position = 0;
++ for (const h of page.hits) {
++ position++;
++ const found = await store.byRid(h.rid);
++ // Defence in depth: the engine already filtered; re-check against the stored row.
++ if (!found || !canView(viewer, found.row, found.doc)) continue;
++ const key = dedupeKey(found.doc);
++ if (seen.has(key)) continue;
++ seen.add(key);
++ results.push(projection(found.doc, found.row, h.snip, { keyword: position, vector: null }));
++ }
++ next = page.next ? encodeCursor({ k: 'r', r: page.next.rank, m: page.next.importance, s: page.next.recency, i: page.next.rid, n: refNow, h: hash, ...x }) : null;
++ explain = { ranking: 'keyword', reason: state.reason };
++ }
++ return { results, next, explain };
++ }
++
async function run({ text = '', query = {}, filters = null, viewer, limit, cursor, facetKeys = [], now = clock(), since = null }) {
if (text.length > 500) throw new QueryError('search.bad_query', 'q is longer than 500 characters');
filters = filters || parseFilters(query, config);
@@@ -254,7 +317,11 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
if (facetKeys.length > 5 || facetKeys.some(k => !FACET_KEY_RE.test(k))) throw new QueryError('search.bad_query', 'facets: at most 5 well-formed keys');
const trimmed = text.trim();
-- const match = trimmed ? toMatch(text, { maxTerms: config.query.maxTerms }) : null;
++ const exact = trimmed ? toMatch(text, { maxTerms: config.query.maxTerms }) : null;
++ const relaxedMatch = exact && exact.words.length > 1 ? { ...exact, any: true } : null;
++ // A later page of a relaxed answer says so in its cursor (x), and continues that query.
++ const relaxedCursor = Boolean(relaxedMatch && cursor && cursorFlag(cursor, 'x'));
++ const match = relaxedCursor ? relaxedMatch : exact;
const hash = queryHash([match, filters]);
const after = decodeCursor(cursor, hash);
if (trimmed && !match) {
@@@ -266,55 +333,19 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
};
}
-- const results = [];
-- const seen = new Set();
++ let results = [];
let next = null;
let explain;
-- let refNow = now;
++ let used = match;
if (match) {
const state = await semanticState(trimmed);
-- if (state.ranking === 'hybrid') {
-- if (after && after.offset === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');
-- if (after && after.now != null) refNow = after.now;
-- let fused = await hybridCandidates({ match, filters, viewer, now: refNow, vector: state.vector });
-- // A plain query's tiers: site first, then title > summary > body, thin VODs last within a tier.
-- fused = await rankOrder(fused, { match, store });
-- // Deals: the product the person asked for (its title or category) beats a description-only match.
-- if (filters.owner === 'deals') fused = await dealsOrder(fused, { match, store });
-- let consumed = after ? after.offset : 0;
-- let lastRid = after ? after.rid : null;
-- for (let i = consumed; i < fused.length && results.length < limit; i++) {
-- const f = fused[i];
-- consumed = i + 1;
-- lastRid = f.rid;
-- // Defence in depth: the vector store is public-only, the keyword SQL is ACL-aware.
-- const found = f.found !== undefined ? f.found : await store.byRid(f.rid);
-- if (!found || !canView(viewer, found.row, found.doc)) continue;
-- const key = dedupeKey(found.doc);
-- if (seen.has(key)) continue;
-- seen.add(key);
-- results.push(projection(found.doc, found.row, f.snip, f.ranks));
-- }
-- if (consumed < fused.length) next = encodeCursor({ k: 'f', o: consumed, i: lastRid, n: refNow, h: hash });
-- explain = { ranking: 'hybrid' };
-- } else {
-- if (after && after.rank === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');
-- if (after && after.now != null) refNow = after.now;
-- const page = await engine.search({ match, filters, viewer, limit, after, now: refNow });
-- let position = 0;
-- for (const h of page.hits) {
-- position++;
-- const found = await store.byRid(h.rid);
-- // Defence in depth: the engine already filtered; re-check against the stored row.
-- if (!found || !canView(viewer, found.row, found.doc)) continue;
-- const key = dedupeKey(found.doc);
-- if (seen.has(key)) continue;
-- seen.add(key);
-- results.push(projection(found.doc, found.row, h.snip, { keyword: position, vector: null }));
-- }
-- next = page.next ? encodeCursor({ k: 'r', r: page.next.rank, m: page.next.importance, s: page.next.recency, i: page.next.rid, n: refNow, h: hash }) : null;
-- explain = { ranking: 'keyword', reason: state.reason };
++ let page = await matchedPage({ match, hash, after, state, filters, viewer, limit, now, relaxed: relaxedCursor });
++ // The first page of several words that found little: every document with any of the words instead.
++ if (!cursor && relaxedMatch && !relaxedCursor && page.results.length < RELAX_BELOW) {
++ const loose = await matchedPage({ match: relaxedMatch, hash: queryHash([relaxedMatch, filters]), after: null, state, filters, viewer, limit, now, relaxed: true });
++ if (loose.results.length > page.results.length) { page = loose; used = relaxedMatch; }
}
++ ({ results, next, explain } = page);
} else {
if (after && after.sort_at === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');
const page = await engine.browse({ filters, viewer, limit, after });
@@@ -329,7 +360,7 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
}
const body = { results, next_cursor: next, explain };
-- if (facetKeys.length) body.facets = await engine.facetCounts({ match, filters, viewer, keys: facetKeys });
++ if (facetKeys.length) body.facets = await engine.facetCounts({ match: used, filters, viewer, keys: facetKeys });
return body;
}
ddiff --git a/server/engine/pg.js b/server/engine/pg.js
iindex d101925..b5c3bc8 100644
---- a/server/engine/pg.js
++++ b/server/engine/pg.js
@@@ -342,13 +342,16 @@ function toMatch(q, { maxTerms = 16, column = null, prefixLast = false } = {}) {
return { words, prefix, column };
}
--/** toMatch()'s query → to_tsquery text: 'w1' & 'w2' & 'w3':* (and :A on each word for the title). */
++/**
++ * toMatch()'s query → to_tsquery text: 'w1' & 'w2' & 'w3':* (and :A on each word for the title). any: true (the
++ * relaxed answer to several words that found little, server/api/query.js) joins them with | instead.
++ */
function tsquery(m) {
const weight = m.column === 'title' ? 'A' : '';
return m.words.map((w, i) => {
const star = m.prefix && i === m.words.length - 1 ? '*' : '';
return `'${w}'${star || weight ? `:${star}${weight}` : ''}`;
-- }).join(' & ');
++ }).join(m.any ? ' | ' : ' & ');
}
/** Plain text with the engine's snippet marks → HTML with <mark>, everything else escaped. */
ddiff --git a/test/relaxed-words.test.js b/test/relaxed-words.test.js
nnew file mode 100644
iindex 0000000..f911441
---- /dev/null
++++ b/test/relaxed-words.test.js
@@@ -0,0 +1,70 @@
++'use strict';
++/**
++ * Several words that find little: "sony headphones" found nothing on production (2026-11-02) while "headphones" found
++ * deals, because every word had to match. A query of several words whose first page finds fewer than five documents
++ * answers with the documents holding any of the words instead, ranked by the same tiers (a title with a word before a
++ * body that mentions one); one with every word still comes first. Its next page continues the relaxed query (the
++ * cursor's x). A query whose every word finds enough is answered exactly as before.
++ */
++const assert = require('assert');
++const { boot, request, doc, suite } = require('./helpers');
++
++const t = suite('relaxed words');
++let svc;
++const AT = '2026-11-01T00:00:00Z';
++const put = (overrides) => svc.store.apply(doc({ published_at: AT, updated_at: AT, summary: '', body: '', ...overrides }));
++const ids = (r) => r.body.results.map((x) => x.id);
++const q = (text, extra = '') => request(svc.base, 'GET', `/api/v1/search?q=${encodeURIComponent(text)}${extra}`);
++
++t('boot and seed', async () => {
++ svc = await boot();
++ for (let i = 1; i <= 7; i++) {
++ await put({ owner: 'deals', type: 'offer', id: `phones-${i}`, title: `Wireless headphones model ${i}`, summary: 'Over-ear, 30 hours of battery.', canonical_url: `https://openvibe.deals/d/phones-${i}` });
++ }
++ await put({ owner: 'deals', type: 'offer', id: 'sony-phones', title: 'Sony WH-1000XM5 wireless noise cancelling headphones', summary: 'Noise cancelling, 30 hours of battery.', canonical_url: 'https://openvibe.deals/d/sony-phones' });
++ await put({ owner: 'live', type: 'vod', id: 'vod-mentions', title: 'Evening stream', summary: 'We talked about cheap tickets.', body: 'cheap', canonical_url: 'https://openvibe.video/v/7001' });
++ for (let i = 1; i <= 6; i++) {
++ await put({ owner: 'wiki', type: 'page', id: `garden-${i}`, title: `Garden notes ${i}`, summary: 'Tomato and basil, side by side.', canonical_url: `https://openvibe.wiki/p/garden-${i}` });
++ }
++});
++
++t('several words that find little answer with any of the words, every word first, titles before bodies', async () => {
++ const r = await q('sony headphones', '&limit=5');
++ assert.strictEqual(r.status, 200);
++ const got = ids(r);
++ assert.strictEqual(got.length, 5, 'a full page');
++ assert.strictEqual(got[0], 'sony-phones', 'the one with both words first');
++ assert.ok(got.every((id) => id.includes('phones')), 'then headphones by their titles');
++ assert.ok(r.body.next_cursor, 'there is more');
++ const cursor = JSON.parse(Buffer.from(r.body.next_cursor, 'base64url').toString('utf8'));
++ assert.strictEqual(cursor.x, 1, 'the cursor says the answer was relaxed');
++ const page2 = await q('sony headphones', `&limit=5&cursor=${r.body.next_cursor}`);
++ assert.strictEqual(page2.status, 200, JSON.stringify(page2.body));
++ const all = [...got, ...ids(page2)];
++ assert.strictEqual(new Set(all).size, all.length, 'no document twice across pages');
++ assert.ok(all.includes('phones-7'), 'the second page continues the relaxed query');
++});
++
++t('a word that only a body mentions ranks after the titles that hold the other', async () => {
++ const r = await q('cheap headphones', '&limit=20');
++ const got = ids(r);
++ assert.ok(got.length >= 8, `headphones are found (${got.length})`);
++ assert.ok(got.indexOf('vod-mentions') > got.indexOf('phones-1'), 'a body mention ranks after a title');
++});
++
++t('a query whose every word finds enough is answered exactly as before', async () => {
++ const r = await q('garden notes', '&limit=10');
++ const got = ids(r);
++ assert.strictEqual(got.length, 6);
++ assert.ok(got.every((id) => id.startsWith('garden-')), 'only documents with both words');
++ const none = await q('garden tomato basil quantum');
++ assert.ok(ids(none).every((id) => id.startsWith('garden-')), 'relaxed, the garden pages still answer');
++});
++
++t('a single word that finds nothing still finds nothing', async () => {
++ assert.deepStrictEqual(ids(await q('zzzunknownword')), []);
++});
++
++t('close', async () => { await svc.stop(); });
++
++t.run();