openvibers/openvibe.search Commits
Bound semantic search latency and measure 50 network queries (#282)
Changes
nnew file mode 100644@@@ -0,0 +1,39 @@++'use strict';++// Fixed public network-search probes. `topic` is a deliberately broad relevance label, not a click judgement.++const probes = [++ ['openvibe live', 'live'], ['openvibe chat', 'chat'], ['openvibe codes', 'codes'], ['openvibe design', 'design'], ['openvibe exchange', 'exchange'],++ ['openvibe search', 'search'], ['openvibe wiki', 'wiki'], ['openvibe video', 'video'], ['openvibe pics', 'pics'], ['openvibe music', 'music'],++ ['live channels', 'channel'], ['live recordings', 'record'], ['stream clips', 'clip'], ['video uploads', 'video'], ['community forum', 'forum'],++ ['AI models', 'model'], ['AI providers', 'provider'], ['model benchmarks', 'benchmark'], ['image search', 'image'], ['web search', 'search'],++ ['developer API', 'api'], ['software development', 'software'], ['robot controls', 'robot'], ['open source projects', 'open source'], ['privacy policy', 'privacy'],++ ['live streaming', 'stream'], ['group chat', 'chat'], ['search indexing', 'search'], ['code repositories', 'code'], ['music playlists', 'music'],++ ['video thumbnails', 'thumbnail'], ['user profiles', 'profile'], ['channels on openvibe', 'channel'], ['openvibe network', 'network'], ['openvibe media', 'media'],++ ['openvibe actors', 'actor'], ['openvibe report', 'report'], ['openvibe bench', 'bench'], ['openvibe compare', 'compare'], ['openvibe link', 'link'],++ ['openvib live', 'live'], ['opnvibe chat', 'chat'], ['vidoe thumbnails', 'thumbnail'], ['benchmrk models', 'model'], ['commmunity forums', 'forum'],++ ['busqueda de imagenes', 'imagen'], ['recherche images', 'image'], ['suche modelle', 'modell'], ['動画 検索', '動画'], ['поиск видео', 'видео'],++];++if (probes.length !== 50) throw new Error('Benchmark must contain 50 fixed queries');++const base = process.env.SEARCH_BENCH_URL || 'http://127.0.0.1:14710';++const percentile = (values, p) => values.slice().sort((a, b) => a - b)[Math.ceil(values.length * p) - 1];++async function main() {++ const rows = [];++ for (const [q, topic] of probes) {++ const start = performance.now();++ try {++ const res = await fetch(`${base}/api/v1/search?q=${encodeURIComponent(q)}&limit=10`, { signal: AbortSignal.timeout(5000) });++ if (!res.ok) throw new Error(`HTTP ${res.status}`);++ const body = await res.json();++ const ms = Math.round(performance.now() - start);++ const hits = body.results || [];++ const rank = hits.findIndex((r) => `${r.title || ''} ${r.summary || ''}`.toLowerCase().includes(topic.toLowerCase()));++ rows.push({ q, ms, hits: hits.length, topic_rank: rank < 0 ? null : rank + 1 });++ } catch (err) {++ rows.push({ q, ms: Math.round(performance.now() - start), hits: 0, topic_rank: null, error: err.name || 'error' });++ }++ }++ const times = rows.map((r) => r.ms);++ const summary = { queries: rows.length, p50_ms: percentile(times, .5), p95_ms: percentile(times, .95), timeouts: rows.filter((r) => r.error).length, hit_at_10: rows.filter((r) => r.hits).length,++ topic_at_10: rows.filter((r) => r.topic_rank).length, mean_reciprocal_topic_rank: Number((rows.reduce((n, r) => n + (r.topic_rank ? 1 / r.topic_rank : 0), 0) / rows.length).toFixed(3)) };++ process.stdout.write(JSON.stringify({ summary, rows }) + '\n');++}++main().catch((err) => { process.stderr.write(String(err.message || err) + '\n'); process.exitCode = 1; });@@@ -241,7 +241,14 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddingsif (unavailable) return { ranking: 'keyword', reason: unavailable };if (String(text).trim().length < 2) return { ranking: 'keyword', reason: 'short_query' };if (!queryEmbeddings) return { ranking: 'keyword', reason: 'ai_unavailable' };-- const embedded = await queryEmbeddings.get(text);++ let timer;++ let embedded;++ try {++ embedded = await Promise.race([++ queryEmbeddings.get(text, { timeoutMs: 750 }),++ new Promise((resolve) => { timer = setTimeout(() => resolve({ ok: false, error: 'embedding deadline' }), 750); }),++ ]);++ } finally { clearTimeout(timer); }if (!embedded.ok) return { ranking: 'keyword', reason: 'ai_unavailable' };if (vectorStore.dim && embedded.vector.length !== vectorStore.dim) return { ranking: 'keyword', reason: 'ai_unavailable' };return { ranking: 'hybrid', vector: embedded.vector };@@@ -298,9 +305,10 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddingsif (after && after.now != null) refNow = after.now;const page = await engine.search({ match, filters, viewer, limit, after, now: refNow });let position = 0;-- for (const h of page.hits) {++ const foundPage = await Promise.all(page.hits.map((h) => store.byRid(h.rid)));++ for (const [i, h] of page.hits.entries()) {position++;-- const found = await store.byRid(h.rid);++ const found = foundPage[i];// Defence in depth: the engine already filtered; re-check against the stored row.if (!found || !canView(viewer, found.row, found.doc)) continue;const key = dedupeKey(found.doc);@@@ -359,8 +367,9 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings} else {if (after && after.sort_at === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');const page = await engine.browse({ filters, viewer, limit, after });-- for (const h of page.hits) {-- const found = await store.byRid(h.rid);++ const foundPage = await Promise.all(page.hits.map((h) => store.byRid(h.rid)));++ for (const [i, h] of page.hits.entries()) {++ const found = foundPage[i];if (!found || !canView(viewer, found.row, found.doc)) continue;results.push(projection(found.doc, found.row));}@@@ -54,7 +54,7 @@ function createEmbeddingClient({ config, fetchImpl = globalThis.fetch, tokenClie}/** Embed 1..64 texts. → { ok, vectors, dimensions, model, ms } | { ok: false, error, ms }. */-- async function embed(texts) {++ async function embed(texts, { timeoutMs = AI_TIMEOUT_MS } = {}) {const started = now();const ms = () => Math.max(0, Math.round(now() - started));const fail = (error) => ({ ok: false, error, ms: ms() });@@@ -65,10 +65,12 @@ function createEmbeddingClient({ config, fetchImpl = globalThis.fetch, tokenClieconst state = available();if (!state.configured) return fail(state.reason);++ const bounded = Math.max(100, Math.min(AI_TIMEOUT_MS, timeoutMs));++ const timeout = AbortSignal.timeout(bounded);try {const token = await client().getToken({ audience: 'openvibe.ai' });-- const timeout = AbortSignal.timeout(AI_TIMEOUT_MS);-- const res = await fetchImpl(`${aiUrl}/api/v1/embed?wait=30000`, {++ if (timeout.aborted) return fail('embedding deadline');++ const res = await fetchImpl(`${aiUrl}/api/v1/embed?wait=${bounded}`, {method: 'POST',headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', Accept: 'application/json' },body: JSON.stringify({ input: inputs }),@@@ -109,7 +111,7 @@ function createEmbeddingClient({ config, fetchImpl = globalThis.fetch, tokenCliefunction createQueryEmbeddingCache({ client, now = () => Date.now(), ttlMs = 10 * 60 * 1000, max = 500 }) {const entries = new Map(); // insertion order is the LRU order-- async function get(text) {++ async function get(text, { timeoutMs } = {}) {const key = normalizeQuery(text);if (!key) return { ok: false, error: 'empty query' };const hit = entries.get(key);@@@ -121,7 +123,7 @@ function createQueryEmbeddingCache({ client, now = () => Date.now(), ttlMs = 10}entries.delete(key);}-- const r = await client.embed([key]);++ const r = await client.embed([key], { timeoutMs });if (!r.ok) return { ok: false, error: r.error };const vector = r.vectors && r.vectors[0];if (!Array.isArray(vector) || !vector.length) return { ok: false, error: 'AI answered no vector' };@@@ -227,6 +227,21 @@ t('query falls back to keyword with no_vectors, short_query and ai_unavailable',}});++t('a stuck semantic call answers from the keyword index within one second', async () => {++ const vectors = new FakeVectors();++ const embeddings = { ...fakeEmbeddings(), embed: () => new Promise(() => {}) };++ const svc = await boot({ vectorStore: vectors, embeddingClient: embeddings });++ try {++ await svc.store.apply(doc({ id: 'quick', title: 'Search speed' }));++ const start = Date.now();++ const r = await request(svc.base, 'GET', '/api/v1/search?q=search%20speed');++ assert.strictEqual(r.status, 200, r.text);++ assert.ok(Date.now() - start < 1500, 'semantic work has a deadline');++ assert.deepStrictEqual(r.body.results.map((x) => x.id), ['quick']);++ assert.deepStrictEqual(r.body.explain, { ranking: 'keyword', reason: 'ai_unavailable' });++ } finally { await svc.stop(); }++});++t('hybrid: keyword and vector top-100 are fused, ranks are per hit, and a hidden vector hit is dropped', async () => {const vectors = new FakeVectors();const embeddings = fakeEmbeddings();
new file mode 100644
new file mode 100644
@@ -0,0 +1,39 @@
@@ -0,0 +1,39 @@
+'use strict';
+// Fixed public network-search probes. `topic` is a deliberately broad relevance label, not a click judgement.
+const probes = [
+ ['openvibe live', 'live'], ['openvibe chat', 'chat'], ['openvibe codes', 'codes'], ['openvibe design', 'design'], ['openvibe exchange', 'exchange'],
+ ['openvibe search', 'search'], ['openvibe wiki', 'wiki'], ['openvibe video', 'video'], ['openvibe pics', 'pics'], ['openvibe music', 'music'],
+ ['live channels', 'channel'], ['live recordings', 'record'], ['stream clips', 'clip'], ['video uploads', 'video'], ['community forum', 'forum'],
+ ['AI models', 'model'], ['AI providers', 'provider'], ['model benchmarks', 'benchmark'], ['image search', 'image'], ['web search', 'search'],
+ ['developer API', 'api'], ['software development', 'software'], ['robot controls', 'robot'], ['open source projects', 'open source'], ['privacy policy', 'privacy'],
+ ['live streaming', 'stream'], ['group chat', 'chat'], ['search indexing', 'search'], ['code repositories', 'code'], ['music playlists', 'music'],
+ ['video thumbnails', 'thumbnail'], ['user profiles', 'profile'], ['channels on openvibe', 'channel'], ['openvibe network', 'network'], ['openvibe media', 'media'],
+ ['openvibe actors', 'actor'], ['openvibe report', 'report'], ['openvibe bench', 'bench'], ['openvibe compare', 'compare'], ['openvibe link', 'link'],
+ ['openvib live', 'live'], ['opnvibe chat', 'chat'], ['vidoe thumbnails', 'thumbnail'], ['benchmrk models', 'model'], ['commmunity forums', 'forum'],
+ ['busqueda de imagenes', 'imagen'], ['recherche images', 'image'], ['suche modelle', 'modell'], ['動画 検索', '動画'], ['поиск видео', 'видео'],
+];
+if (probes.length !== 50) throw new Error('Benchmark must contain 50 fixed queries');
+const base = process.env.SEARCH_BENCH_URL || 'http://127.0.0.1:14710';
+const percentile = (values, p) => values.slice().sort((a, b) => a - b)[Math.ceil(values.length * p) - 1];
+async function main() {
+ const rows = [];
+ for (const [q, topic] of probes) {
+ const start = performance.now();
+ try {
+ const res = await fetch(`${base}/api/v1/search?q=${encodeURIComponent(q)}&limit=10`, { signal: AbortSignal.timeout(5000) });
+ if (!res.ok) throw new Error(`HTTP ${res.status}`);
+ const body = await res.json();
+ const ms = Math.round(performance.now() - start);
+ const hits = body.results || [];
+ const rank = hits.findIndex((r) => `${r.title || ''} ${r.summary || ''}`.toLowerCase().includes(topic.toLowerCase()));
+ rows.push({ q, ms, hits: hits.length, topic_rank: rank < 0 ? null : rank + 1 });
+ } catch (err) {
+ rows.push({ q, ms: Math.round(performance.now() - start), hits: 0, topic_rank: null, error: err.name || 'error' });
+ }
+ }
+ const times = rows.map((r) => r.ms);
+ const summary = { queries: rows.length, p50_ms: percentile(times, .5), p95_ms: percentile(times, .95), timeouts: rows.filter((r) => r.error).length, hit_at_10: rows.filter((r) => r.hits).length,
+ topic_at_10: rows.filter((r) => r.topic_rank).length, mean_reciprocal_topic_rank: Number((rows.reduce((n, r) => n + (r.topic_rank ? 1 / r.topic_rank : 0), 0) / rows.length).toFixed(3)) };
+ process.stdout.write(JSON.stringify({ summary, rows }) + '\n');
+}
+main().catch((err) => { process.stderr.write(String(err.message || err) + '\n'); process.exitCode = 1; });
@@ -241,7 +241,14 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
@@ -241,7 +241,14 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
if (unavailable) return { ranking: 'keyword', reason: unavailable };
if (unavailable) return { ranking: 'keyword', reason: unavailable };
if (String(text).trim().length < 2) return { ranking: 'keyword', reason: 'short_query' };
if (String(text).trim().length < 2) return { ranking: 'keyword', reason: 'short_query' };
if (!queryEmbeddings) return { ranking: 'keyword', reason: 'ai_unavailable' };
if (!queryEmbeddings) return { ranking: 'keyword', reason: 'ai_unavailable' };
- const embedded = await queryEmbeddings.get(text);
+ let timer;
+ let embedded;
+ try {
+ embedded = await Promise.race([
+ queryEmbeddings.get(text, { timeoutMs: 750 }),
+ new Promise((resolve) => { timer = setTimeout(() => resolve({ ok: false, error: 'embedding deadline' }), 750); }),
+ ]);
+ } finally { clearTimeout(timer); }
if (!embedded.ok) return { ranking: 'keyword', reason: 'ai_unavailable' };
if (!embedded.ok) return { ranking: 'keyword', reason: 'ai_unavailable' };
if (vectorStore.dim && embedded.vector.length !== vectorStore.dim) return { ranking: 'keyword', reason: 'ai_unavailable' };
if (vectorStore.dim && embedded.vector.length !== vectorStore.dim) return { ranking: 'keyword', reason: 'ai_unavailable' };
return { ranking: 'hybrid', vector: embedded.vector };
return { ranking: 'hybrid', vector: embedded.vector };
@@ -298,9 +305,10 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
@@ -298,9 +305,10 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
if (after && after.now != null) refNow = after.now;
if (after && after.now != null) refNow = after.now;
const page = await engine.search({ match, filters, viewer, limit, after, now: refNow });
const page = await engine.search({ match, filters, viewer, limit, after, now: refNow });
let position = 0;
let position = 0;
- for (const h of page.hits) {
+ const foundPage = await Promise.all(page.hits.map((h) => store.byRid(h.rid)));
+ for (const [i, h] of page.hits.entries()) {
position++;
position++;
- const found = await store.byRid(h.rid);
+ const found = foundPage[i];
// Defence in depth: the engine already filtered; re-check against the stored row.
// Defence in depth: the engine already filtered; re-check against the stored row.
if (!found || !canView(viewer, found.row, found.doc)) continue;
if (!found || !canView(viewer, found.row, found.doc)) continue;
const key = dedupeKey(found.doc);
const key = dedupeKey(found.doc);
@@ -359,8 +367,9 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
@@ -359,8 +367,9 @@ function createSearcher({ config, store, engine, vectorStore = null, embeddings
} else {
} else {
if (after && after.sort_at === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');
if (after && after.sort_at === undefined) throw new QueryError('search.bad_cursor', 'cursor does not belong to this query');
const page = await engine.browse({ filters, viewer, limit, after });
const page = await engine.browse({ filters, viewer, limit, after });
- for (const h of page.hits) {
- const found = await store.byRid(h.rid);
+ const foundPage = await Promise.all(page.hits.map((h) => store.byRid(h.rid)));
+ for (const [i, h] of page.hits.entries()) {
+ const found = foundPage[i];
if (!found || !canView(viewer, found.row, found.doc)) continue;
if (!found || !canView(viewer, found.row, found.doc)) continue;
results.push(projection(found.doc, found.row));
results.push(projection(found.doc, found.row));
}
}
@@ -54,7 +54,7 @@ function createEmbeddingClient({ config, fetchImpl = globalThis.fetch, tokenClie
@@ -54,7 +54,7 @@ function createEmbeddingClient({ config, fetchImpl = globalThis.fetch, tokenClie
}
}
/** Embed 1..64 texts. → { ok, vectors, dimensions, model, ms } | { ok: false, error, ms }. */
/** Embed 1..64 texts. → { ok, vectors, dimensions, model, ms } | { ok: false, error, ms }. */
- async function embed(texts) {
+ async function embed(texts, { timeoutMs = AI_TIMEOUT_MS } = {}) {
const started = now();
const started = now();
const ms = () => Math.max(0, Math.round(now() - started));
const ms = () => Math.max(0, Math.round(now() - started));
const fail = (error) => ({ ok: false, error, ms: ms() });
const fail = (error) => ({ ok: false, error, ms: ms() });
@@ -65,10 +65,12 @@ function createEmbeddingClient({ config, fetchImpl = globalThis.fetch, tokenClie
@@ -65,10 +65,12 @@ function createEmbeddingClient({ config, fetchImpl = globalThis.fetch, tokenClie
const state = available();
const state = available();
if (!state.configured) return fail(state.reason);
if (!state.configured) return fail(state.reason);
+ const bounded = Math.max(100, Math.min(AI_TIMEOUT_MS, timeoutMs));
+ const timeout = AbortSignal.timeout(bounded);
try {
try {
const token = await client().getToken({ audience: 'openvibe.ai' });
const token = await client().getToken({ audience: 'openvibe.ai' });
- const timeout = AbortSignal.timeout(AI_TIMEOUT_MS);
- const res = await fetchImpl(`${aiUrl}/api/v1/embed?wait=30000`, {
+ if (timeout.aborted) return fail('embedding deadline');
+ const res = await fetchImpl(`${aiUrl}/api/v1/embed?wait=${bounded}`, {
method: 'POST',
method: 'POST',
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', Accept: 'application/json' },
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', Accept: 'application/json' },
body: JSON.stringify({ input: inputs }),
body: JSON.stringify({ input: inputs }),
@@ -109,7 +111,7 @@ function createEmbeddingClient({ config, fetchImpl = globalThis.fetch, tokenClie
@@ -109,7 +111,7 @@ function createEmbeddingClient({ config, fetchImpl = globalThis.fetch, tokenClie
function createQueryEmbeddingCache({ client, now = () => Date.now(), ttlMs = 10 * 60 * 1000, max = 500 }) {
function createQueryEmbeddingCache({ client, now = () => Date.now(), ttlMs = 10 * 60 * 1000, max = 500 }) {
const entries = new Map(); // insertion order is the LRU order
const entries = new Map(); // insertion order is the LRU order
- async function get(text) {
+ async function get(text, { timeoutMs } = {}) {
const key = normalizeQuery(text);
const key = normalizeQuery(text);
if (!key) return { ok: false, error: 'empty query' };
if (!key) return { ok: false, error: 'empty query' };
const hit = entries.get(key);
const hit = entries.get(key);
@@ -121,7 +123,7 @@ function createQueryEmbeddingCache({ client, now = () => Date.now(), ttlMs = 10
@@ -121,7 +123,7 @@ function createQueryEmbeddingCache({ client, now = () => Date.now(), ttlMs = 10
}
}
entries.delete(key);
entries.delete(key);
}
}
- const r = await client.embed([key]);
+ const r = await client.embed([key], { timeoutMs });
if (!r.ok) return { ok: false, error: r.error };
if (!r.ok) return { ok: false, error: r.error };
const vector = r.vectors && r.vectors[0];
const vector = r.vectors && r.vectors[0];
if (!Array.isArray(vector) || !vector.length) return { ok: false, error: 'AI answered no vector' };
if (!Array.isArray(vector) || !vector.length) return { ok: false, error: 'AI answered no vector' };
@@ -227,6 +227,21 @@ t('query falls back to keyword with no_vectors, short_query and ai_unavailable',
@@ -227,6 +227,21 @@ t('query falls back to keyword with no_vectors, short_query and ai_unavailable',
}
}
});
});
+t('a stuck semantic call answers from the keyword index within one second', async () => {
+ const vectors = new FakeVectors();
+ const embeddings = { ...fakeEmbeddings(), embed: () => new Promise(() => {}) };
+ const svc = await boot({ vectorStore: vectors, embeddingClient: embeddings });
+ try {
+ await svc.store.apply(doc({ id: 'quick', title: 'Search speed' }));
+ const start = Date.now();
+ const r = await request(svc.base, 'GET', '/api/v1/search?q=search%20speed');
+ assert.strictEqual(r.status, 200, r.text);
+ assert.ok(Date.now() - start < 1500, 'semantic work has a deadline');
+ assert.deepStrictEqual(r.body.results.map((x) => x.id), ['quick']);
+ assert.deepStrictEqual(r.body.explain, { ranking: 'keyword', reason: 'ai_unavailable' });
+ } finally { await svc.stop(); }
+});
+
t('hybrid: keyword and vector top-100 are fused, ranks are per hit, and a hidden vector hit is dropped', async () => {
t('hybrid: keyword and vector top-100 are fused, ranks are per hit, and a hidden vector hit is dropped', async () => {
const vectors = new FakeVectors();
const vectors = new FakeVectors();
const embeddings = fakeEmbeddings();
const embeddings = fakeEmbeddings();