openvibers/openvibe.ai Commits

MegaSource: drop DeepSeek's '#n' hit fragments from source URLs (#150)

OP OpenVibers committed 19 days ago · 4dd4568d0a · 2 files, +3 −2

Changes

ddiff --git a/server/research/deepseek.js b/server/research/deepseek.js
iindex b771ede..f0f51d4 100644
---- a/server/research/deepseek.js
++++ b/server/research/deepseek.js
@@@ -49,8 +49,9 @@ function createDeepseek({ http, key, base = 'https://api.deepseek.com', dailyCap
const body = { query: String(query).slice(0, 128), count, need_content: needContent };
if (question) body.ori_question = String(question).slice(0, 128);
const j = await call('/v1/tools/search', body, signal);
++ // DeepSeek numbers repeated hits on one page with a fragment ("…#1"): the page is the same without it.
return (j.results || []).filter((r) => r && canonicalUrl(r.url)).map((r) => ({
-- url: r.url,
++ url: String(r.url).replace(/#\d+$/, ''),
title: String(r.title || '').slice(0, 300),
site: r.site_name || null,
snippet: String(r.snippet || '').slice(0, 400),
ddiff --git a/test/fixtures/research-fetch.js b/test/fixtures/research-fetch.js
iindex 863b3fd..214404e 100644
---- a/test/fixtures/research-fetch.js
++++ b/test/fixtures/research-fetch.js
@@@ -52,7 +52,7 @@ function createResearchFetch({ deepseek = { status: 200 }, fandom = false } = {}
if (u.pathname === '/v1/tools/search') return json({ results: [
{ title: 'SQLite Home Page', url: 'https://sqlite.org/index.html?utm_source=a', snippet: 'Small. Fast. Reliable.', summary: 'SQLite is a C library.', content: 'SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured SQL database engine. '.repeat(6), site_name: 'SQLite', date: '2026-01-01T00:00:00Z', authority: 4, rerank_score: 2.8 },
{ title: 'SQLite — Wikipedia mirror', url: 'https://en.m.wikipedia.org/wiki/SQLite', snippet: 'dup', summary: 'dup', content: 'duplicate of the encyclopedia page', authority: 3, rerank_score: 2.0 },
-- { title: 'A thin page', url: 'https://example.com/thin', snippet: 'short', summary: '', content: '', authority: 1, rerank_score: 1.2 },
++ { title: 'A thin page', url: 'https://example.com/thin#1', snippet: 'short', summary: '', content: '', authority: 1, rerank_score: 1.2 },
] });
if (u.pathname === '/v1/tools/fetch') return json({ title: 'A thin page', content: 'The full text DeepSeek fetched for the thin page.' });
}