openvibers/openvibe.ai Commits
MegaSource: drop DeepSeek's '#n' hit fragments from source URLs (#150)
Changes
@@@ -49,8 +49,9 @@ function createDeepseek({ http, key, base = 'https://api.deepseek.com', dailyCapconst body = { query: String(query).slice(0, 128), count, need_content: needContent };if (question) body.ori_question = String(question).slice(0, 128);const j = await call('/v1/tools/search', body, signal);++ // DeepSeek numbers repeated hits on one page with a fragment ("…#1"): the page is the same without it.return (j.results || []).filter((r) => r && canonicalUrl(r.url)).map((r) => ({-- url: r.url,++ url: String(r.url).replace(/#\d+$/, ''),title: String(r.title || '').slice(0, 300),site: r.site_name || null,snippet: String(r.snippet || '').slice(0, 400),@@@ -52,7 +52,7 @@ function createResearchFetch({ deepseek = { status: 200 }, fandom = false } = {}if (u.pathname === '/v1/tools/search') return json({ results: [{ title: 'SQLite Home Page', url: 'https://sqlite.org/index.html?utm_source=a', snippet: 'Small. Fast. Reliable.', summary: 'SQLite is a C library.', content: 'SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured SQL database engine. '.repeat(6), site_name: 'SQLite', date: '2026-01-01T00:00:00Z', authority: 4, rerank_score: 2.8 },{ title: 'SQLite — Wikipedia mirror', url: 'https://en.m.wikipedia.org/wiki/SQLite', snippet: 'dup', summary: 'dup', content: 'duplicate of the encyclopedia page', authority: 3, rerank_score: 2.0 },-- { title: 'A thin page', url: 'https://example.com/thin', snippet: 'short', summary: '', content: '', authority: 1, rerank_score: 1.2 },++ { title: 'A thin page', url: 'https://example.com/thin#1', snippet: 'short', summary: '', content: '', authority: 1, rerank_score: 1.2 },] });if (u.pathname === '/v1/tools/fetch') return json({ title: 'A thin page', content: 'The full text DeepSeek fetched for the thin page.' });}
@@ -49,8 +49,9 @@ function createDeepseek({ http, key, base = 'https://api.deepseek.com', dailyCap
@@ -49,8 +49,9 @@ function createDeepseek({ http, key, base = 'https://api.deepseek.com', dailyCap
const body = { query: String(query).slice(0, 128), count, need_content: needContent };
const body = { query: String(query).slice(0, 128), count, need_content: needContent };
if (question) body.ori_question = String(question).slice(0, 128);
if (question) body.ori_question = String(question).slice(0, 128);
const j = await call('/v1/tools/search', body, signal);
const j = await call('/v1/tools/search', body, signal);
+ // DeepSeek numbers repeated hits on one page with a fragment ("…#1"): the page is the same without it.
return (j.results || []).filter((r) => r && canonicalUrl(r.url)).map((r) => ({
return (j.results || []).filter((r) => r && canonicalUrl(r.url)).map((r) => ({
- url: r.url,
+ url: String(r.url).replace(/#\d+$/, ''),
title: String(r.title || '').slice(0, 300),
title: String(r.title || '').slice(0, 300),
site: r.site_name || null,
site: r.site_name || null,
snippet: String(r.snippet || '').slice(0, 400),
snippet: String(r.snippet || '').slice(0, 400),
@@ -52,7 +52,7 @@ function createResearchFetch({ deepseek = { status: 200 }, fandom = false } = {}
@@ -52,7 +52,7 @@ function createResearchFetch({ deepseek = { status: 200 }, fandom = false } = {}
if (u.pathname === '/v1/tools/search') return json({ results: [
if (u.pathname === '/v1/tools/search') return json({ results: [
{ title: 'SQLite Home Page', url: 'https://sqlite.org/index.html?utm_source=a', snippet: 'Small. Fast. Reliable.', summary: 'SQLite is a C library.', content: 'SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured SQL database engine. '.repeat(6), site_name: 'SQLite', date: '2026-01-01T00:00:00Z', authority: 4, rerank_score: 2.8 },
{ title: 'SQLite Home Page', url: 'https://sqlite.org/index.html?utm_source=a', snippet: 'Small. Fast. Reliable.', summary: 'SQLite is a C library.', content: 'SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured SQL database engine. '.repeat(6), site_name: 'SQLite', date: '2026-01-01T00:00:00Z', authority: 4, rerank_score: 2.8 },
{ title: 'SQLite — Wikipedia mirror', url: 'https://en.m.wikipedia.org/wiki/SQLite', snippet: 'dup', summary: 'dup', content: 'duplicate of the encyclopedia page', authority: 3, rerank_score: 2.0 },
{ title: 'SQLite — Wikipedia mirror', url: 'https://en.m.wikipedia.org/wiki/SQLite', snippet: 'dup', summary: 'dup', content: 'duplicate of the encyclopedia page', authority: 3, rerank_score: 2.0 },
- { title: 'A thin page', url: 'https://example.com/thin', snippet: 'short', summary: '', content: '', authority: 1, rerank_score: 1.2 },
+ { title: 'A thin page', url: 'https://example.com/thin#1', snippet: 'short', summary: '', content: '', authority: 1, rerank_score: 1.2 },
] });
] });
if (u.pathname === '/v1/tools/fetch') return json({ title: 'A thin page', content: 'The full text DeepSeek fetched for the thin page.' });
if (u.pathname === '/v1/tools/fetch') return json({ title: 'A thin page', content: 'The full text DeepSeek fetched for the thin page.' });
}
}