openvibers/openvibe.community Commits
Search: a paste whose slug starts with '-' or '_' is indexed as paste_<row id> (a Search document id starts with a letter or digit; Search refused -wgXuYn0); the refused id is forgotten, with no tombstone
Changes
@@@ -21,6 +21,10 @@ const events = require('../events');const SCAN_MS = 60 * 1000;const REFRESH_MS = 60 * 60 * 1000;const SLUG_RE = /^[A-Za-z0-9_-]{1,80}$/;++// A Search document id starts with a letter or digit (search.index-document@1): a paste whose slug starts++// with '-' or '_' is indexed as paste_<row id> instead.++const DOC_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,79}$/;++const pasteDocId = (p) => (DOC_ID_RE.test(String(p.slug || '')) ? p.slug : `paste_${p.id}`);const iso = (v) => {if (!v) return null;@@@ -93,7 +97,7 @@ function createSearchDocuments({ db }) {if (!shot && p.language) facets.syntax = String(p.language).slice(0, 200);if (tags.length) facets.tags = tags;const doc = {-- owner: 'community', type: 'paste', id: p.slug, deleted: false, visibility: 'public',++ owner: 'community', type: 'paste', id: pasteDocId(p), deleted: false, visibility: 'public',canonical_url: `${config.baseUrl}/p/${encodeURIComponent(p.slug)}`,title: clean(p.title, 500) || (shot ? 'Screenshot' : 'Untitled'),summary: clean(p.ai_summary, 4000) || clean(shot ? '' : p.content, 300) || `A ${shot ? 'screenshot' : 'paste'} on OpenVibe.Community.`,@@@ -129,7 +133,7 @@ function createSearchDocuments({ db }) {const publishThread = (threadId) => send('thread', String(threadId), threadDocument(threadId));function publishPaste(row) {if (!row || !SLUG_RE.test(String(row.slug || ''))) return 'skipped';-- return send('paste', row.slug, pasteDocument(row));++ return send('paste', pasteDocId(row), pasteDocument(row));}const guard = (fn) => { try { fn(); } catch (err) { stats.lastError = err.message; } };@@@ -150,9 +154,12 @@ function createSearchDocuments({ db }) {if (!enabled()) return 0;const rows = db.prepare('SELECT * FROM pastes').all();const seen = new Set();-- for (const p of rows) { seen.add(p.slug); guard(() => publishPaste(p)); }++ for (const p of rows) { seen.add(pasteDocId(p)); guard(() => publishPaste(p)); }for (const r of db.prepare("SELECT id FROM search_doc_pushes WHERE type = 'paste' AND deleted = 0").all()) {-- if (!seen.has(r.id)) guard(() => send('paste', r.id, { deleted: true }));++ if (seen.has(r.id)) continue;++ // An id Search could never accept (sent before paste_<id> existed): nothing to remove there.++ if (!DOC_ID_RE.test(r.id)) db.prepare("DELETE FROM search_doc_pushes WHERE type = 'paste' AND id = ?").run(r.id);++ else guard(() => send('paste', r.id, { deleted: true }));}return rows.length;}@@@ -96,5 +96,18 @@ check('the scan finds a paste changed since the last scan; the refresh removes oassert.deepStrictEqual(last.payload, { type: 'paste', id: 'green-elk-3', revision: 2 });});++check('a slug starting with a dash is indexed as paste_<id> (a document id starts with a letter or digit)', () => {++ const p = pastes.insertPaste(db, { slug: '-wgXuYn0', owner_subject: USR, type: 'paste', title: 'Dash', content: 'dash body', visibility: 'public' });++ db.prepare("INSERT INTO search_doc_pushes (type, id, hash, revision) VALUES ('paste', '-wgXuYn0', 'x', 1)").run();++ assert.strictEqual(docs.publishPaste(db.prepare('SELECT * FROM pastes WHERE id = ?').get(p.id)), 'sent');++ const ev = index().at(-1); ok(ev);++ assert.strictEqual(ev.payload.id, `paste_${p.id}`); assert.strictEqual(ev.payload.canonical_url, `${base}/p/-wgXuYn0`);++ const r = contracts.validate('search.index-document@1', ev.payload); assert.ok(r.valid, JSON.stringify(r.errors));++ const before = index().length;++ docs.refresh();++ assert.ok(!db.prepare("SELECT 1 FROM search_doc_pushes WHERE id = '-wgXuYn0'").get(), 'the unacceptable id is forgotten');++ assert.ok(!index().slice(before).some((e) => e.payload.id === '-wgXuYn0'), 'and no tombstone is sent for it');++});++events._reset();console.log(`search documents: ${n} checks passed`);
@@ -21,6 +21,10 @@ const events = require('../events');
@@ -21,6 +21,10 @@ const events = require('../events');
const SCAN_MS = 60 * 1000;
const SCAN_MS = 60 * 1000;
const REFRESH_MS = 60 * 60 * 1000;
const REFRESH_MS = 60 * 60 * 1000;
const SLUG_RE = /^[A-Za-z0-9_-]{1,80}$/;
const SLUG_RE = /^[A-Za-z0-9_-]{1,80}$/;
+// A Search document id starts with a letter or digit (search.index-document@1): a paste whose slug starts
+// with '-' or '_' is indexed as paste_<row id> instead.
+const DOC_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,79}$/;
+const pasteDocId = (p) => (DOC_ID_RE.test(String(p.slug || '')) ? p.slug : `paste_${p.id}`);
const iso = (v) => {
const iso = (v) => {
if (!v) return null;
if (!v) return null;
@@ -93,7 +97,7 @@ function createSearchDocuments({ db }) {
@@ -93,7 +97,7 @@ function createSearchDocuments({ db }) {
if (!shot && p.language) facets.syntax = String(p.language).slice(0, 200);
if (!shot && p.language) facets.syntax = String(p.language).slice(0, 200);
if (tags.length) facets.tags = tags;
if (tags.length) facets.tags = tags;
const doc = {
const doc = {
- owner: 'community', type: 'paste', id: p.slug, deleted: false, visibility: 'public',
+ owner: 'community', type: 'paste', id: pasteDocId(p), deleted: false, visibility: 'public',
canonical_url: `${config.baseUrl}/p/${encodeURIComponent(p.slug)}`,
canonical_url: `${config.baseUrl}/p/${encodeURIComponent(p.slug)}`,
title: clean(p.title, 500) || (shot ? 'Screenshot' : 'Untitled'),
title: clean(p.title, 500) || (shot ? 'Screenshot' : 'Untitled'),
summary: clean(p.ai_summary, 4000) || clean(shot ? '' : p.content, 300) || `A ${shot ? 'screenshot' : 'paste'} on OpenVibe.Community.`,
summary: clean(p.ai_summary, 4000) || clean(shot ? '' : p.content, 300) || `A ${shot ? 'screenshot' : 'paste'} on OpenVibe.Community.`,
@@ -129,7 +133,7 @@ function createSearchDocuments({ db }) {
@@ -129,7 +133,7 @@ function createSearchDocuments({ db }) {
const publishThread = (threadId) => send('thread', String(threadId), threadDocument(threadId));
const publishThread = (threadId) => send('thread', String(threadId), threadDocument(threadId));
function publishPaste(row) {
function publishPaste(row) {
if (!row || !SLUG_RE.test(String(row.slug || ''))) return 'skipped';
if (!row || !SLUG_RE.test(String(row.slug || ''))) return 'skipped';
- return send('paste', row.slug, pasteDocument(row));
+ return send('paste', pasteDocId(row), pasteDocument(row));
}
}
const guard = (fn) => { try { fn(); } catch (err) { stats.lastError = err.message; } };
const guard = (fn) => { try { fn(); } catch (err) { stats.lastError = err.message; } };
@@ -150,9 +154,12 @@ function createSearchDocuments({ db }) {
@@ -150,9 +154,12 @@ function createSearchDocuments({ db }) {
if (!enabled()) return 0;
if (!enabled()) return 0;
const rows = db.prepare('SELECT * FROM pastes').all();
const rows = db.prepare('SELECT * FROM pastes').all();
const seen = new Set();
const seen = new Set();
- for (const p of rows) { seen.add(p.slug); guard(() => publishPaste(p)); }
+ for (const p of rows) { seen.add(pasteDocId(p)); guard(() => publishPaste(p)); }
for (const r of db.prepare("SELECT id FROM search_doc_pushes WHERE type = 'paste' AND deleted = 0").all()) {
for (const r of db.prepare("SELECT id FROM search_doc_pushes WHERE type = 'paste' AND deleted = 0").all()) {
- if (!seen.has(r.id)) guard(() => send('paste', r.id, { deleted: true }));
+ if (seen.has(r.id)) continue;
+ // An id Search could never accept (sent before paste_<id> existed): nothing to remove there.
+ if (!DOC_ID_RE.test(r.id)) db.prepare("DELETE FROM search_doc_pushes WHERE type = 'paste' AND id = ?").run(r.id);
+ else guard(() => send('paste', r.id, { deleted: true }));
}
}
return rows.length;
return rows.length;
}
}
@@ -96,5 +96,18 @@ check('the scan finds a paste changed since the last scan; the refresh removes o
@@ -96,5 +96,18 @@ check('the scan finds a paste changed since the last scan; the refresh removes o
assert.deepStrictEqual(last.payload, { type: 'paste', id: 'green-elk-3', revision: 2 });
assert.deepStrictEqual(last.payload, { type: 'paste', id: 'green-elk-3', revision: 2 });
});
});
+check('a slug starting with a dash is indexed as paste_<id> (a document id starts with a letter or digit)', () => {
+ const p = pastes.insertPaste(db, { slug: '-wgXuYn0', owner_subject: USR, type: 'paste', title: 'Dash', content: 'dash body', visibility: 'public' });
+ db.prepare("INSERT INTO search_doc_pushes (type, id, hash, revision) VALUES ('paste', '-wgXuYn0', 'x', 1)").run();
+ assert.strictEqual(docs.publishPaste(db.prepare('SELECT * FROM pastes WHERE id = ?').get(p.id)), 'sent');
+ const ev = index().at(-1); ok(ev);
+ assert.strictEqual(ev.payload.id, `paste_${p.id}`); assert.strictEqual(ev.payload.canonical_url, `${base}/p/-wgXuYn0`);
+ const r = contracts.validate('search.index-document@1', ev.payload); assert.ok(r.valid, JSON.stringify(r.errors));
+ const before = index().length;
+ docs.refresh();
+ assert.ok(!db.prepare("SELECT 1 FROM search_doc_pushes WHERE id = '-wgXuYn0'").get(), 'the unacceptable id is forgotten');
+ assert.ok(!index().slice(before).some((e) => e.payload.id === '-wgXuYn0'), 'and no tombstone is sent for it');
+});
+
events._reset();
events._reset();
console.log(`search documents: ${n} checks passed`);
console.log(`search documents: ${n} checks passed`);