openvibers/openvibe.wiki Commits
Vibe: source numbers a writer leaves in the text ([0][2]) are removed; a picture that is not free reads '© its owners'; a job's lease is renewed every minute (3 min), so a restart resumes it quickly (#106)
Changes
@@@ -26,7 +26,9 @@ const { fetchImage } = require('../integrations/image-fetch');const SIZES = ['quick', 'standard', 'big'];const ACTIVE = ['queued', 'researching', 'planning', 'writing'];--const LEASE_MS = 10 * 60 * 1000;++// A job's lease is renewed every minute while it runs, so a restart frees it within LEASE_MS.++const LEASE_MS = 3 * 60 * 1000;++const HEARTBEAT_MS = 60 * 1000;const DAY_MS = 24 * 60 * 60 * 1000;const MAX_IMAGES_PER_PAGE = 3;// How the main page groups the rest, by what the plan says each page is about.@@@ -102,18 +104,25 @@ function creditOf(img) {let site = null;try { site = img.source_url ? new URL(img.source_url).hostname.replace(/^www\./, '') : null; } catch { /* no site */ }const who = oneLine(img.author, 80) || site || oneLine(img.provider, 40);-- return [who ? `Picture: ${who}` : 'Picture', img.license ? oneLine(img.license, 80) : null].filter(Boolean).join(', ');++ // A picture that is not freely licensed is shown to identify the subject; its owners keep the copyright.++ const terms = img.free ? oneLine(img.license, 60) : '© its owners';++ return [who ? `Picture: ${who}` : 'Picture', terms || null].filter(Boolean).join(', ');++}++++/** Source numbers a writer left in the text ("…stories.[0][2]"): the page lists its sources, so they go. Links stay. */++function stripMarkers(text) {++ return String(text || '').replace(/(?:\s?\[\d{1,2}(?:\s*[,–-]\s*\d{1,2})*\])+(?![(:])/g, '').replace(/[ \t]+([.,;:!?])/g, '$1');}/** The Markdown of one page from the writer's output: lead, sections with their pictures, and the main page's contents. */function composeBody(out, { placed = new Map(), contents = null }) {const parts = [];-- const lead = String(out.lead || '').trim();++ const lead = stripMarkers(out.lead).trim();if (lead) parts.push(lead);const used = new Set();for (const sec of Array.isArray(out.sections) ? out.sections : []) {const heading = oneLine(sec.heading, 120).replace(/^#+\s*/, '');-- const body = String(sec.body || '').trim().replace(/^#{1,2}\s/gm, '### ');++ const body = stripMarkers(sec.body).trim().replace(/^#{1,2}\s/gm, '### ');if (!heading || !body) continue;const key = heading.toLowerCase();const pic = placed.get(key);@@@ -260,7 +269,9 @@ function createVibes({ db, svc, platform, config, now = () => Date.now(), log =try {const v = await q.claim.get({ owner, until: now() + LEASE_MS, now: now() });if (!v) return false;-- await runJob(v);++ const beat = setInterval(() => { q.renew.run(now() + LEASE_MS, v.id, owner).catch(() => {}); }, HEARTBEAT_MS);++ if (beat.unref) beat.unref();++ try { await runJob(v); } finally { clearInterval(beat); }return true;} finally { running = false; }}@@@ -449,4 +460,4 @@ function createVibes({ db, svc, platform, config, now = () => Date.now(), log =return { start, get, mine, recent, tick, startWorker, stopWorker, SIZES, ACTIVE, pagesFor };}--module.exports = { createVibes, VibeError, normalizePlan, composeBody, contentsOf, citedIndices, creditOf, spaceSlugFor, SIZES };++module.exports = { createVibes, VibeError, normalizePlan, composeBody, contentsOf, citedIndices, creditOf, spaceSlugFor, stripMarkers, SIZES };@@@ -10,7 +10,7 @@ const assert = require('assert');const H = require('./helpers');const content = require('../server/wiki/content');const { checkUrl, sniff } = require('../server/integrations/image-fetch');--const { normalizePlan, spaceSlugFor, contentsOf } = require('../server/wiki/vibe');++const { normalizePlan, spaceSlugFor, contentsOf, stripMarkers, creditOf } = require('../server/wiki/vibe');const MED = (n) => `med_01J8Z6Q3KX0000000000000${String(n).padStart(3, '0')}`;const PNG = Buffer.from('89504e470d0a1a0a0000000d4948445200000001000000010806000000', 'hex');@@@ -33,6 +33,9 @@ const LONG = 'Kim Possible is a teenager who saves the world between classes, wiconst plan = normalizePlan({ title: 'KP', summary: 's', pages: [{ title: 'Ron', kind: 'character', outline: ['A'] }, { title: 'Kim Possible', kind: 'overview' }, { title: 'ron', kind: 'person' }, { title: 'Bad [[x]]|', kind: 'weird' }] }, 'Kim Possible', 3);assert.deepStrictEqual(plan.pages.map((p) => [p.title, p.kind]), [['Kim Possible', 'overview'], ['Ron', 'character'], ['Bad x', 'other']], 'main page first, duplicates out, kinds known, link syntax stripped');assert.match(contentsOf(plan), /### Characters\n\n- \[\[Ron\]\]/);++ assert.strictEqual(stripMarkers('Her stories.[0][2][4] See [the site](https://x.test) and [[Ron]] [3, 5].'), 'Her stories. See [the site](https://x.test) and [[Ron]].');++ assert.strictEqual(creditOf({ author: null, source_url: 'https://kimpossible.fandom.com/wiki/Shego', free: false, license: 'Fandom wiki picture' }), 'Picture: kimpossible.fandom.com, © its owners');++ assert.strictEqual(creditOf({ author: 'Jane', free: true, license: 'CC BY 2.0' }), 'Picture: Jane, CC BY 2.0');}(async () => {@@@ -51,7 +54,7 @@ const LONG = 'Kim Possible is a teenager who saves the world between classes, wifacts: [], topic_wiki: { name: 'Kim Possible Wiki', url: 'https://kimpossible.fandom.com' }, providers: { wikipedia: { ok: true, count: 1 }, fandom: { ok: true, count: 1 }, deepseek: { ok: false, skipped: 'not configured' } }, usage: {},});const page = (title) => ({-- title, summary: `${title} is part of Kim Possible.`, lead: `**${title}** is part of the show. ${title === 'Kim Possible' ? 'Her friend is [[Ron Stoppable]].' : 'He is friends with [[Kim Possible]].'} ${LONG}`,++ title, summary: `${title} is part of Kim Possible.`, lead: `**${title}** is part of the show.[0][1] ${title === 'Kim Possible' ? 'Her friend is [[Ron Stoppable]].' : 'He is friends with [[Kim Possible]].'} ${LONG}`,sections: [{ heading: 'Premise', body: LONG, citations: [0] }, { heading: 'Legacy', body: LONG, citations: [1] }],infobox: [{ label: 'Created by', value: 'Bob Schooley and Mark McCorkle', citations: [0] }, { label: 'Network', value: 'Disney Channel', citations: [0] }],images: [{ index: 0, caption: `${title} on screen`, section: 'lead' }, { index: 7, caption: 'out of range', section: 'Premise' }],@@@ -160,6 +163,7 @@ const LONG = 'Kim Possible is a teenager who saves the world between classes, wiassert.match(r.text, new RegExp(`class="wk-ib-pic"[\\s\\S]*?<img src="https://openvibe.media/o/${MED(1)}"`), 'the lead picture in the infobox');assert.match(r.text, /Kim Possible on screen \(Picture: Jane Doe, CC BY-SA 4.0\)/, 'captioned with its credit');assert.match(r.text, /<th scope="row">Created by<\/th><td>Bob Schooley and Mark McCorkle<\/td>/);++ assert.ok(!/\[0\]\[1\]/.test(r.text), 'source numbers left in the text are removed');assert.match(r.text, /<h2[^>]*>In this wiki<\/h2>/);assert.match(r.text, /<a class="ov-wikilink" href="\/w\/kim-possible\/ron-stoppable">Ron Stoppable<\/a>/, 'a blue link to the page written after it');assert.match(r.text, /href="https:\/\/en.wikipedia.org\/wiki\/Kim_Possible"[^>]*>Kim Possible — Wikipedia<\/a>, retrieved/);
@@ -26,7 +26,9 @@ const { fetchImage } = require('../integrations/image-fetch');
@@ -26,7 +26,9 @@ const { fetchImage } = require('../integrations/image-fetch');
const SIZES = ['quick', 'standard', 'big'];
const SIZES = ['quick', 'standard', 'big'];
const ACTIVE = ['queued', 'researching', 'planning', 'writing'];
const ACTIVE = ['queued', 'researching', 'planning', 'writing'];
-const LEASE_MS = 10 * 60 * 1000;
+// A job's lease is renewed every minute while it runs, so a restart frees it within LEASE_MS.
+const LEASE_MS = 3 * 60 * 1000;
+const HEARTBEAT_MS = 60 * 1000;
const DAY_MS = 24 * 60 * 60 * 1000;
const DAY_MS = 24 * 60 * 60 * 1000;
const MAX_IMAGES_PER_PAGE = 3;
const MAX_IMAGES_PER_PAGE = 3;
// How the main page groups the rest, by what the plan says each page is about.
// How the main page groups the rest, by what the plan says each page is about.
@@ -102,18 +104,25 @@ function creditOf(img) {
@@ -102,18 +104,25 @@ function creditOf(img) {
let site = null;
let site = null;
try { site = img.source_url ? new URL(img.source_url).hostname.replace(/^www\./, '') : null; } catch { /* no site */ }
try { site = img.source_url ? new URL(img.source_url).hostname.replace(/^www\./, '') : null; } catch { /* no site */ }
const who = oneLine(img.author, 80) || site || oneLine(img.provider, 40);
const who = oneLine(img.author, 80) || site || oneLine(img.provider, 40);
- return [who ? `Picture: ${who}` : 'Picture', img.license ? oneLine(img.license, 80) : null].filter(Boolean).join(', ');
+ // A picture that is not freely licensed is shown to identify the subject; its owners keep the copyright.
+ const terms = img.free ? oneLine(img.license, 60) : '© its owners';
+ return [who ? `Picture: ${who}` : 'Picture', terms || null].filter(Boolean).join(', ');
+}
+
+/** Source numbers a writer left in the text ("…stories.[0][2]"): the page lists its sources, so they go. Links stay. */
+function stripMarkers(text) {
+ return String(text || '').replace(/(?:\s?\[\d{1,2}(?:\s*[,–-]\s*\d{1,2})*\])+(?![(:])/g, '').replace(/[ \t]+([.,;:!?])/g, '$1');
}
}
/** The Markdown of one page from the writer's output: lead, sections with their pictures, and the main page's contents. */
/** The Markdown of one page from the writer's output: lead, sections with their pictures, and the main page's contents. */
function composeBody(out, { placed = new Map(), contents = null }) {
function composeBody(out, { placed = new Map(), contents = null }) {
const parts = [];
const parts = [];
- const lead = String(out.lead || '').trim();
+ const lead = stripMarkers(out.lead).trim();
if (lead) parts.push(lead);
if (lead) parts.push(lead);
const used = new Set();
const used = new Set();
for (const sec of Array.isArray(out.sections) ? out.sections : []) {
for (const sec of Array.isArray(out.sections) ? out.sections : []) {
const heading = oneLine(sec.heading, 120).replace(/^#+\s*/, '');
const heading = oneLine(sec.heading, 120).replace(/^#+\s*/, '');
- const body = String(sec.body || '').trim().replace(/^#{1,2}\s/gm, '### ');
+ const body = stripMarkers(sec.body).trim().replace(/^#{1,2}\s/gm, '### ');
if (!heading || !body) continue;
if (!heading || !body) continue;
const key = heading.toLowerCase();
const key = heading.toLowerCase();
const pic = placed.get(key);
const pic = placed.get(key);
@@ -260,7 +269,9 @@ function createVibes({ db, svc, platform, config, now = () => Date.now(), log =
@@ -260,7 +269,9 @@ function createVibes({ db, svc, platform, config, now = () => Date.now(), log =
try {
try {
const v = await q.claim.get({ owner, until: now() + LEASE_MS, now: now() });
const v = await q.claim.get({ owner, until: now() + LEASE_MS, now: now() });
if (!v) return false;
if (!v) return false;
- await runJob(v);
+ const beat = setInterval(() => { q.renew.run(now() + LEASE_MS, v.id, owner).catch(() => {}); }, HEARTBEAT_MS);
+ if (beat.unref) beat.unref();
+ try { await runJob(v); } finally { clearInterval(beat); }
return true;
return true;
} finally { running = false; }
} finally { running = false; }
}
}
@@ -449,4 +460,4 @@ function createVibes({ db, svc, platform, config, now = () => Date.now(), log =
@@ -449,4 +460,4 @@ function createVibes({ db, svc, platform, config, now = () => Date.now(), log =
return { start, get, mine, recent, tick, startWorker, stopWorker, SIZES, ACTIVE, pagesFor };
return { start, get, mine, recent, tick, startWorker, stopWorker, SIZES, ACTIVE, pagesFor };
}
}
-module.exports = { createVibes, VibeError, normalizePlan, composeBody, contentsOf, citedIndices, creditOf, spaceSlugFor, SIZES };
+module.exports = { createVibes, VibeError, normalizePlan, composeBody, contentsOf, citedIndices, creditOf, spaceSlugFor, stripMarkers, SIZES };
@@ -10,7 +10,7 @@ const assert = require('assert');
@@ -10,7 +10,7 @@ const assert = require('assert');
const H = require('./helpers');
const H = require('./helpers');
const content = require('../server/wiki/content');
const content = require('../server/wiki/content');
const { checkUrl, sniff } = require('../server/integrations/image-fetch');
const { checkUrl, sniff } = require('../server/integrations/image-fetch');
-const { normalizePlan, spaceSlugFor, contentsOf } = require('../server/wiki/vibe');
+const { normalizePlan, spaceSlugFor, contentsOf, stripMarkers, creditOf } = require('../server/wiki/vibe');
const MED = (n) => `med_01J8Z6Q3KX0000000000000${String(n).padStart(3, '0')}`;
const MED = (n) => `med_01J8Z6Q3KX0000000000000${String(n).padStart(3, '0')}`;
const PNG = Buffer.from('89504e470d0a1a0a0000000d4948445200000001000000010806000000', 'hex');
const PNG = Buffer.from('89504e470d0a1a0a0000000d4948445200000001000000010806000000', 'hex');
@@ -33,6 +33,9 @@ const LONG = 'Kim Possible is a teenager who saves the world between classes, wi
@@ -33,6 +33,9 @@ const LONG = 'Kim Possible is a teenager who saves the world between classes, wi
const plan = normalizePlan({ title: 'KP', summary: 's', pages: [{ title: 'Ron', kind: 'character', outline: ['A'] }, { title: 'Kim Possible', kind: 'overview' }, { title: 'ron', kind: 'person' }, { title: 'Bad [[x]]|', kind: 'weird' }] }, 'Kim Possible', 3);
const plan = normalizePlan({ title: 'KP', summary: 's', pages: [{ title: 'Ron', kind: 'character', outline: ['A'] }, { title: 'Kim Possible', kind: 'overview' }, { title: 'ron', kind: 'person' }, { title: 'Bad [[x]]|', kind: 'weird' }] }, 'Kim Possible', 3);
assert.deepStrictEqual(plan.pages.map((p) => [p.title, p.kind]), [['Kim Possible', 'overview'], ['Ron', 'character'], ['Bad x', 'other']], 'main page first, duplicates out, kinds known, link syntax stripped');
assert.deepStrictEqual(plan.pages.map((p) => [p.title, p.kind]), [['Kim Possible', 'overview'], ['Ron', 'character'], ['Bad x', 'other']], 'main page first, duplicates out, kinds known, link syntax stripped');
assert.match(contentsOf(plan), /### Characters\n\n- \[\[Ron\]\]/);
assert.match(contentsOf(plan), /### Characters\n\n- \[\[Ron\]\]/);
+ assert.strictEqual(stripMarkers('Her stories.[0][2][4] See [the site](https://x.test) and [[Ron]] [3, 5].'), 'Her stories. See [the site](https://x.test) and [[Ron]].');
+ assert.strictEqual(creditOf({ author: null, source_url: 'https://kimpossible.fandom.com/wiki/Shego', free: false, license: 'Fandom wiki picture' }), 'Picture: kimpossible.fandom.com, © its owners');
+ assert.strictEqual(creditOf({ author: 'Jane', free: true, license: 'CC BY 2.0' }), 'Picture: Jane, CC BY 2.0');
}
}
(async () => {
(async () => {
@@ -51,7 +54,7 @@ const LONG = 'Kim Possible is a teenager who saves the world between classes, wi
@@ -51,7 +54,7 @@ const LONG = 'Kim Possible is a teenager who saves the world between classes, wi
facts: [], topic_wiki: { name: 'Kim Possible Wiki', url: 'https://kimpossible.fandom.com' }, providers: { wikipedia: { ok: true, count: 1 }, fandom: { ok: true, count: 1 }, deepseek: { ok: false, skipped: 'not configured' } }, usage: {},
facts: [], topic_wiki: { name: 'Kim Possible Wiki', url: 'https://kimpossible.fandom.com' }, providers: { wikipedia: { ok: true, count: 1 }, fandom: { ok: true, count: 1 }, deepseek: { ok: false, skipped: 'not configured' } }, usage: {},
});
});
const page = (title) => ({
const page = (title) => ({
- title, summary: `${title} is part of Kim Possible.`, lead: `**${title}** is part of the show. ${title === 'Kim Possible' ? 'Her friend is [[Ron Stoppable]].' : 'He is friends with [[Kim Possible]].'} ${LONG}`,
+ title, summary: `${title} is part of Kim Possible.`, lead: `**${title}** is part of the show.[0][1] ${title === 'Kim Possible' ? 'Her friend is [[Ron Stoppable]].' : 'He is friends with [[Kim Possible]].'} ${LONG}`,
sections: [{ heading: 'Premise', body: LONG, citations: [0] }, { heading: 'Legacy', body: LONG, citations: [1] }],
sections: [{ heading: 'Premise', body: LONG, citations: [0] }, { heading: 'Legacy', body: LONG, citations: [1] }],
infobox: [{ label: 'Created by', value: 'Bob Schooley and Mark McCorkle', citations: [0] }, { label: 'Network', value: 'Disney Channel', citations: [0] }],
infobox: [{ label: 'Created by', value: 'Bob Schooley and Mark McCorkle', citations: [0] }, { label: 'Network', value: 'Disney Channel', citations: [0] }],
images: [{ index: 0, caption: `${title} on screen`, section: 'lead' }, { index: 7, caption: 'out of range', section: 'Premise' }],
images: [{ index: 0, caption: `${title} on screen`, section: 'lead' }, { index: 7, caption: 'out of range', section: 'Premise' }],
@@ -160,6 +163,7 @@ const LONG = 'Kim Possible is a teenager who saves the world between classes, wi
@@ -160,6 +163,7 @@ const LONG = 'Kim Possible is a teenager who saves the world between classes, wi
assert.match(r.text, new RegExp(`class="wk-ib-pic"[\\s\\S]*?<img src="https://openvibe.media/o/${MED(1)}"`), 'the lead picture in the infobox');
assert.match(r.text, new RegExp(`class="wk-ib-pic"[\\s\\S]*?<img src="https://openvibe.media/o/${MED(1)}"`), 'the lead picture in the infobox');
assert.match(r.text, /Kim Possible on screen \(Picture: Jane Doe, CC BY-SA 4.0\)/, 'captioned with its credit');
assert.match(r.text, /Kim Possible on screen \(Picture: Jane Doe, CC BY-SA 4.0\)/, 'captioned with its credit');
assert.match(r.text, /<th scope="row">Created by<\/th><td>Bob Schooley and Mark McCorkle<\/td>/);
assert.match(r.text, /<th scope="row">Created by<\/th><td>Bob Schooley and Mark McCorkle<\/td>/);
+ assert.ok(!/\[0\]\[1\]/.test(r.text), 'source numbers left in the text are removed');
assert.match(r.text, /<h2[^>]*>In this wiki<\/h2>/);
assert.match(r.text, /<h2[^>]*>In this wiki<\/h2>/);
assert.match(r.text, /<a class="ov-wikilink" href="\/w\/kim-possible\/ron-stoppable">Ron Stoppable<\/a>/, 'a blue link to the page written after it');
assert.match(r.text, /<a class="ov-wikilink" href="\/w\/kim-possible\/ron-stoppable">Ron Stoppable<\/a>/, 'a blue link to the page written after it');
assert.match(r.text, /href="https:\/\/en.wikipedia.org\/wiki\/Kim_Possible"[^>]*>Kim Possible — Wikipedia<\/a>, retrieved/);
assert.match(r.text, /href="https:\/\/en.wikipedia.org\/wiki\/Kim_Possible"[^>]*>Kim Possible — Wikipedia<\/a>, retrieved/);