openvibers/openvibe.ai Commits

content.wiki: the wiki writer workflow, research sources for later steps, and citations beside the run (#404)

OP OpenVibers committed 4 days ago · 83df17f947 · 5 files, +89 −9

* content.wiki: the wiki writer workflow and research sources for the steps after it

* Merge origin/main into w1/ai-content-wiki

* Test the run's citations beside its answer

* Research with no new sources keeps the writer's own

* Feed the research facts to the wiki writer's infobox

* content.wiki takes the article language for its research

Changes

ddiff --git a/server/api/runs.js b/server/api/runs.js
iindex e778d23..34f6991 100644
---- a/server/api/runs.js
++++ b/server/api/runs.js
@@@ -73,7 +73,9 @@ function runsRouter({ choices = null, runs, registry, auth, config, log = consol
const run = await runs.explained(await runs.wait(created, waitMs));
const status = !created.created ? 200 : (runs.TERMINAL.has(run.status) ? 201 : 202);
if (status === 202) res.setHeader('Location', `/api/v1/runs/${run.id}`);
-- return res.status(status).json({ run, replayed: !created.created || undefined });
++ // The run's numbered sources with it: a product that files the output (a wiki draft) needs their URLs and
++ // licences to attach the citations its sentences used, without a second call.
++ return res.status(status).json({ run, citations: await runs.citations(run.id), replayed: !created.created || undefined });
}
/**
ddiff --git a/server/workflows/engine.js b/server/workflows/engine.js
iindex 7cd87ed..a66e382 100644
---- a/server/workflows/engine.js
++++ b/server/workflows/engine.js
@@@ -466,7 +466,11 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
if (err && err.status === 422) throw new AiError(422, 'input.invalid', err.message);
throw err;
}
-- if (!output.sources.length && !output.images.length) throw new AiError(502, 'source.unavailable', `no source answered for "${output.query}"`, { providers: output.providers });
++ if (!output.sources.length && !output.images.length) {
++ // A writer already has its article's sources: web search finding nothing new is not a failed run.
++ if (Array.isArray(input.sources) && input.sources.length) output = { ...output, sources: input.sources };
++ else throw new AiError(502, 'source.unavailable', `no source answered for "${output.query}"`, { providers: output.providers });
++ }
// Recorded like a provider call: DeepSeek's per-call price counts against the cost quotas and the daily usage.
const exec = { provider: 'research', model: 'megasource', usage: { input: 0, output: 0, cached: 0 }, cost: Number(output.usage && output.usage.cost_usd) || 0, synthetic: false, fallbackUsed: false, explain: ctx.taskChoice ? {
// The contract's explain needs an objective, a reason, what was selected and the candidates; a search
@@@ -514,10 +518,22 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
let saved = 0;
let exec = null;
let jsonSchema = null;
++ let researchOut = null;
for (const step of wf.steps) {
const fn = STEP[step.kind];
if (!fn) throw new AiError(500, 'workflow.broken', `unknown step kind ${step.kind}`);
last = await fn(step, input, wf, run, ctx);
++ // A research step feeds the steps after it: what it read joins input.sources, with the caller's own
++ // sources first and no URL twice, so the writer cites the numbered list it was shown.
++ if (step.kind === 'research' && last && last.output) {
++ researchOut = last.output;
++ const before = Array.isArray(input.sources) ? input.sources : [];
++ const seen = new Set(before.map((s) => s && s.url).filter(Boolean));
++ const added = (Array.isArray(last.output.sources) ? last.output.sources : []).filter((s) => s && s.url && !seen.has(s.url));
++ if (added.length) input = { ...input, sources: [...before, ...added] };
++ // The structured facts research read (Wikidata and the sources) fill a wiki article's infobox.
++ if (Array.isArray(last.output.facts) && last.output.facts.length) input = { ...input, facts: last.output.facts };
++ }
if (last.exec) {
exec = last.exec;
usage.input += exec.usage.input || 0;
@@@ -541,7 +557,16 @@ function createEngine({ registry, pool, fetcher, quotas = null, config = {}, spa
}));
return {
output: last.output, citations, jsonSchema,
-- grounding: wf.steps.some((s) => s.kind === 'research') ? researchGrounding(last.output) : groundingOf(last.output, sources.length, citeStep),
++ grounding: researchOut
++ ? (wf.steps.length === 1
++ ? researchGrounding(researchOut)
++ : (() => {
++ // A workflow that researches and then writes: what was read, and what the answer cites.
++ const read = researchGrounding(researchOut);
++ const wrote = groundingOf(last.output, sources.length, citeStep);
++ return { ...read, cited: wrote.cited, gaps: [...read.gaps, ...wrote.gaps].slice(0, 20) };
++ })())
++ : groundingOf(last.output, sources.length, citeStep),
synthetic: Boolean(exec && exec.synthetic),
provider: exec ? exec.provider : null, model: exec ? exec.model : null, fallbackUsed: Boolean(exec && exec.fallbackUsed),
...(exec && exec.result && exec.result.label ? { contentPolicy: exec.result.policy || 'uncensored', label: exec.result.label } : {}),
ddiff --git a/server/workflows/hooks.js b/server/workflows/hooks.js
iindex fef82a8..8f8fa0c 100644
---- a/server/workflows/hooks.js
++++ b/server/workflows/hooks.js
@@@ -328,6 +328,25 @@ const PREPARE = {
},
};
},
++ // The wiki writer (content.wiki): the numbered sources, the outline and the pages it may link.
++ 'content.wiki'(input) {
++ const outline = (Array.isArray(input.outline) ? input.outline : []).map((h) => clean(h, 300)).filter(Boolean).slice(0, 30);
++ const pages = (Array.isArray(input.pages) ? input.pages : []).map((t) => clean(t, 200)).filter(Boolean).slice(0, 60);
++ const facts = (Array.isArray(input.facts) ? input.facts : []).map((f) => ({ label: clean(f && f.label, 80), value: clean(f && f.value, 400) })).filter((f) => f.label && f.value).slice(0, 40);
++ return {
++ vars: {
++ ...input,
++ topic: input.topic || input.title,
++ space: input.space || input.topic || input.title,
++ instruction: input.instruction ? clean(input.instruction, 4000) : '',
++ outline_block: outline.map((h, i) => `${i + 1}. ${h}`).join('\n'),
++ pages_block: pages.map((t) => `- ${t}`).join('\n'),
++ facts_block: facts.map((f) => `- ${f.label}: ${f.value}`).join('\n'),
++ sources_block: sourcesBlock(input.sources),
++ source_count: (input.sources || []).length,
++ },
++ };
++ },
'pics.describe_image'(input) {
return { vars: { title: clean(input.title, 120) || '' } };
},
ddiff --git a/server/workflows/products.js b/server/workflows/products.js
iindex 212225d..07b4fc1 100644
---- a/server/workflows/products.js
++++ b/server/workflows/products.js
@@@ -17,6 +17,9 @@ const STR = (n) => ({ type: 'string', maxLength: n });
const OBJ = (props, required) => ({ type: 'object', additionalProperties: false, required: required || Object.keys(props), properties: props });
const ARR = (items, max = 20) => ({ type: 'array', items, maxItems: max });
const CLAIM = OBJ({ text: STR(600), citations: CITES });
++// A sentence with at least one source: the wiki writer's promise, checked by the output schema.
++const CITED = { ...CITES, minItems: 1 };
++const SENTENCE = OBJ({ text: STR(2000), citations: CITED });
// What a wiki page is about (wiki.generate_space): the wiki's own page groups follow from it.
const PAGE_KINDS = ['overview', 'character', 'person', 'episode', 'season', 'work', 'place', 'group', 'object', 'concept', 'event', 'production', 'reception', 'list', 'other'];
@@@ -37,6 +40,8 @@ const templates = [
'Topic: {{topic}}{{#audience}}\nAudience: {{audience}}{{/audience}}\nPages at most: {{max_pages}}\n\n'),
productTemplate('wiki.generate_page', 'Wiki: write a page', 'Write one page of the wiki from the numbered sources, encyclopedic and engaging, the way a well-kept wiki reads. `summary`: one or two plain sentences saying what the subject is. `lead`: the opening paragraph or two, in Markdown. `sections`: follow the requested outline where the sources support it (skip or merge headings they do not), each with a `heading` and a Markdown `body` of one or more full paragraphs (a list where it reads better), each citing its sources; about 500 to 1500 words in all when the sources allow, shorter when they do not. Link other pages of this wiki, and only titles from the list of pages, as [[Exact Title]] the first time each is mentioned. `infobox`: up to twelve key facts as `label` and `value` (for example First appearance, Voiced by, Created by, Release date, Location), each cited; the Wikidata facts are good material. `images`: up to three pictures from the numbered list of pictures that clearly show this page\'s subject, judged by their title and description (never a picture of something or someone else, and none when no picture fits), each with its `index`, a short factual `caption`, and `section`: the heading it illustrates, or lead for the page\'s main picture. Never write source numbers or "[n]" in the text.', 'wiki.write',
'Page title: {{title}}\nWiki topic: {{topic}}{{#outline}}\nRequested outline: {{json outline}}{{/outline}}\n\nPages of this wiki (link only these, as [[Title]]):\n{{pages_block}}\n\n{{#images_block}}Pictures (by number):\n{{images_block}}\n\n{{/images_block}}'),
++ productTemplate('content.wiki', 'Wiki: write or improve an article', 'Write or improve one wiki article from the numbered sources — the sources the article already cites, then what web search found. Every sentence of the article must cite the source numbers it rests on in its `citations` array (integers, e.g. [0, 2]); a sentence with no source does not go in. `lead` is the opening sentences; `sections` are the article\'s headings, each with its sentences; `infobox` is up to twelve key facts as label, type (text, number, date, url, boolean, page, media), value and citations, taken from the structured facts (Wikidata and the sources) and never guessed. `summary` is one or two plain sentences saying what the subject is. Link other pages of this wiki only as [[Exact Title]] from the list of pages. Never write source numbers or "[n]" in the text itself: the citations arrays carry them. encyclopedic and plain, the way a well-kept wiki reads.', 'wiki.write',
++ 'Article: {{title}}\nWiki: {{space}}\nTopic: {{topic}}{{#instruction}}\nRequest: {{instruction}}{{/instruction}}{{#outline_block}}\nOutline to follow where the sources support it:\n{{outline_block}}{{/outline_block}}{{#pages_block}}\nPages of this wiki (link only these, as [[Title]]):\n{{pages_block}}{{/pages_block}}{{#facts_block}}\nStructured facts (use them for the infobox; cite their source):\n{{facts_block}}{{/facts_block}}\n\n'),
productTemplate('blog.draft_post', 'Blog: draft a post', 'Draft a blog post in Markdown for the brief. Claims of fact must cite sources; opinion and framing need none. If there are no sources, write only what needs no facts and list the facts you would need in gaps.', 'background.json',
'Topic: {{topic}}{{#brief}}\nBrief: {{brief}}{{/brief}}{{#tone}}\nTone: {{tone}}{{/tone}}{{#audience}}\nAudience: {{audience}}{{/audience}}\n\n'),
productTemplate('blog.write_digest', 'Blog: write a desk digest', 'Write one digest post for an OpenVibe.Blog desk from the numbered sources (the items its sources published in the period). Pick the items a reader of this desk would most want to know about, about twelve at most, and skip routine, minor or duplicate ones. Group them into two to five sections, most important first. Each section is a list of short paragraphs, one story per paragraph, two to four plain sentences saying what happened and why it matters to this desk\'s readers, exactly as the sources state it; each paragraph lists the numbers of the sources it rests on in its `citations`. Never write source numbers, "(Source n)" or brackets in the text itself. Neutral and readable: no opinion stated as fact, no hype, no advice, no predictions. Name prices, dates and numbers only as the sources give them. List in gaps only what a reader would miss (an item you could not make sense of, a missing date or price). title: a headline naming the one or two biggest stories, at most 90 characters, without the words digest, desk, roundup or snapshot and without a date. dek: one sentence about the period.', 'background.json',
@@@ -95,6 +100,29 @@ const workflows = [
default_route: 'wiki.write',
steps: [{ kind: 'llm', template: 'wiki.generate_page', operation: 'generate', output: 'json', prepare: 'wiki.page', cite: true, params: { max_tokens: 7000, temperature: 0.4, timeout_ms: 170000 } }],
},
++ {
++ // The wiki writer (rich-content plan §5): research, then every sentence cited and the infobox from facts.
++ // OpenVibe.Wiki files the answer as a pending proposal; the workflow itself never publishes.
++ key: 'content.wiki', name: 'Write or improve a wiki article', namespace: 'wiki',
++ input_schema: OBJ({
++ title: STR(300), topic: STR(300), space: STR(200), instruction: STR(4000),
++ outline: ARR(STR(300), 30), pages: ARR(STR(200), 60), query: STR(200),
++ lang: { type: 'string', pattern: '^[a-z]{2,3}$' },
++ sources: SOURCES(0),
++ }, ['title']),
++ output_schema: OBJ({
++ title: STR(300), summary: STR(500),
++ lead: ARR(SENTENCE, 12),
++ sections: ARR(OBJ({ heading: STR(160), sentences: ARR(SENTENCE, 40) }), 14),
++ infobox: ARR(OBJ({ label: STR(80), type: { enum: ['text', 'number', 'date', 'url', 'boolean', 'page', 'media'] }, value: STR(500), citations: CITED }), 14),
++ citations: CITES, gaps: GAPS,
++ }),
++ default_route: 'wiki.write',
++ steps: [
++ { kind: 'research' },
++ { kind: 'llm', template: 'content.wiki', operation: 'generate', output: 'json', prepare: 'content.wiki', cite: true, params: { max_tokens: 8000, temperature: 0.4, timeout_ms: 180000 } },
++ ],
++ },
{
key: 'blog.draft_post', name: 'Draft a blog post', namespace: 'blog',
input_schema: OBJ({ topic: STR(300), brief: STR(4000), tone: STR(200), audience: STR(300), sources: SOURCES(0) }, ['topic']),
ddiff --git a/test/workflows.test.js b/test/workflows.test.js
iindex 2a3b135..06c5f1e 100644
---- a/test/workflows.test.js
++++ b/test/workflows.test.js
@@@ -8,6 +8,7 @@ const nodeHttp = require('http');
const { boot, request, token, suite, ALL } = require('./helpers');
const schemas = require('../server/schemas');
const { KIND_TO_WORKFLOW } = require('../server/workflows/live');
++require('../server/api/runs'); // the run route this test posts to (and its citations beside the run)
const PNG = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==';
const SRC = (n) => Array.from({ length: n }, (_, i) => ({ source_type: 'news.article', source_id: `a${i}`, title: `Source ${i}`, url: `https://example.org/${i}`, content: `Fact number ${i} happened on day ${i}.`, published_at: '2026-09-20' }));
@@@ -38,6 +39,7 @@ const EXAMPLES = {
'ai.embed': { input: ['one', 'two'] },
'ai.image': { prompt: 'A goose eating croutons, watercolour.', size: '1024x1024', n: 1 },
'wiki.generate_space': { topic: 'SQLite', max_pages: 6, sources: SRC(2) },
++ 'content.wiki': { title: 'SQLite', topic: 'SQLite', space: 'SQLite', query: 'SQLite', instruction: 'Write the article from its sources.', outline: ['History'], pages: ['SQLite'], sources: SRC(2) },
'wiki.generate_page': { title: 'WAL mode', topic: 'SQLite', pages: ['SQLite', 'WAL mode', 'Journal'], images: [{ title: 'WAL diagram', description: 'How WAL writes' }], sources: SRC(3) },
'research.gather': { query: 'SQLite', max_images: 4 },
'blog.draft_post': { topic: 'Why we self-host', sources: SRC(1) },
@@@ -104,7 +106,7 @@ t.test('boot seeds every required workflow key', async () => {
// AI_IMAGE_MODEL seeds default.image, so ai.image runs end to end on the stub's PNG like every other workflow.
h = await boot({ env: { OV_MEDIA_INTERNAL_URL: mediaBase, AI_IMAGE_MODEL: 'gpt-image-1' } });
const keys = (await h.registry.listWorkflows()).map(w => w.key);
-- for (const k of ['wiki.generate_space', 'wiki.generate_page', 'blog.draft_post', 'blog.write_digest', 'blog.write_satire', 'news.summarize_story', 'news.compare_perspectives', 'reviews.summarize_entity', 'deals.enrich_deal',
++ for (const k of ['content.wiki', 'wiki.generate_space', 'wiki.generate_page', 'blog.draft_post', 'blog.write_digest', 'blog.write_satire', 'news.summarize_story', 'news.compare_perspectives', 'reviews.summarize_entity', 'deals.enrich_deal',
'coupons.extract_coupon', 'trade.summarize_market_context', 'codes.generate_docs', 'games.generate_lore', 'moderation.classify', 'tools.describe', 'food.suggest_recipes', 'food.read_pantry_photo', 'pics.describe_image', 'rent.screen_listing', 'chat.member.reply', 'chat.member.turn', 'chat.vibe.listen',
'live.translate', 'live.stream.describe_frame', 'live.stream.summarize', 'live.viewers.reply', 'live.chat.insight', 'live.hero.slogans', 'live.stream.recap', 'network.site_copy']) {
assert.ok(keys.includes(k), `missing ${k}`);
@@@ -144,20 +146,23 @@ t.test('every workflow runs end to end on the stub and returns schema-valid outp
if (w.key === 'content.devblog') { assert.strictEqual(run.status, 'failed', 'synthetic prose fails the evidence gate'); continue; }
assert.strictEqual(run.status, 'succeeded', `${w.key}: ${JSON.stringify(run.error)}`);
// Research calls no model: it reads (here, the canned web of test/fixtures/research-fetch.js) and is never synthetic.
-- const reads = w.steps.some((s) => s.kind === 'research');
-- assert.strictEqual(run.synthetic, !reads, `${w.key} should ${reads ? 'not ' : ''}be marked synthetic`);
++ const researchOnly = w.steps.length === 1 && w.steps[0].kind === 'research';
++ assert.strictEqual(run.synthetic, !researchOnly, `${w.key} should ${researchOnly ? 'not ' : ''}be marked synthetic`);
assert.strictEqual(run.provenance.origin, 'ai');
assert.strictEqual(run.provenance.workflow, w.key);
assert.ok(run.provenance.model, `${w.key} has a model`);
assert.ok(schemas.validate(w.output_schema, run.output).valid, `${w.key} output invalid`);
-- if (reads) { assert.ok(run.output.sources.length && Array.isArray(run.grounding.gaps)); continue; }
++ if (researchOnly) { assert.ok(run.output.sources.length && Array.isArray(run.grounding.gaps)); continue; }
// WS-O task 3: every output says what backs it: cited sources, or explicit gaps.
assert.ok(run.grounding && Array.isArray(run.grounding.cited) && Array.isArray(run.grounding.gaps), `${w.key} has grounding`);
assert.ok(run.grounding.cited.length || run.grounding.gaps.length, `${w.key}: citations or explicit gaps`);
if (EXAMPLES[w.key].sources) {
-- assert.strictEqual(run.citations_count, EXAMPLES[w.key].sources.length, `${w.key} citations`);
++ // A research step adds what it read to the caller's sources; a writing run's list only grows.
++ const researches = w.steps.some((s) => s.kind === 'research');
++ if (researches) assert.ok(run.citations_count >= EXAMPLES[w.key].sources.length, `${w.key} citations`);
++ else assert.strictEqual(run.citations_count, EXAMPLES[w.key].sources.length, `${w.key} citations`);
const d = await request(h.base, 'GET', `/api/v1/runs/${run.id}`, { tok });
-- assert.strictEqual(d.body.citations.length, EXAMPLES[w.key].sources.length);
++ assert.ok(d.body.citations.length >= EXAMPLES[w.key].sources.length);
assert.ok(d.body.citations.every(c => c.source_type && c.content_hash), 'citations carry type and content hash');
}
}
@@@ -175,6 +180,7 @@ t.test('grounding: cited sources, the output\'s own gaps, or a gap written for i
t.test('citations index only the provided sources', async () => {
const r = await request(h.base, 'POST', '/api/v1/runs?wait=5000', { tok, body: { workflow: 'news.summarize_story', input: { sources: SRC(2) } } });
++ assert.ok(Array.isArray(r.body.citations) && r.body.citations.length === 2, 'the run answer carries its numbered sources');
const seen = JSON.stringify(r.body.run.output).match(/"citations":\[([0-9,]*)\]/g) || [];
for (const m of seen) for (const n of m.replace(/\D+/g, ' ').trim().split(' ').filter(Boolean)) assert.ok(Number(n) < 2, `citation ${n} out of range`);
});