openvibers/openvibe.tours Commits

Wikivoyage text keeps its inline templates' words (#43)

OP OpenVibers committed 11 days ago · bec3fec307 · 4 files, +48 −5

Every template was removed from Wikivoyage text, so a sentence that held a measure lost it:
Chongqing's lead read "…and covers .". The templates that carry words are now kept as text:
{{convert}} (with units and ranges), {{lang}}, {{nowrap}}, {{w}}, {{IATA}} and {{marker}}; the
rest still go, and a space left before punctuation or an empty bracket pair is removed. The
cities job names a version (inline-templates-1), so the first boot with it rewrites the summaries.

Changes

ddiff --git a/server/jobs/cities.js b/server/jobs/cities.js
iindex eaa9e66..dbbb39e 100644
---- a/server/jobs/cities.js
++++ b/server/jobs/cities.js
@@@ -30,13 +30,17 @@ function fileTitle(name) {
return /^File:/i.test(s) ? s : `File:${s}`;
}
++// A run under older rules is not fresh (jobs/index.js): inline-templates-1 keeps {{convert}}, {{lang}} and the like in the
++// summaries, so the first boot with it rewrites them.
++const CITIES_VERSION = 'inline-templates-1';
++
function createCitiesJob({ config, s, store, fetchImpl, log = console }) {
const wikivoyage = createWikivoyage({ config, fetchImpl, pace: s.pace });
const wikidata = createWikidata({ config, fetchImpl, pace: s.pace, log });
async function runOnce({ limit = config.openData.cityLimit } = {}) {
const catalog = loadCatalog(limit);
-- const result = { catalog: catalog.length, cities: 0, withSummary: 0, withSee: 0, districts: 0, skipped: [], errors: [] };
++ const result = { catalog: catalog.length, cities: 0, withSummary: 0, withSee: 0, districts: 0, skipped: [], errors: [], version: CITIES_VERSION };
const qidMap = await wikidata.resolveQids(catalog.map((c) => c.wikipedia || c.name));
const qids = [...qidMap.values()].map((v) => v.qid);
@@@ -134,7 +138,7 @@ function createCitiesJob({ config, s, store, fetchImpl, log = console }) {
return result;
}
-- return { runOnce, loadCatalog, pageUrl };
++ return { runOnce, loadCatalog, pageUrl, version: CITIES_VERSION };
}
module.exports = { createCitiesJob, loadCatalog };
ddiff --git a/server/jobs/index.js b/server/jobs/index.js
iindex 9fd453c..9e889cb 100644
---- a/server/jobs/index.js
++++ b/server/jobs/index.js
@@@ -25,7 +25,7 @@ function createJobs({ config, s, store, fetchImpl = globalThis.fetch, log = cons
};
const jobs = {
-- cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs },
++ cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs, version: creators.cities.version },
places: { run: creators.places.runOnce, maxAgeMs: config.jobs.placesEveryMs, intervalMs: config.jobs.placesEveryMs, version: creators.places.version },
routes: { run: creators.routes.runOnce, maxAgeMs: config.jobs.routesEveryMs, intervalMs: config.jobs.routesEveryMs, version: creators.routes.version },
live: { run: creators.live.runOnce, maxAgeMs: config.jobs.liveEveryMs, intervalMs: config.jobs.liveEveryMs },
ddiff --git a/server/sources/wikivoyage.js b/server/sources/wikivoyage.js
iindex da66297..a2ff83a 100644
---- a/server/sources/wikivoyage.js
++++ b/server/sources/wikivoyage.js
@@@ -22,12 +22,44 @@ function matchingBrace(text, start) {
return -1;
}
++// Inline templates that carry the sentence's words are kept as their text; every other template goes. Removing
++// {{convert|82400|km2}} left Chongqing's lead "…and covers ." (production, 2026-11-02).
++const UNITS = { km2: 'km²', 'km²': 'km²', sqkm: 'km²', sqmi: 'sq mi', mi2: 'sq mi', km: 'km', mi: 'mi', m: 'm', ft: 'ft', ha: 'ha',
++ acre: 'acres', acres: 'acres', c: '°C', f: '°F', '°c': '°C', '°f': '°F', kg: 'kg', lb: 'lb', 'km/h': 'km/h', mph: 'mph', l: 'L' };
++const number = (v) => (/^-?\d+(\.\d+)?$/.test(String(v).trim()) ? Number(v).toLocaleString('en-US', { maximumFractionDigits: 2 }) : String(v).trim());
++function inlineTemplate(inner) {
++ const parts = splitParams(inner).map((x) => x.trim());
++ const name = String(parts[0] || '').toLowerCase().replace(/_/g, ' ').trim();
++ const pos = parts.slice(1).filter((x) => !/^[a-z_ -]+=/i.test(x));
++ const named = paramsOf(inner);
++ if (name === 'convert' || name === 'cvt') {
++ if (!pos[0]) return '';
++ // {{convert|20|-|25|km}}, {{convert|5|to|10|mi}}: a range
++ if (['-', '–', 'to', 'and'].includes(String(pos[1]).toLowerCase()) && pos[2]) {
++ const unit = UNITS[String(pos[3] || '').toLowerCase()] || pos[3] || '';
++ return `${number(pos[0])}${pos[1] === 'to' || pos[1] === 'and' ? ` ${pos[1]} ` : '–'}${number(pos[2])}${unit ? ` ${unit}` : ''}`;
++ }
++ const unit = UNITS[String(pos[1] || '').toLowerCase()] || pos[1] || '';
++ return `${number(pos[0])}${unit ? ` ${unit}` : ''}`;
++ }
++ if (name === 'lang' || name === 'lang-zh') return pos[pos.length - 1] || '';
++ if (name === 'nowrap' || name === 'nobr' || name === 'small' || name === 'iata' || name === 'icao') return pos[0] || '';
++ if (name === 'w' || name === 'wp') return pos[1] || pos[0] || '';
++ if (name === 'marker') return named.name || '';
++ return null;
++}
++
function removeTemplates(text) {
let out = '';
for (let i = 0; i < text.length; i += 1) {
if (text[i] === '{' && text[i + 1] === '{') {
const end = matchingBrace(text, i);
-- if (end > 0) { i = end - 1; continue; }
++ if (end > 0) {
++ const kept = inlineTemplate(text.slice(i + 2, end - 2));
++ if (kept) out += removeTemplates(kept);
++ i = end - 1;
++ continue;
++ }
}
out += text[i];
}
@@@ -64,7 +96,9 @@ function stripWiki(value, { max = 1200 } = {}) {
s = s.replace(/<\/?[a-z][^>]*>/gi, ' ');
s = s.replace(/\{\{|\}\}/g, ' ');
for (const [k, v] of Object.entries(ENTITIES)) s = s.split(k).join(v);
-- s = s.replace(/[ \t]+/g, ' ').replace(/\s*\n\s*/g, ' ').trim();
++ s = s.replace(/[ \t]+/g, ' ').replace(/\s*\n\s*/g, ' ');
++ // A space a removed template left before punctuation, and an empty bracket pair.
++ s = s.replace(/\(\s*\)/g, '').replace(/\s+([.,;:!?])/g, '$1').replace(/[ ]{2,}/g, ' ').trim();
return s.slice(0, max);
}
ddiff --git a/test/units.test.js b/test/units.test.js
iindex d0d5108..ab188d9 100644
---- a/test/units.test.js
++++ b/test/units.test.js
@@@ -18,6 +18,11 @@ check('stripWiki keeps the last character, follows links and removes templates,
assert.strictEqual(stripWiki('Ueno Park'), 'Ueno Park');
assert.strictEqual(stripWiki("[[Tokyo/Asakusa|Asakusa]]'s '''Senso-ji''' is [[Buddhist]].", {}), "Asakusa's Senso-ji is Buddhist.");
assert.strictEqual(stripWiki('A {{m|28}} statue<ref>source</ref>'), 'A statue');
++ // Inline templates that carry words are kept as text (Chongqing's lead ended "covers ." on 2026-11-02).
++ assert.strictEqual(stripWiki('It covers {{convert|82400|km2|sqmi}}.'), 'It covers 82,400 km².');
++ assert.strictEqual(stripWiki('{{convert|20|-|25|km}} away, {{convert|5|to|10|mi}}'), '20–25 km away, 5 to 10 mi');
++ assert.strictEqual(stripWiki('Chongqing ({{lang|zh|重庆}}) on {{w|Yangtze River|the Yangtze}} at {{IATA|CKG}}.'), 'Chongqing (重庆) on the Yangtze at CKG.');
++ assert.strictEqual(stripWiki('Left {{unknown|template}} out .'), 'Left out.');
assert.strictEqual(stripWiki('[[File:Photo.jpg|thumb|Caption]] A temple.'), 'A temple.');
});