openvibers/openvibe.tours Commits
Wikivoyage text keeps its inline templates' words (#43)
Changes
@@@ -30,13 +30,17 @@ function fileTitle(name) {return /^File:/i.test(s) ? s : `File:${s}`;}++// A run under older rules is not fresh (jobs/index.js): inline-templates-1 keeps {{convert}}, {{lang}} and the like in the++// summaries, so the first boot with it rewrites them.++const CITIES_VERSION = 'inline-templates-1';++function createCitiesJob({ config, s, store, fetchImpl, log = console }) {const wikivoyage = createWikivoyage({ config, fetchImpl, pace: s.pace });const wikidata = createWikidata({ config, fetchImpl, pace: s.pace, log });async function runOnce({ limit = config.openData.cityLimit } = {}) {const catalog = loadCatalog(limit);-- const result = { catalog: catalog.length, cities: 0, withSummary: 0, withSee: 0, districts: 0, skipped: [], errors: [] };++ const result = { catalog: catalog.length, cities: 0, withSummary: 0, withSee: 0, districts: 0, skipped: [], errors: [], version: CITIES_VERSION };const qidMap = await wikidata.resolveQids(catalog.map((c) => c.wikipedia || c.name));const qids = [...qidMap.values()].map((v) => v.qid);@@@ -134,7 +138,7 @@ function createCitiesJob({ config, s, store, fetchImpl, log = console }) {return result;}-- return { runOnce, loadCatalog, pageUrl };++ return { runOnce, loadCatalog, pageUrl, version: CITIES_VERSION };}module.exports = { createCitiesJob, loadCatalog };@@@ -25,7 +25,7 @@ function createJobs({ config, s, store, fetchImpl = globalThis.fetch, log = cons};const jobs = {-- cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs },++ cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs, version: creators.cities.version },places: { run: creators.places.runOnce, maxAgeMs: config.jobs.placesEveryMs, intervalMs: config.jobs.placesEveryMs, version: creators.places.version },routes: { run: creators.routes.runOnce, maxAgeMs: config.jobs.routesEveryMs, intervalMs: config.jobs.routesEveryMs, version: creators.routes.version },live: { run: creators.live.runOnce, maxAgeMs: config.jobs.liveEveryMs, intervalMs: config.jobs.liveEveryMs },@@@ -22,12 +22,44 @@ function matchingBrace(text, start) {return -1;}++// Inline templates that carry the sentence's words are kept as their text; every other template goes. Removing++// {{convert|82400|km2}} left Chongqing's lead "…and covers ." (production, 2026-11-02).++const UNITS = { km2: 'km²', 'km²': 'km²', sqkm: 'km²', sqmi: 'sq mi', mi2: 'sq mi', km: 'km', mi: 'mi', m: 'm', ft: 'ft', ha: 'ha',++ acre: 'acres', acres: 'acres', c: '°C', f: '°F', '°c': '°C', '°f': '°F', kg: 'kg', lb: 'lb', 'km/h': 'km/h', mph: 'mph', l: 'L' };++const number = (v) => (/^-?\d+(\.\d+)?$/.test(String(v).trim()) ? Number(v).toLocaleString('en-US', { maximumFractionDigits: 2 }) : String(v).trim());++function inlineTemplate(inner) {++ const parts = splitParams(inner).map((x) => x.trim());++ const name = String(parts[0] || '').toLowerCase().replace(/_/g, ' ').trim();++ const pos = parts.slice(1).filter((x) => !/^[a-z_ -]+=/i.test(x));++ const named = paramsOf(inner);++ if (name === 'convert' || name === 'cvt') {++ if (!pos[0]) return '';++ // {{convert|20|-|25|km}}, {{convert|5|to|10|mi}}: a range++ if (['-', '–', 'to', 'and'].includes(String(pos[1]).toLowerCase()) && pos[2]) {++ const unit = UNITS[String(pos[3] || '').toLowerCase()] || pos[3] || '';++ return `${number(pos[0])}${pos[1] === 'to' || pos[1] === 'and' ? ` ${pos[1]} ` : '–'}${number(pos[2])}${unit ? ` ${unit}` : ''}`;++ }++ const unit = UNITS[String(pos[1] || '').toLowerCase()] || pos[1] || '';++ return `${number(pos[0])}${unit ? ` ${unit}` : ''}`;++ }++ if (name === 'lang' || name === 'lang-zh') return pos[pos.length - 1] || '';++ if (name === 'nowrap' || name === 'nobr' || name === 'small' || name === 'iata' || name === 'icao') return pos[0] || '';++ if (name === 'w' || name === 'wp') return pos[1] || pos[0] || '';++ if (name === 'marker') return named.name || '';++ return null;++}++function removeTemplates(text) {let out = '';for (let i = 0; i < text.length; i += 1) {if (text[i] === '{' && text[i + 1] === '{') {const end = matchingBrace(text, i);-- if (end > 0) { i = end - 1; continue; }++ if (end > 0) {++ const kept = inlineTemplate(text.slice(i + 2, end - 2));++ if (kept) out += removeTemplates(kept);++ i = end - 1;++ continue;++ }}out += text[i];}@@@ -64,7 +96,9 @@ function stripWiki(value, { max = 1200 } = {}) {s = s.replace(/<\/?[a-z][^>]*>/gi, ' ');s = s.replace(/\{\{|\}\}/g, ' ');for (const [k, v] of Object.entries(ENTITIES)) s = s.split(k).join(v);-- s = s.replace(/[ \t]+/g, ' ').replace(/\s*\n\s*/g, ' ').trim();++ s = s.replace(/[ \t]+/g, ' ').replace(/\s*\n\s*/g, ' ');++ // A space a removed template left before punctuation, and an empty bracket pair.++ s = s.replace(/\(\s*\)/g, '').replace(/\s+([.,;:!?])/g, '$1').replace(/[ ]{2,}/g, ' ').trim();return s.slice(0, max);}@@@ -18,6 +18,11 @@ check('stripWiki keeps the last character, follows links and removes templates,assert.strictEqual(stripWiki('Ueno Park'), 'Ueno Park');assert.strictEqual(stripWiki("[[Tokyo/Asakusa|Asakusa]]'s '''Senso-ji''' is [[Buddhist]].", {}), "Asakusa's Senso-ji is Buddhist.");assert.strictEqual(stripWiki('A {{m|28}} statue<ref>source</ref>'), 'A statue');++ // Inline templates that carry words are kept as text (Chongqing's lead ended "covers ." on 2026-11-02).++ assert.strictEqual(stripWiki('It covers {{convert|82400|km2|sqmi}}.'), 'It covers 82,400 km².');++ assert.strictEqual(stripWiki('{{convert|20|-|25|km}} away, {{convert|5|to|10|mi}}'), '20–25 km away, 5 to 10 mi');++ assert.strictEqual(stripWiki('Chongqing ({{lang|zh|重庆}}) on {{w|Yangtze River|the Yangtze}} at {{IATA|CKG}}.'), 'Chongqing (重庆) on the Yangtze at CKG.');++ assert.strictEqual(stripWiki('Left {{unknown|template}} out .'), 'Left out.');assert.strictEqual(stripWiki('[[File:Photo.jpg|thumb|Caption]] A temple.'), 'A temple.');});
@@ -30,13 +30,17 @@ function fileTitle(name) {
@@ -30,13 +30,17 @@ function fileTitle(name) {
return /^File:/i.test(s) ? s : `File:${s}`;
return /^File:/i.test(s) ? s : `File:${s}`;
}
}
+// A run under older rules is not fresh (jobs/index.js): inline-templates-1 keeps {{convert}}, {{lang}} and the like in the
+// summaries, so the first boot with it rewrites them.
+const CITIES_VERSION = 'inline-templates-1';
+
function createCitiesJob({ config, s, store, fetchImpl, log = console }) {
function createCitiesJob({ config, s, store, fetchImpl, log = console }) {
const wikivoyage = createWikivoyage({ config, fetchImpl, pace: s.pace });
const wikivoyage = createWikivoyage({ config, fetchImpl, pace: s.pace });
const wikidata = createWikidata({ config, fetchImpl, pace: s.pace, log });
const wikidata = createWikidata({ config, fetchImpl, pace: s.pace, log });
async function runOnce({ limit = config.openData.cityLimit } = {}) {
async function runOnce({ limit = config.openData.cityLimit } = {}) {
const catalog = loadCatalog(limit);
const catalog = loadCatalog(limit);
- const result = { catalog: catalog.length, cities: 0, withSummary: 0, withSee: 0, districts: 0, skipped: [], errors: [] };
+ const result = { catalog: catalog.length, cities: 0, withSummary: 0, withSee: 0, districts: 0, skipped: [], errors: [], version: CITIES_VERSION };
const qidMap = await wikidata.resolveQids(catalog.map((c) => c.wikipedia || c.name));
const qidMap = await wikidata.resolveQids(catalog.map((c) => c.wikipedia || c.name));
const qids = [...qidMap.values()].map((v) => v.qid);
const qids = [...qidMap.values()].map((v) => v.qid);
@@ -134,7 +138,7 @@ function createCitiesJob({ config, s, store, fetchImpl, log = console }) {
@@ -134,7 +138,7 @@ function createCitiesJob({ config, s, store, fetchImpl, log = console }) {
return result;
return result;
}
}
- return { runOnce, loadCatalog, pageUrl };
+ return { runOnce, loadCatalog, pageUrl, version: CITIES_VERSION };
}
}
module.exports = { createCitiesJob, loadCatalog };
module.exports = { createCitiesJob, loadCatalog };
@@ -25,7 +25,7 @@ function createJobs({ config, s, store, fetchImpl = globalThis.fetch, log = cons
@@ -25,7 +25,7 @@ function createJobs({ config, s, store, fetchImpl = globalThis.fetch, log = cons
};
};
const jobs = {
const jobs = {
- cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs },
+ cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs, version: creators.cities.version },
places: { run: creators.places.runOnce, maxAgeMs: config.jobs.placesEveryMs, intervalMs: config.jobs.placesEveryMs, version: creators.places.version },
places: { run: creators.places.runOnce, maxAgeMs: config.jobs.placesEveryMs, intervalMs: config.jobs.placesEveryMs, version: creators.places.version },
routes: { run: creators.routes.runOnce, maxAgeMs: config.jobs.routesEveryMs, intervalMs: config.jobs.routesEveryMs, version: creators.routes.version },
routes: { run: creators.routes.runOnce, maxAgeMs: config.jobs.routesEveryMs, intervalMs: config.jobs.routesEveryMs, version: creators.routes.version },
live: { run: creators.live.runOnce, maxAgeMs: config.jobs.liveEveryMs, intervalMs: config.jobs.liveEveryMs },
live: { run: creators.live.runOnce, maxAgeMs: config.jobs.liveEveryMs, intervalMs: config.jobs.liveEveryMs },
@@ -22,12 +22,44 @@ function matchingBrace(text, start) {
@@ -22,12 +22,44 @@ function matchingBrace(text, start) {
return -1;
return -1;
}
}
+// Inline templates that carry the sentence's words are kept as their text; every other template goes. Removing
+// {{convert|82400|km2}} left Chongqing's lead "…and covers ." (production, 2026-11-02).
+const UNITS = { km2: 'km²', 'km²': 'km²', sqkm: 'km²', sqmi: 'sq mi', mi2: 'sq mi', km: 'km', mi: 'mi', m: 'm', ft: 'ft', ha: 'ha',
+ acre: 'acres', acres: 'acres', c: '°C', f: '°F', '°c': '°C', '°f': '°F', kg: 'kg', lb: 'lb', 'km/h': 'km/h', mph: 'mph', l: 'L' };
+const number = (v) => (/^-?\d+(\.\d+)?$/.test(String(v).trim()) ? Number(v).toLocaleString('en-US', { maximumFractionDigits: 2 }) : String(v).trim());
+function inlineTemplate(inner) {
+ const parts = splitParams(inner).map((x) => x.trim());
+ const name = String(parts[0] || '').toLowerCase().replace(/_/g, ' ').trim();
+ const pos = parts.slice(1).filter((x) => !/^[a-z_ -]+=/i.test(x));
+ const named = paramsOf(inner);
+ if (name === 'convert' || name === 'cvt') {
+ if (!pos[0]) return '';
+ // {{convert|20|-|25|km}}, {{convert|5|to|10|mi}}: a range
+ if (['-', '–', 'to', 'and'].includes(String(pos[1]).toLowerCase()) && pos[2]) {
+ const unit = UNITS[String(pos[3] || '').toLowerCase()] || pos[3] || '';
+ return `${number(pos[0])}${pos[1] === 'to' || pos[1] === 'and' ? ` ${pos[1]} ` : '–'}${number(pos[2])}${unit ? ` ${unit}` : ''}`;
+ }
+ const unit = UNITS[String(pos[1] || '').toLowerCase()] || pos[1] || '';
+ return `${number(pos[0])}${unit ? ` ${unit}` : ''}`;
+ }
+ if (name === 'lang' || name === 'lang-zh') return pos[pos.length - 1] || '';
+ if (name === 'nowrap' || name === 'nobr' || name === 'small' || name === 'iata' || name === 'icao') return pos[0] || '';
+ if (name === 'w' || name === 'wp') return pos[1] || pos[0] || '';
+ if (name === 'marker') return named.name || '';
+ return null;
+}
+
function removeTemplates(text) {
function removeTemplates(text) {
let out = '';
let out = '';
for (let i = 0; i < text.length; i += 1) {
for (let i = 0; i < text.length; i += 1) {
if (text[i] === '{' && text[i + 1] === '{') {
if (text[i] === '{' && text[i + 1] === '{') {
const end = matchingBrace(text, i);
const end = matchingBrace(text, i);
- if (end > 0) { i = end - 1; continue; }
+ if (end > 0) {
+ const kept = inlineTemplate(text.slice(i + 2, end - 2));
+ if (kept) out += removeTemplates(kept);
+ i = end - 1;
+ continue;
+ }
}
}
out += text[i];
out += text[i];
}
}
@@ -64,7 +96,9 @@ function stripWiki(value, { max = 1200 } = {}) {
@@ -64,7 +96,9 @@ function stripWiki(value, { max = 1200 } = {}) {
s = s.replace(/<\/?[a-z][^>]*>/gi, ' ');
s = s.replace(/<\/?[a-z][^>]*>/gi, ' ');
s = s.replace(/\{\{|\}\}/g, ' ');
s = s.replace(/\{\{|\}\}/g, ' ');
for (const [k, v] of Object.entries(ENTITIES)) s = s.split(k).join(v);
for (const [k, v] of Object.entries(ENTITIES)) s = s.split(k).join(v);
- s = s.replace(/[ \t]+/g, ' ').replace(/\s*\n\s*/g, ' ').trim();
+ s = s.replace(/[ \t]+/g, ' ').replace(/\s*\n\s*/g, ' ');
+ // A space a removed template left before punctuation, and an empty bracket pair.
+ s = s.replace(/\(\s*\)/g, '').replace(/\s+([.,;:!?])/g, '$1').replace(/[ ]{2,}/g, ' ').trim();
return s.slice(0, max);
return s.slice(0, max);
}
}
@@ -18,6 +18,11 @@ check('stripWiki keeps the last character, follows links and removes templates,
@@ -18,6 +18,11 @@ check('stripWiki keeps the last character, follows links and removes templates,
assert.strictEqual(stripWiki('Ueno Park'), 'Ueno Park');
assert.strictEqual(stripWiki('Ueno Park'), 'Ueno Park');
assert.strictEqual(stripWiki("[[Tokyo/Asakusa|Asakusa]]'s '''Senso-ji''' is [[Buddhist]].", {}), "Asakusa's Senso-ji is Buddhist.");
assert.strictEqual(stripWiki("[[Tokyo/Asakusa|Asakusa]]'s '''Senso-ji''' is [[Buddhist]].", {}), "Asakusa's Senso-ji is Buddhist.");
assert.strictEqual(stripWiki('A {{m|28}} statue<ref>source</ref>'), 'A statue');
assert.strictEqual(stripWiki('A {{m|28}} statue<ref>source</ref>'), 'A statue');
+ // Inline templates that carry words are kept as text (Chongqing's lead ended "covers ." on 2026-11-02).
+ assert.strictEqual(stripWiki('It covers {{convert|82400|km2|sqmi}}.'), 'It covers 82,400 km².');
+ assert.strictEqual(stripWiki('{{convert|20|-|25|km}} away, {{convert|5|to|10|mi}}'), '20–25 km away, 5 to 10 mi');
+ assert.strictEqual(stripWiki('Chongqing ({{lang|zh|重庆}}) on {{w|Yangtze River|the Yangtze}} at {{IATA|CKG}}.'), 'Chongqing (重庆) on the Yangtze at CKG.');
+ assert.strictEqual(stripWiki('Left {{unknown|template}} out .'), 'Left out.');
assert.strictEqual(stripWiki('[[File:Photo.jpg|thumb|Caption]] A temple.'), 'A temple.');
assert.strictEqual(stripWiki('[[File:Photo.jpg|thumb|Caption]] A temple.'), 'A temple.');
});
});