openvibers/openvibe.tours Commits
Place photos at the size they are drawn; text never stops mid-sentence (#113)
Changes
@@@ -1,4 +1,5 @@'use strict';++const { clipSentences } = require('../places/clip');/*** The places job, in two passes:@@@ -48,7 +49,7 @@ function rankPlace(place, entity, views = 0) {function whyAfterLead(text) {const sentences = String(text || '').match(/[^.!?]+[.!?]+(?:\s|$)/g)?.map((s) => s.trim()) || [];-- return sentences.slice(1, 4).join(' ').slice(0, 600) || null;++ return clipSentences(sentences.slice(1, 4).join(' '), 600) || null;}function createPlacesJob({ config, s, store, fetchImpl, log = console }) {nnew file mode 100644@@@ -0,0 +1,37 @@++'use strict';++++// Text shown to people is cut at a sentence, never in the middle of one. Fixed slices cut Wikivoyage, Wikidata and OSM++// descriptions mid-sentence, and the place page printed them as written: the Space Needle's "Why go" ended++// "…but the wine bar is still" (11-09).++++// A sentence ends at . ! ? or … , maybe followed by a closing quote or bracket, then a space or the end.++const END = /[.!?…]["'”’)\]]?(?=\s|$)/g;++++/**++ * `text` in at most `max` characters, ending at a sentence end. When even the first sentence is longer than `max`,++ * it is cut at the last word that fits and marked with "…". Short text is returned unchanged.++ */++function clipSentences(text, max) {++ const t = String(text ?? '').replace(/\s+/g, ' ').trim();++ if (!t || t.length <= max) return t;++ const head = t.slice(0, max + 1); // + 1: a sentence that ends exactly at max still counts++ let cut = -1;++ for (const m of head.matchAll(END)) if (m.index + m[0].length <= max) cut = m.index + m[0].length;++ if (cut > 0) return t.slice(0, cut).trim();++ const space = t.lastIndexOf(' ', max - 1);++ return `${t.slice(0, space > 0 ? space : max - 1).replace(/[\s,;:–—-]+$/, '')}…`;++}++++/**++ * Text already stored cut mid-sentence: drop the unfinished tail when a finished sentence comes before it, so a page++ * never ends in the middle of a sentence. Text that is one unfinished sentence keeps its words and gains "…".++ */++function finishedSentences(text) {++ const t = String(text ?? '').replace(/\s+/g, ' ').trim();++ if (!t || /[.!?…]["'”’)\]]?$/.test(t)) return t;++ let cut = -1;++ for (const m of t.matchAll(END)) cut = m.index + m[0].length;++ return cut > 0 ? t.slice(0, cut).trim() : `${t.replace(/[\s,;:–—-]+$/, '')}…`;++}++++module.exports = { clipSentences, finishedSentences };@@@ -1,4 +1,5 @@'use strict';++const { clipSentences } = require('./clip');const crypto = require('crypto');@@@ -24,7 +25,7 @@ function createNarrative({ config, s, fetchImpl = globalThis.fetch, log = consolconst r = await ai.run('ai.summarize', {text: `${place.name}: ${place.description}`, max_words: 70, style: 'a short, inviting visitor guide using only the source facts',}, { target: { service: 'tours', type: 'place', id: place.id }, idempotencyKey: `tours-place-${key}` });-- const summary = String(r.output?.text || '').trim().slice(0, 800);++ const summary = clipSentences(r.output?.text || '', 800);if (!summary || r.synthetic) return null;await s.db.exec(`INSERT INTO tours_place_summaries(place_id, source_fetched_at, summary, run_id)VALUES ($1,$2::timestamptz,$3,$4) ON CONFLICT(place_id) DO UPDATE SET@@@ -1,4 +1,5 @@'use strict';++const { clipSentences } = require('../places/clip');/*** OpenStreetMap through Overpass: one bounded `around:` query per city, named destinations only.@@@ -64,7 +65,7 @@ function placeFromElement(el, cityId, fetchedAt) {imageUrl: null,imageCredit: null,imageLicense: null,-- description: tags.description ? String(tags.description).slice(0, 800) : null,++ description: tags.description ? clipSentences(tags.description, 800) : null,source: 'openstreetmap',sourceUrl: `https://www.openstreetmap.org/${osmType}/${el.id}`,fetchedAt,@@@ -1,4 +1,5 @@'use strict';++const { clipSentences } = require('../places/clip');/*** Wikidata, Wikipedia and Wikimedia Commons (CC0 / CC BY-SA; each image keeps its own licence).@@@ -62,7 +63,7 @@ function extractGuide(extract, title) {const sentences = lead.match(/[^.!?]+[.!?]+(?:\s|$)/g)?.map((s) => s.trim()) || [];const intro = sentences[0] || '';if (!intro || intro.length < 20) return null;-- const why = sentences.slice(1, 4).join(' ').slice(0, 600) || null;++ const why = clipSentences(sentences.slice(1, 4).join(' '), 600) || null;const section = (name) => text.match(new RegExp(`== ${name} ==\\n([\\s\\S]*?)(?=\\n== |$)`, 'i'))?.[1] || '';const features = section('Features and programs').match(/Major features[^.]*\./i)?.[0] || null;const visit = text.match(/[^.\n]*(?:public tour|guided tour|free tour)[^.\n]*\./i)?.[0]?.trim() || null;@@@ -71,8 +72,8 @@ function extractGuide(extract, title) {const seasonal = /\b(?:spring|summer|fall|autumn|winter|bloom(?:ing|s)?|foliage|season(?:al|s)?|January|February|March|April|May|June|July|August|September|October|November|December)\b/i;const season = text.split(/(?<=[.!?])\s+|\n+/).map((s) => s.trim()).find((s) => seasonal.test(s) && new RegExp(`\\b${named}\\b`, 'i').test(s)) || null;-- return { intro: intro.slice(0, 600), why, features: features?.slice(0, 400) || null,-- visit: visit?.slice(0, 400) || null, season: season?.slice(0, 400) || null,++ return { intro: clipSentences(intro, 600), why, features: features ? clipSentences(features, 400) : null,++ visit: visit ? clipSentences(visit, 400) : null, season: season ? clipSentences(season, 400) : null,url: `https://en.wikipedia.org/wiki/${encodeURIComponent(String(title).replace(/ /g, '_'))}` };}@@@ -1,4 +1,5 @@'use strict';++const { clipSentences } = require('../places/clip');/*** Wikivoyage parsing (CC BY-SA). A city page gives the lead summary and its "See" listings; a districted city keeps@@@ -118,7 +119,7 @@ function leadSummary(wikitext) {const line = block.trim();if (!line || line.startsWith('|') || line.startsWith('=') || line.startsWith('*') || line.startsWith('#')) continue;const plain = stripWiki(line, { max: 1400 });-- if (plain.length >= 80) return plain.slice(0, 1000);++ if (plain.length >= 80) return clipSentences(plain, 1000);}return '';}@@@ -1,4 +1,5 @@'use strict';++const { finishedSentences } = require('../places/clip');/*** Tours' server-rendered pages. They work without JavaScript: every link is real, every action is a plain form.@@@ -50,6 +51,14 @@ function imageWidth(u, w) {url.searchParams.set('width', String(w));return url.toString();}++ // Media serves a public picture at a smaller width (GET /o/:id?w=, WebP, cached; its ladder stops at 640). A++ // card drawn 180 px wide was downloading the whole copy (912 KB for Lumen Field, 11-09; 37 KB at w=480).++ // Wider than 640 (the cover on a desktop) takes the stored copy itself.++ if (url.hostname === 'openvibe.media' && /^\/o\/[A-Za-z0-9_-]{6,64}$/.test(url.pathname) && !url.search) {++ if (w > 640) return url.toString();++ url.searchParams.set('w', String(w));++ return url.toString();++ }if (/(^|\.)wikimedia\.org$/i.test(url.hostname)) {// Commons only guarantees the exact thumbnail URL returned by imageinfo. Asking for a new width// can answer 400 for a file whose derivatives have not been rendered yet.@@@ -373,7 +382,7 @@ ${stays.length ? `<section class="tr-sec"><div class="tr-sec-head"><h2>Stays</h2}function placePage({ place, routeList = [], trips = [], nearby = [], live = [], summary = null } = {}) {-- const sentences = (place.description || '').split(/(?<=[.!?])\s+/).filter(Boolean);++ const sentences = finishedSentences(place.description || '').split(/(?<=[.!?…])\s+/).filter(Boolean);const seasonal = sentences.find((s) => /\b(spring|summer|autumn|fall|winter|bloom|blossom|season)\b/i.test(s));const guided = sentences.find((s) => /\b(guided tour|free tour)\b/i.test(s));const now = new Date();@@@ -382,7 +391,7 @@ function placePage({ place, routeList = [], trips = [], nearby = [], live = [],const name = placeName(place);const lede = sentences[0] || `${name} in ${place.city_name}.`;// Why go never repeats the lede: the AI summary when there is one, else the description after its first sentence.-- const why = summary?.text || sentences.slice(1).join(' ');++ const why = finishedSentences(summary?.text || '') || sentences.slice(1).join(' ');const lookFor = place.guide_features && place.guide_features !== place.description ? place.guide_features : '';const season = place.guide_season || seasonal || '';const visit = place.guide_visit || guided || '';nnew file mode 100644@@@ -0,0 +1,42 @@++'use strict';++// Text people read is cut at a sentence, never in the middle of one (11-09: the Space Needle's "Why go" ended++// "…but the wine bar is still"). server/places/clip.js; the sources, the places job, the AI summary and the place++// page all use it.++// node test/clip-sentences.test.js++const assert = require('assert');++const fs = require('fs');++const path = require('path');++const { clipSentences, finishedSentences } = require('../server/places/clip');++++const needle = 'The view is spectacular. There are floor-to-ceiling windows on both levels. The revolving Sky City restaurant has been closed due to renovations, but the wine bar is still open in the afternoon.';++// A cut inside the third sentence keeps the first two.++assert.strictEqual(clipSentences(needle, 120), 'The view is spectacular. There are floor-to-ceiling windows on both levels.');++// Short text is unchanged (whitespace tidied).++assert.strictEqual(clipSentences(' One. Two. ', 100), 'One. Two.');++// A sentence ending exactly at the limit counts.++assert.strictEqual(clipSentences('Abc. Def.', 4), 'Abc.');++// One long first sentence: cut at a word, marked.++const long = clipSentences('A very long first sentence that keeps going and going without any end in sight at all', 30);++assert.ok(long.endsWith('…') && long.length <= 31 && !/\s…$/.test(long), long);++// Closing quotes and brackets stay with their sentence.++assert.strictEqual(clipSentences('He said "go there." Then more words follow here for a while.', 25), 'He said "go there."');++assert.strictEqual(clipSentences('', 10), '');++assert.strictEqual(clipSentences(null, 10), '');++++// Stored text that was cut mid-sentence ends at its last finished sentence.++assert.strictEqual(finishedSentences('The view is spectacular. The wine bar is still'), 'The view is spectacular.');++assert.strictEqual(finishedSentences('Complete sentence.'), 'Complete sentence.');++assert.strictEqual(finishedSentences('Only an unfinished one,'), 'Only an unfinished one…');++assert.strictEqual(finishedSentences('Ends with a question?'), 'Ends with a question?');++assert.strictEqual(finishedSentences(''), '');++++// Every place that used to slice shown text now clips at a sentence.++const root = path.join(__dirname, '..', 'server');++const read = (f) => fs.readFileSync(path.join(root, f), 'utf8');++assert.match(read('sources/wikivoyage.js'), /clipSentences\(plain, 1000\)/);++assert.match(read('sources/wikidata.js'), /clipSentences\(sentences\.slice\(1, 4\)\.join\(' '\), 600\)/);++assert.match(read('sources/osm.js'), /clipSentences\(tags\.description, 800\)/);++assert.match(read('jobs/places.js'), /clipSentences\(sentences\.slice\(1, 4\)\.join\(' '\), 600\)/);++assert.match(read('places/narrative.js'), /clipSentences\(r\.output\?\.text \|\| '', 800\)/);++assert.match(read('views/index.js'), /finishedSentences\(place\.description \|\| ''\)/);++console.log('clip sentences: never mid-sentence, at the source or on the page');@@@ -22,9 +22,16 @@ const thumbUrl = 'https://thumb.wikimedia.org/wikipedia/commons/thumb/a/ab/Foo.jassert.strictEqual(imageWidth(thumbUrl, 330), null, 'a Commons thumbnail keeps the verified imageinfo URL');assert.strictEqual(imageWidth('https://upload.wikimedia.org/wikipedia/commons/5/51/Foo.png?utm_source=x', 640), null, 'an unscaled original keeps its URL because Wikimedia may reject an enlarged thumb');assert.strictEqual(imageWidth('https://commons.wikimedia.org/wiki/Special:FilePath/Foo.jpg?width=1000', 330), 'https://commons.wikimedia.org/wiki/Special:FilePath/Foo.jpg?width=330', 'Special:FilePath takes a width');--assert.strictEqual(imageWidth('https://openvibe.media/t/x.jpg', 330), null, 'a non-Wikimedia picture is not rewritten');++assert.strictEqual(imageWidth('https://openvibe.media/t/x.jpg', 330), null, 'a Media path that is not a public object is not rewritten');assert.strictEqual(imageWidth('https://upload.wikimedia.org/wikipedia/commons/a/ab/Foo.jpg', 330), null, 'an original with a two-letter hash prefix keeps its URL');assert.strictEqual(imageWidth(null, 330), null);++// Media's public objects take ?w= (WebP at its ladder, at most 640): a card no longer downloads the whole copy++// (11-09: 912 KB for one Nearby card, 37 KB at w=480). Wider than 640, the stored copy itself.++const obj = 'https://openvibe.media/o/med_01M6YPV5FV0XE5FV2EJW0TB9AA';++assert.strictEqual(imageWidth(obj, 330), obj + '?w=330', 'a Media picture at the card width');++assert.strictEqual(imageWidth(obj, 640), obj + '?w=640');++assert.strictEqual(imageWidth(obj, 1200), obj, 'wider than Media resizes: the stored copy');++assert.strictEqual(imageWidth(obj + '?w=160', 330), null, 'a URL that already asks for a size is left alone');assert.strictEqual(safeImage('https://photos.app.goo.gl/A7PbTNW1RbBbZUrTA'), null, 'a share link is not a picture');assert.ok(safeImage('https://upload.wikimedia.org/wikipedia/commons/a/ab/Hachiko.jpg'), 'Commons uploads');
@@ -1,4 +1,5 @@
@@ -1,4 +1,5 @@
'use strict';
'use strict';
+const { clipSentences } = require('../places/clip');
/**
/**
* The places job, in two passes:
* The places job, in two passes:
@@ -48,7 +49,7 @@ function rankPlace(place, entity, views = 0) {
@@ -48,7 +49,7 @@ function rankPlace(place, entity, views = 0) {
function whyAfterLead(text) {
function whyAfterLead(text) {
const sentences = String(text || '').match(/[^.!?]+[.!?]+(?:\s|$)/g)?.map((s) => s.trim()) || [];
const sentences = String(text || '').match(/[^.!?]+[.!?]+(?:\s|$)/g)?.map((s) => s.trim()) || [];
- return sentences.slice(1, 4).join(' ').slice(0, 600) || null;
+ return clipSentences(sentences.slice(1, 4).join(' '), 600) || null;
}
}
function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
new file mode 100644
new file mode 100644
@@ -0,0 +1,37 @@
@@ -0,0 +1,37 @@
+'use strict';
+
+// Text shown to people is cut at a sentence, never in the middle of one. Fixed slices cut Wikivoyage, Wikidata and OSM
+// descriptions mid-sentence, and the place page printed them as written: the Space Needle's "Why go" ended
+// "…but the wine bar is still" (11-09).
+
+// A sentence ends at . ! ? or … , maybe followed by a closing quote or bracket, then a space or the end.
+const END = /[.!?…]["'”’)\]]?(?=\s|$)/g;
+
+/**
+ * `text` in at most `max` characters, ending at a sentence end. When even the first sentence is longer than `max`,
+ * it is cut at the last word that fits and marked with "…". Short text is returned unchanged.
+ */
+function clipSentences(text, max) {
+ const t = String(text ?? '').replace(/\s+/g, ' ').trim();
+ if (!t || t.length <= max) return t;
+ const head = t.slice(0, max + 1); // + 1: a sentence that ends exactly at max still counts
+ let cut = -1;
+ for (const m of head.matchAll(END)) if (m.index + m[0].length <= max) cut = m.index + m[0].length;
+ if (cut > 0) return t.slice(0, cut).trim();
+ const space = t.lastIndexOf(' ', max - 1);
+ return `${t.slice(0, space > 0 ? space : max - 1).replace(/[\s,;:–—-]+$/, '')}…`;
+}
+
+/**
+ * Text already stored cut mid-sentence: drop the unfinished tail when a finished sentence comes before it, so a page
+ * never ends in the middle of a sentence. Text that is one unfinished sentence keeps its words and gains "…".
+ */
+function finishedSentences(text) {
+ const t = String(text ?? '').replace(/\s+/g, ' ').trim();
+ if (!t || /[.!?…]["'”’)\]]?$/.test(t)) return t;
+ let cut = -1;
+ for (const m of t.matchAll(END)) cut = m.index + m[0].length;
+ return cut > 0 ? t.slice(0, cut).trim() : `${t.replace(/[\s,;:–—-]+$/, '')}…`;
+}
+
+module.exports = { clipSentences, finishedSentences };
@@ -1,4 +1,5 @@
@@ -1,4 +1,5 @@
'use strict';
'use strict';
+const { clipSentences } = require('./clip');
const crypto = require('crypto');
const crypto = require('crypto');
@@ -24,7 +25,7 @@ function createNarrative({ config, s, fetchImpl = globalThis.fetch, log = consol
@@ -24,7 +25,7 @@ function createNarrative({ config, s, fetchImpl = globalThis.fetch, log = consol
const r = await ai.run('ai.summarize', {
const r = await ai.run('ai.summarize', {
text: `${place.name}: ${place.description}`, max_words: 70, style: 'a short, inviting visitor guide using only the source facts',
text: `${place.name}: ${place.description}`, max_words: 70, style: 'a short, inviting visitor guide using only the source facts',
}, { target: { service: 'tours', type: 'place', id: place.id }, idempotencyKey: `tours-place-${key}` });
}, { target: { service: 'tours', type: 'place', id: place.id }, idempotencyKey: `tours-place-${key}` });
- const summary = String(r.output?.text || '').trim().slice(0, 800);
+ const summary = clipSentences(r.output?.text || '', 800);
if (!summary || r.synthetic) return null;
if (!summary || r.synthetic) return null;
await s.db.exec(`INSERT INTO tours_place_summaries(place_id, source_fetched_at, summary, run_id)
await s.db.exec(`INSERT INTO tours_place_summaries(place_id, source_fetched_at, summary, run_id)
VALUES ($1,$2::timestamptz,$3,$4) ON CONFLICT(place_id) DO UPDATE SET
VALUES ($1,$2::timestamptz,$3,$4) ON CONFLICT(place_id) DO UPDATE SET
@@ -1,4 +1,5 @@
@@ -1,4 +1,5 @@
'use strict';
'use strict';
+const { clipSentences } = require('../places/clip');
/**
/**
* OpenStreetMap through Overpass: one bounded `around:` query per city, named destinations only.
* OpenStreetMap through Overpass: one bounded `around:` query per city, named destinations only.
@@ -64,7 +65,7 @@ function placeFromElement(el, cityId, fetchedAt) {
@@ -64,7 +65,7 @@ function placeFromElement(el, cityId, fetchedAt) {
imageUrl: null,
imageUrl: null,
imageCredit: null,
imageCredit: null,
imageLicense: null,
imageLicense: null,
- description: tags.description ? String(tags.description).slice(0, 800) : null,
+ description: tags.description ? clipSentences(tags.description, 800) : null,
source: 'openstreetmap',
source: 'openstreetmap',
sourceUrl: `https://www.openstreetmap.org/${osmType}/${el.id}`,
sourceUrl: `https://www.openstreetmap.org/${osmType}/${el.id}`,
fetchedAt,
fetchedAt,
@@ -1,4 +1,5 @@
@@ -1,4 +1,5 @@
'use strict';
'use strict';
+const { clipSentences } = require('../places/clip');
/**
/**
* Wikidata, Wikipedia and Wikimedia Commons (CC0 / CC BY-SA; each image keeps its own licence).
* Wikidata, Wikipedia and Wikimedia Commons (CC0 / CC BY-SA; each image keeps its own licence).
@@ -62,7 +63,7 @@ function extractGuide(extract, title) {
@@ -62,7 +63,7 @@ function extractGuide(extract, title) {
const sentences = lead.match(/[^.!?]+[.!?]+(?:\s|$)/g)?.map((s) => s.trim()) || [];
const sentences = lead.match(/[^.!?]+[.!?]+(?:\s|$)/g)?.map((s) => s.trim()) || [];
const intro = sentences[0] || '';
const intro = sentences[0] || '';
if (!intro || intro.length < 20) return null;
if (!intro || intro.length < 20) return null;
- const why = sentences.slice(1, 4).join(' ').slice(0, 600) || null;
+ const why = clipSentences(sentences.slice(1, 4).join(' '), 600) || null;
const section = (name) => text.match(new RegExp(`== ${name} ==\\n([\\s\\S]*?)(?=\\n== |$)`, 'i'))?.[1] || '';
const section = (name) => text.match(new RegExp(`== ${name} ==\\n([\\s\\S]*?)(?=\\n== |$)`, 'i'))?.[1] || '';
const features = section('Features and programs').match(/Major features[^.]*\./i)?.[0] || null;
const features = section('Features and programs').match(/Major features[^.]*\./i)?.[0] || null;
const visit = text.match(/[^.\n]*(?:public tour|guided tour|free tour)[^.\n]*\./i)?.[0]?.trim() || null;
const visit = text.match(/[^.\n]*(?:public tour|guided tour|free tour)[^.\n]*\./i)?.[0]?.trim() || null;
@@ -71,8 +72,8 @@ function extractGuide(extract, title) {
@@ -71,8 +72,8 @@ function extractGuide(extract, title) {
const seasonal = /\b(?:spring|summer|fall|autumn|winter|bloom(?:ing|s)?|foliage|season(?:al|s)?|January|February|March|April|May|June|July|August|September|October|November|December)\b/i;
const seasonal = /\b(?:spring|summer|fall|autumn|winter|bloom(?:ing|s)?|foliage|season(?:al|s)?|January|February|March|April|May|June|July|August|September|October|November|December)\b/i;
const season = text.split(/(?<=[.!?])\s+|\n+/).map((s) => s.trim())
const season = text.split(/(?<=[.!?])\s+|\n+/).map((s) => s.trim())
.find((s) => seasonal.test(s) && new RegExp(`\\b${named}\\b`, 'i').test(s)) || null;
.find((s) => seasonal.test(s) && new RegExp(`\\b${named}\\b`, 'i').test(s)) || null;
- return { intro: intro.slice(0, 600), why, features: features?.slice(0, 400) || null,
- visit: visit?.slice(0, 400) || null, season: season?.slice(0, 400) || null,
+ return { intro: clipSentences(intro, 600), why, features: features ? clipSentences(features, 400) : null,
+ visit: visit ? clipSentences(visit, 400) : null, season: season ? clipSentences(season, 400) : null,
url: `https://en.wikipedia.org/wiki/${encodeURIComponent(String(title).replace(/ /g, '_'))}` };
url: `https://en.wikipedia.org/wiki/${encodeURIComponent(String(title).replace(/ /g, '_'))}` };
}
}
@@ -1,4 +1,5 @@
@@ -1,4 +1,5 @@
'use strict';
'use strict';
+const { clipSentences } = require('../places/clip');
/**
/**
* Wikivoyage parsing (CC BY-SA). A city page gives the lead summary and its "See" listings; a districted city keeps
* Wikivoyage parsing (CC BY-SA). A city page gives the lead summary and its "See" listings; a districted city keeps
@@ -118,7 +119,7 @@ function leadSummary(wikitext) {
@@ -118,7 +119,7 @@ function leadSummary(wikitext) {
const line = block.trim();
const line = block.trim();
if (!line || line.startsWith('|') || line.startsWith('=') || line.startsWith('*') || line.startsWith('#')) continue;
if (!line || line.startsWith('|') || line.startsWith('=') || line.startsWith('*') || line.startsWith('#')) continue;
const plain = stripWiki(line, { max: 1400 });
const plain = stripWiki(line, { max: 1400 });
- if (plain.length >= 80) return plain.slice(0, 1000);
+ if (plain.length >= 80) return clipSentences(plain, 1000);
}
}
return '';
return '';
}
}
@@ -1,4 +1,5 @@
@@ -1,4 +1,5 @@
'use strict';
'use strict';
+const { finishedSentences } = require('../places/clip');
/**
/**
* Tours' server-rendered pages. They work without JavaScript: every link is real, every action is a plain form.
* Tours' server-rendered pages. They work without JavaScript: every link is real, every action is a plain form.
@@ -50,6 +51,14 @@ function imageWidth(u, w) {
@@ -50,6 +51,14 @@ function imageWidth(u, w) {
url.searchParams.set('width', String(w));
url.searchParams.set('width', String(w));
return url.toString();
return url.toString();
}
}
+ // Media serves a public picture at a smaller width (GET /o/:id?w=, WebP, cached; its ladder stops at 640). A
+ // card drawn 180 px wide was downloading the whole copy (912 KB for Lumen Field, 11-09; 37 KB at w=480).
+ // Wider than 640 (the cover on a desktop) takes the stored copy itself.
+ if (url.hostname === 'openvibe.media' && /^\/o\/[A-Za-z0-9_-]{6,64}$/.test(url.pathname) && !url.search) {
+ if (w > 640) return url.toString();
+ url.searchParams.set('w', String(w));
+ return url.toString();
+ }
if (/(^|\.)wikimedia\.org$/i.test(url.hostname)) {
if (/(^|\.)wikimedia\.org$/i.test(url.hostname)) {
// Commons only guarantees the exact thumbnail URL returned by imageinfo. Asking for a new width
// Commons only guarantees the exact thumbnail URL returned by imageinfo. Asking for a new width
// can answer 400 for a file whose derivatives have not been rendered yet.
// can answer 400 for a file whose derivatives have not been rendered yet.
@@ -373,7 +382,7 @@ ${stays.length ? `<section class="tr-sec"><div class="tr-sec-head"><h2>Stays</h2
@@ -373,7 +382,7 @@ ${stays.length ? `<section class="tr-sec"><div class="tr-sec-head"><h2>Stays</h2
}
}
function placePage({ place, routeList = [], trips = [], nearby = [], live = [], summary = null } = {}) {
function placePage({ place, routeList = [], trips = [], nearby = [], live = [], summary = null } = {}) {
- const sentences = (place.description || '').split(/(?<=[.!?])\s+/).filter(Boolean);
+ const sentences = finishedSentences(place.description || '').split(/(?<=[.!?…])\s+/).filter(Boolean);
const seasonal = sentences.find((s) => /\b(spring|summer|autumn|fall|winter|bloom|blossom|season)\b/i.test(s));
const seasonal = sentences.find((s) => /\b(spring|summer|autumn|fall|winter|bloom|blossom|season)\b/i.test(s));
const guided = sentences.find((s) => /\b(guided tour|free tour)\b/i.test(s));
const guided = sentences.find((s) => /\b(guided tour|free tour)\b/i.test(s));
const now = new Date();
const now = new Date();
@@ -382,7 +391,7 @@ function placePage({ place, routeList = [], trips = [], nearby = [], live = [],
@@ -382,7 +391,7 @@ function placePage({ place, routeList = [], trips = [], nearby = [], live = [],
const name = placeName(place);
const name = placeName(place);
const lede = sentences[0] || `${name} in ${place.city_name}.`;
const lede = sentences[0] || `${name} in ${place.city_name}.`;
// Why go never repeats the lede: the AI summary when there is one, else the description after its first sentence.
// Why go never repeats the lede: the AI summary when there is one, else the description after its first sentence.
- const why = summary?.text || sentences.slice(1).join(' ');
+ const why = finishedSentences(summary?.text || '') || sentences.slice(1).join(' ');
const lookFor = place.guide_features && place.guide_features !== place.description ? place.guide_features : '';
const lookFor = place.guide_features && place.guide_features !== place.description ? place.guide_features : '';
const season = place.guide_season || seasonal || '';
const season = place.guide_season || seasonal || '';
const visit = place.guide_visit || guided || '';
const visit = place.guide_visit || guided || '';
new file mode 100644
new file mode 100644
@@ -0,0 +1,42 @@
@@ -0,0 +1,42 @@
+'use strict';
+// Text people read is cut at a sentence, never in the middle of one (11-09: the Space Needle's "Why go" ended
+// "…but the wine bar is still"). server/places/clip.js; the sources, the places job, the AI summary and the place
+// page all use it.
+// node test/clip-sentences.test.js
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { clipSentences, finishedSentences } = require('../server/places/clip');
+
+const needle = 'The view is spectacular. There are floor-to-ceiling windows on both levels. The revolving Sky City restaurant has been closed due to renovations, but the wine bar is still open in the afternoon.';
+// A cut inside the third sentence keeps the first two.
+assert.strictEqual(clipSentences(needle, 120), 'The view is spectacular. There are floor-to-ceiling windows on both levels.');
+// Short text is unchanged (whitespace tidied).
+assert.strictEqual(clipSentences(' One. Two. ', 100), 'One. Two.');
+// A sentence ending exactly at the limit counts.
+assert.strictEqual(clipSentences('Abc. Def.', 4), 'Abc.');
+// One long first sentence: cut at a word, marked.
+const long = clipSentences('A very long first sentence that keeps going and going without any end in sight at all', 30);
+assert.ok(long.endsWith('…') && long.length <= 31 && !/\s…$/.test(long), long);
+// Closing quotes and brackets stay with their sentence.
+assert.strictEqual(clipSentences('He said "go there." Then more words follow here for a while.', 25), 'He said "go there."');
+assert.strictEqual(clipSentences('', 10), '');
+assert.strictEqual(clipSentences(null, 10), '');
+
+// Stored text that was cut mid-sentence ends at its last finished sentence.
+assert.strictEqual(finishedSentences('The view is spectacular. The wine bar is still'), 'The view is spectacular.');
+assert.strictEqual(finishedSentences('Complete sentence.'), 'Complete sentence.');
+assert.strictEqual(finishedSentences('Only an unfinished one,'), 'Only an unfinished one…');
+assert.strictEqual(finishedSentences('Ends with a question?'), 'Ends with a question?');
+assert.strictEqual(finishedSentences(''), '');
+
+// Every place that used to slice shown text now clips at a sentence.
+const root = path.join(__dirname, '..', 'server');
+const read = (f) => fs.readFileSync(path.join(root, f), 'utf8');
+assert.match(read('sources/wikivoyage.js'), /clipSentences\(plain, 1000\)/);
+assert.match(read('sources/wikidata.js'), /clipSentences\(sentences\.slice\(1, 4\)\.join\(' '\), 600\)/);
+assert.match(read('sources/osm.js'), /clipSentences\(tags\.description, 800\)/);
+assert.match(read('jobs/places.js'), /clipSentences\(sentences\.slice\(1, 4\)\.join\(' '\), 600\)/);
+assert.match(read('places/narrative.js'), /clipSentences\(r\.output\?\.text \|\| '', 800\)/);
+assert.match(read('views/index.js'), /finishedSentences\(place\.description \|\| ''\)/);
+console.log('clip sentences: never mid-sentence, at the source or on the page');
@@ -22,9 +22,16 @@ const thumbUrl = 'https://thumb.wikimedia.org/wikipedia/commons/thumb/a/ab/Foo.j
@@ -22,9 +22,16 @@ const thumbUrl = 'https://thumb.wikimedia.org/wikipedia/commons/thumb/a/ab/Foo.j
assert.strictEqual(imageWidth(thumbUrl, 330), null, 'a Commons thumbnail keeps the verified imageinfo URL');
assert.strictEqual(imageWidth(thumbUrl, 330), null, 'a Commons thumbnail keeps the verified imageinfo URL');
assert.strictEqual(imageWidth('https://upload.wikimedia.org/wikipedia/commons/5/51/Foo.png?utm_source=x', 640), null, 'an unscaled original keeps its URL because Wikimedia may reject an enlarged thumb');
assert.strictEqual(imageWidth('https://upload.wikimedia.org/wikipedia/commons/5/51/Foo.png?utm_source=x', 640), null, 'an unscaled original keeps its URL because Wikimedia may reject an enlarged thumb');
assert.strictEqual(imageWidth('https://commons.wikimedia.org/wiki/Special:FilePath/Foo.jpg?width=1000', 330), 'https://commons.wikimedia.org/wiki/Special:FilePath/Foo.jpg?width=330', 'Special:FilePath takes a width');
assert.strictEqual(imageWidth('https://commons.wikimedia.org/wiki/Special:FilePath/Foo.jpg?width=1000', 330), 'https://commons.wikimedia.org/wiki/Special:FilePath/Foo.jpg?width=330', 'Special:FilePath takes a width');
-assert.strictEqual(imageWidth('https://openvibe.media/t/x.jpg', 330), null, 'a non-Wikimedia picture is not rewritten');
+assert.strictEqual(imageWidth('https://openvibe.media/t/x.jpg', 330), null, 'a Media path that is not a public object is not rewritten');
assert.strictEqual(imageWidth('https://upload.wikimedia.org/wikipedia/commons/a/ab/Foo.jpg', 330), null, 'an original with a two-letter hash prefix keeps its URL');
assert.strictEqual(imageWidth('https://upload.wikimedia.org/wikipedia/commons/a/ab/Foo.jpg', 330), null, 'an original with a two-letter hash prefix keeps its URL');
assert.strictEqual(imageWidth(null, 330), null);
assert.strictEqual(imageWidth(null, 330), null);
+// Media's public objects take ?w= (WebP at its ladder, at most 640): a card no longer downloads the whole copy
+// (11-09: 912 KB for one Nearby card, 37 KB at w=480). Wider than 640, the stored copy itself.
+const obj = 'https://openvibe.media/o/med_01M6YPV5FV0XE5FV2EJW0TB9AA';
+assert.strictEqual(imageWidth(obj, 330), obj + '?w=330', 'a Media picture at the card width');
+assert.strictEqual(imageWidth(obj, 640), obj + '?w=640');
+assert.strictEqual(imageWidth(obj, 1200), obj, 'wider than Media resizes: the stored copy');
+assert.strictEqual(imageWidth(obj + '?w=160', 330), null, 'a URL that already asks for a size is left alone');
assert.strictEqual(safeImage('https://photos.app.goo.gl/A7PbTNW1RbBbZUrTA'), null, 'a share link is not a picture');
assert.strictEqual(safeImage('https://photos.app.goo.gl/A7PbTNW1RbBbZUrTA'), null, 'a share link is not a picture');
assert.ok(safeImage('https://upload.wikimedia.org/wikipedia/commons/a/ab/Hachiko.jpg'), 'Commons uploads');
assert.ok(safeImage('https://upload.wikimedia.org/wikipedia/commons/a/ab/Hachiko.jpg'), 'Commons uploads');