openvibers/openvibe.tours Commits

Place photos at the size they are drawn; text never stops mid-sentence (#113)

OP OpenVibers committed 4 days ago · 8ced16dee6 · 9 files, +110 −10

Photos: Tours drew Media's copies at full size, 912 KB for one Nearby card shown 180 px wide (61 on the Seattle
page). imageWidth now asks Media for the drawn width (GET /o/:id?w=, WebP, cached; 37 KB at 480); wider than 640
the stored copy itself, so the cover stays sharp.
Text: Wikivoyage, Wikidata, OSM, the places job and the AI summary sliced text at a fixed length, and the place page
printed it as cut: the Space Needle's Why go ended "…but the wine bar is still". clipSentences cuts at the last
sentence end within the limit (a word and an ellipsis when the first sentence is too long), and the page drops an
unfinished tail from text already stored.

Changes

ddiff --git a/server/jobs/places.js b/server/jobs/places.js
iindex 58935cf..407a027 100644
---- a/server/jobs/places.js
++++ b/server/jobs/places.js
@@@ -1,4 +1,5 @@
'use strict';
++const { clipSentences } = require('../places/clip');
/**
* The places job, in two passes:
@@@ -48,7 +49,7 @@ function rankPlace(place, entity, views = 0) {
function whyAfterLead(text) {
const sentences = String(text || '').match(/[^.!?]+[.!?]+(?:\s|$)/g)?.map((s) => s.trim()) || [];
-- return sentences.slice(1, 4).join(' ').slice(0, 600) || null;
++ return clipSentences(sentences.slice(1, 4).join(' '), 600) || null;
}
function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
ddiff --git a/server/places/clip.js b/server/places/clip.js
nnew file mode 100644
iindex 0000000..5f23b54
---- /dev/null
++++ b/server/places/clip.js
@@@ -0,0 +1,37 @@
++'use strict';
++
++// Text shown to people is cut at a sentence, never in the middle of one. Fixed slices cut Wikivoyage, Wikidata and OSM
++// descriptions mid-sentence, and the place page printed them as written: the Space Needle's "Why go" ended
++// "…but the wine bar is still" (11-09).
++
++// A sentence ends at . ! ? or … , maybe followed by a closing quote or bracket, then a space or the end.
++const END = /[.!?…]["'”’)\]]?(?=\s|$)/g;
++
++/**
++ * `text` in at most `max` characters, ending at a sentence end. When even the first sentence is longer than `max`,
++ * it is cut at the last word that fits and marked with "…". Short text is returned unchanged.
++ */
++function clipSentences(text, max) {
++ const t = String(text ?? '').replace(/\s+/g, ' ').trim();
++ if (!t || t.length <= max) return t;
++ const head = t.slice(0, max + 1); // + 1: a sentence that ends exactly at max still counts
++ let cut = -1;
++ for (const m of head.matchAll(END)) if (m.index + m[0].length <= max) cut = m.index + m[0].length;
++ if (cut > 0) return t.slice(0, cut).trim();
++ const space = t.lastIndexOf(' ', max - 1);
++ return `${t.slice(0, space > 0 ? space : max - 1).replace(/[\s,;:–—-]+$/, '')}…`;
++}
++
++/**
++ * Text already stored cut mid-sentence: drop the unfinished tail when a finished sentence comes before it, so a page
++ * never ends in the middle of a sentence. Text that is one unfinished sentence keeps its words and gains "…".
++ */
++function finishedSentences(text) {
++ const t = String(text ?? '').replace(/\s+/g, ' ').trim();
++ if (!t || /[.!?…]["'”’)\]]?$/.test(t)) return t;
++ let cut = -1;
++ for (const m of t.matchAll(END)) cut = m.index + m[0].length;
++ return cut > 0 ? t.slice(0, cut).trim() : `${t.replace(/[\s,;:–—-]+$/, '')}…`;
++}
++
++module.exports = { clipSentences, finishedSentences };
ddiff --git a/server/places/narrative.js b/server/places/narrative.js
iindex 27b4167..5999b44 100644
---- a/server/places/narrative.js
++++ b/server/places/narrative.js
@@@ -1,4 +1,5 @@
'use strict';
++const { clipSentences } = require('./clip');
const crypto = require('crypto');
@@@ -24,7 +25,7 @@ function createNarrative({ config, s, fetchImpl = globalThis.fetch, log = consol
const r = await ai.run('ai.summarize', {
text: `${place.name}: ${place.description}`, max_words: 70, style: 'a short, inviting visitor guide using only the source facts',
}, { target: { service: 'tours', type: 'place', id: place.id }, idempotencyKey: `tours-place-${key}` });
-- const summary = String(r.output?.text || '').trim().slice(0, 800);
++ const summary = clipSentences(r.output?.text || '', 800);
if (!summary || r.synthetic) return null;
await s.db.exec(`INSERT INTO tours_place_summaries(place_id, source_fetched_at, summary, run_id)
VALUES ($1,$2::timestamptz,$3,$4) ON CONFLICT(place_id) DO UPDATE SET
ddiff --git a/server/sources/osm.js b/server/sources/osm.js
iindex 5f36a4a..e35cc2f 100644
---- a/server/sources/osm.js
++++ b/server/sources/osm.js
@@@ -1,4 +1,5 @@
'use strict';
++const { clipSentences } = require('../places/clip');
/**
* OpenStreetMap through Overpass: one bounded `around:` query per city, named destinations only.
@@@ -64,7 +65,7 @@ function placeFromElement(el, cityId, fetchedAt) {
imageUrl: null,
imageCredit: null,
imageLicense: null,
-- description: tags.description ? String(tags.description).slice(0, 800) : null,
++ description: tags.description ? clipSentences(tags.description, 800) : null,
source: 'openstreetmap',
sourceUrl: `https://www.openstreetmap.org/${osmType}/${el.id}`,
fetchedAt,
ddiff --git a/server/sources/wikidata.js b/server/sources/wikidata.js
iindex 8daf4da..de4251a 100644
---- a/server/sources/wikidata.js
++++ b/server/sources/wikidata.js
@@@ -1,4 +1,5 @@
'use strict';
++const { clipSentences } = require('../places/clip');
/**
* Wikidata, Wikipedia and Wikimedia Commons (CC0 / CC BY-SA; each image keeps its own licence).
@@@ -62,7 +63,7 @@ function extractGuide(extract, title) {
const sentences = lead.match(/[^.!?]+[.!?]+(?:\s|$)/g)?.map((s) => s.trim()) || [];
const intro = sentences[0] || '';
if (!intro || intro.length < 20) return null;
-- const why = sentences.slice(1, 4).join(' ').slice(0, 600) || null;
++ const why = clipSentences(sentences.slice(1, 4).join(' '), 600) || null;
const section = (name) => text.match(new RegExp(`== ${name} ==\\n([\\s\\S]*?)(?=\\n== |$)`, 'i'))?.[1] || '';
const features = section('Features and programs').match(/Major features[^.]*\./i)?.[0] || null;
const visit = text.match(/[^.\n]*(?:public tour|guided tour|free tour)[^.\n]*\./i)?.[0]?.trim() || null;
@@@ -71,8 +72,8 @@ function extractGuide(extract, title) {
const seasonal = /\b(?:spring|summer|fall|autumn|winter|bloom(?:ing|s)?|foliage|season(?:al|s)?|January|February|March|April|May|June|July|August|September|October|November|December)\b/i;
const season = text.split(/(?<=[.!?])\s+|\n+/).map((s) => s.trim())
.find((s) => seasonal.test(s) && new RegExp(`\\b${named}\\b`, 'i').test(s)) || null;
-- return { intro: intro.slice(0, 600), why, features: features?.slice(0, 400) || null,
-- visit: visit?.slice(0, 400) || null, season: season?.slice(0, 400) || null,
++ return { intro: clipSentences(intro, 600), why, features: features ? clipSentences(features, 400) : null,
++ visit: visit ? clipSentences(visit, 400) : null, season: season ? clipSentences(season, 400) : null,
url: `https://en.wikipedia.org/wiki/${encodeURIComponent(String(title).replace(/ /g, '_'))}` };
}
ddiff --git a/server/sources/wikivoyage.js b/server/sources/wikivoyage.js
iindex 3b389e9..6ea5e56 100644
---- a/server/sources/wikivoyage.js
++++ b/server/sources/wikivoyage.js
@@@ -1,4 +1,5 @@
'use strict';
++const { clipSentences } = require('../places/clip');
/**
* Wikivoyage parsing (CC BY-SA). A city page gives the lead summary and its "See" listings; a districted city keeps
@@@ -118,7 +119,7 @@ function leadSummary(wikitext) {
const line = block.trim();
if (!line || line.startsWith('|') || line.startsWith('=') || line.startsWith('*') || line.startsWith('#')) continue;
const plain = stripWiki(line, { max: 1400 });
-- if (plain.length >= 80) return plain.slice(0, 1000);
++ if (plain.length >= 80) return clipSentences(plain, 1000);
}
return '';
}
ddiff --git a/server/views/index.js b/server/views/index.js
iindex c64958c..30eddb1 100644
---- a/server/views/index.js
++++ b/server/views/index.js
@@@ -1,4 +1,5 @@
'use strict';
++const { finishedSentences } = require('../places/clip');
/**
* Tours' server-rendered pages. They work without JavaScript: every link is real, every action is a plain form.
@@@ -50,6 +51,14 @@ function imageWidth(u, w) {
url.searchParams.set('width', String(w));
return url.toString();
}
++ // Media serves a public picture at a smaller width (GET /o/:id?w=, WebP, cached; its ladder stops at 640). A
++ // card drawn 180 px wide was downloading the whole copy (912 KB for Lumen Field, 11-09; 37 KB at w=480).
++ // Wider than 640 (the cover on a desktop) takes the stored copy itself.
++ if (url.hostname === 'openvibe.media' && /^\/o\/[A-Za-z0-9_-]{6,64}$/.test(url.pathname) && !url.search) {
++ if (w > 640) return url.toString();
++ url.searchParams.set('w', String(w));
++ return url.toString();
++ }
if (/(^|\.)wikimedia\.org$/i.test(url.hostname)) {
// Commons only guarantees the exact thumbnail URL returned by imageinfo. Asking for a new width
// can answer 400 for a file whose derivatives have not been rendered yet.
@@@ -373,7 +382,7 @@ ${stays.length ? `<section class="tr-sec"><div class="tr-sec-head"><h2>Stays</h2
}
function placePage({ place, routeList = [], trips = [], nearby = [], live = [], summary = null } = {}) {
-- const sentences = (place.description || '').split(/(?<=[.!?])\s+/).filter(Boolean);
++ const sentences = finishedSentences(place.description || '').split(/(?<=[.!?…])\s+/).filter(Boolean);
const seasonal = sentences.find((s) => /\b(spring|summer|autumn|fall|winter|bloom|blossom|season)\b/i.test(s));
const guided = sentences.find((s) => /\b(guided tour|free tour)\b/i.test(s));
const now = new Date();
@@@ -382,7 +391,7 @@ function placePage({ place, routeList = [], trips = [], nearby = [], live = [],
const name = placeName(place);
const lede = sentences[0] || `${name} in ${place.city_name}.`;
// Why go never repeats the lede: the AI summary when there is one, else the description after its first sentence.
-- const why = summary?.text || sentences.slice(1).join(' ');
++ const why = finishedSentences(summary?.text || '') || sentences.slice(1).join(' ');
const lookFor = place.guide_features && place.guide_features !== place.description ? place.guide_features : '';
const season = place.guide_season || seasonal || '';
const visit = place.guide_visit || guided || '';
ddiff --git a/test/clip-sentences.test.js b/test/clip-sentences.test.js
nnew file mode 100644
iindex 0000000..520b5d7
---- /dev/null
++++ b/test/clip-sentences.test.js
@@@ -0,0 +1,42 @@
++'use strict';
++// Text people read is cut at a sentence, never in the middle of one (11-09: the Space Needle's "Why go" ended
++// "…but the wine bar is still"). server/places/clip.js; the sources, the places job, the AI summary and the place
++// page all use it.
++// node test/clip-sentences.test.js
++const assert = require('assert');
++const fs = require('fs');
++const path = require('path');
++const { clipSentences, finishedSentences } = require('../server/places/clip');
++
++const needle = 'The view is spectacular. There are floor-to-ceiling windows on both levels. The revolving Sky City restaurant has been closed due to renovations, but the wine bar is still open in the afternoon.';
++// A cut inside the third sentence keeps the first two.
++assert.strictEqual(clipSentences(needle, 120), 'The view is spectacular. There are floor-to-ceiling windows on both levels.');
++// Short text is unchanged (whitespace tidied).
++assert.strictEqual(clipSentences(' One. Two. ', 100), 'One. Two.');
++// A sentence ending exactly at the limit counts.
++assert.strictEqual(clipSentences('Abc. Def.', 4), 'Abc.');
++// One long first sentence: cut at a word, marked.
++const long = clipSentences('A very long first sentence that keeps going and going without any end in sight at all', 30);
++assert.ok(long.endsWith('…') && long.length <= 31 && !/\s…$/.test(long), long);
++// Closing quotes and brackets stay with their sentence.
++assert.strictEqual(clipSentences('He said "go there." Then more words follow here for a while.', 25), 'He said "go there."');
++assert.strictEqual(clipSentences('', 10), '');
++assert.strictEqual(clipSentences(null, 10), '');
++
++// Stored text that was cut mid-sentence ends at its last finished sentence.
++assert.strictEqual(finishedSentences('The view is spectacular. The wine bar is still'), 'The view is spectacular.');
++assert.strictEqual(finishedSentences('Complete sentence.'), 'Complete sentence.');
++assert.strictEqual(finishedSentences('Only an unfinished one,'), 'Only an unfinished one…');
++assert.strictEqual(finishedSentences('Ends with a question?'), 'Ends with a question?');
++assert.strictEqual(finishedSentences(''), '');
++
++// Every place that used to slice shown text now clips at a sentence.
++const root = path.join(__dirname, '..', 'server');
++const read = (f) => fs.readFileSync(path.join(root, f), 'utf8');
++assert.match(read('sources/wikivoyage.js'), /clipSentences\(plain, 1000\)/);
++assert.match(read('sources/wikidata.js'), /clipSentences\(sentences\.slice\(1, 4\)\.join\(' '\), 600\)/);
++assert.match(read('sources/osm.js'), /clipSentences\(tags\.description, 800\)/);
++assert.match(read('jobs/places.js'), /clipSentences\(sentences\.slice\(1, 4\)\.join\(' '\), 600\)/);
++assert.match(read('places/narrative.js'), /clipSentences\(r\.output\?\.text \|\| '', 800\)/);
++assert.match(read('views/index.js'), /finishedSentences\(place\.description \|\| ''\)/);
++console.log('clip sentences: never mid-sentence, at the source or on the page');
ddiff --git a/test/safe-image.test.js b/test/safe-image.test.js
iindex b86b590..07265f1 100644
---- a/test/safe-image.test.js
++++ b/test/safe-image.test.js
@@@ -22,9 +22,16 @@ const thumbUrl = 'https://thumb.wikimedia.org/wikipedia/commons/thumb/a/ab/Foo.j
assert.strictEqual(imageWidth(thumbUrl, 330), null, 'a Commons thumbnail keeps the verified imageinfo URL');
assert.strictEqual(imageWidth('https://upload.wikimedia.org/wikipedia/commons/5/51/Foo.png?utm_source=x', 640), null, 'an unscaled original keeps its URL because Wikimedia may reject an enlarged thumb');
assert.strictEqual(imageWidth('https://commons.wikimedia.org/wiki/Special:FilePath/Foo.jpg?width=1000', 330), 'https://commons.wikimedia.org/wiki/Special:FilePath/Foo.jpg?width=330', 'Special:FilePath takes a width');
--assert.strictEqual(imageWidth('https://openvibe.media/t/x.jpg', 330), null, 'a non-Wikimedia picture is not rewritten');
++assert.strictEqual(imageWidth('https://openvibe.media/t/x.jpg', 330), null, 'a Media path that is not a public object is not rewritten');
assert.strictEqual(imageWidth('https://upload.wikimedia.org/wikipedia/commons/a/ab/Foo.jpg', 330), null, 'an original with a two-letter hash prefix keeps its URL');
assert.strictEqual(imageWidth(null, 330), null);
++// Media's public objects take ?w= (WebP at its ladder, at most 640): a card no longer downloads the whole copy
++// (11-09: 912 KB for one Nearby card, 37 KB at w=480). Wider than 640, the stored copy itself.
++const obj = 'https://openvibe.media/o/med_01M6YPV5FV0XE5FV2EJW0TB9AA';
++assert.strictEqual(imageWidth(obj, 330), obj + '?w=330', 'a Media picture at the card width');
++assert.strictEqual(imageWidth(obj, 640), obj + '?w=640');
++assert.strictEqual(imageWidth(obj, 1200), obj, 'wider than Media resizes: the stored copy');
++assert.strictEqual(imageWidth(obj + '?w=160', 330), null, 'a URL that already asks for a size is left alone');
assert.strictEqual(safeImage('https://photos.app.goo.gl/A7PbTNW1RbBbZUrTA'), null, 'a share link is not a picture');
assert.ok(safeImage('https://upload.wikimedia.org/wikipedia/commons/a/ab/Hachiko.jpg'), 'Commons uploads');