openvibers/openvibe.tours Commits

Place photos: their own job every 10 minutes, newest cities first (#109)

OP OpenVibers committed 5 days ago · 674d39463b · 4 files, +30 −21

The photo pass ran inside the places job, which runs once a week: 60 places a week would
take more than a year for the 4,040 places still without a photo. It is a job of its own
now (TOURS_PHOTOS_INTERVAL_MS, default 10 minutes), 60 places a run with uploads paced
under Media's limit, the newest cities first (a city someone just looked up, like
Seattle), then the best ranked places.

Changes

ddiff --git a/server/config.js b/server/config.js
iindex c3b63ca..f897da0 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -98,6 +98,7 @@ function load(env = process.env) {
intervalMs: Math.max(60_000, int(env.TOURS_JOBS_INTERVAL_MS, 6 * 60 * 60 * 1000)),
citiesEveryMs: Math.max(60_000, int(env.TOURS_CITIES_INTERVAL_MS, 24 * 60 * 60 * 1000)),
placesEveryMs: Math.max(60_000, int(env.TOURS_PLACES_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),
++ photosEveryMs: Math.max(60_000, int(env.TOURS_PHOTOS_INTERVAL_MS, 10 * 60 * 1000)),
routesEveryMs: Math.max(60_000, int(env.TOURS_ROUTES_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),
liveEveryMs: Math.max(30_000, int(env.TOURS_LIVE_INTERVAL_MS, 2 * 60 * 1000)),
staysEveryMs: Math.max(60_000, int(env.TOURS_STAYS_INTERVAL_MS, 6 * 60 * 60 * 1000)),
ddiff --git a/server/jobs/index.js b/server/jobs/index.js
iindex f14aeb5..31b745c 100644
---- a/server/jobs/index.js
++++ b/server/jobs/index.js
@@@ -27,6 +27,8 @@ function createJobs({ config, s, store, fetchImpl = globalThis.fetch, log = cons
const jobs = {
cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs, version: creators.cities.version },
places: { run: creators.places.runOnce, maxAgeMs: config.jobs.placesEveryMs, intervalMs: config.jobs.placesEveryMs, version: creators.places.version },
++ // Place photos the places job could not copy (Media's upload limit, a refused thumbnail): a batch every few minutes.
++ photos: { run: creators.places.photoPass, maxAgeMs: config.jobs.photosEveryMs, intervalMs: config.jobs.photosEveryMs },
routes: { run: creators.routes.runOnce, maxAgeMs: config.jobs.routesEveryMs, intervalMs: config.jobs.routesEveryMs, version: creators.routes.version },
live: { run: creators.live.runOnce, maxAgeMs: config.jobs.liveEveryMs, intervalMs: config.jobs.liveEveryMs },
stays: { run: creators.stays.runOnce, maxAgeMs: config.jobs.staysEveryMs, intervalMs: config.jobs.staysEveryMs },
ddiff --git a/server/jobs/places.js b/server/jobs/places.js
iindex 7d5103e..58935cf 100644
---- a/server/jobs/places.js
++++ b/server/jobs/places.js
@@@ -231,6 +231,27 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
return { seeded, notable, osm: count };
}
++ /**
++ * The photo pass (its own job, every few minutes): places stored without a photo (a copy refused, or a photo added to
++ * Commons since), the newest cities first, a bounded batch per run; a place with no photo is asked again after a
++ * week, one whose copy failed at the next run.
++ */
++ async function photoPass({ limit = 60 } = {}) {
++ const result = { photos: 0, photosDeferred: 0, noPhoto: 0, errors: [], version: PLACES_VERSION };
++ try {
++ const bare = await store.placesWithoutPhoto({ limit });
++ if (!bare.length) return result;
++ const pictured = await photosFor(bare.map((r) => ({ id: r.id, wikidataId: r.wikidata_id, name: r.name })));
++ for (const p of pictured) {
++ if (p.photoDeferred) { result.photosDeferred += 1; continue; }
++ if (await store.setPlacePhoto(p.id, p)) result.photos += 1; else result.noPhoto += 1;
++ }
++ } catch (err) {
++ result.errors.push(`photos: ${err && err.message}`);
++ }
++ return result;
++ }
++
async function runOnce({ limit = config.openData.cityLimit, osmLimit = config.openData.osmCitiesPerRun } = {}) {
const result = { cities: 0, wikivoyagePlaces: 0, places: 0, osmCities: 0, empty: [], errors: [], version: PLACES_VERSION };
@@@ -245,22 +266,6 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
}
}
-- // 1b. Places stored without a photo (a copy refused on 11-08, or a photo added to Commons since): a bounded
-- // batch per run, tried again after a week.
-- try {
-- const bare = await store.placesWithoutPhoto({ limit: 60 });
-- if (bare.length) {
-- const pictured = await photosFor(bare.map((r) => ({ id: r.id, wikidataId: r.wikidata_id, name: r.name })));
-- result.photos = 0;
-- for (const p of pictured) {
-- if (p.photoDeferred) { result.photosDeferred = (result.photosDeferred || 0) + 1; continue; }
-- if (await store.setPlacePhoto(p.id, p)) result.photos += 1;
-- }
-- }
-- } catch (err) {
-- result.errors.push(`photos: ${err && err.message}`);
-- }
--
// 2. OpenStreetMap for the cities whose OSM data is oldest, bounded per run.
const want = Math.min(100, Math.max(0, Number(osmLimit) || 0));
if (!want) return result;
@@@ -281,7 +286,7 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
return result;
}
-- return { runOnce, seedWikivoyage, populateCity, photosFor, distanceM, version: PLACES_VERSION };
++ return { runOnce, photoPass, seedWikivoyage, populateCity, photosFor, distanceM, version: PLACES_VERSION };
}
module.exports = { createPlacesJob, distanceM, rankPlace };
ddiff --git a/server/store.js b/server/store.js
iindex 5db8b8f..4b3bae6 100644
---- a/server/store.js
++++ b/server/store.js
@@@ -687,10 +687,11 @@ function createStore(s, { config = {}, sharing = null } = {}) {
/** Places with a Wikidata item and no photo, not tried in the last week: the best ranked first. */
async function placesWithoutPhoto({ limit = 60 } = {}) {
-- return db.many(`SELECT id, wikidata_id, name FROM tours_places
-- WHERE image_url IS NULL AND wikidata_id IS NOT NULL
-- AND (photo_tried_at IS NULL OR photo_tried_at < now() - interval '7 days')
-- ORDER BY rank_score DESC NULLS LAST, id LIMIT $1`, [Math.max(1, Math.min(200, Number(limit) || 60))]);
++ // The newest cities first (a city someone just looked up), then the best ranked places.
++ return db.many(`SELECT p.id, p.wikidata_id, p.name FROM tours_places p JOIN tours_cities c ON c.id = p.city_id
++ WHERE p.image_url IS NULL AND p.wikidata_id IS NOT NULL
++ AND (p.photo_tried_at IS NULL OR p.photo_tried_at < now() - interval '7 days')
++ ORDER BY c.created_at DESC, p.rank_score DESC NULLS LAST, p.id LIMIT $1`, [Math.max(1, Math.min(200, Number(limit) || 60))]);
}
/** A photo found for a stored place (credited), or only the time it was looked for. */
async function setPlacePhoto(id, p) {