openvibers/openvibe.tours Commits
Place photos: their own job every 10 minutes, newest cities first (#109)
Changes
@@@ -98,6 +98,7 @@ function load(env = process.env) {intervalMs: Math.max(60_000, int(env.TOURS_JOBS_INTERVAL_MS, 6 * 60 * 60 * 1000)),citiesEveryMs: Math.max(60_000, int(env.TOURS_CITIES_INTERVAL_MS, 24 * 60 * 60 * 1000)),placesEveryMs: Math.max(60_000, int(env.TOURS_PLACES_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),++ photosEveryMs: Math.max(60_000, int(env.TOURS_PHOTOS_INTERVAL_MS, 10 * 60 * 1000)),routesEveryMs: Math.max(60_000, int(env.TOURS_ROUTES_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),liveEveryMs: Math.max(30_000, int(env.TOURS_LIVE_INTERVAL_MS, 2 * 60 * 1000)),staysEveryMs: Math.max(60_000, int(env.TOURS_STAYS_INTERVAL_MS, 6 * 60 * 60 * 1000)),@@@ -27,6 +27,8 @@ function createJobs({ config, s, store, fetchImpl = globalThis.fetch, log = consconst jobs = {cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs, version: creators.cities.version },places: { run: creators.places.runOnce, maxAgeMs: config.jobs.placesEveryMs, intervalMs: config.jobs.placesEveryMs, version: creators.places.version },++ // Place photos the places job could not copy (Media's upload limit, a refused thumbnail): a batch every few minutes.++ photos: { run: creators.places.photoPass, maxAgeMs: config.jobs.photosEveryMs, intervalMs: config.jobs.photosEveryMs },routes: { run: creators.routes.runOnce, maxAgeMs: config.jobs.routesEveryMs, intervalMs: config.jobs.routesEveryMs, version: creators.routes.version },live: { run: creators.live.runOnce, maxAgeMs: config.jobs.liveEveryMs, intervalMs: config.jobs.liveEveryMs },stays: { run: creators.stays.runOnce, maxAgeMs: config.jobs.staysEveryMs, intervalMs: config.jobs.staysEveryMs },@@@ -231,6 +231,27 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {return { seeded, notable, osm: count };}++ /**++ * The photo pass (its own job, every few minutes): places stored without a photo (a copy refused, or a photo added to++ * Commons since), the newest cities first, a bounded batch per run; a place with no photo is asked again after a++ * week, one whose copy failed at the next run.++ */++ async function photoPass({ limit = 60 } = {}) {++ const result = { photos: 0, photosDeferred: 0, noPhoto: 0, errors: [], version: PLACES_VERSION };++ try {++ const bare = await store.placesWithoutPhoto({ limit });++ if (!bare.length) return result;++ const pictured = await photosFor(bare.map((r) => ({ id: r.id, wikidataId: r.wikidata_id, name: r.name })));++ for (const p of pictured) {++ if (p.photoDeferred) { result.photosDeferred += 1; continue; }++ if (await store.setPlacePhoto(p.id, p)) result.photos += 1; else result.noPhoto += 1;++ }++ } catch (err) {++ result.errors.push(`photos: ${err && err.message}`);++ }++ return result;++ }++async function runOnce({ limit = config.openData.cityLimit, osmLimit = config.openData.osmCitiesPerRun } = {}) {const result = { cities: 0, wikivoyagePlaces: 0, places: 0, osmCities: 0, empty: [], errors: [], version: PLACES_VERSION };@@@ -245,22 +266,6 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {}}-- // 1b. Places stored without a photo (a copy refused on 11-08, or a photo added to Commons since): a bounded-- // batch per run, tried again after a week.-- try {-- const bare = await store.placesWithoutPhoto({ limit: 60 });-- if (bare.length) {-- const pictured = await photosFor(bare.map((r) => ({ id: r.id, wikidataId: r.wikidata_id, name: r.name })));-- result.photos = 0;-- for (const p of pictured) {-- if (p.photoDeferred) { result.photosDeferred = (result.photosDeferred || 0) + 1; continue; }-- if (await store.setPlacePhoto(p.id, p)) result.photos += 1;-- }-- }-- } catch (err) {-- result.errors.push(`photos: ${err && err.message}`);-- }--// 2. OpenStreetMap for the cities whose OSM data is oldest, bounded per run.const want = Math.min(100, Math.max(0, Number(osmLimit) || 0));if (!want) return result;@@@ -281,7 +286,7 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {return result;}-- return { runOnce, seedWikivoyage, populateCity, photosFor, distanceM, version: PLACES_VERSION };++ return { runOnce, photoPass, seedWikivoyage, populateCity, photosFor, distanceM, version: PLACES_VERSION };}module.exports = { createPlacesJob, distanceM, rankPlace };@@@ -687,10 +687,11 @@ function createStore(s, { config = {}, sharing = null } = {}) {/** Places with a Wikidata item and no photo, not tried in the last week: the best ranked first. */async function placesWithoutPhoto({ limit = 60 } = {}) {-- return db.many(`SELECT id, wikidata_id, name FROM tours_places-- WHERE image_url IS NULL AND wikidata_id IS NOT NULL-- AND (photo_tried_at IS NULL OR photo_tried_at < now() - interval '7 days')-- ORDER BY rank_score DESC NULLS LAST, id LIMIT $1`, [Math.max(1, Math.min(200, Number(limit) || 60))]);++ // The newest cities first (a city someone just looked up), then the best ranked places.++ return db.many(`SELECT p.id, p.wikidata_id, p.name FROM tours_places p JOIN tours_cities c ON c.id = p.city_id++ WHERE p.image_url IS NULL AND p.wikidata_id IS NOT NULL++ AND (p.photo_tried_at IS NULL OR p.photo_tried_at < now() - interval '7 days')++ ORDER BY c.created_at DESC, p.rank_score DESC NULLS LAST, p.id LIMIT $1`, [Math.max(1, Math.min(200, Number(limit) || 60))]);}/** A photo found for a stored place (credited), or only the time it was looked for. */async function setPlacePhoto(id, p) {
@@ -98,6 +98,7 @@ function load(env = process.env) {
@@ -98,6 +98,7 @@ function load(env = process.env) {
intervalMs: Math.max(60_000, int(env.TOURS_JOBS_INTERVAL_MS, 6 * 60 * 60 * 1000)),
intervalMs: Math.max(60_000, int(env.TOURS_JOBS_INTERVAL_MS, 6 * 60 * 60 * 1000)),
citiesEveryMs: Math.max(60_000, int(env.TOURS_CITIES_INTERVAL_MS, 24 * 60 * 60 * 1000)),
citiesEveryMs: Math.max(60_000, int(env.TOURS_CITIES_INTERVAL_MS, 24 * 60 * 60 * 1000)),
placesEveryMs: Math.max(60_000, int(env.TOURS_PLACES_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),
placesEveryMs: Math.max(60_000, int(env.TOURS_PLACES_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),
+ photosEveryMs: Math.max(60_000, int(env.TOURS_PHOTOS_INTERVAL_MS, 10 * 60 * 1000)),
routesEveryMs: Math.max(60_000, int(env.TOURS_ROUTES_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),
routesEveryMs: Math.max(60_000, int(env.TOURS_ROUTES_INTERVAL_MS, 7 * 24 * 60 * 60 * 1000)),
liveEveryMs: Math.max(30_000, int(env.TOURS_LIVE_INTERVAL_MS, 2 * 60 * 1000)),
liveEveryMs: Math.max(30_000, int(env.TOURS_LIVE_INTERVAL_MS, 2 * 60 * 1000)),
staysEveryMs: Math.max(60_000, int(env.TOURS_STAYS_INTERVAL_MS, 6 * 60 * 60 * 1000)),
staysEveryMs: Math.max(60_000, int(env.TOURS_STAYS_INTERVAL_MS, 6 * 60 * 60 * 1000)),
@@ -27,6 +27,8 @@ function createJobs({ config, s, store, fetchImpl = globalThis.fetch, log = cons
@@ -27,6 +27,8 @@ function createJobs({ config, s, store, fetchImpl = globalThis.fetch, log = cons
const jobs = {
const jobs = {
cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs, version: creators.cities.version },
cities: { run: creators.cities.runOnce, maxAgeMs: config.jobs.citiesEveryMs, intervalMs: config.jobs.citiesEveryMs, version: creators.cities.version },
places: { run: creators.places.runOnce, maxAgeMs: config.jobs.placesEveryMs, intervalMs: config.jobs.placesEveryMs, version: creators.places.version },
places: { run: creators.places.runOnce, maxAgeMs: config.jobs.placesEveryMs, intervalMs: config.jobs.placesEveryMs, version: creators.places.version },
+ // Place photos the places job could not copy (Media's upload limit, a refused thumbnail): a batch every few minutes.
+ photos: { run: creators.places.photoPass, maxAgeMs: config.jobs.photosEveryMs, intervalMs: config.jobs.photosEveryMs },
routes: { run: creators.routes.runOnce, maxAgeMs: config.jobs.routesEveryMs, intervalMs: config.jobs.routesEveryMs, version: creators.routes.version },
routes: { run: creators.routes.runOnce, maxAgeMs: config.jobs.routesEveryMs, intervalMs: config.jobs.routesEveryMs, version: creators.routes.version },
live: { run: creators.live.runOnce, maxAgeMs: config.jobs.liveEveryMs, intervalMs: config.jobs.liveEveryMs },
live: { run: creators.live.runOnce, maxAgeMs: config.jobs.liveEveryMs, intervalMs: config.jobs.liveEveryMs },
stays: { run: creators.stays.runOnce, maxAgeMs: config.jobs.staysEveryMs, intervalMs: config.jobs.staysEveryMs },
stays: { run: creators.stays.runOnce, maxAgeMs: config.jobs.staysEveryMs, intervalMs: config.jobs.staysEveryMs },
@@ -231,6 +231,27 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
@@ -231,6 +231,27 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
return { seeded, notable, osm: count };
return { seeded, notable, osm: count };
}
}
+ /**
+ * The photo pass (its own job, every few minutes): places stored without a photo (a copy refused, or a photo added to
+ * Commons since), the newest cities first, a bounded batch per run; a place with no photo is asked again after a
+ * week, one whose copy failed at the next run.
+ */
+ async function photoPass({ limit = 60 } = {}) {
+ const result = { photos: 0, photosDeferred: 0, noPhoto: 0, errors: [], version: PLACES_VERSION };
+ try {
+ const bare = await store.placesWithoutPhoto({ limit });
+ if (!bare.length) return result;
+ const pictured = await photosFor(bare.map((r) => ({ id: r.id, wikidataId: r.wikidata_id, name: r.name })));
+ for (const p of pictured) {
+ if (p.photoDeferred) { result.photosDeferred += 1; continue; }
+ if (await store.setPlacePhoto(p.id, p)) result.photos += 1; else result.noPhoto += 1;
+ }
+ } catch (err) {
+ result.errors.push(`photos: ${err && err.message}`);
+ }
+ return result;
+ }
+
async function runOnce({ limit = config.openData.cityLimit, osmLimit = config.openData.osmCitiesPerRun } = {}) {
async function runOnce({ limit = config.openData.cityLimit, osmLimit = config.openData.osmCitiesPerRun } = {}) {
const result = { cities: 0, wikivoyagePlaces: 0, places: 0, osmCities: 0, empty: [], errors: [], version: PLACES_VERSION };
const result = { cities: 0, wikivoyagePlaces: 0, places: 0, osmCities: 0, empty: [], errors: [], version: PLACES_VERSION };
@@ -245,22 +266,6 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
@@ -245,22 +266,6 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
}
}
}
}
- // 1b. Places stored without a photo (a copy refused on 11-08, or a photo added to Commons since): a bounded
- // batch per run, tried again after a week.
- try {
- const bare = await store.placesWithoutPhoto({ limit: 60 });
- if (bare.length) {
- const pictured = await photosFor(bare.map((r) => ({ id: r.id, wikidataId: r.wikidata_id, name: r.name })));
- result.photos = 0;
- for (const p of pictured) {
- if (p.photoDeferred) { result.photosDeferred = (result.photosDeferred || 0) + 1; continue; }
- if (await store.setPlacePhoto(p.id, p)) result.photos += 1;
- }
- }
- } catch (err) {
- result.errors.push(`photos: ${err && err.message}`);
- }
-
// 2. OpenStreetMap for the cities whose OSM data is oldest, bounded per run.
// 2. OpenStreetMap for the cities whose OSM data is oldest, bounded per run.
const want = Math.min(100, Math.max(0, Number(osmLimit) || 0));
const want = Math.min(100, Math.max(0, Number(osmLimit) || 0));
if (!want) return result;
if (!want) return result;
@@ -281,7 +286,7 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
@@ -281,7 +286,7 @@ function createPlacesJob({ config, s, store, fetchImpl, log = console }) {
return result;
return result;
}
}
- return { runOnce, seedWikivoyage, populateCity, photosFor, distanceM, version: PLACES_VERSION };
+ return { runOnce, photoPass, seedWikivoyage, populateCity, photosFor, distanceM, version: PLACES_VERSION };
}
}
module.exports = { createPlacesJob, distanceM, rankPlace };
module.exports = { createPlacesJob, distanceM, rankPlace };
@@ -687,10 +687,11 @@ function createStore(s, { config = {}, sharing = null } = {}) {
@@ -687,10 +687,11 @@ function createStore(s, { config = {}, sharing = null } = {}) {
/** Places with a Wikidata item and no photo, not tried in the last week: the best ranked first. */
/** Places with a Wikidata item and no photo, not tried in the last week: the best ranked first. */
async function placesWithoutPhoto({ limit = 60 } = {}) {
async function placesWithoutPhoto({ limit = 60 } = {}) {
- return db.many(`SELECT id, wikidata_id, name FROM tours_places
- WHERE image_url IS NULL AND wikidata_id IS NOT NULL
- AND (photo_tried_at IS NULL OR photo_tried_at < now() - interval '7 days')
- ORDER BY rank_score DESC NULLS LAST, id LIMIT $1`, [Math.max(1, Math.min(200, Number(limit) || 60))]);
+ // The newest cities first (a city someone just looked up), then the best ranked places.
+ return db.many(`SELECT p.id, p.wikidata_id, p.name FROM tours_places p JOIN tours_cities c ON c.id = p.city_id
+ WHERE p.image_url IS NULL AND p.wikidata_id IS NOT NULL
+ AND (p.photo_tried_at IS NULL OR p.photo_tried_at < now() - interval '7 days')
+ ORDER BY c.created_at DESC, p.rank_score DESC NULLS LAST, p.id LIMIT $1`, [Math.max(1, Math.min(200, Number(limit) || 60))]);
}
}
/** A photo found for a stored place (credited), or only the time it was looked for. */
/** A photo found for a stored place (credited), or only the time it was looked for. */
async function setPlacePhoto(id, p) {
async function setPlacePhoto(id, p) {