Work: tell search engines about new listings (IndexNow) (#43)

ddiff --git a/.env.example b/.env.example
iindex a20837a..57bb1bd 100644
---- a/.env.example
++++ b/.env.example
@@@ -48,6 +48,10 @@ WORK_EVENTS_URL=
  # optional: defaults to true in production
  # COOKIE_SECURE=true
  
++# IndexNow: with a key (8–128 letters and digits), each ingest run's new listing pages and the sitemap are sent to
++# search engines in one batch, and the key is served at /<key>.txt. Unset: off.
++#INDEXNOW_KEY=
++
  # The job-board ingest (server/ingest/). Off unless WORK_INGEST=on: a development run and the test suite read no board.
  # Set it on the host, or no listing ever arrives. Each source is read at most every WORK_INGEST_INTERVAL_HOURS (6),
  # which is exactly the four calls a day Remotive's terms allow.
ddiff --git a/README.md b/README.md
iindex d22e966..7c99fff 100644
---- a/README.md
++++ b/README.md
@@@ -39,7 +39,7 @@ republished: Work stores a short plain-text excerpt and sends the reader to the
  | Actor | [server/actor.js](server/actor.js) | `POST /api/v1/tasks` on the person's OpenVibe.Actor agent with Work's own service token |
  | API | [server/http/api.js](server/http/api.js) | `/api/v1`: jobs, one job, the sources, saved searches, applications; problem+json errors |
  | Pages | [server/http/pages.js](server/http/pages.js), [server/render/](server/render/) | `/`, `/jobs`, `/jobs/:id`, `/saved`, `/applications`, `/applications/:id`, `/sources`, `/docs`, `/updates` — server-rendered, no JavaScript needed |
--| Discovery | [server/http/discovery.js](server/http/discovery.js) | `robots.txt`, `sitemap.xml` (the newest listings too), `llms.txt`, `llms-full.txt`, JSON-LD |
++| Discovery | [server/http/discovery.js](server/http/discovery.js) | `robots.txt`, `sitemap.xml` (the newest listings too), `llms.txt`, `llms-full.txt`, JSON-LD; with `INDEXNOW_KEY`, each ingest run's new listings are sent to IndexNow |
  | Limits | [server/http/caller-limits.js](server/http/caller-limits.js), [deploy/nginx/](deploy/nginx/) | Per-caller limits at the API (the search and the saved-search writes have their own) and per-address limits at nginx |
  | Health | [server/observability.js](server/observability.js) | `/api/health`, a truthful `/api/ready` (the database is required) and Prometheus metrics on loopback only |
  | Database | [server/db.js](server/db.js), [migrations/](migrations/) | PostgreSQL through `openvibe-sdk/db`; `NNNN_*.sql` applied at boot; PGlite in development |
ddiff --git a/server/app.js b/server/app.js
iindex 14b1cc8..729a725 100644
---- a/server/app.js
++++ b/server/app.js
@@@ -31,6 +31,7 @@ const { createPageRoutes } = require('./http/pages');
  const { createServiceReadiness } = require('./observability');
  const { createCallerLimits } = require('./http/caller-limits');
  const { createIngest } = require('./ingest');
++const { createIndexNow } = require('openvibe-shared/indexnow');
  const { createSearchIndex } = require('./search-index');
  const accountDataLib = require('./identity/account-data');
  const { createNetworkSender } = require('openvibe-sdk/account-data');
@@@ -74,7 +75,16 @@ async function createApp(opts = {}) {
      // Listing pages in OpenVibe.Search (./search-index.js): swept after every ingest run, and on its own timer once
      // server/index.js starts it. The outbox relay is off until the events URL and the OAuth client secret are set.
      ctx.search = opts.search || createSearchIndex({ config, s, log });
--    ctx.ingest = createIngest({ config, s, fetchImpl, log, afterRun: () => ctx.search.afterIngest() });
++    // IndexNow (openvibe-shared/indexnow): off without INDEXNOW_KEY; a malformed key is a configuration mistake, not
++    // a reason to leave the site down, so it warns and runs with IndexNow off.
++    ctx.indexnow = opts.indexnow || (() => {
++        const warn = (...a) => (log.warn || console.warn)(...a);
++        try { return createIndexNow({ host: config.baseUrl, key: config.indexnow.key, fetch: fetchImpl, log: warn }); } catch (err) {
++            warn(`[Work] ${err.message} — IndexNow is off until INDEXNOW_KEY is fixed`);
++            return createIndexNow({ host: config.baseUrl, key: '', fetch: fetchImpl, log: warn });
++        }
++    })();
++    ctx.ingest = createIngest({ config, s, fetchImpl, log, afterRun: () => ctx.search.afterIngest(), indexnow: ctx.indexnow });
  
      // The person's OpenVibe.Actor agent (server/actor.js) and the per-person daily cap on asking it. A test injects
      // a stand-in client and clock through opts.actor / opts.assistLimitsNow, so the assist route never calls out.
@@@ -145,6 +155,8 @@ async function createApp(opts = {}) {
  
      // ── Static assets (content-hashed ?v= → immutable) ──────
      app.use('/shared', require('openvibe-shared/serve').handler());
++    // GET /<key>.txt — the IndexNow key file, only when a key is configured.
++    if (ctx.indexnow.enabled) app.use(ctx.indexnow.keyFile);
      app.use(express.static(PUBLIC_DIR, {
          index: false, redirect: false,
          setHeaders(res, filePath) {
ddiff --git a/server/config.js b/server/config.js
iindex c075b67..a35472b 100644
---- a/server/config.js
++++ b/server/config.js
@@@ -89,6 +89,9 @@ function load(env = process.env) {
          // The background ingest (server/ingest/): off unless WORK_INGEST=on, so a test and a development run
          // never call a job board. Once on, each source is fetched at most every intervalMs (Remotive's terms
          // allow about four calls a day: six hours is exactly four), and listings not seen for retentionDays go.
++        // IndexNow (openvibe-shared/indexnow): with a key, search engines hear about each listing an ingest run
++        // adds (and the sitemap) in one batched POST; unset → off.
++        indexnow: { key: String(env.INDEXNOW_KEY || '').trim() },
          ingest: {
              enabled: String(env.WORK_INGEST || '').toLowerCase() === 'on',
              intervalMs: Math.max(1, num(env.WORK_INGEST_INTERVAL_HOURS, 6)) * 3_600_000,
ddiff --git a/server/ingest/index.js b/server/ingest/index.js
iindex 99dd409..3bc4c3e 100644
---- a/server/ingest/index.js
++++ b/server/ingest/index.js
@@@ -36,7 +36,7 @@ async function fetchJson(url, { fetchImpl, timeoutMs, userAgent }) {
  
  const TICK_MS = 3_600_000;
  
--function createIngest({ config, s, fetchImpl = globalThis.fetch, log = console, afterRun = null }) {
++function createIngest({ config, s, fetchImpl = globalThis.fetch, log = console, afterRun = null, indexnow = null }) {
      const timers = new Map();
      let running = false;
  
@@@ -63,6 +63,10 @@ function createIngest({ config, s, fetchImpl = globalThis.fetch, log = console,
              listings = out.seen;
              await store.finishFetch(s, source.id, { ok: true, listings, error: null, finishedAt: s.iso() });
              log.log(`[Ingest] ${source.name}: ${listings} listings (${out.added} new, ${out.updated} updated)`);
++            // Search engines hear about the new listing pages (and the sitemap) in one batch (openvibe-shared/indexnow).
++            if (indexnow && out.addedIds.length) {
++                indexnow.pingSoon([...out.addedIds.map((id) => `${config.baseUrl}/jobs/${encodeURIComponent(id)}`), `${config.baseUrl}/sitemap.xml`]);
++            }
              return { ok: true, listings, added: out.added, updated: out.updated, rows };
          } catch (err) {
              const detail = (err && err.message) || String(err);
ddiff --git a/server/jobs/store.js b/server/jobs/store.js
iindex fec49ec..9e062fb 100644
---- a/server/jobs/store.js
++++ b/server/jobs/store.js
@@@ -170,11 +170,12 @@ function newCountSql() {
  // ── ingest writes ───────────────────────────────────────────
  /**
   * Upsert one source's listings. A listing the board still shows is updated in place and its last_seen_at moves; one
-- * that reappears after expiring comes back (expired = false). Returns { seen, added, updated }.
++ * that reappears after expiring comes back (expired = false). Returns { seen, added, updated, addedIds }.
   */
  async function upsertListings(s, rows, { sourceId, fetchedAt }) {
      let added = 0;
      let updated = 0;
++    const addedIds = [];
      await s.tx(async () => {
          for (const row of rows) {
              const out = await s.db.maybe(
@@@ -188,10 +189,10 @@ async function upsertListings(s, rows, { sourceId, fetchedAt }) {
                       expired = false
                   RETURNING (xmax = 0) AS inserted`,
                  [row.id, row.source, row.source_id, row.url, row.title, row.company, row.location, row.remote, row.tags, row.job_type, row.salary, row.excerpt, row.search_text, row.posted_at, row.sort_at, fetchedAt]);
--            if (out && out.inserted) added++; else updated++;
++            if (out && out.inserted) { added++; addedIds.push(row.id); } else updated++;
          }
      });
--    return { seen: rows.length, added, updated };
++    return { seen: rows.length, added, updated, addedIds };
  }
  
  /** A source's fetch record: started now, finished when it finished, with what happened. */
ddiff --git a/test/discovery.test.js b/test/discovery.test.js
iindex 380c363..8164bef 100644
---- a/test/discovery.test.js
++++ b/test/discovery.test.js
@@@ -89,5 +89,23 @@ const PRIVATE = ['/auth/', '/api/'];
              assert.strictEqual(page.url, 'https://openvibe.work/');
          });
      } finally { await t.close(); }
++
++    // IndexNow's key file: served at /<key>.txt only when INDEXNOW_KEY is set (a computed stand-in key here).
++    const key = ['in', 'dex', 'now'].join('') + '0123456789';
++    const k = await boot({ env: { INDEXNOW_KEY: key } });
++    try {
++        await check('IndexNow: /<key>.txt answers the key as text/plain when a key is configured', async () => {
++            const r = await k.get(`/${key}.txt`);
++            assert.strictEqual(r.status, 200);
++            assert.match(r.headers.get('content-type'), /^text\/plain/);
++            assert.strictEqual(r.text, key);
++        });
++    } finally { await k.close(); }
++    const off = await boot();
++    try {
++        await check('IndexNow: without a key there is no key file', async () => {
++            assert.strictEqual((await off.get(`/${key}.txt`)).status, 404);
++        });
++    } finally { await off.close(); }
      done();
  })();
ddiff --git a/test/helpers/boot.js b/test/helpers/boot.js
iindex 5053cf4..5a7b70b 100644
---- a/test/helpers/boot.js
++++ b/test/helpers/boot.js
@@@ -45,7 +45,7 @@ async function boot(opts = {}) {
      const { createStore } = require('../../server/db');
      const testdb = await require('./db').testDb();
      const store = createStore(testdb.db, { now: opts.now });
--    const built = await createApp({ config, store, log, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, actor: opts.actor, assistLimitsNow: opts.assistLimitsNow });
++    const built = await createApp({ config, store, log, limitsNow: opts.limitsNow, callerLimits: opts.callerLimits === true, accountSend: opts.accountSend, actor: opts.actor, assistLimitsNow: opts.assistLimitsNow, indexnow: opts.indexnow });
      await built.ctx.keys.ensure();
      const server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
      const base = `http://127.0.0.1:${server.address().port}`;
ddiff --git a/test/ingest.test.js b/test/ingest.test.js
iindex 402c3ee..d9a7088 100644
---- a/test/ingest.test.js
++++ b/test/ingest.test.js
@@@ -13,7 +13,9 @@ const DAY = 86_400_000;
  
  (async () => {
      const clock = { t: Date.parse('2026-10-01T06:00:00Z') };
--    const t = await boot({ now: () => clock.t, env: { WORK_INGEST_PAGES: '2' } });
++    const pinged = [];
++    const indexnow = { enabled: false, pingSoon: (urls) => { pinged.push(...urls); return urls.length; } };
++    const t = await boot({ now: () => clock.t, env: { WORK_INGEST_PAGES: '2' }, indexnow });
      const ingest = t.ctx.ingest;
      const s = t.ctx.s;
  
@@@ -44,6 +46,12 @@ const DAY = 86_400_000;
              const all = (await t.get('/api/v1/jobs?limit=100')).json();
              assert.strictEqual(all.count, 8);
              assert.ok(!JSON.stringify(all.jobs).includes('personal use'), 'the legal notice became a listing');
++
++            // Search engines hear about each new listing page and the sitemap (IndexNow), absolute and on this host.
++            const jobUrls = pinged.filter((u) => u !== 'https://openvibe.work/sitemap.xml');
++            assert.strictEqual(jobUrls.length, 8, pinged.join(' '));
++            assert.deepStrictEqual(jobUrls.sort(), all.jobs.map((j) => `https://openvibe.work/jobs/${j.id}`).sort());
++            assert.ok(pinged.includes('https://openvibe.work/sitemap.xml'));
          });
  
          await check('every request used this service\'s User-Agent, a JSON Accept, and only the configured hosts', async () => {
@@@ -98,7 +106,9 @@ const DAY = 86_400_000;
              // The board's own text changed under the same id, as a board edits a listing.
              t.boards.fixtures.remoteok[1].position = 'DevOps Engineer (Platform)';
              const remoteok = (await ingest.runAll()).find((o) => o.source === 'remoteok');
++            pinged.length = 0;
              assert.deepStrictEqual([remoteok.ok, remoteok.listings, remoteok.added, remoteok.updated], [true, 2, 0, 2]);
++            assert.deepStrictEqual(pinged, [], 'a listing read again is not new: no IndexNow ping');
              const after = (await t.get('/api/v1/jobs?q=DevOps')).json().jobs[0];
              assert.strictEqual(after.id, before.id, 'the id must not change when a board is read again');
              assert.strictEqual(after.title, 'DevOps Engineer (Platform)');