openvibers/openvibe-git Commits
Seed the official mirrors from the GitHub organisation (#3)
Changes
@@@ -42,6 +42,12 @@ GIT_MAX_FILE_BYTES=2097152GIT_MIRROR_POLL_MS=600000GIT_MIRROR_TIMEOUT_MS=600000++# Official mirrors seed: the GitHub organisation to mirror, the OpenVibe owner to create the mirrors under++# (default: the first of GIT_OFFICIAL_OWNERS), and how often a run is due. Empty GIT_OFFICIAL_ORG turns it off.++GIT_OFFICIAL_ORG=++GIT_OFFICIAL_OWNER=openvibers++GIT_OFFICIAL_SYNC_MS=21600000++# Account export/deletion events from OpenVibe.Network (loopback only).GIT_EVENTS_SECRET=@@@ -19,11 +19,30 @@ npm run test:pg # the same through PostgreSQL + PgBouncerProduction reads `/etc/openvibe/git.env`; see `.env.example`. `DATABASE_URL` serves, `DATABASE_DIRECT_URL` migrates, and`GIT_DATA_DIR` holds the bare repositories (`DATA_DIR/repos/<owner>/<name>.git`).++## Official mirrors++++The Forgejo cut-over keeps the mirror list and nothing else, so the service rebuilds it from GitHub itself. When++`GIT_OFFICIAL_ORG` is set, a due-based background job mirrors that organisation's public repositories under++`GIT_OFFICIAL_OWNER` (by default the first of `GIT_OFFICIAL_OWNERS`, `openvibers`):++++- it lists every page of the organisation through GitHub's REST API, with `GIT_GITHUB_TOKEN` when set;++- it skips forks and archived repositories, and creates each missing repository as a GitHub mirror through the++ normal repository code, one at a time and with the mirror timeout;++- names follow this service's rule, so `OpenVibe.Live` becomes `openvibe.live`; a name that cannot be used here is++ logged and counted as failed;++- it never deletes anything, and one log line per run reports created, existing and failed.++++A run is due when `GIT_OFFICIAL_SYNC_MS` (default six hours) has passed since the last finished run; the last run++is a row in the database, so a deploy does not reset the interval. The first check is shortly after boot and the++job ticks hourly, in the background: `/api/ready` never waits on it, and a failed run waits for the next due time.++An empty `GIT_OFFICIAL_ORG` turns the job off, and so does drill mode.++## Layout- `server/git/` — repositories on disk: creation, refs, trees, blobs, commits, diffs, bunches, and `http-backend`.- `server/http/` — the pages and `/api/v1`.- `server/mirror.js` — GitHub mirrors: the push webhook, the ten-minute poll, and the push mirror back.++- `server/official.js` — the due-based seed of the official mirrors from a GitHub organisation.- `server/insights.js` — views, unique visitors, clones, stars and commits per week.- `contracts` — `git.repo@1`, `git.commit@1`, `git.tree@1`, `git.insights@1` (OpenVibe.Contracts).- `deploy/` — the systemd unit and the nginx vhost for `git.openvibe.codes`.nnew file mode 100644@@@ -0,0 +1,11 @@++-- phase: expand++-- The official mirrors seed's own state (server/official.js): when it last finished and what it found.++-- The job is due-based: a row read at each tick decides whether it runs, so a deploy does not reset the++-- interval the way an in-process timer would.++++CREATE TABLE git_official_state (++ id text PRIMARY KEY, -- official-mirrors++ last_run_at timestamptz,++ last_result jsonb NOT NULL DEFAULT '{}'::jsonb, -- { listed, created, existing, failed, ok, error }++ updated_at timestamptz NOT NULL DEFAULT now()++);@@@ -27,6 +27,7 @@ const { createGithubClient } = require('./github');const { createInsights } = require('./insights');const { createRepos } = require('./repos');const { createMirrors } = require('./mirror');++const { createOfficialMirrors } = require('./official');const { createSmartHttp, parseGitPath } = require('./git/http-backend');const { identify } = require('./auth/git-auth');const { createApi } = require('./http/api');@@@ -47,13 +48,14 @@ function createApp(opts = {}) {const insights = opts.insights || createInsights({ store: s, log });const repos = opts.repos || createRepos({ store: s, config, git, github, insights, log });const mirrors = opts.mirrors || createMirrors({ store: s, repos, config, git, github, log });++ const official = opts.official || createOfficialMirrors({ store: s, repos, config, github, log });const smartHttp = opts.smartHttp || createSmartHttp({ config, repos, mirrors, insights, log });const sso = opts.sso || createSsoClient({site: 'git', baseUrl: config.baseUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,networkUrl: config.networkUrl, networkInternalUrl: config.networkInternalUrl,secureCookies: config.cookies.secure, fetch: fetchImpl, log,});-- const ctx = { config, store: s, s, git, github, insights, repos, mirrors, smartHttp, sso, log };++ const ctx = { config, store: s, s, git, github, insights, repos, mirrors, official, smartHttp, sso, log };const release = opts.release || require('openvibe-shared/release').createRelease({ service: 'git', root: path.join(__dirname, '..') });setRelease(release.release);@@@ -18,6 +18,7 @@ function load(env = process.env) {const port = int(env.PORT, 5270);const baseUrl = trim(env.BASE_URL || (isProduction ? 'https://git.openvibe.codes' : `http://localhost:${port}`));const networkUrl = trim(env.OV_NETWORK_URL || 'https://openvibe.network');++ const officialOwners = list(env.GIT_OFFICIAL_OWNERS || 'openvibers').map((o) => o.toLowerCase());return {service: 'git',@@@ -79,7 +80,15 @@ function load(env = process.env) {},// Repositories under these owners are "official" (the OpenVibers projects).-- officialOwners: list(env.GIT_OFFICIAL_OWNERS || 'openvibers'),++ officialOwners,++++ // The official mirrors seed (server/official.js): the GitHub organisation to mirror, the OpenVibe owner++ // the mirrors are created under, and how often a run is due. An empty GIT_OFFICIAL_ORG turns it off.++ official: {++ org: trim(env.GIT_OFFICIAL_ORG || ''),++ owner: String(env.GIT_OFFICIAL_OWNER || officialOwners[0] || 'openvibers').trim().toLowerCase(),++ syncMs: Math.max(60_000, int(env.GIT_OFFICIAL_SYNC_MS, 6 * 60 * 60 * 1000)),++ },apiRateLimitPerMinute: Math.max(1, int(env.GIT_API_RATE_LIMIT_PER_MIN, 180)),pageRateLimitPerMinute: Math.max(1, int(env.GIT_PAGE_RATE_LIMIT_PER_MIN, 300)),@@@ -90,6 +90,41 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>return meta;}++ /** The `rel="next"` URL in a GitHub Link header, if it stays on the configured API origin. */++ function nextLink(header) {++ if (!header) return null;++ const match = /<([^>]+)>\s*;\s*rel="next"/i.exec(String(header));++ if (!match) return null;++ try {++ const next = new URL(match[1], config.github.apiUrl);++ // The token is on every request: never follow a page onto another host.++ if (next.origin !== new URL(config.github.apiUrl).origin) return null;++ return next.toString();++ } catch { return null; }++ }++++ /** Every public repository of an organisation, following GitHub's Link pagination. */++ async function orgRepos(org, { perPage = 100, maxPages = 100 } = {}) {++ const apiOrigin = new URL(config.github.apiUrl).origin;++ let url = `${config.github.apiUrl}/orgs/${encodeURIComponent(org)}/repos?type=public&per_page=${perPage}`;++ const all = [];++ for (let page = 0; url && page < maxPages; page += 1) {++ const current = new URL(url);++ if (current.origin !== apiOrigin) throw new Error('GitHub pagination left the API origin');++ const res = await fetchImpl(current.toString(), { headers: headers(), signal: AbortSignal.timeout(15_000) });++ if (!res.ok) {++ const err = new Error(`GitHub ${res.status} for ${current.pathname}`);++ err.status = res.status;++ throw err;++ }++ const batch = await res.json();++ if (!Array.isArray(batch)) throw new Error('GitHub returned no repository list');++ all.push(...batch);++ url = nextLink(res.headers && typeof res.headers.get === 'function' ? res.headers.get('link') : null);++ }++ return all;++ }++/** GitHub's X-Hub-Signature-256 over the raw body, constant time. */function signatureOk(rawBody, signature) {const secret = config.github.webhookSecret;@@@ -100,7 +135,7 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>return a.length === b.length && crypto.timingSafeEqual(a, b);}-- return { parseGithubUrl, card, meta, signatureOk, _cache: cache };++ return { parseGithubUrl, card, meta, orgRepos, signatureOk, _cache: cache };}module.exports = { createGithubClient, parseGithubUrl, FULL_RE };@@@ -2,8 +2,8 @@/*** OpenVibe.Git — process entry. `node server/index.js` listens on PORT (5270) behind nginx (deploy/), starts the-- * mirror poll, and stops in order on SIGTERM: the HTTP drain, the mirror poll, the cat-file processes, the-- * database.++ * mirror poll and the due-based official mirrors seed, and stops in order on SIGTERM: the HTTP drain, the timers,++ * the cat-file processes, the database.*/const { createApp } = require('./app');const { gracefulStop } = require('openvibe-sdk/service');@@@ -11,6 +11,7 @@ const { catFiles } = require('./git/store');function createLifecycle({ server, exit, signals, ctx } = {}) {const mirrors = ctx && ctx.mirrors;++ const official = ctx && ctx.official;const store = ctx && ctx.store;return gracefulStop({name: 'OpenVibe.Git',@@@ -19,7 +20,7 @@ function createLifecycle({ server, exit, signals, ctx } = {}) {exit,signals,deadlineMs: 10_000,-- stop: [() => (mirrors ? mirrors.stop() : undefined)],++ stop: [() => (mirrors ? mirrors.stop() : undefined), () => (official ? official.stop() : undefined)],close: [() => catFiles.close(), () => (store ? store.close() : undefined)],});}@@@ -28,6 +29,8 @@ async function start() {const { app, ctx, release } = await createApp();const { config } = ctx;if (config.mirrors.pollMs) ctx.mirrors.start();++ // Due-based and off unless GIT_OFFICIAL_ORG is set: the first check is shortly after boot, never in this path.++ ctx.official.start();const server = app.listen(config.port, config.host, () => {console.log(`[OpenVibe.Git] ${config.nodeEnv} on http://${config.host}:${config.port} → ${config.baseUrl} (release ${release.release})`);});nnew file mode 100644@@@ -0,0 +1,142 @@++'use strict';++++/**++ * The official mirrors seed: every public repository in the GitHub organisation named by GIT_OFFICIAL_ORG becomes++ * a mirror under the official OpenVibe owner (GIT_OFFICIAL_OWNER, by default the first of GIT_OFFICIAL_OWNERS).++ * This is the one thing the Forgejo cut-over keeps: the mirror list is rebuilt from GitHub itself.++ *++ * The job is due-based, not timer-based: the last finished run is a row in the database (git_official_state,++ * migration 0002), so a deploy — and this service is deployed much more often than every six hours — does not++ * reset the interval. A tick runs shortly after boot and then hourly, and only does the work when the interval++ * has passed.++ *++ * A run lists all pages of the organisation's public repositories, skips forks and archived repositories, and++ * creates each missing one through repos.create (which clones with the mirror timeout). Names follow this++ * service's rule: lower-case, no leading dot, never ending in .git — "OpenVibe.Live" becomes "openvibe.live".++ * Nothing is ever deleted. One line per run says how many were created, already existed and failed.++ *++ * It runs in the background: /api/ready never waits on it, and a failed run is logged and retried at the next++ * due time. An empty GIT_OFFICIAL_ORG keeps it off, and so does drill mode.++ */++const { validName } = require('./git/refs');++++const STATE_ID = 'official-mirrors';++const FIRST_RUN_MS = 30_000; // shortly after boot, not in the boot path++const TICK_MS = 60 * 60_000; // how often the due check is asked++++function createOfficialMirrors({ store, repos, config, github, log = console, clock = null }) {++ const settings = config.official;++ const now = clock && clock.now ? () => clock.now() : () => store.now();++ const iso = () => new Date(now()).toISOString();++ let running = false;++ let timer = null;++ let first = null;++++ const enabled = () => Boolean(settings.org) && !config.drill;++++ /** When the last run finished and what it found. */++ async function lastRun() {++ const row = await store.maybe('SELECT last_run_at, last_result FROM git_official_state WHERE id = ?', [STATE_ID]);++ if (!row || !row.last_run_at) return { at: null, result: null };++ const at = new Date(row.last_run_at).getTime();++ const result = typeof row.last_result === 'string' ? JSON.parse(row.last_result || '{}') : (row.last_result || null);++ return { at: Number.isFinite(at) ? at : null, result };++ }++++ async function recordRun(summary) {++ await store.execute(++ `INSERT INTO git_official_state (id, last_run_at, last_result, updated_at) VALUES (?, ?, ?, now())++ ON CONFLICT (id) DO UPDATE SET last_run_at = EXCLUDED.last_run_at,++ last_result = EXCLUDED.last_result, updated_at = now()`,++ [STATE_ID, iso(), JSON.stringify(summary)],++ );++ }++++ /** One pass over the organisation: create what is missing, count what is already there, delete nothing. */++ async function run() {++ const summary = { org: settings.org, owner: settings.owner, listed: 0, created: 0, existing: 0, failed: 0, ok: false };++ if (!settings.org) {++ summary.off = true;++ log.log('[OpenVibe.Git] official mirrors: off (GIT_OFFICIAL_ORG is empty)');++ return summary;++ }++ let listed;++ try {++ listed = await github.orgRepos(settings.org);++ } catch (err) {++ summary.failed = 1;++ summary.error = String(err.message || err).slice(0, 300);++ log.warn(`[OpenVibe.Git] official mirrors: listing ${settings.org}: ${summary.error}`);++ log.log(`[OpenVibe.Git] official mirrors: created ${summary.created}, existing ${summary.existing}, failed ${summary.failed}`);++ return summary;++ }++ summary.listed = listed.length;++ for (const repo of listed) {++ if (!repo || repo.fork || repo.archived || repo.private) continue;++ const name = String(repo.name || '').toLowerCase();++ if (!validName(name)) {++ summary.failed += 1;++ log.warn(`[OpenVibe.Git] official mirrors: ${repo.full_name || repo.name} is not a valid repository name here (${name})`);++ continue;++ }++ if (await repos.getRow(settings.owner, name)) {++ summary.existing += 1;++ continue;++ }++ // repos.create checks the GitHub URL, reads the metadata and clones with config.mirrors.timeoutMs.++ try {++ await repos.create({++ actor: { username: settings.owner, subject: null },++ name,++ mirrorUrl: repo.clone_url || `https://github.com/${settings.org}/${repo.name}.git`,++ });++ summary.created += 1;++ } catch (err) {++ if (err && err.status === 409) { summary.existing += 1; continue; }++ summary.failed += 1;++ log.warn(`[OpenVibe.Git] official mirror ${settings.org}/${repo.name}: ${String(err.message || err).slice(0, 300)}`);++ }++ }++ summary.ok = summary.failed === 0;++ log.log(`[OpenVibe.Git] official mirrors: created ${summary.created}, existing ${summary.existing}, failed ${summary.failed}`);++ return summary;++ }++++ /** Run when due. The in-process guard keeps the hourly tick from stacking runs; the state row spans restarts. */++ async function tick() {++ if (!enabled()) return { ran: false, reason: config.drill ? 'drill' : 'off' };++ if (running) return { ran: false, reason: 'running' };++ running = true;++ try {++ const last = await lastRun();++ const dueAt = (last.at || 0) + settings.syncMs;++ if (last.at && now() < dueAt) return { ran: false, reason: 'not due', due_at: new Date(dueAt).toISOString() };++ const summary = await run();++ await recordRun(summary);++ return { ran: true, ...summary };++ } finally {++ running = false;++ }++ }++++ function start() {++ if (!enabled() || timer || first) return false;++ const check = () => tick().catch((err) => log.warn('[OpenVibe.Git] official mirrors:', err.message));++ first = setTimeout(() => { first = null; check(); }, FIRST_RUN_MS);++ first.unref();++ timer = setInterval(check, Math.max(60_000, Math.min(TICK_MS, settings.syncMs)));++ timer.unref();++ return true;++ }++++ function stop() {++ if (first) clearTimeout(first);++ if (timer) clearInterval(timer);++ first = null;++ timer = null;++ }++++ return { run, tick, start, stop, enabled, lastRun, STATE_ID };++}++++module.exports = { createOfficialMirrors, STATE_ID, FIRST_RUN_MS, TICK_MS };nnew file mode 100644@@@ -0,0 +1,128 @@++'use strict';++const assert = require('assert');++const { boot } = require('./helpers/boot');++const { createOfficialMirrors } = require('../server/official');++++// A GitHub stand-in: two pages of the organisation's repositories (Link pagination), the metadata repo.create++// reads for a mirror, a fork, an archived repository and a name this service cannot use. Nothing touches the++// network, and the clone itself is stubbed below.++const response = (body, link = null) => ({++ ok: true,++ status: 200,++ headers: { get: (name) => (String(name).toLowerCase() === 'link' ? link : null) },++ json: async () => body,++});++++const LISTING = {++ 1: [++ { name: 'OpenVibe.Live', full_name: 'OpenVibers/OpenVibe.Live', fork: false, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Live.git' },++ { name: 'OpenVibe.Archived', full_name: 'OpenVibers/OpenVibe.Archived', fork: false, archived: true, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Archived.git' },++ ],++ 2: [++ { name: 'OpenVibe.Fork', full_name: 'OpenVibers/OpenVibe.Fork', fork: true, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Fork.git' },++ { name: 'OpenVibe.Contracts', full_name: 'OpenVibers/OpenVibe.Contracts', fork: false, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Contracts.git' },++ { name: '.github', full_name: 'OpenVibers/.github', fork: false, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/.github.git' },++ ],++};++++(async () => {++ let nowMs = Date.now();++ const calls = [];++ const fetchImpl = async (url, opts = {}) => {++ calls.push({ url: String(url), authorization: opts.headers && opts.headers.authorization });++ const target = new URL(url);++ if (target.pathname === '/orgs/OpenVibers/repos') {++ const page = Number(target.searchParams.get('page') || '1');++ const link = page === 1 ? '<https://api.github.com/orgs/OpenVibers/repos?type=public&per_page=100&page=2>; rel="next"' : null;++ return response(LISTING[page] || [], link);++ }++ const meta = /^\/repos\/OpenVibers\/([^/]+)$/.exec(target.pathname);++ if (meta) return response({ full_name: `OpenVibers/${meta[1]}`, description: '', default_branch: 'main', private: false, stargazers_count: 0 });++ throw new Error(`unexpected GitHub request ${url}`);++ };++++ const logs = [];++ const log = { log: (m) => logs.push(String(m)), warn: (m) => logs.push(String(m)), error: (m) => logs.push(String(m)) };++ const t = await boot({++ env: { GIT_DRILL: '', GIT_OFFICIAL_ORG: '', GIT_GITHUB_TOKEN: 'test-token' },++ fetchImpl,++ now: () => nowMs,++ });++ const clones = [];++ t.ctx.git.cloneMirror = async (owner, name) => { clones.push(`${owner}/${name}`); return true; };++ t.ctx.git.languages = async () => ({});++ const job = createOfficialMirrors({ store: t.store, repos: t.ctx.repos, config: t.config, github: t.ctx.github, log, clock: { now: () => nowMs } });++ try {++ // Off without GIT_OFFICIAL_ORG, and off in drill mode: no GitHub request either way.++ let result = await job.tick();++ assert.equal(result.ran, false);++ assert.equal(result.reason, 'off');++ assert.equal(calls.length, 0);++ t.config.drill = true;++ result = await job.tick();++ assert.equal(result.reason, 'drill');++ assert.equal(calls.length, 0);++ t.config.drill = false;++++ // The first due run: page 1 and page 2, fork and archived skipped, a lower-case mirror per valid name.++ t.config.official.org = 'OpenVibers';++ result = await job.tick();++ assert.equal(result.ran, true);++ assert.equal(result.listed, 5);++ assert.equal(result.created, 2);++ assert.equal(result.existing, 0);++ assert.equal(result.failed, 1);++ assert.deepEqual(clones.sort(), ['openvibers/openvibe.contracts', 'openvibers/openvibe.live']);++ assert.ok(calls.some((c) => c.url.includes('page=2')), 'page 2 was read');++ assert.ok(calls.filter((c) => c.url.includes('/orgs/')).every((c) => c.authorization === 'Bearer test-token'));++++ const rows = await t.store.many("SELECT name, official, mirror_url, mirror_state FROM git_repos WHERE owner_username = 'openvibers' ORDER BY name");++ assert.deepEqual(rows.map((r) => r.name), ['openvibe.contracts', 'openvibe.live']);++ assert.ok(rows.every((r) => r.official === true));++ assert.equal(rows[0].mirror_url, 'https://github.com/OpenVibers/OpenVibe.Contracts.git');++ assert.equal(rows[1].mirror_url, 'https://github.com/OpenVibers/OpenVibe.Live.git');++ assert.ok(rows.every((r) => r.mirror_state === 'ready'));++ assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE name IN ('openvibe.fork', 'openvibe.archived')")), 0);++++ const summary = logs.filter((l) => l.includes('official mirrors: created'));++ assert.equal(summary.length, 1);++ assert.match(summary[0], /created 2, existing 0, failed 1/);++++ // Not due right away: no second listing and no second clone.++ calls.length = 0;++ result = await job.tick();++ assert.equal(result.ran, false);++ assert.equal(result.reason, 'not due');++ assert.equal(calls.length, 0);++++ // Due again: idempotent. The two mirrors exist, so nothing is created and nothing is cloned.++ nowMs += t.config.official.syncMs;++ result = await job.tick();++ assert.equal(result.ran, true);++ assert.equal(result.created, 0);++ assert.equal(result.existing, 2);++ assert.equal(result.failed, 1);++ assert.equal(clones.length, 2);++ assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE owner_username = 'openvibers'")), 2);++ const state = await job.lastRun();++ assert.ok(state.at);++ assert.equal(state.result.created, 0);++++ // A long run is background work: /api/ready answers while the listing is still in flight.++ nowMs += t.config.official.syncMs;++ let release;++ const realOrgRepos = t.ctx.github.orgRepos;++ t.ctx.github.orgRepos = () => new Promise((resolve) => { release = resolve; });++ const pending = job.tick();++ for (let i = 0; i < 200 && !release; i += 1) await new Promise((r) => setTimeout(r, 5));++ assert.ok(release, 'the run reached the listing');++ const ready = await t.get('/api/ready');++ assert.equal(ready.status, 200);++ assert.equal(ready.json().ok, true);++ release([]);++ assert.equal((await pending).ran, true);++ t.ctx.github.orgRepos = realOrgRepos;++++ console.log('official: 30 checks passed');++ } finally { await t.close(); }++})().catch((err) => { console.error(err); process.exit(1); });
@@ -42,6 +42,12 @@ GIT_MAX_FILE_BYTES=2097152
@@ -42,6 +42,12 @@ GIT_MAX_FILE_BYTES=2097152
GIT_MIRROR_POLL_MS=600000
GIT_MIRROR_POLL_MS=600000
GIT_MIRROR_TIMEOUT_MS=600000
GIT_MIRROR_TIMEOUT_MS=600000
+# Official mirrors seed: the GitHub organisation to mirror, the OpenVibe owner to create the mirrors under
+# (default: the first of GIT_OFFICIAL_OWNERS), and how often a run is due. Empty GIT_OFFICIAL_ORG turns it off.
+GIT_OFFICIAL_ORG=
+GIT_OFFICIAL_OWNER=openvibers
+GIT_OFFICIAL_SYNC_MS=21600000
+
# Account export/deletion events from OpenVibe.Network (loopback only).
# Account export/deletion events from OpenVibe.Network (loopback only).
GIT_EVENTS_SECRET=
GIT_EVENTS_SECRET=
@@ -19,11 +19,30 @@ npm run test:pg # the same through PostgreSQL + PgBouncer
@@ -19,11 +19,30 @@ npm run test:pg # the same through PostgreSQL + PgBouncer
Production reads `/etc/openvibe/git.env`; see `.env.example`. `DATABASE_URL` serves, `DATABASE_DIRECT_URL` migrates, and
Production reads `/etc/openvibe/git.env`; see `.env.example`. `DATABASE_URL` serves, `DATABASE_DIRECT_URL` migrates, and
`GIT_DATA_DIR` holds the bare repositories (`DATA_DIR/repos/<owner>/<name>.git`).
`GIT_DATA_DIR` holds the bare repositories (`DATA_DIR/repos/<owner>/<name>.git`).
+## Official mirrors
+
+The Forgejo cut-over keeps the mirror list and nothing else, so the service rebuilds it from GitHub itself. When
+`GIT_OFFICIAL_ORG` is set, a due-based background job mirrors that organisation's public repositories under
+`GIT_OFFICIAL_OWNER` (by default the first of `GIT_OFFICIAL_OWNERS`, `openvibers`):
+
+- it lists every page of the organisation through GitHub's REST API, with `GIT_GITHUB_TOKEN` when set;
+- it skips forks and archived repositories, and creates each missing repository as a GitHub mirror through the
+ normal repository code, one at a time and with the mirror timeout;
+- names follow this service's rule, so `OpenVibe.Live` becomes `openvibe.live`; a name that cannot be used here is
+ logged and counted as failed;
+- it never deletes anything, and one log line per run reports created, existing and failed.
+
+A run is due when `GIT_OFFICIAL_SYNC_MS` (default six hours) has passed since the last finished run; the last run
+is a row in the database, so a deploy does not reset the interval. The first check is shortly after boot and the
+job ticks hourly, in the background: `/api/ready` never waits on it, and a failed run waits for the next due time.
+An empty `GIT_OFFICIAL_ORG` turns the job off, and so does drill mode.
+
## Layout
## Layout
- `server/git/` — repositories on disk: creation, refs, trees, blobs, commits, diffs, bunches, and `http-backend`.
- `server/git/` — repositories on disk: creation, refs, trees, blobs, commits, diffs, bunches, and `http-backend`.
- `server/http/` — the pages and `/api/v1`.
- `server/http/` — the pages and `/api/v1`.
- `server/mirror.js` — GitHub mirrors: the push webhook, the ten-minute poll, and the push mirror back.
- `server/mirror.js` — GitHub mirrors: the push webhook, the ten-minute poll, and the push mirror back.
+- `server/official.js` — the due-based seed of the official mirrors from a GitHub organisation.
- `server/insights.js` — views, unique visitors, clones, stars and commits per week.
- `server/insights.js` — views, unique visitors, clones, stars and commits per week.
- `contracts` — `git.repo@1`, `git.commit@1`, `git.tree@1`, `git.insights@1` (OpenVibe.Contracts).
- `contracts` — `git.repo@1`, `git.commit@1`, `git.tree@1`, `git.insights@1` (OpenVibe.Contracts).
- `deploy/` — the systemd unit and the nginx vhost for `git.openvibe.codes`.
- `deploy/` — the systemd unit and the nginx vhost for `git.openvibe.codes`.
new file mode 100644
new file mode 100644
@@ -0,0 +1,11 @@
@@ -0,0 +1,11 @@
+-- phase: expand
+-- The official mirrors seed's own state (server/official.js): when it last finished and what it found.
+-- The job is due-based: a row read at each tick decides whether it runs, so a deploy does not reset the
+-- interval the way an in-process timer would.
+
+CREATE TABLE git_official_state (
+ id text PRIMARY KEY, -- official-mirrors
+ last_run_at timestamptz,
+ last_result jsonb NOT NULL DEFAULT '{}'::jsonb, -- { listed, created, existing, failed, ok, error }
+ updated_at timestamptz NOT NULL DEFAULT now()
+);
@@ -27,6 +27,7 @@ const { createGithubClient } = require('./github');
@@ -27,6 +27,7 @@ const { createGithubClient } = require('./github');
const { createInsights } = require('./insights');
const { createInsights } = require('./insights');
const { createRepos } = require('./repos');
const { createRepos } = require('./repos');
const { createMirrors } = require('./mirror');
const { createMirrors } = require('./mirror');
+const { createOfficialMirrors } = require('./official');
const { createSmartHttp, parseGitPath } = require('./git/http-backend');
const { createSmartHttp, parseGitPath } = require('./git/http-backend');
const { identify } = require('./auth/git-auth');
const { identify } = require('./auth/git-auth');
const { createApi } = require('./http/api');
const { createApi } = require('./http/api');
@@ -47,13 +48,14 @@ function createApp(opts = {}) {
@@ -47,13 +48,14 @@ function createApp(opts = {}) {
const insights = opts.insights || createInsights({ store: s, log });
const insights = opts.insights || createInsights({ store: s, log });
const repos = opts.repos || createRepos({ store: s, config, git, github, insights, log });
const repos = opts.repos || createRepos({ store: s, config, git, github, insights, log });
const mirrors = opts.mirrors || createMirrors({ store: s, repos, config, git, github, log });
const mirrors = opts.mirrors || createMirrors({ store: s, repos, config, git, github, log });
+ const official = opts.official || createOfficialMirrors({ store: s, repos, config, github, log });
const smartHttp = opts.smartHttp || createSmartHttp({ config, repos, mirrors, insights, log });
const smartHttp = opts.smartHttp || createSmartHttp({ config, repos, mirrors, insights, log });
const sso = opts.sso || createSsoClient({
const sso = opts.sso || createSsoClient({
site: 'git', baseUrl: config.baseUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,
site: 'git', baseUrl: config.baseUrl, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret,
networkUrl: config.networkUrl, networkInternalUrl: config.networkInternalUrl,
networkUrl: config.networkUrl, networkInternalUrl: config.networkInternalUrl,
secureCookies: config.cookies.secure, fetch: fetchImpl, log,
secureCookies: config.cookies.secure, fetch: fetchImpl, log,
});
});
- const ctx = { config, store: s, s, git, github, insights, repos, mirrors, smartHttp, sso, log };
+ const ctx = { config, store: s, s, git, github, insights, repos, mirrors, official, smartHttp, sso, log };
const release = opts.release || require('openvibe-shared/release').createRelease({ service: 'git', root: path.join(__dirname, '..') });
const release = opts.release || require('openvibe-shared/release').createRelease({ service: 'git', root: path.join(__dirname, '..') });
setRelease(release.release);
setRelease(release.release);
@@ -18,6 +18,7 @@ function load(env = process.env) {
@@ -18,6 +18,7 @@ function load(env = process.env) {
const port = int(env.PORT, 5270);
const port = int(env.PORT, 5270);
const baseUrl = trim(env.BASE_URL || (isProduction ? 'https://git.openvibe.codes' : `http://localhost:${port}`));
const baseUrl = trim(env.BASE_URL || (isProduction ? 'https://git.openvibe.codes' : `http://localhost:${port}`));
const networkUrl = trim(env.OV_NETWORK_URL || 'https://openvibe.network');
const networkUrl = trim(env.OV_NETWORK_URL || 'https://openvibe.network');
+ const officialOwners = list(env.GIT_OFFICIAL_OWNERS || 'openvibers').map((o) => o.toLowerCase());
return {
return {
service: 'git',
service: 'git',
@@ -79,7 +80,15 @@ function load(env = process.env) {
@@ -79,7 +80,15 @@ function load(env = process.env) {
},
},
// Repositories under these owners are "official" (the OpenVibers projects).
// Repositories under these owners are "official" (the OpenVibers projects).
- officialOwners: list(env.GIT_OFFICIAL_OWNERS || 'openvibers'),
+ officialOwners,
+
+ // The official mirrors seed (server/official.js): the GitHub organisation to mirror, the OpenVibe owner
+ // the mirrors are created under, and how often a run is due. An empty GIT_OFFICIAL_ORG turns it off.
+ official: {
+ org: trim(env.GIT_OFFICIAL_ORG || ''),
+ owner: String(env.GIT_OFFICIAL_OWNER || officialOwners[0] || 'openvibers').trim().toLowerCase(),
+ syncMs: Math.max(60_000, int(env.GIT_OFFICIAL_SYNC_MS, 6 * 60 * 60 * 1000)),
+ },
apiRateLimitPerMinute: Math.max(1, int(env.GIT_API_RATE_LIMIT_PER_MIN, 180)),
apiRateLimitPerMinute: Math.max(1, int(env.GIT_API_RATE_LIMIT_PER_MIN, 180)),
pageRateLimitPerMinute: Math.max(1, int(env.GIT_PAGE_RATE_LIMIT_PER_MIN, 300)),
pageRateLimitPerMinute: Math.max(1, int(env.GIT_PAGE_RATE_LIMIT_PER_MIN, 300)),
@@ -90,6 +90,41 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
@@ -90,6 +90,41 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
return meta;
return meta;
}
}
+ /** The `rel="next"` URL in a GitHub Link header, if it stays on the configured API origin. */
+ function nextLink(header) {
+ if (!header) return null;
+ const match = /<([^>]+)>\s*;\s*rel="next"/i.exec(String(header));
+ if (!match) return null;
+ try {
+ const next = new URL(match[1], config.github.apiUrl);
+ // The token is on every request: never follow a page onto another host.
+ if (next.origin !== new URL(config.github.apiUrl).origin) return null;
+ return next.toString();
+ } catch { return null; }
+ }
+
+ /** Every public repository of an organisation, following GitHub's Link pagination. */
+ async function orgRepos(org, { perPage = 100, maxPages = 100 } = {}) {
+ const apiOrigin = new URL(config.github.apiUrl).origin;
+ let url = `${config.github.apiUrl}/orgs/${encodeURIComponent(org)}/repos?type=public&per_page=${perPage}`;
+ const all = [];
+ for (let page = 0; url && page < maxPages; page += 1) {
+ const current = new URL(url);
+ if (current.origin !== apiOrigin) throw new Error('GitHub pagination left the API origin');
+ const res = await fetchImpl(current.toString(), { headers: headers(), signal: AbortSignal.timeout(15_000) });
+ if (!res.ok) {
+ const err = new Error(`GitHub ${res.status} for ${current.pathname}`);
+ err.status = res.status;
+ throw err;
+ }
+ const batch = await res.json();
+ if (!Array.isArray(batch)) throw new Error('GitHub returned no repository list');
+ all.push(...batch);
+ url = nextLink(res.headers && typeof res.headers.get === 'function' ? res.headers.get('link') : null);
+ }
+ return all;
+ }
+
/** GitHub's X-Hub-Signature-256 over the raw body, constant time. */
/** GitHub's X-Hub-Signature-256 over the raw body, constant time. */
function signatureOk(rawBody, signature) {
function signatureOk(rawBody, signature) {
const secret = config.github.webhookSecret;
const secret = config.github.webhookSecret;
@@ -100,7 +135,7 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
@@ -100,7 +135,7 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
return a.length === b.length && crypto.timingSafeEqual(a, b);
return a.length === b.length && crypto.timingSafeEqual(a, b);
}
}
- return { parseGithubUrl, card, meta, signatureOk, _cache: cache };
+ return { parseGithubUrl, card, meta, orgRepos, signatureOk, _cache: cache };
}
}
module.exports = { createGithubClient, parseGithubUrl, FULL_RE };
module.exports = { createGithubClient, parseGithubUrl, FULL_RE };
@@ -2,8 +2,8 @@
@@ -2,8 +2,8 @@
/**
/**
* OpenVibe.Git — process entry. `node server/index.js` listens on PORT (5270) behind nginx (deploy/), starts the
* OpenVibe.Git — process entry. `node server/index.js` listens on PORT (5270) behind nginx (deploy/), starts the
- * mirror poll, and stops in order on SIGTERM: the HTTP drain, the mirror poll, the cat-file processes, the
- * database.
+ * mirror poll and the due-based official mirrors seed, and stops in order on SIGTERM: the HTTP drain, the timers,
+ * the cat-file processes, the database.
*/
*/
const { createApp } = require('./app');
const { createApp } = require('./app');
const { gracefulStop } = require('openvibe-sdk/service');
const { gracefulStop } = require('openvibe-sdk/service');
@@ -11,6 +11,7 @@ const { catFiles } = require('./git/store');
@@ -11,6 +11,7 @@ const { catFiles } = require('./git/store');
function createLifecycle({ server, exit, signals, ctx } = {}) {
function createLifecycle({ server, exit, signals, ctx } = {}) {
const mirrors = ctx && ctx.mirrors;
const mirrors = ctx && ctx.mirrors;
+ const official = ctx && ctx.official;
const store = ctx && ctx.store;
const store = ctx && ctx.store;
return gracefulStop({
return gracefulStop({
name: 'OpenVibe.Git',
name: 'OpenVibe.Git',
@@ -19,7 +20,7 @@ function createLifecycle({ server, exit, signals, ctx } = {}) {
@@ -19,7 +20,7 @@ function createLifecycle({ server, exit, signals, ctx } = {}) {
exit,
exit,
signals,
signals,
deadlineMs: 10_000,
deadlineMs: 10_000,
- stop: [() => (mirrors ? mirrors.stop() : undefined)],
+ stop: [() => (mirrors ? mirrors.stop() : undefined), () => (official ? official.stop() : undefined)],
close: [() => catFiles.close(), () => (store ? store.close() : undefined)],
close: [() => catFiles.close(), () => (store ? store.close() : undefined)],
});
});
}
}
@@ -28,6 +29,8 @@ async function start() {
@@ -28,6 +29,8 @@ async function start() {
const { app, ctx, release } = await createApp();
const { app, ctx, release } = await createApp();
const { config } = ctx;
const { config } = ctx;
if (config.mirrors.pollMs) ctx.mirrors.start();
if (config.mirrors.pollMs) ctx.mirrors.start();
+ // Due-based and off unless GIT_OFFICIAL_ORG is set: the first check is shortly after boot, never in this path.
+ ctx.official.start();
const server = app.listen(config.port, config.host, () => {
const server = app.listen(config.port, config.host, () => {
console.log(`[OpenVibe.Git] ${config.nodeEnv} on http://${config.host}:${config.port} → ${config.baseUrl} (release ${release.release})`);
console.log(`[OpenVibe.Git] ${config.nodeEnv} on http://${config.host}:${config.port} → ${config.baseUrl} (release ${release.release})`);
});
});
new file mode 100644
new file mode 100644
@@ -0,0 +1,142 @@
@@ -0,0 +1,142 @@
+'use strict';
+
+/**
+ * The official mirrors seed: every public repository in the GitHub organisation named by GIT_OFFICIAL_ORG becomes
+ * a mirror under the official OpenVibe owner (GIT_OFFICIAL_OWNER, by default the first of GIT_OFFICIAL_OWNERS).
+ * This is the one thing the Forgejo cut-over keeps: the mirror list is rebuilt from GitHub itself.
+ *
+ * The job is due-based, not timer-based: the last finished run is a row in the database (git_official_state,
+ * migration 0002), so a deploy — and this service is deployed much more often than every six hours — does not
+ * reset the interval. A tick runs shortly after boot and then hourly, and only does the work when the interval
+ * has passed.
+ *
+ * A run lists all pages of the organisation's public repositories, skips forks and archived repositories, and
+ * creates each missing one through repos.create (which clones with the mirror timeout). Names follow this
+ * service's rule: lower-case, no leading dot, never ending in .git — "OpenVibe.Live" becomes "openvibe.live".
+ * Nothing is ever deleted. One line per run says how many were created, already existed and failed.
+ *
+ * It runs in the background: /api/ready never waits on it, and a failed run is logged and retried at the next
+ * due time. An empty GIT_OFFICIAL_ORG keeps it off, and so does drill mode.
+ */
+const { validName } = require('./git/refs');
+
+const STATE_ID = 'official-mirrors';
+const FIRST_RUN_MS = 30_000; // shortly after boot, not in the boot path
+const TICK_MS = 60 * 60_000; // how often the due check is asked
+
+function createOfficialMirrors({ store, repos, config, github, log = console, clock = null }) {
+ const settings = config.official;
+ const now = clock && clock.now ? () => clock.now() : () => store.now();
+ const iso = () => new Date(now()).toISOString();
+ let running = false;
+ let timer = null;
+ let first = null;
+
+ const enabled = () => Boolean(settings.org) && !config.drill;
+
+ /** When the last run finished and what it found. */
+ async function lastRun() {
+ const row = await store.maybe('SELECT last_run_at, last_result FROM git_official_state WHERE id = ?', [STATE_ID]);
+ if (!row || !row.last_run_at) return { at: null, result: null };
+ const at = new Date(row.last_run_at).getTime();
+ const result = typeof row.last_result === 'string' ? JSON.parse(row.last_result || '{}') : (row.last_result || null);
+ return { at: Number.isFinite(at) ? at : null, result };
+ }
+
+ async function recordRun(summary) {
+ await store.execute(
+ `INSERT INTO git_official_state (id, last_run_at, last_result, updated_at) VALUES (?, ?, ?, now())
+ ON CONFLICT (id) DO UPDATE SET last_run_at = EXCLUDED.last_run_at,
+ last_result = EXCLUDED.last_result, updated_at = now()`,
+ [STATE_ID, iso(), JSON.stringify(summary)],
+ );
+ }
+
+ /** One pass over the organisation: create what is missing, count what is already there, delete nothing. */
+ async function run() {
+ const summary = { org: settings.org, owner: settings.owner, listed: 0, created: 0, existing: 0, failed: 0, ok: false };
+ if (!settings.org) {
+ summary.off = true;
+ log.log('[OpenVibe.Git] official mirrors: off (GIT_OFFICIAL_ORG is empty)');
+ return summary;
+ }
+ let listed;
+ try {
+ listed = await github.orgRepos(settings.org);
+ } catch (err) {
+ summary.failed = 1;
+ summary.error = String(err.message || err).slice(0, 300);
+ log.warn(`[OpenVibe.Git] official mirrors: listing ${settings.org}: ${summary.error}`);
+ log.log(`[OpenVibe.Git] official mirrors: created ${summary.created}, existing ${summary.existing}, failed ${summary.failed}`);
+ return summary;
+ }
+ summary.listed = listed.length;
+ for (const repo of listed) {
+ if (!repo || repo.fork || repo.archived || repo.private) continue;
+ const name = String(repo.name || '').toLowerCase();
+ if (!validName(name)) {
+ summary.failed += 1;
+ log.warn(`[OpenVibe.Git] official mirrors: ${repo.full_name || repo.name} is not a valid repository name here (${name})`);
+ continue;
+ }
+ if (await repos.getRow(settings.owner, name)) {
+ summary.existing += 1;
+ continue;
+ }
+ // repos.create checks the GitHub URL, reads the metadata and clones with config.mirrors.timeoutMs.
+ try {
+ await repos.create({
+ actor: { username: settings.owner, subject: null },
+ name,
+ mirrorUrl: repo.clone_url || `https://github.com/${settings.org}/${repo.name}.git`,
+ });
+ summary.created += 1;
+ } catch (err) {
+ if (err && err.status === 409) { summary.existing += 1; continue; }
+ summary.failed += 1;
+ log.warn(`[OpenVibe.Git] official mirror ${settings.org}/${repo.name}: ${String(err.message || err).slice(0, 300)}`);
+ }
+ }
+ summary.ok = summary.failed === 0;
+ log.log(`[OpenVibe.Git] official mirrors: created ${summary.created}, existing ${summary.existing}, failed ${summary.failed}`);
+ return summary;
+ }
+
+ /** Run when due. The in-process guard keeps the hourly tick from stacking runs; the state row spans restarts. */
+ async function tick() {
+ if (!enabled()) return { ran: false, reason: config.drill ? 'drill' : 'off' };
+ if (running) return { ran: false, reason: 'running' };
+ running = true;
+ try {
+ const last = await lastRun();
+ const dueAt = (last.at || 0) + settings.syncMs;
+ if (last.at && now() < dueAt) return { ran: false, reason: 'not due', due_at: new Date(dueAt).toISOString() };
+ const summary = await run();
+ await recordRun(summary);
+ return { ran: true, ...summary };
+ } finally {
+ running = false;
+ }
+ }
+
+ function start() {
+ if (!enabled() || timer || first) return false;
+ const check = () => tick().catch((err) => log.warn('[OpenVibe.Git] official mirrors:', err.message));
+ first = setTimeout(() => { first = null; check(); }, FIRST_RUN_MS);
+ first.unref();
+ timer = setInterval(check, Math.max(60_000, Math.min(TICK_MS, settings.syncMs)));
+ timer.unref();
+ return true;
+ }
+
+ function stop() {
+ if (first) clearTimeout(first);
+ if (timer) clearInterval(timer);
+ first = null;
+ timer = null;
+ }
+
+ return { run, tick, start, stop, enabled, lastRun, STATE_ID };
+}
+
+module.exports = { createOfficialMirrors, STATE_ID, FIRST_RUN_MS, TICK_MS };
new file mode 100644
new file mode 100644
@@ -0,0 +1,128 @@
@@ -0,0 +1,128 @@
+'use strict';
+const assert = require('assert');
+const { boot } = require('./helpers/boot');
+const { createOfficialMirrors } = require('../server/official');
+
+// A GitHub stand-in: two pages of the organisation's repositories (Link pagination), the metadata repo.create
+// reads for a mirror, a fork, an archived repository and a name this service cannot use. Nothing touches the
+// network, and the clone itself is stubbed below.
+const response = (body, link = null) => ({
+ ok: true,
+ status: 200,
+ headers: { get: (name) => (String(name).toLowerCase() === 'link' ? link : null) },
+ json: async () => body,
+});
+
+const LISTING = {
+ 1: [
+ { name: 'OpenVibe.Live', full_name: 'OpenVibers/OpenVibe.Live', fork: false, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Live.git' },
+ { name: 'OpenVibe.Archived', full_name: 'OpenVibers/OpenVibe.Archived', fork: false, archived: true, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Archived.git' },
+ ],
+ 2: [
+ { name: 'OpenVibe.Fork', full_name: 'OpenVibers/OpenVibe.Fork', fork: true, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Fork.git' },
+ { name: 'OpenVibe.Contracts', full_name: 'OpenVibers/OpenVibe.Contracts', fork: false, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Contracts.git' },
+ { name: '.github', full_name: 'OpenVibers/.github', fork: false, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/.github.git' },
+ ],
+};
+
+(async () => {
+ let nowMs = Date.now();
+ const calls = [];
+ const fetchImpl = async (url, opts = {}) => {
+ calls.push({ url: String(url), authorization: opts.headers && opts.headers.authorization });
+ const target = new URL(url);
+ if (target.pathname === '/orgs/OpenVibers/repos') {
+ const page = Number(target.searchParams.get('page') || '1');
+ const link = page === 1 ? '<https://api.github.com/orgs/OpenVibers/repos?type=public&per_page=100&page=2>; rel="next"' : null;
+ return response(LISTING[page] || [], link);
+ }
+ const meta = /^\/repos\/OpenVibers\/([^/]+)$/.exec(target.pathname);
+ if (meta) return response({ full_name: `OpenVibers/${meta[1]}`, description: '', default_branch: 'main', private: false, stargazers_count: 0 });
+ throw new Error(`unexpected GitHub request ${url}`);
+ };
+
+ const logs = [];
+ const log = { log: (m) => logs.push(String(m)), warn: (m) => logs.push(String(m)), error: (m) => logs.push(String(m)) };
+ const t = await boot({
+ env: { GIT_DRILL: '', GIT_OFFICIAL_ORG: '', GIT_GITHUB_TOKEN: 'test-token' },
+ fetchImpl,
+ now: () => nowMs,
+ });
+ const clones = [];
+ t.ctx.git.cloneMirror = async (owner, name) => { clones.push(`${owner}/${name}`); return true; };
+ t.ctx.git.languages = async () => ({});
+ const job = createOfficialMirrors({ store: t.store, repos: t.ctx.repos, config: t.config, github: t.ctx.github, log, clock: { now: () => nowMs } });
+ try {
+ // Off without GIT_OFFICIAL_ORG, and off in drill mode: no GitHub request either way.
+ let result = await job.tick();
+ assert.equal(result.ran, false);
+ assert.equal(result.reason, 'off');
+ assert.equal(calls.length, 0);
+ t.config.drill = true;
+ result = await job.tick();
+ assert.equal(result.reason, 'drill');
+ assert.equal(calls.length, 0);
+ t.config.drill = false;
+
+ // The first due run: page 1 and page 2, fork and archived skipped, a lower-case mirror per valid name.
+ t.config.official.org = 'OpenVibers';
+ result = await job.tick();
+ assert.equal(result.ran, true);
+ assert.equal(result.listed, 5);
+ assert.equal(result.created, 2);
+ assert.equal(result.existing, 0);
+ assert.equal(result.failed, 1);
+ assert.deepEqual(clones.sort(), ['openvibers/openvibe.contracts', 'openvibers/openvibe.live']);
+ assert.ok(calls.some((c) => c.url.includes('page=2')), 'page 2 was read');
+ assert.ok(calls.filter((c) => c.url.includes('/orgs/')).every((c) => c.authorization === 'Bearer test-token'));
+
+ const rows = await t.store.many("SELECT name, official, mirror_url, mirror_state FROM git_repos WHERE owner_username = 'openvibers' ORDER BY name");
+ assert.deepEqual(rows.map((r) => r.name), ['openvibe.contracts', 'openvibe.live']);
+ assert.ok(rows.every((r) => r.official === true));
+ assert.equal(rows[0].mirror_url, 'https://github.com/OpenVibers/OpenVibe.Contracts.git');
+ assert.equal(rows[1].mirror_url, 'https://github.com/OpenVibers/OpenVibe.Live.git');
+ assert.ok(rows.every((r) => r.mirror_state === 'ready'));
+ assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE name IN ('openvibe.fork', 'openvibe.archived')")), 0);
+
+ const summary = logs.filter((l) => l.includes('official mirrors: created'));
+ assert.equal(summary.length, 1);
+ assert.match(summary[0], /created 2, existing 0, failed 1/);
+
+ // Not due right away: no second listing and no second clone.
+ calls.length = 0;
+ result = await job.tick();
+ assert.equal(result.ran, false);
+ assert.equal(result.reason, 'not due');
+ assert.equal(calls.length, 0);
+
+ // Due again: idempotent. The two mirrors exist, so nothing is created and nothing is cloned.
+ nowMs += t.config.official.syncMs;
+ result = await job.tick();
+ assert.equal(result.ran, true);
+ assert.equal(result.created, 0);
+ assert.equal(result.existing, 2);
+ assert.equal(result.failed, 1);
+ assert.equal(clones.length, 2);
+ assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE owner_username = 'openvibers'")), 2);
+ const state = await job.lastRun();
+ assert.ok(state.at);
+ assert.equal(state.result.created, 0);
+
+ // A long run is background work: /api/ready answers while the listing is still in flight.
+ nowMs += t.config.official.syncMs;
+ let release;
+ const realOrgRepos = t.ctx.github.orgRepos;
+ t.ctx.github.orgRepos = () => new Promise((resolve) => { release = resolve; });
+ const pending = job.tick();
+ for (let i = 0; i < 200 && !release; i += 1) await new Promise((r) => setTimeout(r, 5));
+ assert.ok(release, 'the run reached the listing');
+ const ready = await t.get('/api/ready');
+ assert.equal(ready.status, 200);
+ assert.equal(ready.json().ok, true);
+ release([]);
+ assert.equal((await pending).ran, true);
+ t.ctx.github.orgRepos = realOrgRepos;
+
+ console.log('official: 30 checks passed');
+ } finally { await t.close(); }
+})().catch((err) => { console.error(err); process.exit(1); });