openvibers/openvibe-git Commits
Official mirrors without GitHub's REST quota (#12)
Changes
@@@ -43,10 +43,12 @@ GIT_MIRROR_POLL_MS=600000GIT_MIRROR_TIMEOUT_MS=600000# Official mirrors seed: the GitHub organisation to mirror, the OpenVibe owner to create the mirrors under--# (default: the first of GIT_OFFICIAL_OWNERS), and how often a run is due. Empty GIT_OFFICIAL_ORG turns it off.++# (default: the first of GIT_OFFICIAL_OWNERS), how often a run is due and how many clones run at once. Empty++# GIT_OFFICIAL_ORG turns it off. When GitHub's REST listing is refused the list comes from openvibe-contracts.GIT_OFFICIAL_ORG=GIT_OFFICIAL_OWNER=openvibersGIT_OFFICIAL_SYNC_MS=21600000++GIT_OFFICIAL_CONCURRENCY=3# Account export/deletion events from OpenVibe.Network (loopback only).GIT_EVENTS_SECRET=@@@ -25,12 +25,24 @@ The Forgejo cut-over keeps the mirror list and nothing else, so the service rebu`GIT_OFFICIAL_ORG` is set, a due-based background job mirrors that organisation's public repositories under`GIT_OFFICIAL_OWNER` (by default the first of `GIT_OFFICIAL_OWNERS`, `openvibers`):--- it lists every page of the organisation through GitHub's REST API, with `GIT_GITHUB_TOKEN` when set;++- it lists every page of the organisation through GitHub's REST API, with `GIT_GITHUB_TOKEN` when set. When that++ listing is refused for any reason (403/429/5xx/network — the host shares GitHub's 60 anonymous calls an hour), it++ falls back to the repositories named in `openvibe-contracts` (`manifests/services/*.json` and++ `manifests/repositories/*.json`), deduplicated, and the run says which list it used;++- each candidate is checked as public and existing through git's protocol (protocol-v2 `ls-refs` over++ `github.com`, which spends no REST quota); private and missing repositories are skipped and counted with the++ reason, and the answer names the default branch;- it skips forks and archived repositories, and creates each missing repository as a GitHub mirror through the-- normal repository code, one at a time and with the mirror timeout;++ normal repository code, a few clones at a time (`GIT_OFFICIAL_CONCURRENCY`, default 3) and with the mirror++ timeout. When the API refuses the metadata for quota, the clone still runs and the card is marked pending; a++ later run fills the description, topics and default branch;- names follow this service's rule, so `OpenVibe.Live` becomes `openvibe.live`; a name that cannot be used here is-- logged and counted as failed;--- it never deletes anything, and one log line per run reports created, existing and failed.++ skipped and counted as invalid;++- it never deletes anything, and one log line per run reports the list's source, created, existing, skipped (with++ each reason) and failed.++++The ten-minute mirror poll reads the remote's refs with `git ls-remote` (also `ls-refs`, also no REST quota) and++fetches only when something changed.A run is due when `GIT_OFFICIAL_SYNC_MS` (default six hours) has passed since the last finished run; the last runis a row in the database, so a deploy does not reset the interval. The first check is shortly after boot and the@@@ -42,7 +54,8 @@ An empty `GIT_OFFICIAL_ORG` turns the job off, and so does drill mode.- `server/git/` — repositories on disk: creation, refs, trees, blobs, commits, diffs, bunches, and `http-backend`.- `server/http/` — the pages and `/api/v1`.- `server/mirror.js` — GitHub mirrors: the push webhook, the ten-minute poll, and the push mirror back.--- `server/official.js` — the due-based seed of the official mirrors from a GitHub organisation.++- `server/official.js` — the due-based seed of the official mirrors from a GitHub organisation or the contracts.++- `server/contracts.js` — the repository list named by openvibe-contracts when GitHub's listing is refused.- `server/insights.js` — views, unique visitors, clones, stars and commits per week.- `contracts` — `git.repo@1`, `git.commit@1`, `git.tree@1`, `git.insights@1` (OpenVibe.Contracts).- `deploy/` — the systemd unit and the nginx vhost for `git.openvibe.codes`.nnew file mode 100644@@@ -0,0 +1,7 @@++-- phase: expand++-- A mirror's card can be made without GitHub's REST metadata: the clone needs no quota, so when the API answers++-- 403/429 the mirror is created and marked 'pending'; a later official-mirrors run fills the description, topics++-- and default branch when the quota is back. 'ready' is every mirror whose metadata was read.++++ALTER TABLE git_repos ADD COLUMN mirror_meta_state text NOT NULL DEFAULT 'ready'++ CHECK (mirror_meta_state IN ('ready', 'pending'));@@@ -47,7 +47,7 @@ function createApp(opts = {}) {const fetchImpl = opts.fetchImpl || globalThis.fetch;const s = opts.store || await openStore(config, { now: opts.now, log });-- const github = opts.github || createGithubClient({ config, fetchImpl, now: s.now, log });++ const github = opts.github || createGithubClient({ config, fetchImpl, now: s.now, log, lookup: opts.lookup, guard: opts.guard });const insights = opts.insights || createInsights({ store: s, log });const repos = opts.repos || createRepos({ store: s, config, git, github, insights, log });const mirrors = opts.mirrors || createMirrors({ store: s, repos, config, git, github, log });@@@ -83,11 +83,13 @@ function load(env = process.env) {officialOwners,// The official mirrors seed (server/official.js): the GitHub organisation to mirror, the OpenVibe owner-- // the mirrors are created under, and how often a run is due. An empty GIT_OFFICIAL_ORG turns it off.++ // the mirrors are created under, how often a run is due and how many clones may run at once. An empty++ // GIT_OFFICIAL_ORG turns it off.official: {org: trim(env.GIT_OFFICIAL_ORG || ''),owner: String(env.GIT_OFFICIAL_OWNER || officialOwners[0] || 'openvibers').trim().toLowerCase(),syncMs: Math.max(60_000, int(env.GIT_OFFICIAL_SYNC_MS, 6 * 60 * 60 * 1000)),++ concurrency: Math.max(1, Math.min(8, int(env.GIT_OFFICIAL_CONCURRENCY, 3))),},apiRateLimitPerMinute: Math.max(1, int(env.GIT_API_RATE_LIMIT_PER_MIN, 180)),nnew file mode 100644@@@ -0,0 +1,72 @@++'use strict';++++/**++ * The repositories the network's own contracts name (openvibe-contracts manifests/repositories/*.json and++ * manifests/services/*.json). When GitHub's REST listing is refused for quota (the host has 60 anonymous calls an++ * hour, shared by every service), the official mirror seed still knows what to mirror: each repository manifest's++ * `repository`, or OpenVibers/<name> when it is null, plus each service manifest's `repository`.++ *++ * Read-only at boot from the installed package; no network, no REST quota, no cache to go stale.++ */++const fs = require('node:fs');++const path = require('node:path');++++const FULL_RE = /^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/;++++/** The installed openvibe-contracts manifests directory, or null when the package cannot be found. */++function manifestsRoot() {++ try {++ return path.join(path.dirname(require.resolve('openvibe-contracts/package.json')), 'manifests');++ } catch {++ return null;++ }++}++++function jsonFiles(dir) {++ try {++ return fs.readdirSync(dir).filter((f) => f.endsWith('.json')).sort();++ } catch {++ return [];++ }++}++++function readJson(file) {++ try {++ const value = JSON.parse(fs.readFileSync(file, 'utf8'));++ return value && typeof value === 'object' ? value : null;++ } catch {++ return null;++ }++}++++/**++ * [{ full_name, name, clone_url }], deduplicated by lower-case full name, in manifest order. A repository manifest++ * without `repository` (AFResume today) is OpenVibers/<name>; a service manifest without one is named but not++ * mirrored.++ */++function contractRepositories({ root = manifestsRoot() } = {}) {++ const found = new Map();++ if (!root) return [];++++ const add = (repository, fallbackName) => {++ const full = String(repository || '').trim() || (fallbackName ? `OpenVibers/${fallbackName}` : '');++ if (!FULL_RE.test(full)) return;++ const [owner, name] = full.split('/');++ if (!owner || !name || name === '.' || name === '..') return;++ const key = full.toLowerCase();++ if (found.has(key)) return;++ found.set(key, { full_name: full, name, clone_url: `https://github.com/${full}.git` });++ };++++ for (const dir of ['repositories', 'services']) {++ const base = path.join(root, dir);++ for (const file of jsonFiles(base)) {++ const manifest = readJson(path.join(base, file));++ if (!manifest) continue;++ add(manifest.repository, dir === 'repositories' ? manifest.name : '');++ }++ }++ return [...found.values()];++}++++module.exports = { contractRepositories, manifestsRoot, FULL_RE };@@@ -439,6 +439,41 @@ async function fetchMirror(owner, name, { timeoutMs = 600_000 } = {}) {await runGit(['fetch', '--prune', '--tags', '--quiet', 'origin'], { cwd: dir, timeoutMs, maxBuffer: 16 * 1024 * 1024 });}++function parseRefList(out, separator) {++ const refs = new Map();++ for (const line of String(out || '').split('\n')) {++ const trimmed = line.trim();++ if (!trimmed) continue;++ const at = trimmed.indexOf(separator);++ if (at < 0) continue;++ const sha = trimmed.slice(0, at).trim();++ const ref = trimmed.slice(at + separator.length).trim();++ if (!/^[0-9a-f]{40,64}$/.test(sha) || !ref.startsWith('refs/')) continue;++ refs.set(ref, sha);++ }++ return refs;++}++++/** The heads and tags the remote advertises, read with git's own ls-refs (no REST quota). */++async function remoteRefs(owner, name, { timeoutMs = 60_000 } = {}) {++ const out = await runText(['ls-remote', '--refs', '--heads', '--tags', 'origin'], { cwd: repoDir(owner, name), timeoutMs, maxBuffer: 16 * 1024 * 1024 });++ return parseRefList(out, '\t');++}++++/** The heads and tags this mirror already has. */++async function localRefs(owner, name) {++ const out = await runText(['for-each-ref', '--format=%(objectname) %(refname)', 'refs/heads', 'refs/tags'], { cwd: repoDir(owner, name), maxBuffer: 16 * 1024 * 1024 });++ return parseRefList(out, ' ');++}++++/** Whether the remote advertises anything this mirror does not have: the poll reads refs before it fetches. */++async function mirrorChanged(owner, name, { timeoutMs = 60_000 } = {}) {++ const [remote, local] = await Promise.all([remoteRefs(owner, name, { timeoutMs }), localRefs(owner, name)]);++ if (remote.size !== local.size) return true;++ for (const [ref, sha] of remote) if (local.get(ref) !== sha) return true;++ return false;++}++/** Push a repository mirror to another URL, the token in the environment (never argv, never the log). */async function pushMirror(owner, name, url, { token, timeoutMs = 600_000 } = {}) {const dir = repoDir(owner, name);@@@ -454,4 +489,5 @@ module.exports = {repoDir, ensureDataDirs, repoExists, repoBytes, initRepo, setRepoConfig, setRemote, resolveCommit, defaultRef,readTree, readBlob, rawBlob, blobBytes, history, commitDetail, branches, tags, contributors, commitsPerWeek, languages, commitFiles,writeInitialCommit, cloneMirror, cloneLocal, fetchMirror, pushMirror, removeRepo, parseLsTree, catFiles,++ remoteRefs, localRefs, mirrorChanged,};@@@ -9,9 +9,35 @@* api.github.com; without one the public limit applies.*/const crypto = require('node:crypto');++const { assertPublicUrl } = require('openvibe-shared/egress');const GITHUB_HOSTS = new Set(['github.com', 'www.github.com']);const FULL_RE = /^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/;++const GITHUB_WEB = 'https://github.com';++++// ls-refs for HEAD in git's protocol v2, with symrefs so the answer names the default branch. Each pkt-line carries++// its own length in hex, four bytes included; 0001 is the delimiter between the command and its arguments.++const pktLine = (text) => (Buffer.byteLength(text, 'utf8') + 4).toString(16).padStart(4, '0') + text;++const LS_REFS_HEAD = pktLine('command=ls-refs\n') + '0001' + pktLine('symrefs\n') + pktLine('ref-prefix HEAD\n') + '0000';++++/** The refs in an ls-refs answer: [{ sha, ref, attrs }]. Lengths, delimiters and flushes are skipped. */++function parseRefs(text) {++ const refs = [];++ const body = String(text || '');++ let i = 0;++ while (i + 4 <= body.length) {++ const size = parseInt(body.slice(i, i + 4), 16);++ if (size === 0 || size === 1 || size === 2) { i += 4; continue; }++ if (!Number.isFinite(size) || size < 4 || i + size > body.length) break;++ const line = body.slice(i + 4, i + size).replace(/\n$/, '');++ i += size;++ if (!line) continue;++ const parts = line.split(' ');++ if (parts.length < 2) continue;++ refs.push({ sha: parts[0], ref: parts[1], attrs: parts.slice(2) });++ }++ return refs;++}/** A public GitHub repository URL → { full_name, clone_url }, or null. No credentials, no other host. */function parseGithubUrl(input) {@@@ -27,9 +53,10 @@ function parseGithubUrl(input) {return { full_name: `${parts[0]}/${name}`, clone_url: `https://github.com/${parts[0]}/${name}.git` };}--function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console }) {++function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console, lookup = null, guard = null }) {const cache = new Map(); // full_name (lower) → { at, card }const metaCache = new Map();++ const probeCache = new Map();const headers = () => {const h = { accept: 'application/vnd.github+json', 'user-agent': 'openvibe-git' };if (config.github.token) h.authorization = `Bearer ${config.github.token}`;@@@ -85,11 +112,61 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>const meta = {full_name: repo.full_name, description: repo.description || '', default_branch: repo.default_branch || 'main',language: repo.language || null, stars: Number(repo.stargazers_count) || 0, private: !!repo.private,++ topics: Array.isArray(repo.topics) ? repo.topics.map(String).slice(0, 20) : [],};metaCache.set(key, { at: now(), meta });return meta;}++ /**++ * Whether a GitHub repository is public, and its default branch, through git's own protocol: a protocol-v2++ * ls-refs POST needs no REST quota (git clone and fetch need none either). GitHub answers 401 for a private++ * repository and 404 for a missing one; both are skipped by the caller. Through the egress guard like every++ * other connection to github.com.++ */++ async function probe(full) {++ const name = String(full || '');++ if (!FULL_RE.test(name)) return { state: 'invalid', default_branch: null };++ const key = name.toLowerCase();++ const hit = probeCache.get(key);++ if (hit && now() - hit.at < 60_000) return hit.value;++ const url = `${GITHUB_WEB}/${name}.git/git-upload-pack`;++ let res;++ try {++ if (guard) await guard(url);++ else await assertPublicUrl(url, lookup ? { lookup } : undefined);++ res = await fetchImpl(url, {++ method: 'POST',++ headers: {++ 'content-type': 'application/x-git-upload-pack-request',++ accept: 'application/x-git-upload-pack-result',++ 'git-protocol': 'version=2',++ 'accept-encoding': 'identity',++ 'user-agent': 'openvibe-git',++ },++ body: LS_REFS_HEAD,++ signal: AbortSignal.timeout(8000),++ });++ } catch (err) {++ return { state: 'unknown', default_branch: null, error: String(err.message || err).slice(0, 200) };++ }++ let value;++ if (res.status === 401) value = { state: 'private', default_branch: null };++ else if (res.status === 404) value = { state: 'missing', default_branch: null };++ else if (!res.ok) value = { state: 'unknown', default_branch: null, error: `GitHub ${res.status}` };++ else {++ let text = '';++ try { text = typeof res.text === 'function' ? await res.text() : ''; } catch { text = ''; }++ const refs = parseRefs(text);++ const head = refs.find((r) => r.ref === 'HEAD');++ const target = head && head.attrs.map((a) => (a.startsWith('symref-target:') ? a.slice('symref-target:'.length) : '')).find(Boolean);++ const branch = target && target.startsWith('refs/heads/') ? target.slice('refs/heads/'.length) : 'main';++ value = { state: 'public', default_branch: branch, head: head ? head.sha : null };++ }++ probeCache.set(key, { at: now(), value });++ return value;++ }++/** The `rel="next"` URL in a GitHub Link header, if it stays on the configured API origin. */function nextLink(header) {if (!header) return null;@@@ -138,6 +215,7 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>metaCache.set(String(repo.full_name).toLowerCase(), { at: now(), meta: {full_name: repo.full_name, description: repo.description || '', default_branch: repo.default_branch || 'main',language: repo.language || null, stars: Number(repo.stargazers_count) || 0, private: !!repo.private,++ topics: Array.isArray(repo.topics) ? repo.topics.map(String).slice(0, 20) : [],} });}all.push(...batch);@@@ -156,7 +234,7 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>return a.length === b.length && crypto.timingSafeEqual(a, b);}-- return { parseGithubUrl, card, meta, orgRepos, signatureOk, _cache: cache };++ return { parseGithubUrl, card, meta, probe, orgRepos, signatureOk, _cache: cache };}--module.exports = { createGithubClient, parseGithubUrl, FULL_RE };++module.exports = { createGithubClient, parseGithubUrl, parseRefs, pktLine, FULL_RE };@@@ -35,7 +35,15 @@ function createMirrors({ store, repos, config, git, github, log = console }) {if (initial || !(await git.repoExists(row.owner_username, row.name))) {await git.cloneMirror(row.owner_username, row.name, row.mirror_url, { timeoutMs: config.mirrors.timeoutMs });} else {-- await git.fetchMirror(row.owner_username, row.name, { timeoutMs: config.mirrors.timeoutMs });++ // Read the remote's refs through git's own ls-refs first: an unchanged repository costs no fetch and,++ // like every git transport here, no GitHub REST quota.++ let changed = true;++ try {++ changed = await git.mirrorChanged(row.owner_username, row.name, { timeoutMs: config.mirrors.timeoutMs });++ } catch (err) {++ log.warn(`[OpenVibe.Git] mirror refs ${row.owner_username}/${row.name}: ${String(err.message || err).slice(0, 200)}`);++ }++ if (changed) await git.fetchMirror(row.owner_username, row.name, { timeoutMs: config.mirrors.timeoutMs });}await store.execute("UPDATE git_repos SET mirror_state = 'ready', mirror_error = NULL, mirror_last_synced_at = now(), pushed_at = now(), updated_at = now() WHERE id = ?", [row.id]);await repos.refreshLanguages(row);@@@ -3,22 +3,31 @@/*** The official mirrors seed: every public repository in the GitHub organisation named by GIT_OFFICIAL_ORG becomes* a mirror under the official OpenVibe owner (GIT_OFFICIAL_OWNER, by default the first of GIT_OFFICIAL_OWNERS).-- * This is the one thing the Forgejo cut-over keeps: the mirror list is rebuilt from GitHub itself.++ * This is the one thing the Forgejo cut-over keeps: the mirror list is rebuilt from the network's own contracts.** The job is due-based, not timer-based: the last finished run is a row in the database (git_official_state,* migration 0002), so a deploy — and this service is deployed much more often than every six hours — does not* reset the interval. A tick runs shortly after boot and then hourly, and only does the work when the interval* has passed.*-- * A run lists all pages of the organisation's public repositories, skips forks and archived repositories, and-- * creates each missing one through repos.create (which clones with the mirror timeout). Names follow this-- * service's rule: lower-case, no leading dot, never ending in .git — "OpenVibe.Live" becomes "openvibe.live".-- * Nothing is ever deleted. One line per run says how many were created, already existed and failed.++ * No REST quota is needed:++ * - the list comes from GitHub's REST API when it answers, and from openvibe-contracts manifests/services and++ * manifests/repositories when it does not (403/429/5xx/network); the run's summary says which;++ * - each candidate is checked as public and existing through git's protocol (ls-refs over github.com, which++ * spends no REST quota) and names the default branch; private and missing repositories are skipped and counted;++ * - a mirror is created through repos.create, which clones with the mirror timeout even when the metadata call++ * is refused, marking the card pending for a later run.++ *++ * Names follow this service's rule: lower-case, no leading dot, never ending in .git — "OpenVibe.Live" becomes++ * "openvibe.live". Nothing is ever deleted. Clones run a few at a time. One line per run says where the list came++ * from, how many were created, already existed, skipped (with the reason) and failed.** It runs in the background: /api/ready never waits on it, and a failed run is logged and retried at the next-- * due time. An empty GIT_OFFICIAL_ORG keeps it off, and so does drill mode.++ * due time. An empty GIT_OFFICIAL_ORG keeps it off, and so does drill mode. stop() ends a run at the next++ * candidate; a run that was stopped is not recorded as finished, so the next boot tries again.*/const { validName } = require('./git/refs');++const { contractRepositories } = require('./contracts');const STATE_ID = 'official-mirrors';const FIRST_RUN_MS = 30_000; // shortly after boot, not in the boot path@@@ -27,15 +36,17 @@ const FIRST_RUN_MS = 30_000; // shortly after boot, not in the boot// other hour, while the host's GitHub quota had come back).const TICK_MS = 15 * 60_000;--function createOfficialMirrors({ store, repos, config, github, log = console, clock = null }) {++function createOfficialMirrors({ store, repos, config, github, log = console, clock = null, contractList = null }) {const settings = config.official;const now = clock && clock.now ? () => clock.now() : () => store.now();const iso = () => new Date(now()).toISOString();let running = false;++ let stopped = false;let timer = null;let first = null;const enabled = () => Boolean(settings.org) && !config.drill;++ const listContracts = contractList || contractRepositories;/** When the last run finished and what it found. */async function lastRun() {@@@ -55,59 +66,161 @@ function createOfficialMirrors({ store, repos, config, github, log = console, cl);}-- /** One pass over the organisation: create what is missing, count what is already there, delete nothing. */++ /** Fill a mirror's card from the REST metadata once the API answers again; a quota refusal stops the attempts. */++ async function fillPending(row, state) {++ if (!row || !row.github_full_name || row.mirror_meta_state !== 'pending' || state.refused) return false;++ try {++ const meta = await github.meta(row.github_full_name);++ const topics = meta.topics && meta.topics.length ? meta.topics : (typeof row.topics === 'string' ? JSON.parse(row.topics || '[]') : (row.topics || []));++ await store.execute(++ `UPDATE git_repos SET description = ?, topics = ?, default_branch = ?, mirror_meta_state = 'ready', updated_at = now() WHERE id = ?`,++ [meta.description || row.description || '', JSON.stringify(topics), meta.default_branch || row.default_branch, row.id],++ );++ return true;++ } catch (err) {++ if (err && (err.status === 403 || err.status === 429)) state.refused = true;++ return false;++ }++ }++++ /** One pass over the list: create what is missing, count what is already there, delete nothing. */async function run() {-- const summary = { org: settings.org, owner: settings.owner, listed: 0, created: 0, existing: 0, failed: 0, ok: false };++ const summary = {++ org: settings.org, owner: settings.owner, source: null, listed: 0, created: 0, existing: 0, failed: 0,++ skipped: { private: 0, missing: 0, invalid: 0, fork: 0, archived: 0 }, metadata_pending: 0, ok: false,++ };if (!settings.org) {summary.off = true;log.log('[OpenVibe.Git] official mirrors: off (GIT_OFFICIAL_ORG is empty)');return summary;}-- let listed;++ if (stopped) {++ summary.stopped = true;++ return summary;++ }++++ // Where the list comes from: GitHub's REST API, or the network's own contracts when that is refused.++ let listed = [];++ let listError = null;++ const state = { refused: false };try {listed = await github.orgRepos(settings.org);++ summary.source = 'github';} catch (err) {++ listError = err;++ summary.source = 'contracts';++ if (err && (err.status === 403 || err.status === 429)) state.refused = true;++ try {++ listed = listContracts();++ } catch (listErr) {++ listed = [];++ summary.contract_error = String(listErr.message || listErr).slice(0, 200);++ }++ summary.listing_error = String(err.message || err).slice(0, 300);++ log.warn(`[OpenVibe.Git] official mirrors: GitHub listing ${settings.org} refused (${summary.listing_error}); `++ + `using ${listed.length} repositories from the contracts`);++ }++ summary.listed = Array.isArray(listed) ? listed.length : 0;++ if (!summary.listed) {summary.failed = 1;-- summary.error = String(err.message || err).slice(0, 300);-- log.warn(`[OpenVibe.Git] official mirrors: listing ${settings.org}: ${summary.error}`);-- log.log(`[OpenVibe.Git] official mirrors: created ${summary.created}, existing ${summary.existing}, failed ${summary.failed}`);++ summary.error = listError ? String(listError.message || listError).slice(0, 300) : 'no repositories listed';++ log.log(`[OpenVibe.Git] official mirrors: source ${summary.source}, created 0, existing 0, failed 1`);return summary;}-- summary.listed = listed.length;-- for (const repo of listed) {-- if (!repo || repo.fork || repo.archived || repo.private) continue;-- const name = String(repo.name || '').toLowerCase();++++ async function one(repo) {++ if (stopped || !repo) return;++ if (repo.fork) { summary.skipped.fork += 1; return; }++ if (repo.archived) { summary.skipped.archived += 1; return; }++ if (repo.private) {++ summary.skipped.private += 1;++ log.warn(`[OpenVibe.Git] official mirrors: ${repo.full_name || repo.name} is private (skipped)`);++ return;++ }++ const full = String(repo.full_name || '');++ const name = String(repo.name || full.split('/')[1] || '').toLowerCase();if (!validName(name)) {-- summary.failed += 1;-- log.warn(`[OpenVibe.Git] official mirrors: ${repo.full_name || repo.name} is not a valid repository name here (${name})`);-- continue;++ summary.skipped.invalid += 1;++ log.warn(`[OpenVibe.Git] official mirrors: ${full || name} is not a valid repository name here (${name})`);++ return;}-- if (await repos.getRow(settings.owner, name)) {++ const existing = await repos.getRow(settings.owner, name);++ if (existing) {summary.existing += 1;-- continue;++ if (existing.mirror_meta_state === 'pending') await fillPending(existing, state);++ return;++ }++++ // Public and existing through git's own protocol: no REST quota, and the default branch comes with it.++ const info = await github.probe(full).catch(() => null);++ if (info && info.state === 'private') {++ summary.skipped.private += 1;++ log.warn(`[OpenVibe.Git] official mirrors: ${full} is private (skipped)`);++ return;++ }++ if (info && info.state === 'missing') {++ summary.skipped.missing += 1;++ log.warn(`[OpenVibe.Git] official mirrors: ${full} does not exist (skipped)`);++ return;}-- // repos.create checks the GitHub URL, reads the metadata and clones with config.mirrors.timeoutMs.++ if (info && info.state === 'invalid') {++ summary.skipped.invalid += 1;++ log.warn(`[OpenVibe.Git] official mirrors: ${full} is not a valid GitHub repository name (skipped)`);++ return;++ }++ if (stopped) return;++++ // repos.create checks the GitHub URL, reads the metadata (pending when refused) and clones with the++ // mirror timeout. The probe's answer saves the second ls-refs and names the default branch.try {await repos.create({actor: { username: settings.owner, subject: null },name,-- mirrorUrl: repo.clone_url || `https://github.com/${settings.org}/${repo.name}.git`,++ mirrorUrl: repo.clone_url || `https://github.com/${full}.git`,++ mirrorInfo: info,});summary.created += 1;} catch (err) {-- if (err && err.status === 409) { summary.existing += 1; continue; }++ if (err && err.status === 409) { summary.existing += 1; return; }summary.failed += 1;-- log.warn(`[OpenVibe.Git] official mirror ${settings.org}/${repo.name}: ${String(err.message || err).slice(0, 300)}`);++ log.warn(`[OpenVibe.Git] official mirror ${full || name}: ${String(err.message || err).slice(0, 300)}`);++ }++ }++++ // A few clones at once, not the whole list: GitHub sees a handful of connections and the host stays usable.++ const limit = Math.max(1, Math.min(8, Number(settings.concurrency) || 3));++ let cursor = 0;++ const workers = Array.from({ length: Math.min(limit, summary.listed) }, async () => {++ while (!stopped) {++ const index = cursor;++ cursor += 1;++ if (index >= summary.listed) return;++ await one(listed[index]);}++ });++ await Promise.all(workers);++++ if (stopped) {++ summary.stopped = true;++ } else {++ summary.metadata_pending = Number(await store.value(++ "SELECT count(*)::int FROM git_repos WHERE mirror_url IS NOT NULL AND mirror_meta_state = 'pending'",++ )) || 0;}summary.ok = summary.failed === 0;-- log.log(`[OpenVibe.Git] official mirrors: created ${summary.created}, existing ${summary.existing}, failed ${summary.failed}`);++ const skipped = summary.skipped;++ const skippedTotal = skipped.private + skipped.missing + skipped.invalid + skipped.fork + skipped.archived;++ log.log(`[OpenVibe.Git] official mirrors: source ${summary.source}, created ${summary.created}, existing ${summary.existing}, `++ + `skipped ${skippedTotal} (private ${skipped.private}, missing ${skipped.missing}, invalid ${skipped.invalid}, `++ + `fork ${skipped.fork}, archived ${skipped.archived}), failed ${summary.failed}`++ + (summary.metadata_pending ? `, metadata pending ${summary.metadata_pending}` : ''));return summary;}/** Run when due. The in-process guard keeps the hourly tick from stacking runs; the state row spans restarts. */async function tick() {if (!enabled()) return { ran: false, reason: config.drill ? 'drill' : 'off' };++ if (stopped) return { ran: false, reason: 'stopped' };if (running) return { ran: false, reason: 'running' };running = true;try {@@@ -115,7 +228,7 @@ function createOfficialMirrors({ store, repos, config, github, log = console, clconst dueAt = (last.at || 0) + settings.syncMs;if (last.at && now() < dueAt) return { ran: false, reason: 'not due', due_at: new Date(dueAt).toISOString() };const summary = await run();-- await recordRun(summary);++ if (!summary.stopped) await recordRun(summary);return { ran: true, ...summary };} finally {running = false;@@@ -133,6 +246,7 @@ function createOfficialMirrors({ store, repos, config, github, log = console, cl}function stop() {++ stopped = true;if (first) clearTimeout(first);if (timer) clearInterval(timer);first = null;@@@ -95,7 +95,7 @@ function createRepos({ store, config, git, github, insights, log = console }) {} catch (err) { log.warn('[OpenVibe.Git] audit not recorded:', err.message); }}-- async function create({ actor, name, description = '', visibility = 'public', topics = [], starter = 'empty', mirrorUrl = null }, { address = null } = {}) {++ async function create({ actor, name, description = '', visibility = 'public', topics = [], starter = 'empty', mirrorUrl = null, mirrorInfo = null }, { address = null } = {}) {const owner = String(actor.username || '').toLowerCase();const repoName = String(name || '').toLowerCase();if (!validName(owner)) throw Object.assign(new Error('A signed-in OpenVibe username is required.'), { status: 403 });@@@ -126,13 +126,30 @@ function createRepos({ store, config, git, github, insights, log = console }) {const row = await byId(id);try {if (mirror) {-- let meta = { description: '', default_branch: 'main', private: false };-- try { meta = await github.meta(mirror.full_name); } catch { /* the clone still works without the card */ }-- if (meta.private) throw Object.assign(new Error('That GitHub repository is private.'), { status: 422 });++ let meta = null;++ let metaPending = false;++ try {++ meta = await github.meta(mirror.full_name);++ } catch (err) {++ // Quota refused (403/429): the clone needs none, so make the mirror now and fill the card later.++ metaPending = Boolean(err && (err.status === 403 || err.status === 429));++ }++ if (meta && meta.private) throw Object.assign(new Error('That GitHub repository is private.'), { status: 422 });++ // The default branch comes from the REST metadata, or from git's own ls-refs when REST is out.++ let defaultBranch = meta && meta.default_branch ? meta.default_branch : 'main';++ if (!meta) {++ const info = mirrorInfo || await github.probe(mirror.full_name).catch(() => null);++ if (info && info.state === 'private') throw Object.assign(new Error('That GitHub repository is private.'), { status: 422 });++ if (info && info.state === 'missing') throw Object.assign(new Error('That GitHub repository does not exist.'), { status: 422 });++ if (info && info.default_branch) defaultBranch = info.default_branch;++ }++ const metaTopics = meta && Array.isArray(meta.topics) ? meta.topics : [];await git.cloneMirror(owner, repoName, mirror.clone_url, { timeoutMs: config.mirrors.timeoutMs });await store.execute(-- `UPDATE git_repos SET description = ?, default_branch = ?, mirror_state = 'ready', mirror_last_synced_at = now(), pushed_at = now(), updated_at = now() WHERE id = ?`,-- [description || meta.description || '', meta.default_branch || 'main', id],++ `UPDATE git_repos SET description = ?, default_branch = ?, topics = ?, mirror_state = 'ready',++ mirror_meta_state = ?, mirror_last_synced_at = now(), pushed_at = now(), updated_at = now() WHERE id = ?`,++ [description || (meta && meta.description) || '', defaultBranch, JSON.stringify(topics.length ? topics : metaTopics),++ metaPending ? 'pending' : 'ready', id],);} else {const files = starterFiles(starter, { full: `${owner}/${repoName}` });@@@ -44,7 +44,11 @@ async function boot(opts = {}) {const testdb = await require('./db').testDb();const store = createStore(testdb.db, { now: opts.now });const log = { log: () => {}, warn: (...a) => captured.push(a.map(String).join(' ')), error: (...a) => captured.push(a.map(String).join(' ')), info: () => {} };-- const { app, ctx } = await createApp({ config, store, log, reviewServices: opts.reviewServices, fetchImpl: typeof opts.fetchImpl === 'function' ? opts.fetchImpl : undefined });++ const { app, ctx } = await createApp({++ config, store, log, reviewServices: opts.reviewServices,++ fetchImpl: typeof opts.fetchImpl === 'function' ? opts.fetchImpl : undefined,++ lookup: opts.lookup, guard: opts.guard,++ });const server = await new Promise((resolve) => { const s = http.createServer(app); s.listen(0, '127.0.0.1', () => resolve(s)); });const port = server.address().port;const base = `http://127.0.0.1:${port}`;nnew file mode 100644@@@ -0,0 +1,185 @@++'use strict';++const assert = require('assert');++const { boot } = require('./helpers/boot');++const { createOfficialMirrors } = require('../server/official');++const { contractRepositories } = require('../server/contracts');++const { createMirrors } = require('../server/mirror');++const { createRepos } = require('../server/repos');++++// The quota-free path, with no live GitHub: a 403 listing falls back to the contracts, a private or missing++// repository is skipped through ls-refs, a refused metadata call still clones and marks the card pending, and the++// poll reads refs before it fetches and never asks REST. The git binary and fetch are both stubbed.++const pkt = (text) => (Buffer.byteLength(text) + 4).toString(16).padStart(4, '0') + text;++const publicRefs = (branch = 'main') => ({++ ok: true, status: 200, headers: { get: () => null }, json: async () => ({}),++ text: async () => pkt(`${'b'.repeat(40)} HEAD symref-target:refs/heads/${branch}\n`) + '0000',++});++const status = (code) => ({ ok: false, status: code, headers: { get: () => null }, json: async () => ({}), text: async () => '' });++const ok = (body) => ({ ok: true, status: 200, headers: { get: () => null }, json: async () => body, text: async () => '' });++const lookup = (host, options, callback) => callback(null, [{ address: '140.82.112.3', family: 4 }]);++++const CONTRACTS = [++ { full_name: 'OpenVibers/OpenVibe.Live', name: 'OpenVibe.Live', clone_url: 'https://github.com/OpenVibers/OpenVibe.Live.git' },++ { full_name: 'OpenVibers/OpenVibe.Private', name: 'OpenVibe.Private', clone_url: 'https://github.com/OpenVibers/OpenVibe.Private.git' },++ { full_name: 'OpenVibers/OpenVibe.Gone', name: 'OpenVibe.Gone', clone_url: 'https://github.com/OpenVibers/OpenVibe.Gone.git' },++ { full_name: 'OpenVibers/.github', name: '.github', clone_url: 'https://github.com/OpenVibers/.github.git' },++];++++(async () => {++ // The contracts themselves name the mirror list, deduplicated and with the OpenVibers fallback for a null++ // repository. Read from the installed openvibe-contracts package, no network.++ const fromContracts = contractRepositories();++ assert.ok(fromContracts.length >= 60, `contracts list has ${fromContracts.length} repositories`);++ assert.ok(fromContracts.every((r) => /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(r.full_name)));++ const keys = new Set(fromContracts.map((r) => r.full_name.toLowerCase()));++ assert.equal(keys.size, fromContracts.length, 'deduplicated');++ assert.ok(fromContracts.some((r) => r.full_name === 'OpenVibers/AFResume'), 'a null repository uses OpenVibers/<name>');++++ // ── A refused listing falls back to the contracts; private and missing names are skipped, not failed. ──++ {++ const calls = [];++ const fetchImpl = async (url) => {++ const target = new URL(url);++ calls.push(target.toString());++ if (target.hostname === 'api.github.com' && target.pathname.startsWith('/orgs/')) return status(403);++ if (target.hostname === 'github.com') {++ if (target.pathname.includes('/OpenVibe.Live.')) return publicRefs('trunk');++ if (target.pathname.includes('/OpenVibe.Private.')) return status(401);++ if (target.pathname.includes('/OpenVibe.Gone.')) return status(404);++ }++ if (target.hostname === 'api.github.com' && target.pathname.startsWith('/repos/')) return status(403);++ throw new Error(`unexpected request ${url}`);++ };++ const t = await boot({ env: { GIT_DRILL: '' }, fetchImpl, lookup });++ const clones = [];++ t.ctx.git.cloneMirror = async (owner, name) => { clones.push(`${owner}/${name}`); return true; };++ t.ctx.git.languages = async () => ({});++ const repos = createRepos({ store: t.store, config: t.config, git: t.ctx.git, github: t.ctx.github, insights: t.ctx.insights, log: { warn() {} } });++ const job = createOfficialMirrors({++ store: t.store, repos, config: t.config, github: t.ctx.github,++ log: { log() {}, warn() {}, error() {} }, contractList: () => CONTRACTS,++ });++ try {++ t.config.official.org = 'OpenVibers';++ const summary = await job.run();++ assert.equal(summary.source, 'contracts');++ assert.equal(summary.listed, CONTRACTS.length);++ assert.equal(summary.created, 1);++ assert.equal(summary.skipped.private, 1);++ assert.equal(summary.skipped.missing, 1);++ assert.equal(summary.skipped.invalid, 1);++ assert.equal(summary.failed, 0);++ assert.deepEqual(clones, ['openvibers/openvibe.live']);++++ // The metadata call was refused: the clone still happened, and the card is pending with the branch++ // ls-refs named.++ const row = await t.ctx.repos.getRow('openvibers', 'openvibe.live');++ assert.equal(row.mirror_state, 'ready');++ assert.equal(row.mirror_meta_state, 'pending');++ assert.equal(row.default_branch, 'trunk');++ assert.equal(row.mirror_url, 'https://github.com/OpenVibers/OpenVibe.Live.git');++++ // A later run with REST answering fills the card and clears pending.++ const rest = async (url) => {++ const target = new URL(url);++ if (target.hostname === 'github.com') return publicRefs('trunk');++ if (target.pathname === '/repos/OpenVibers/OpenVibe.Live') {++ return ok({ full_name: 'OpenVibers/OpenVibe.Live', description: 'Live', default_branch: 'trunk', topics: ['live', 'video'], private: false });++ }++ if (target.pathname.startsWith('/orgs/')) {++ return ok([{ name: 'OpenVibe.Live', full_name: 'OpenVibers/OpenVibe.Live', private: false, fork: false, archived: false, description: 'Live', default_branch: 'trunk', topics: ['live', 'video'] }]);++ }++ throw new Error(`unexpected request ${url}`);++ };++ const later = await boot({ env: { GIT_DRILL: '' }, fetchImpl: rest, lookup });++ try {++ await later.store.execute(++ `INSERT INTO git_repos (id, owner_username, name, description, visibility, topics, official, github_full_name,++ mirror_url, mirror_state, mirror_meta_state)++ VALUES ('repo_later', 'openvibers', 'openvibe.live', '', 'public', '[]', true, 'OpenVibers/OpenVibe.Live',++ 'https://github.com/OpenVibers/OpenVibe.Live.git', 'ready', 'pending')`,++ );++ later.ctx.git.cloneMirror = async () => true;++ later.ctx.git.languages = async () => ({});++ const job2 = createOfficialMirrors({++ store: later.store, repos: later.ctx.repos, config: later.config, github: later.ctx.github,++ log: { log() {}, warn() {}, error() {} }, contractList: () => [CONTRACTS[0]],++ });++ later.config.official.org = 'OpenVibers';++ const again = await job2.run();++ assert.equal(again.existing, 1);++ const filled = await later.ctx.repos.getRow('openvibers', 'openvibe.live');++ assert.equal(filled.mirror_meta_state, 'ready');++ assert.equal(filled.description, 'Live');++ assert.deepEqual(typeof filled.topics === 'string' ? JSON.parse(filled.topics) : filled.topics, ['live', 'video']);++ } finally { await later.close(); }++ } finally { await t.close(); }++ }++++ // ── At most a few clones at once. ──++ {++ const many = Array.from({ length: 6 }, (_, i) => ({++ full_name: `OpenVibers/OpenVibe.Many${i}`, name: `OpenVibe.Many${i}`, clone_url: `https://github.com/OpenVibers/OpenVibe.Many${i}.git`,++ }));++ const fetchImpl = async (url) => {++ const target = new URL(url);++ if (target.hostname === 'github.com') return publicRefs('main');++ if (target.pathname.startsWith('/orgs/')) return status(403);++ if (target.pathname.startsWith('/repos/')) return status(403);++ throw new Error(`unexpected request ${url}`);++ };++ const t = await boot({ env: { GIT_DRILL: '' }, fetchImpl, lookup });++ t.config.official.concurrency = 2;++ let active = 0;++ let peak = 0;++ t.ctx.git.cloneMirror = async () => {++ active += 1;++ peak = Math.max(peak, active);++ await new Promise((resolve) => setTimeout(resolve, 15));++ active -= 1;++ return true;++ };++ t.ctx.git.languages = async () => ({});++ const job = createOfficialMirrors({++ store: t.store, repos: t.ctx.repos, config: t.config, github: t.ctx.github,++ log: { log() {}, warn() {}, error() {} }, contractList: () => many,++ });++ try {++ t.config.official.org = 'OpenVibers';++ const summary = await job.run();++ assert.equal(summary.created, 6);++ assert.ok(peak <= 2, `at most two clones at once (peak ${peak})`);++ } finally { await t.close(); }++ }++++ // ── The poll reads refs first: nothing changed means no fetch, and REST is never called. ──++ {++ const restCalls = [];++ const t = await boot({ fetchImpl: async (url) => { restCalls.push(String(url)); throw new Error('REST is not allowed in the poll'); }, lookup });++ try {++ await t.ctx.repos.create({ actor: t.asUser('sam'), name: 'polled', starter: 'empty' });++ await t.store.execute("UPDATE git_repos SET mirror_url = 'https://github.com/OpenVibers/OpenVibe.Live.git' WHERE name = 'polled'");++ let changed = 0;++ let fetched = 0;++ t.ctx.git.mirrorChanged = async () => { changed += 1; return false; };++ t.ctx.git.fetchMirror = async () => { fetched += 1; return true; };++ t.ctx.git.languages = async () => ({});++ const mirrors = createMirrors({ store: t.store, repos: t.ctx.repos, config: t.config, git: t.ctx.git, github: t.ctx.github, log: { warn() {} } });++++ await mirrors.poll();++ assert.equal(changed, 1);++ assert.equal(fetched, 0, 'an unchanged mirror is not fetched');++ const after = await t.ctx.repos.getRow('sam', 'polled');++ assert.equal(after.mirror_state, 'ready');++ assert.ok(after.mirror_last_synced_at);++ assert.equal(restCalls.length, 0, 'the poll spent no REST call');++++ t.ctx.git.mirrorChanged = async () => true;++ await mirrors.poll();++ assert.equal(fetched, 1, 'a changed mirror is fetched');++ assert.equal(restCalls.length, 0);++ } finally { await t.close(); }++ }++++ console.log('mirrors-quota: 27 checks passed');++})().catch((err) => { console.error(err); process.exit(1); });@@@ -13,6 +13,13 @@ const response = (body, link = null) => ({json: async () => body,});++// git's protocol v2 answer to ls-refs for HEAD: one pkt-line naming the default branch, then a flush.++const pkt = (text) => (Buffer.byteLength(text) + 4).toString(16).padStart(4, '0') + text;++const publicRefs = (branch = 'main') => ({++ ok: true, status: 200, headers: { get: () => null }, json: async () => ({}),++ text: async () => pkt(`${'a'.repeat(40)} HEAD symref-target:refs/heads/${branch}\n`) + '0000',++});++const LISTING = {1: [{ name: 'OpenVibe.Live', full_name: 'OpenVibers/OpenVibe.Live', fork: false, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Live.git' },@@@ -28,9 +35,15 @@ const LISTING = {(async () => {let nowMs = Date.now();const calls = [];++ const probes = [];const fetchImpl = async (url, opts = {}) => {calls.push({ url: String(url), authorization: opts.headers && opts.headers.authorization });const target = new URL(url);++ // git's own ls-refs: no REST quota, and the answer names the default branch.++ if (target.hostname === 'github.com' && target.pathname.endsWith('/git-upload-pack')) {++ probes.push(target.pathname);++ return publicRefs('main');++ }if (target.pathname === '/orgs/OpenVibers/repos') {const page = Number(target.searchParams.get('page') || '1');const link = page === 1 ? '<https://api.github.com/orgs/OpenVibers/repos?type=public&per_page=100&page=2>; rel="next"' : null;@@@ -49,6 +62,7 @@ const LISTING = {const t = await boot({env: { GIT_DRILL: '', GIT_OFFICIAL_ORG: '', GIT_GITHUB_TOKEN: 'test-token' },fetchImpl,++ lookup: (host, options, callback) => callback(null, [{ address: '140.82.112.3', family: 4 }]),now: () => nowMs,});const clones = [];@@@ -72,10 +86,15 @@ const LISTING = {result = await job.tick();assert.equal(result.ran, true);assert.equal(result.listed, 5);++ assert.equal(result.source, 'github');assert.equal(result.created, 2);assert.equal(result.existing, 0);-- assert.equal(result.failed, 1);++ assert.equal(result.failed, 0);++ assert.equal(result.skipped.invalid, 1);++ assert.equal(result.skipped.fork, 1);++ assert.equal(result.skipped.archived, 1);assert.deepEqual(clones.sort(), ['openvibers/openvibe.contracts', 'openvibers/openvibe.live']);++ assert.deepEqual(probes.sort(), ['/OpenVibers/OpenVibe.Contracts.git/git-upload-pack', '/OpenVibers/OpenVibe.Live.git/git-upload-pack']);assert.ok(calls.some((c) => c.url.includes('page=2')), 'page 2 was read');assert.ok(calls.filter((c) => c.url.includes('/orgs/')).every((c) => c.authorization === 'Bearer test-token'));@@@ -87,9 +106,9 @@ const LISTING = {assert.ok(rows.every((r) => r.mirror_state === 'ready'));assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE name IN ('openvibe.fork', 'openvibe.archived')")), 0);-- const summary = logs.filter((l) => l.includes('official mirrors: created'));++ const summary = logs.filter((l) => l.includes('official mirrors: source'));assert.equal(summary.length, 1);-- assert.match(summary[0], /created 2, existing 0, failed 1/);++ assert.match(summary[0], /source github, created 2, existing 0, skipped 3 \(private 0, missing 0, invalid 1, fork 1, archived 1\), failed 0/);// Not due right away: no second listing and no second clone.calls.length = 0;@@@ -104,7 +123,8 @@ const LISTING = {assert.equal(result.ran, true);assert.equal(result.created, 0);assert.equal(result.existing, 2);-- assert.equal(result.failed, 1);++ assert.equal(result.failed, 0);++ assert.equal(result.skipped.invalid, 1);assert.equal(clones.length, 2);assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE owner_username = 'openvibers'")), 2);const state = await job.lastRun();
@@ -43,10 +43,12 @@ GIT_MIRROR_POLL_MS=600000
@@ -43,10 +43,12 @@ GIT_MIRROR_POLL_MS=600000
GIT_MIRROR_TIMEOUT_MS=600000
GIT_MIRROR_TIMEOUT_MS=600000
# Official mirrors seed: the GitHub organisation to mirror, the OpenVibe owner to create the mirrors under
# Official mirrors seed: the GitHub organisation to mirror, the OpenVibe owner to create the mirrors under
-# (default: the first of GIT_OFFICIAL_OWNERS), and how often a run is due. Empty GIT_OFFICIAL_ORG turns it off.
+# (default: the first of GIT_OFFICIAL_OWNERS), how often a run is due and how many clones run at once. Empty
+# GIT_OFFICIAL_ORG turns it off. When GitHub's REST listing is refused the list comes from openvibe-contracts.
GIT_OFFICIAL_ORG=
GIT_OFFICIAL_ORG=
GIT_OFFICIAL_OWNER=openvibers
GIT_OFFICIAL_OWNER=openvibers
GIT_OFFICIAL_SYNC_MS=21600000
GIT_OFFICIAL_SYNC_MS=21600000
+GIT_OFFICIAL_CONCURRENCY=3
# Account export/deletion events from OpenVibe.Network (loopback only).
# Account export/deletion events from OpenVibe.Network (loopback only).
GIT_EVENTS_SECRET=
GIT_EVENTS_SECRET=
@@ -25,12 +25,24 @@ The Forgejo cut-over keeps the mirror list and nothing else, so the service rebu
@@ -25,12 +25,24 @@ The Forgejo cut-over keeps the mirror list and nothing else, so the service rebu
`GIT_OFFICIAL_ORG` is set, a due-based background job mirrors that organisation's public repositories under
`GIT_OFFICIAL_ORG` is set, a due-based background job mirrors that organisation's public repositories under
`GIT_OFFICIAL_OWNER` (by default the first of `GIT_OFFICIAL_OWNERS`, `openvibers`):
`GIT_OFFICIAL_OWNER` (by default the first of `GIT_OFFICIAL_OWNERS`, `openvibers`):
-- it lists every page of the organisation through GitHub's REST API, with `GIT_GITHUB_TOKEN` when set;
+- it lists every page of the organisation through GitHub's REST API, with `GIT_GITHUB_TOKEN` when set. When that
+ listing is refused for any reason (403/429/5xx/network — the host shares GitHub's 60 anonymous calls an hour), it
+ falls back to the repositories named in `openvibe-contracts` (`manifests/services/*.json` and
+ `manifests/repositories/*.json`), deduplicated, and the run says which list it used;
+- each candidate is checked as public and existing through git's protocol (protocol-v2 `ls-refs` over
+ `github.com`, which spends no REST quota); private and missing repositories are skipped and counted with the
+ reason, and the answer names the default branch;
- it skips forks and archived repositories, and creates each missing repository as a GitHub mirror through the
- it skips forks and archived repositories, and creates each missing repository as a GitHub mirror through the
- normal repository code, one at a time and with the mirror timeout;
+ normal repository code, a few clones at a time (`GIT_OFFICIAL_CONCURRENCY`, default 3) and with the mirror
+ timeout. When the API refuses the metadata for quota, the clone still runs and the card is marked pending; a
+ later run fills the description, topics and default branch;
- names follow this service's rule, so `OpenVibe.Live` becomes `openvibe.live`; a name that cannot be used here is
- names follow this service's rule, so `OpenVibe.Live` becomes `openvibe.live`; a name that cannot be used here is
- logged and counted as failed;
-- it never deletes anything, and one log line per run reports created, existing and failed.
+ skipped and counted as invalid;
+- it never deletes anything, and one log line per run reports the list's source, created, existing, skipped (with
+ each reason) and failed.
+
+The ten-minute mirror poll reads the remote's refs with `git ls-remote` (also `ls-refs`, also no REST quota) and
+fetches only when something changed.
A run is due when `GIT_OFFICIAL_SYNC_MS` (default six hours) has passed since the last finished run; the last run
A run is due when `GIT_OFFICIAL_SYNC_MS` (default six hours) has passed since the last finished run; the last run
is a row in the database, so a deploy does not reset the interval. The first check is shortly after boot and the
is a row in the database, so a deploy does not reset the interval. The first check is shortly after boot and the
@@ -42,7 +54,8 @@ An empty `GIT_OFFICIAL_ORG` turns the job off, and so does drill mode.
@@ -42,7 +54,8 @@ An empty `GIT_OFFICIAL_ORG` turns the job off, and so does drill mode.
- `server/git/` — repositories on disk: creation, refs, trees, blobs, commits, diffs, bunches, and `http-backend`.
- `server/git/` — repositories on disk: creation, refs, trees, blobs, commits, diffs, bunches, and `http-backend`.
- `server/http/` — the pages and `/api/v1`.
- `server/http/` — the pages and `/api/v1`.
- `server/mirror.js` — GitHub mirrors: the push webhook, the ten-minute poll, and the push mirror back.
- `server/mirror.js` — GitHub mirrors: the push webhook, the ten-minute poll, and the push mirror back.
-- `server/official.js` — the due-based seed of the official mirrors from a GitHub organisation.
+- `server/official.js` — the due-based seed of the official mirrors from a GitHub organisation or the contracts.
+- `server/contracts.js` — the repository list named by openvibe-contracts when GitHub's listing is refused.
- `server/insights.js` — views, unique visitors, clones, stars and commits per week.
- `server/insights.js` — views, unique visitors, clones, stars and commits per week.
- `contracts` — `git.repo@1`, `git.commit@1`, `git.tree@1`, `git.insights@1` (OpenVibe.Contracts).
- `contracts` — `git.repo@1`, `git.commit@1`, `git.tree@1`, `git.insights@1` (OpenVibe.Contracts).
- `deploy/` — the systemd unit and the nginx vhost for `git.openvibe.codes`.
- `deploy/` — the systemd unit and the nginx vhost for `git.openvibe.codes`.
new file mode 100644
new file mode 100644
@@ -0,0 +1,7 @@
@@ -0,0 +1,7 @@
+-- phase: expand
+-- A mirror's card can be made without GitHub's REST metadata: the clone needs no quota, so when the API answers
+-- 403/429 the mirror is created and marked 'pending'; a later official-mirrors run fills the description, topics
+-- and default branch when the quota is back. 'ready' is every mirror whose metadata was read.
+
+ALTER TABLE git_repos ADD COLUMN mirror_meta_state text NOT NULL DEFAULT 'ready'
+ CHECK (mirror_meta_state IN ('ready', 'pending'));
@@ -47,7 +47,7 @@ function createApp(opts = {}) {
@@ -47,7 +47,7 @@ function createApp(opts = {}) {
const fetchImpl = opts.fetchImpl || globalThis.fetch;
const fetchImpl = opts.fetchImpl || globalThis.fetch;
const s = opts.store || await openStore(config, { now: opts.now, log });
const s = opts.store || await openStore(config, { now: opts.now, log });
- const github = opts.github || createGithubClient({ config, fetchImpl, now: s.now, log });
+ const github = opts.github || createGithubClient({ config, fetchImpl, now: s.now, log, lookup: opts.lookup, guard: opts.guard });
const insights = opts.insights || createInsights({ store: s, log });
const insights = opts.insights || createInsights({ store: s, log });
const repos = opts.repos || createRepos({ store: s, config, git, github, insights, log });
const repos = opts.repos || createRepos({ store: s, config, git, github, insights, log });
const mirrors = opts.mirrors || createMirrors({ store: s, repos, config, git, github, log });
const mirrors = opts.mirrors || createMirrors({ store: s, repos, config, git, github, log });
@@ -83,11 +83,13 @@ function load(env = process.env) {
@@ -83,11 +83,13 @@ function load(env = process.env) {
officialOwners,
officialOwners,
// The official mirrors seed (server/official.js): the GitHub organisation to mirror, the OpenVibe owner
// The official mirrors seed (server/official.js): the GitHub organisation to mirror, the OpenVibe owner
- // the mirrors are created under, and how often a run is due. An empty GIT_OFFICIAL_ORG turns it off.
+ // the mirrors are created under, how often a run is due and how many clones may run at once. An empty
+ // GIT_OFFICIAL_ORG turns it off.
official: {
official: {
org: trim(env.GIT_OFFICIAL_ORG || ''),
org: trim(env.GIT_OFFICIAL_ORG || ''),
owner: String(env.GIT_OFFICIAL_OWNER || officialOwners[0] || 'openvibers').trim().toLowerCase(),
owner: String(env.GIT_OFFICIAL_OWNER || officialOwners[0] || 'openvibers').trim().toLowerCase(),
syncMs: Math.max(60_000, int(env.GIT_OFFICIAL_SYNC_MS, 6 * 60 * 60 * 1000)),
syncMs: Math.max(60_000, int(env.GIT_OFFICIAL_SYNC_MS, 6 * 60 * 60 * 1000)),
+ concurrency: Math.max(1, Math.min(8, int(env.GIT_OFFICIAL_CONCURRENCY, 3))),
},
},
apiRateLimitPerMinute: Math.max(1, int(env.GIT_API_RATE_LIMIT_PER_MIN, 180)),
apiRateLimitPerMinute: Math.max(1, int(env.GIT_API_RATE_LIMIT_PER_MIN, 180)),
new file mode 100644
new file mode 100644
@@ -0,0 +1,72 @@
@@ -0,0 +1,72 @@
+'use strict';
+
+/**
+ * The repositories the network's own contracts name (openvibe-contracts manifests/repositories/*.json and
+ * manifests/services/*.json). When GitHub's REST listing is refused for quota (the host has 60 anonymous calls an
+ * hour, shared by every service), the official mirror seed still knows what to mirror: each repository manifest's
+ * `repository`, or OpenVibers/<name> when it is null, plus each service manifest's `repository`.
+ *
+ * Read-only at boot from the installed package; no network, no REST quota, no cache to go stale.
+ */
+const fs = require('node:fs');
+const path = require('node:path');
+
+const FULL_RE = /^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/;
+
+/** The installed openvibe-contracts manifests directory, or null when the package cannot be found. */
+function manifestsRoot() {
+ try {
+ return path.join(path.dirname(require.resolve('openvibe-contracts/package.json')), 'manifests');
+ } catch {
+ return null;
+ }
+}
+
+function jsonFiles(dir) {
+ try {
+ return fs.readdirSync(dir).filter((f) => f.endsWith('.json')).sort();
+ } catch {
+ return [];
+ }
+}
+
+function readJson(file) {
+ try {
+ const value = JSON.parse(fs.readFileSync(file, 'utf8'));
+ return value && typeof value === 'object' ? value : null;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * [{ full_name, name, clone_url }], deduplicated by lower-case full name, in manifest order. A repository manifest
+ * without `repository` (AFResume today) is OpenVibers/<name>; a service manifest without one is named but not
+ * mirrored.
+ */
+function contractRepositories({ root = manifestsRoot() } = {}) {
+ const found = new Map();
+ if (!root) return [];
+
+ const add = (repository, fallbackName) => {
+ const full = String(repository || '').trim() || (fallbackName ? `OpenVibers/${fallbackName}` : '');
+ if (!FULL_RE.test(full)) return;
+ const [owner, name] = full.split('/');
+ if (!owner || !name || name === '.' || name === '..') return;
+ const key = full.toLowerCase();
+ if (found.has(key)) return;
+ found.set(key, { full_name: full, name, clone_url: `https://github.com/${full}.git` });
+ };
+
+ for (const dir of ['repositories', 'services']) {
+ const base = path.join(root, dir);
+ for (const file of jsonFiles(base)) {
+ const manifest = readJson(path.join(base, file));
+ if (!manifest) continue;
+ add(manifest.repository, dir === 'repositories' ? manifest.name : '');
+ }
+ }
+ return [...found.values()];
+}
+
+module.exports = { contractRepositories, manifestsRoot, FULL_RE };
@@ -439,6 +439,41 @@ async function fetchMirror(owner, name, { timeoutMs = 600_000 } = {}) {
@@ -439,6 +439,41 @@ async function fetchMirror(owner, name, { timeoutMs = 600_000 } = {}) {
await runGit(['fetch', '--prune', '--tags', '--quiet', 'origin'], { cwd: dir, timeoutMs, maxBuffer: 16 * 1024 * 1024 });
await runGit(['fetch', '--prune', '--tags', '--quiet', 'origin'], { cwd: dir, timeoutMs, maxBuffer: 16 * 1024 * 1024 });
}
}
+function parseRefList(out, separator) {
+ const refs = new Map();
+ for (const line of String(out || '').split('\n')) {
+ const trimmed = line.trim();
+ if (!trimmed) continue;
+ const at = trimmed.indexOf(separator);
+ if (at < 0) continue;
+ const sha = trimmed.slice(0, at).trim();
+ const ref = trimmed.slice(at + separator.length).trim();
+ if (!/^[0-9a-f]{40,64}$/.test(sha) || !ref.startsWith('refs/')) continue;
+ refs.set(ref, sha);
+ }
+ return refs;
+}
+
+/** The heads and tags the remote advertises, read with git's own ls-refs (no REST quota). */
+async function remoteRefs(owner, name, { timeoutMs = 60_000 } = {}) {
+ const out = await runText(['ls-remote', '--refs', '--heads', '--tags', 'origin'], { cwd: repoDir(owner, name), timeoutMs, maxBuffer: 16 * 1024 * 1024 });
+ return parseRefList(out, '\t');
+}
+
+/** The heads and tags this mirror already has. */
+async function localRefs(owner, name) {
+ const out = await runText(['for-each-ref', '--format=%(objectname) %(refname)', 'refs/heads', 'refs/tags'], { cwd: repoDir(owner, name), maxBuffer: 16 * 1024 * 1024 });
+ return parseRefList(out, ' ');
+}
+
+/** Whether the remote advertises anything this mirror does not have: the poll reads refs before it fetches. */
+async function mirrorChanged(owner, name, { timeoutMs = 60_000 } = {}) {
+ const [remote, local] = await Promise.all([remoteRefs(owner, name, { timeoutMs }), localRefs(owner, name)]);
+ if (remote.size !== local.size) return true;
+ for (const [ref, sha] of remote) if (local.get(ref) !== sha) return true;
+ return false;
+}
+
/** Push a repository mirror to another URL, the token in the environment (never argv, never the log). */
/** Push a repository mirror to another URL, the token in the environment (never argv, never the log). */
async function pushMirror(owner, name, url, { token, timeoutMs = 600_000 } = {}) {
async function pushMirror(owner, name, url, { token, timeoutMs = 600_000 } = {}) {
const dir = repoDir(owner, name);
const dir = repoDir(owner, name);
@@ -454,4 +489,5 @@ module.exports = {
@@ -454,4 +489,5 @@ module.exports = {
repoDir, ensureDataDirs, repoExists, repoBytes, initRepo, setRepoConfig, setRemote, resolveCommit, defaultRef,
repoDir, ensureDataDirs, repoExists, repoBytes, initRepo, setRepoConfig, setRemote, resolveCommit, defaultRef,
readTree, readBlob, rawBlob, blobBytes, history, commitDetail, branches, tags, contributors, commitsPerWeek, languages, commitFiles,
readTree, readBlob, rawBlob, blobBytes, history, commitDetail, branches, tags, contributors, commitsPerWeek, languages, commitFiles,
writeInitialCommit, cloneMirror, cloneLocal, fetchMirror, pushMirror, removeRepo, parseLsTree, catFiles,
writeInitialCommit, cloneMirror, cloneLocal, fetchMirror, pushMirror, removeRepo, parseLsTree, catFiles,
+ remoteRefs, localRefs, mirrorChanged,
};
};
@@ -9,9 +9,35 @@
@@ -9,9 +9,35 @@
* api.github.com; without one the public limit applies.
* api.github.com; without one the public limit applies.
*/
*/
const crypto = require('node:crypto');
const crypto = require('node:crypto');
+const { assertPublicUrl } = require('openvibe-shared/egress');
const GITHUB_HOSTS = new Set(['github.com', 'www.github.com']);
const GITHUB_HOSTS = new Set(['github.com', 'www.github.com']);
const FULL_RE = /^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/;
const FULL_RE = /^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/;
+const GITHUB_WEB = 'https://github.com';
+
+// ls-refs for HEAD in git's protocol v2, with symrefs so the answer names the default branch. Each pkt-line carries
+// its own length in hex, four bytes included; 0001 is the delimiter between the command and its arguments.
+const pktLine = (text) => (Buffer.byteLength(text, 'utf8') + 4).toString(16).padStart(4, '0') + text;
+const LS_REFS_HEAD = pktLine('command=ls-refs\n') + '0001' + pktLine('symrefs\n') + pktLine('ref-prefix HEAD\n') + '0000';
+
+/** The refs in an ls-refs answer: [{ sha, ref, attrs }]. Lengths, delimiters and flushes are skipped. */
+function parseRefs(text) {
+ const refs = [];
+ const body = String(text || '');
+ let i = 0;
+ while (i + 4 <= body.length) {
+ const size = parseInt(body.slice(i, i + 4), 16);
+ if (size === 0 || size === 1 || size === 2) { i += 4; continue; }
+ if (!Number.isFinite(size) || size < 4 || i + size > body.length) break;
+ const line = body.slice(i + 4, i + size).replace(/\n$/, '');
+ i += size;
+ if (!line) continue;
+ const parts = line.split(' ');
+ if (parts.length < 2) continue;
+ refs.push({ sha: parts[0], ref: parts[1], attrs: parts.slice(2) });
+ }
+ return refs;
+}
/** A public GitHub repository URL → { full_name, clone_url }, or null. No credentials, no other host. */
/** A public GitHub repository URL → { full_name, clone_url }, or null. No credentials, no other host. */
function parseGithubUrl(input) {
function parseGithubUrl(input) {
@@ -27,9 +53,10 @@ function parseGithubUrl(input) {
@@ -27,9 +53,10 @@ function parseGithubUrl(input) {
return { full_name: `${parts[0]}/${name}`, clone_url: `https://github.com/${parts[0]}/${name}.git` };
return { full_name: `${parts[0]}/${name}`, clone_url: `https://github.com/${parts[0]}/${name}.git` };
}
}
-function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console }) {
+function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () => Date.now(), log = console, lookup = null, guard = null }) {
const cache = new Map(); // full_name (lower) → { at, card }
const cache = new Map(); // full_name (lower) → { at, card }
const metaCache = new Map();
const metaCache = new Map();
+ const probeCache = new Map();
const headers = () => {
const headers = () => {
const h = { accept: 'application/vnd.github+json', 'user-agent': 'openvibe-git' };
const h = { accept: 'application/vnd.github+json', 'user-agent': 'openvibe-git' };
if (config.github.token) h.authorization = `Bearer ${config.github.token}`;
if (config.github.token) h.authorization = `Bearer ${config.github.token}`;
@@ -85,11 +112,61 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
@@ -85,11 +112,61 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
const meta = {
const meta = {
full_name: repo.full_name, description: repo.description || '', default_branch: repo.default_branch || 'main',
full_name: repo.full_name, description: repo.description || '', default_branch: repo.default_branch || 'main',
language: repo.language || null, stars: Number(repo.stargazers_count) || 0, private: !!repo.private,
language: repo.language || null, stars: Number(repo.stargazers_count) || 0, private: !!repo.private,
+ topics: Array.isArray(repo.topics) ? repo.topics.map(String).slice(0, 20) : [],
};
};
metaCache.set(key, { at: now(), meta });
metaCache.set(key, { at: now(), meta });
return meta;
return meta;
}
}
+ /**
+ * Whether a GitHub repository is public, and its default branch, through git's own protocol: a protocol-v2
+ * ls-refs POST needs no REST quota (git clone and fetch need none either). GitHub answers 401 for a private
+ * repository and 404 for a missing one; both are skipped by the caller. Through the egress guard like every
+ * other connection to github.com.
+ */
+ async function probe(full) {
+ const name = String(full || '');
+ if (!FULL_RE.test(name)) return { state: 'invalid', default_branch: null };
+ const key = name.toLowerCase();
+ const hit = probeCache.get(key);
+ if (hit && now() - hit.at < 60_000) return hit.value;
+ const url = `${GITHUB_WEB}/${name}.git/git-upload-pack`;
+ let res;
+ try {
+ if (guard) await guard(url);
+ else await assertPublicUrl(url, lookup ? { lookup } : undefined);
+ res = await fetchImpl(url, {
+ method: 'POST',
+ headers: {
+ 'content-type': 'application/x-git-upload-pack-request',
+ accept: 'application/x-git-upload-pack-result',
+ 'git-protocol': 'version=2',
+ 'accept-encoding': 'identity',
+ 'user-agent': 'openvibe-git',
+ },
+ body: LS_REFS_HEAD,
+ signal: AbortSignal.timeout(8000),
+ });
+ } catch (err) {
+ return { state: 'unknown', default_branch: null, error: String(err.message || err).slice(0, 200) };
+ }
+ let value;
+ if (res.status === 401) value = { state: 'private', default_branch: null };
+ else if (res.status === 404) value = { state: 'missing', default_branch: null };
+ else if (!res.ok) value = { state: 'unknown', default_branch: null, error: `GitHub ${res.status}` };
+ else {
+ let text = '';
+ try { text = typeof res.text === 'function' ? await res.text() : ''; } catch { text = ''; }
+ const refs = parseRefs(text);
+ const head = refs.find((r) => r.ref === 'HEAD');
+ const target = head && head.attrs.map((a) => (a.startsWith('symref-target:') ? a.slice('symref-target:'.length) : '')).find(Boolean);
+ const branch = target && target.startsWith('refs/heads/') ? target.slice('refs/heads/'.length) : 'main';
+ value = { state: 'public', default_branch: branch, head: head ? head.sha : null };
+ }
+ probeCache.set(key, { at: now(), value });
+ return value;
+ }
+
/** The `rel="next"` URL in a GitHub Link header, if it stays on the configured API origin. */
/** The `rel="next"` URL in a GitHub Link header, if it stays on the configured API origin. */
function nextLink(header) {
function nextLink(header) {
if (!header) return null;
if (!header) return null;
@@ -138,6 +215,7 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
@@ -138,6 +215,7 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
metaCache.set(String(repo.full_name).toLowerCase(), { at: now(), meta: {
metaCache.set(String(repo.full_name).toLowerCase(), { at: now(), meta: {
full_name: repo.full_name, description: repo.description || '', default_branch: repo.default_branch || 'main',
full_name: repo.full_name, description: repo.description || '', default_branch: repo.default_branch || 'main',
language: repo.language || null, stars: Number(repo.stargazers_count) || 0, private: !!repo.private,
language: repo.language || null, stars: Number(repo.stargazers_count) || 0, private: !!repo.private,
+ topics: Array.isArray(repo.topics) ? repo.topics.map(String).slice(0, 20) : [],
} });
} });
}
}
all.push(...batch);
all.push(...batch);
@@ -156,7 +234,7 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
@@ -156,7 +234,7 @@ function createGithubClient({ config, fetchImpl = globalThis.fetch, now = () =>
return a.length === b.length && crypto.timingSafeEqual(a, b);
return a.length === b.length && crypto.timingSafeEqual(a, b);
}
}
- return { parseGithubUrl, card, meta, orgRepos, signatureOk, _cache: cache };
+ return { parseGithubUrl, card, meta, probe, orgRepos, signatureOk, _cache: cache };
}
}
-module.exports = { createGithubClient, parseGithubUrl, FULL_RE };
+module.exports = { createGithubClient, parseGithubUrl, parseRefs, pktLine, FULL_RE };
@@ -35,7 +35,15 @@ function createMirrors({ store, repos, config, git, github, log = console }) {
@@ -35,7 +35,15 @@ function createMirrors({ store, repos, config, git, github, log = console }) {
if (initial || !(await git.repoExists(row.owner_username, row.name))) {
if (initial || !(await git.repoExists(row.owner_username, row.name))) {
await git.cloneMirror(row.owner_username, row.name, row.mirror_url, { timeoutMs: config.mirrors.timeoutMs });
await git.cloneMirror(row.owner_username, row.name, row.mirror_url, { timeoutMs: config.mirrors.timeoutMs });
} else {
} else {
- await git.fetchMirror(row.owner_username, row.name, { timeoutMs: config.mirrors.timeoutMs });
+ // Read the remote's refs through git's own ls-refs first: an unchanged repository costs no fetch and,
+ // like every git transport here, no GitHub REST quota.
+ let changed = true;
+ try {
+ changed = await git.mirrorChanged(row.owner_username, row.name, { timeoutMs: config.mirrors.timeoutMs });
+ } catch (err) {
+ log.warn(`[OpenVibe.Git] mirror refs ${row.owner_username}/${row.name}: ${String(err.message || err).slice(0, 200)}`);
+ }
+ if (changed) await git.fetchMirror(row.owner_username, row.name, { timeoutMs: config.mirrors.timeoutMs });
}
}
await store.execute("UPDATE git_repos SET mirror_state = 'ready', mirror_error = NULL, mirror_last_synced_at = now(), pushed_at = now(), updated_at = now() WHERE id = ?", [row.id]);
await store.execute("UPDATE git_repos SET mirror_state = 'ready', mirror_error = NULL, mirror_last_synced_at = now(), pushed_at = now(), updated_at = now() WHERE id = ?", [row.id]);
await repos.refreshLanguages(row);
await repos.refreshLanguages(row);
@@ -3,22 +3,31 @@
@@ -3,22 +3,31 @@
/**
/**
* The official mirrors seed: every public repository in the GitHub organisation named by GIT_OFFICIAL_ORG becomes
* The official mirrors seed: every public repository in the GitHub organisation named by GIT_OFFICIAL_ORG becomes
* a mirror under the official OpenVibe owner (GIT_OFFICIAL_OWNER, by default the first of GIT_OFFICIAL_OWNERS).
* a mirror under the official OpenVibe owner (GIT_OFFICIAL_OWNER, by default the first of GIT_OFFICIAL_OWNERS).
- * This is the one thing the Forgejo cut-over keeps: the mirror list is rebuilt from GitHub itself.
+ * This is the one thing the Forgejo cut-over keeps: the mirror list is rebuilt from the network's own contracts.
*
*
* The job is due-based, not timer-based: the last finished run is a row in the database (git_official_state,
* The job is due-based, not timer-based: the last finished run is a row in the database (git_official_state,
* migration 0002), so a deploy — and this service is deployed much more often than every six hours — does not
* migration 0002), so a deploy — and this service is deployed much more often than every six hours — does not
* reset the interval. A tick runs shortly after boot and then hourly, and only does the work when the interval
* reset the interval. A tick runs shortly after boot and then hourly, and only does the work when the interval
* has passed.
* has passed.
*
*
- * A run lists all pages of the organisation's public repositories, skips forks and archived repositories, and
- * creates each missing one through repos.create (which clones with the mirror timeout). Names follow this
- * service's rule: lower-case, no leading dot, never ending in .git — "OpenVibe.Live" becomes "openvibe.live".
- * Nothing is ever deleted. One line per run says how many were created, already existed and failed.
+ * No REST quota is needed:
+ * - the list comes from GitHub's REST API when it answers, and from openvibe-contracts manifests/services and
+ * manifests/repositories when it does not (403/429/5xx/network); the run's summary says which;
+ * - each candidate is checked as public and existing through git's protocol (ls-refs over github.com, which
+ * spends no REST quota) and names the default branch; private and missing repositories are skipped and counted;
+ * - a mirror is created through repos.create, which clones with the mirror timeout even when the metadata call
+ * is refused, marking the card pending for a later run.
+ *
+ * Names follow this service's rule: lower-case, no leading dot, never ending in .git — "OpenVibe.Live" becomes
+ * "openvibe.live". Nothing is ever deleted. Clones run a few at a time. One line per run says where the list came
+ * from, how many were created, already existed, skipped (with the reason) and failed.
*
*
* It runs in the background: /api/ready never waits on it, and a failed run is logged and retried at the next
* It runs in the background: /api/ready never waits on it, and a failed run is logged and retried at the next
- * due time. An empty GIT_OFFICIAL_ORG keeps it off, and so does drill mode.
+ * due time. An empty GIT_OFFICIAL_ORG keeps it off, and so does drill mode. stop() ends a run at the next
+ * candidate; a run that was stopped is not recorded as finished, so the next boot tries again.
*/
*/
const { validName } = require('./git/refs');
const { validName } = require('./git/refs');
+const { contractRepositories } = require('./contracts');
const STATE_ID = 'official-mirrors';
const STATE_ID = 'official-mirrors';
const FIRST_RUN_MS = 30_000; // shortly after boot, not in the boot path
const FIRST_RUN_MS = 30_000; // shortly after boot, not in the boot path
@@ -27,15 +36,17 @@ const FIRST_RUN_MS = 30_000; // shortly after boot, not in the boot
@@ -27,15 +36,17 @@ const FIRST_RUN_MS = 30_000; // shortly after boot, not in the boot
// other hour, while the host's GitHub quota had come back).
// other hour, while the host's GitHub quota had come back).
const TICK_MS = 15 * 60_000;
const TICK_MS = 15 * 60_000;
-function createOfficialMirrors({ store, repos, config, github, log = console, clock = null }) {
+function createOfficialMirrors({ store, repos, config, github, log = console, clock = null, contractList = null }) {
const settings = config.official;
const settings = config.official;
const now = clock && clock.now ? () => clock.now() : () => store.now();
const now = clock && clock.now ? () => clock.now() : () => store.now();
const iso = () => new Date(now()).toISOString();
const iso = () => new Date(now()).toISOString();
let running = false;
let running = false;
+ let stopped = false;
let timer = null;
let timer = null;
let first = null;
let first = null;
const enabled = () => Boolean(settings.org) && !config.drill;
const enabled = () => Boolean(settings.org) && !config.drill;
+ const listContracts = contractList || contractRepositories;
/** When the last run finished and what it found. */
/** When the last run finished and what it found. */
async function lastRun() {
async function lastRun() {
@@ -55,59 +66,161 @@ function createOfficialMirrors({ store, repos, config, github, log = console, cl
@@ -55,59 +66,161 @@ function createOfficialMirrors({ store, repos, config, github, log = console, cl
);
);
}
}
- /** One pass over the organisation: create what is missing, count what is already there, delete nothing. */
+ /** Fill a mirror's card from the REST metadata once the API answers again; a quota refusal stops the attempts. */
+ async function fillPending(row, state) {
+ if (!row || !row.github_full_name || row.mirror_meta_state !== 'pending' || state.refused) return false;
+ try {
+ const meta = await github.meta(row.github_full_name);
+ const topics = meta.topics && meta.topics.length ? meta.topics : (typeof row.topics === 'string' ? JSON.parse(row.topics || '[]') : (row.topics || []));
+ await store.execute(
+ `UPDATE git_repos SET description = ?, topics = ?, default_branch = ?, mirror_meta_state = 'ready', updated_at = now() WHERE id = ?`,
+ [meta.description || row.description || '', JSON.stringify(topics), meta.default_branch || row.default_branch, row.id],
+ );
+ return true;
+ } catch (err) {
+ if (err && (err.status === 403 || err.status === 429)) state.refused = true;
+ return false;
+ }
+ }
+
+ /** One pass over the list: create what is missing, count what is already there, delete nothing. */
async function run() {
async function run() {
- const summary = { org: settings.org, owner: settings.owner, listed: 0, created: 0, existing: 0, failed: 0, ok: false };
+ const summary = {
+ org: settings.org, owner: settings.owner, source: null, listed: 0, created: 0, existing: 0, failed: 0,
+ skipped: { private: 0, missing: 0, invalid: 0, fork: 0, archived: 0 }, metadata_pending: 0, ok: false,
+ };
if (!settings.org) {
if (!settings.org) {
summary.off = true;
summary.off = true;
log.log('[OpenVibe.Git] official mirrors: off (GIT_OFFICIAL_ORG is empty)');
log.log('[OpenVibe.Git] official mirrors: off (GIT_OFFICIAL_ORG is empty)');
return summary;
return summary;
}
}
- let listed;
+ if (stopped) {
+ summary.stopped = true;
+ return summary;
+ }
+
+ // Where the list comes from: GitHub's REST API, or the network's own contracts when that is refused.
+ let listed = [];
+ let listError = null;
+ const state = { refused: false };
try {
try {
listed = await github.orgRepos(settings.org);
listed = await github.orgRepos(settings.org);
+ summary.source = 'github';
} catch (err) {
} catch (err) {
+ listError = err;
+ summary.source = 'contracts';
+ if (err && (err.status === 403 || err.status === 429)) state.refused = true;
+ try {
+ listed = listContracts();
+ } catch (listErr) {
+ listed = [];
+ summary.contract_error = String(listErr.message || listErr).slice(0, 200);
+ }
+ summary.listing_error = String(err.message || err).slice(0, 300);
+ log.warn(`[OpenVibe.Git] official mirrors: GitHub listing ${settings.org} refused (${summary.listing_error}); `
+ + `using ${listed.length} repositories from the contracts`);
+ }
+ summary.listed = Array.isArray(listed) ? listed.length : 0;
+ if (!summary.listed) {
summary.failed = 1;
summary.failed = 1;
- summary.error = String(err.message || err).slice(0, 300);
- log.warn(`[OpenVibe.Git] official mirrors: listing ${settings.org}: ${summary.error}`);
- log.log(`[OpenVibe.Git] official mirrors: created ${summary.created}, existing ${summary.existing}, failed ${summary.failed}`);
+ summary.error = listError ? String(listError.message || listError).slice(0, 300) : 'no repositories listed';
+ log.log(`[OpenVibe.Git] official mirrors: source ${summary.source}, created 0, existing 0, failed 1`);
return summary;
return summary;
}
}
- summary.listed = listed.length;
- for (const repo of listed) {
- if (!repo || repo.fork || repo.archived || repo.private) continue;
- const name = String(repo.name || '').toLowerCase();
+
+ async function one(repo) {
+ if (stopped || !repo) return;
+ if (repo.fork) { summary.skipped.fork += 1; return; }
+ if (repo.archived) { summary.skipped.archived += 1; return; }
+ if (repo.private) {
+ summary.skipped.private += 1;
+ log.warn(`[OpenVibe.Git] official mirrors: ${repo.full_name || repo.name} is private (skipped)`);
+ return;
+ }
+ const full = String(repo.full_name || '');
+ const name = String(repo.name || full.split('/')[1] || '').toLowerCase();
if (!validName(name)) {
if (!validName(name)) {
- summary.failed += 1;
- log.warn(`[OpenVibe.Git] official mirrors: ${repo.full_name || repo.name} is not a valid repository name here (${name})`);
- continue;
+ summary.skipped.invalid += 1;
+ log.warn(`[OpenVibe.Git] official mirrors: ${full || name} is not a valid repository name here (${name})`);
+ return;
}
}
- if (await repos.getRow(settings.owner, name)) {
+ const existing = await repos.getRow(settings.owner, name);
+ if (existing) {
summary.existing += 1;
summary.existing += 1;
- continue;
+ if (existing.mirror_meta_state === 'pending') await fillPending(existing, state);
+ return;
+ }
+
+ // Public and existing through git's own protocol: no REST quota, and the default branch comes with it.
+ const info = await github.probe(full).catch(() => null);
+ if (info && info.state === 'private') {
+ summary.skipped.private += 1;
+ log.warn(`[OpenVibe.Git] official mirrors: ${full} is private (skipped)`);
+ return;
+ }
+ if (info && info.state === 'missing') {
+ summary.skipped.missing += 1;
+ log.warn(`[OpenVibe.Git] official mirrors: ${full} does not exist (skipped)`);
+ return;
}
}
- // repos.create checks the GitHub URL, reads the metadata and clones with config.mirrors.timeoutMs.
+ if (info && info.state === 'invalid') {
+ summary.skipped.invalid += 1;
+ log.warn(`[OpenVibe.Git] official mirrors: ${full} is not a valid GitHub repository name (skipped)`);
+ return;
+ }
+ if (stopped) return;
+
+ // repos.create checks the GitHub URL, reads the metadata (pending when refused) and clones with the
+ // mirror timeout. The probe's answer saves the second ls-refs and names the default branch.
try {
try {
await repos.create({
await repos.create({
actor: { username: settings.owner, subject: null },
actor: { username: settings.owner, subject: null },
name,
name,
- mirrorUrl: repo.clone_url || `https://github.com/${settings.org}/${repo.name}.git`,
+ mirrorUrl: repo.clone_url || `https://github.com/${full}.git`,
+ mirrorInfo: info,
});
});
summary.created += 1;
summary.created += 1;
} catch (err) {
} catch (err) {
- if (err && err.status === 409) { summary.existing += 1; continue; }
+ if (err && err.status === 409) { summary.existing += 1; return; }
summary.failed += 1;
summary.failed += 1;
- log.warn(`[OpenVibe.Git] official mirror ${settings.org}/${repo.name}: ${String(err.message || err).slice(0, 300)}`);
+ log.warn(`[OpenVibe.Git] official mirror ${full || name}: ${String(err.message || err).slice(0, 300)}`);
+ }
+ }
+
+ // A few clones at once, not the whole list: GitHub sees a handful of connections and the host stays usable.
+ const limit = Math.max(1, Math.min(8, Number(settings.concurrency) || 3));
+ let cursor = 0;
+ const workers = Array.from({ length: Math.min(limit, summary.listed) }, async () => {
+ while (!stopped) {
+ const index = cursor;
+ cursor += 1;
+ if (index >= summary.listed) return;
+ await one(listed[index]);
}
}
+ });
+ await Promise.all(workers);
+
+ if (stopped) {
+ summary.stopped = true;
+ } else {
+ summary.metadata_pending = Number(await store.value(
+ "SELECT count(*)::int FROM git_repos WHERE mirror_url IS NOT NULL AND mirror_meta_state = 'pending'",
+ )) || 0;
}
}
summary.ok = summary.failed === 0;
summary.ok = summary.failed === 0;
- log.log(`[OpenVibe.Git] official mirrors: created ${summary.created}, existing ${summary.existing}, failed ${summary.failed}`);
+ const skipped = summary.skipped;
+ const skippedTotal = skipped.private + skipped.missing + skipped.invalid + skipped.fork + skipped.archived;
+ log.log(`[OpenVibe.Git] official mirrors: source ${summary.source}, created ${summary.created}, existing ${summary.existing}, `
+ + `skipped ${skippedTotal} (private ${skipped.private}, missing ${skipped.missing}, invalid ${skipped.invalid}, `
+ + `fork ${skipped.fork}, archived ${skipped.archived}), failed ${summary.failed}`
+ + (summary.metadata_pending ? `, metadata pending ${summary.metadata_pending}` : ''));
return summary;
return summary;
}
}
/** Run when due. The in-process guard keeps the hourly tick from stacking runs; the state row spans restarts. */
/** Run when due. The in-process guard keeps the hourly tick from stacking runs; the state row spans restarts. */
async function tick() {
async function tick() {
if (!enabled()) return { ran: false, reason: config.drill ? 'drill' : 'off' };
if (!enabled()) return { ran: false, reason: config.drill ? 'drill' : 'off' };
+ if (stopped) return { ran: false, reason: 'stopped' };
if (running) return { ran: false, reason: 'running' };
if (running) return { ran: false, reason: 'running' };
running = true;
running = true;
try {
try {
@@ -115,7 +228,7 @@ function createOfficialMirrors({ store, repos, config, github, log = console, cl
@@ -115,7 +228,7 @@ function createOfficialMirrors({ store, repos, config, github, log = console, cl
const dueAt = (last.at || 0) + settings.syncMs;
const dueAt = (last.at || 0) + settings.syncMs;
if (last.at && now() < dueAt) return { ran: false, reason: 'not due', due_at: new Date(dueAt).toISOString() };
if (last.at && now() < dueAt) return { ran: false, reason: 'not due', due_at: new Date(dueAt).toISOString() };
const summary = await run();
const summary = await run();
- await recordRun(summary);
+ if (!summary.stopped) await recordRun(summary);
return { ran: true, ...summary };
return { ran: true, ...summary };
} finally {
} finally {
running = false;
running = false;
@@ -133,6 +246,7 @@ function createOfficialMirrors({ store, repos, config, github, log = console, cl
@@ -133,6 +246,7 @@ function createOfficialMirrors({ store, repos, config, github, log = console, cl
}
}
function stop() {
function stop() {
+ stopped = true;
if (first) clearTimeout(first);
if (first) clearTimeout(first);
if (timer) clearInterval(timer);
if (timer) clearInterval(timer);
first = null;
first = null;
@@ -95,7 +95,7 @@ function createRepos({ store, config, git, github, insights, log = console }) {
@@ -95,7 +95,7 @@ function createRepos({ store, config, git, github, insights, log = console }) {
} catch (err) { log.warn('[OpenVibe.Git] audit not recorded:', err.message); }
} catch (err) { log.warn('[OpenVibe.Git] audit not recorded:', err.message); }
}
}
- async function create({ actor, name, description = '', visibility = 'public', topics = [], starter = 'empty', mirrorUrl = null }, { address = null } = {}) {
+ async function create({ actor, name, description = '', visibility = 'public', topics = [], starter = 'empty', mirrorUrl = null, mirrorInfo = null }, { address = null } = {}) {
const owner = String(actor.username || '').toLowerCase();
const owner = String(actor.username || '').toLowerCase();
const repoName = String(name || '').toLowerCase();
const repoName = String(name || '').toLowerCase();
if (!validName(owner)) throw Object.assign(new Error('A signed-in OpenVibe username is required.'), { status: 403 });
if (!validName(owner)) throw Object.assign(new Error('A signed-in OpenVibe username is required.'), { status: 403 });
@@ -126,13 +126,30 @@ function createRepos({ store, config, git, github, insights, log = console }) {
@@ -126,13 +126,30 @@ function createRepos({ store, config, git, github, insights, log = console }) {
const row = await byId(id);
const row = await byId(id);
try {
try {
if (mirror) {
if (mirror) {
- let meta = { description: '', default_branch: 'main', private: false };
- try { meta = await github.meta(mirror.full_name); } catch { /* the clone still works without the card */ }
- if (meta.private) throw Object.assign(new Error('That GitHub repository is private.'), { status: 422 });
+ let meta = null;
+ let metaPending = false;
+ try {
+ meta = await github.meta(mirror.full_name);
+ } catch (err) {
+ // Quota refused (403/429): the clone needs none, so make the mirror now and fill the card later.
+ metaPending = Boolean(err && (err.status === 403 || err.status === 429));
+ }
+ if (meta && meta.private) throw Object.assign(new Error('That GitHub repository is private.'), { status: 422 });
+ // The default branch comes from the REST metadata, or from git's own ls-refs when REST is out.
+ let defaultBranch = meta && meta.default_branch ? meta.default_branch : 'main';
+ if (!meta) {
+ const info = mirrorInfo || await github.probe(mirror.full_name).catch(() => null);
+ if (info && info.state === 'private') throw Object.assign(new Error('That GitHub repository is private.'), { status: 422 });
+ if (info && info.state === 'missing') throw Object.assign(new Error('That GitHub repository does not exist.'), { status: 422 });
+ if (info && info.default_branch) defaultBranch = info.default_branch;
+ }
+ const metaTopics = meta && Array.isArray(meta.topics) ? meta.topics : [];
await git.cloneMirror(owner, repoName, mirror.clone_url, { timeoutMs: config.mirrors.timeoutMs });
await git.cloneMirror(owner, repoName, mirror.clone_url, { timeoutMs: config.mirrors.timeoutMs });
await store.execute(
await store.execute(
- `UPDATE git_repos SET description = ?, default_branch = ?, mirror_state = 'ready', mirror_last_synced_at = now(), pushed_at = now(), updated_at = now() WHERE id = ?`,
- [description || meta.description || '', meta.default_branch || 'main', id],
+ `UPDATE git_repos SET description = ?, default_branch = ?, topics = ?, mirror_state = 'ready',
+ mirror_meta_state = ?, mirror_last_synced_at = now(), pushed_at = now(), updated_at = now() WHERE id = ?`,
+ [description || (meta && meta.description) || '', defaultBranch, JSON.stringify(topics.length ? topics : metaTopics),
+ metaPending ? 'pending' : 'ready', id],
);
);
} else {
} else {
const files = starterFiles(starter, { full: `${owner}/${repoName}` });
const files = starterFiles(starter, { full: `${owner}/${repoName}` });
@@ -44,7 +44,11 @@ async function boot(opts = {}) {
@@ -44,7 +44,11 @@ async function boot(opts = {}) {
const testdb = await require('./db').testDb();
const testdb = await require('./db').testDb();
const store = createStore(testdb.db, { now: opts.now });
const store = createStore(testdb.db, { now: opts.now });
const log = { log: () => {}, warn: (...a) => captured.push(a.map(String).join(' ')), error: (...a) => captured.push(a.map(String).join(' ')), info: () => {} };
const log = { log: () => {}, warn: (...a) => captured.push(a.map(String).join(' ')), error: (...a) => captured.push(a.map(String).join(' ')), info: () => {} };
- const { app, ctx } = await createApp({ config, store, log, reviewServices: opts.reviewServices, fetchImpl: typeof opts.fetchImpl === 'function' ? opts.fetchImpl : undefined });
+ const { app, ctx } = await createApp({
+ config, store, log, reviewServices: opts.reviewServices,
+ fetchImpl: typeof opts.fetchImpl === 'function' ? opts.fetchImpl : undefined,
+ lookup: opts.lookup, guard: opts.guard,
+ });
const server = await new Promise((resolve) => { const s = http.createServer(app); s.listen(0, '127.0.0.1', () => resolve(s)); });
const server = await new Promise((resolve) => { const s = http.createServer(app); s.listen(0, '127.0.0.1', () => resolve(s)); });
const port = server.address().port;
const port = server.address().port;
const base = `http://127.0.0.1:${port}`;
const base = `http://127.0.0.1:${port}`;
new file mode 100644
new file mode 100644
@@ -0,0 +1,185 @@
@@ -0,0 +1,185 @@
+'use strict';
+const assert = require('assert');
+const { boot } = require('./helpers/boot');
+const { createOfficialMirrors } = require('../server/official');
+const { contractRepositories } = require('../server/contracts');
+const { createMirrors } = require('../server/mirror');
+const { createRepos } = require('../server/repos');
+
+// The quota-free path, with no live GitHub: a 403 listing falls back to the contracts, a private or missing
+// repository is skipped through ls-refs, a refused metadata call still clones and marks the card pending, and the
+// poll reads refs before it fetches and never asks REST. The git binary and fetch are both stubbed.
+const pkt = (text) => (Buffer.byteLength(text) + 4).toString(16).padStart(4, '0') + text;
+const publicRefs = (branch = 'main') => ({
+ ok: true, status: 200, headers: { get: () => null }, json: async () => ({}),
+ text: async () => pkt(`${'b'.repeat(40)} HEAD symref-target:refs/heads/${branch}\n`) + '0000',
+});
+const status = (code) => ({ ok: false, status: code, headers: { get: () => null }, json: async () => ({}), text: async () => '' });
+const ok = (body) => ({ ok: true, status: 200, headers: { get: () => null }, json: async () => body, text: async () => '' });
+const lookup = (host, options, callback) => callback(null, [{ address: '140.82.112.3', family: 4 }]);
+
+const CONTRACTS = [
+ { full_name: 'OpenVibers/OpenVibe.Live', name: 'OpenVibe.Live', clone_url: 'https://github.com/OpenVibers/OpenVibe.Live.git' },
+ { full_name: 'OpenVibers/OpenVibe.Private', name: 'OpenVibe.Private', clone_url: 'https://github.com/OpenVibers/OpenVibe.Private.git' },
+ { full_name: 'OpenVibers/OpenVibe.Gone', name: 'OpenVibe.Gone', clone_url: 'https://github.com/OpenVibers/OpenVibe.Gone.git' },
+ { full_name: 'OpenVibers/.github', name: '.github', clone_url: 'https://github.com/OpenVibers/.github.git' },
+];
+
+(async () => {
+ // The contracts themselves name the mirror list, deduplicated and with the OpenVibers fallback for a null
+ // repository. Read from the installed openvibe-contracts package, no network.
+ const fromContracts = contractRepositories();
+ assert.ok(fromContracts.length >= 60, `contracts list has ${fromContracts.length} repositories`);
+ assert.ok(fromContracts.every((r) => /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(r.full_name)));
+ const keys = new Set(fromContracts.map((r) => r.full_name.toLowerCase()));
+ assert.equal(keys.size, fromContracts.length, 'deduplicated');
+ assert.ok(fromContracts.some((r) => r.full_name === 'OpenVibers/AFResume'), 'a null repository uses OpenVibers/<name>');
+
+ // ── A refused listing falls back to the contracts; private and missing names are skipped, not failed. ──
+ {
+ const calls = [];
+ const fetchImpl = async (url) => {
+ const target = new URL(url);
+ calls.push(target.toString());
+ if (target.hostname === 'api.github.com' && target.pathname.startsWith('/orgs/')) return status(403);
+ if (target.hostname === 'github.com') {
+ if (target.pathname.includes('/OpenVibe.Live.')) return publicRefs('trunk');
+ if (target.pathname.includes('/OpenVibe.Private.')) return status(401);
+ if (target.pathname.includes('/OpenVibe.Gone.')) return status(404);
+ }
+ if (target.hostname === 'api.github.com' && target.pathname.startsWith('/repos/')) return status(403);
+ throw new Error(`unexpected request ${url}`);
+ };
+ const t = await boot({ env: { GIT_DRILL: '' }, fetchImpl, lookup });
+ const clones = [];
+ t.ctx.git.cloneMirror = async (owner, name) => { clones.push(`${owner}/${name}`); return true; };
+ t.ctx.git.languages = async () => ({});
+ const repos = createRepos({ store: t.store, config: t.config, git: t.ctx.git, github: t.ctx.github, insights: t.ctx.insights, log: { warn() {} } });
+ const job = createOfficialMirrors({
+ store: t.store, repos, config: t.config, github: t.ctx.github,
+ log: { log() {}, warn() {}, error() {} }, contractList: () => CONTRACTS,
+ });
+ try {
+ t.config.official.org = 'OpenVibers';
+ const summary = await job.run();
+ assert.equal(summary.source, 'contracts');
+ assert.equal(summary.listed, CONTRACTS.length);
+ assert.equal(summary.created, 1);
+ assert.equal(summary.skipped.private, 1);
+ assert.equal(summary.skipped.missing, 1);
+ assert.equal(summary.skipped.invalid, 1);
+ assert.equal(summary.failed, 0);
+ assert.deepEqual(clones, ['openvibers/openvibe.live']);
+
+ // The metadata call was refused: the clone still happened, and the card is pending with the branch
+ // ls-refs named.
+ const row = await t.ctx.repos.getRow('openvibers', 'openvibe.live');
+ assert.equal(row.mirror_state, 'ready');
+ assert.equal(row.mirror_meta_state, 'pending');
+ assert.equal(row.default_branch, 'trunk');
+ assert.equal(row.mirror_url, 'https://github.com/OpenVibers/OpenVibe.Live.git');
+
+ // A later run with REST answering fills the card and clears pending.
+ const rest = async (url) => {
+ const target = new URL(url);
+ if (target.hostname === 'github.com') return publicRefs('trunk');
+ if (target.pathname === '/repos/OpenVibers/OpenVibe.Live') {
+ return ok({ full_name: 'OpenVibers/OpenVibe.Live', description: 'Live', default_branch: 'trunk', topics: ['live', 'video'], private: false });
+ }
+ if (target.pathname.startsWith('/orgs/')) {
+ return ok([{ name: 'OpenVibe.Live', full_name: 'OpenVibers/OpenVibe.Live', private: false, fork: false, archived: false, description: 'Live', default_branch: 'trunk', topics: ['live', 'video'] }]);
+ }
+ throw new Error(`unexpected request ${url}`);
+ };
+ const later = await boot({ env: { GIT_DRILL: '' }, fetchImpl: rest, lookup });
+ try {
+ await later.store.execute(
+ `INSERT INTO git_repos (id, owner_username, name, description, visibility, topics, official, github_full_name,
+ mirror_url, mirror_state, mirror_meta_state)
+ VALUES ('repo_later', 'openvibers', 'openvibe.live', '', 'public', '[]', true, 'OpenVibers/OpenVibe.Live',
+ 'https://github.com/OpenVibers/OpenVibe.Live.git', 'ready', 'pending')`,
+ );
+ later.ctx.git.cloneMirror = async () => true;
+ later.ctx.git.languages = async () => ({});
+ const job2 = createOfficialMirrors({
+ store: later.store, repos: later.ctx.repos, config: later.config, github: later.ctx.github,
+ log: { log() {}, warn() {}, error() {} }, contractList: () => [CONTRACTS[0]],
+ });
+ later.config.official.org = 'OpenVibers';
+ const again = await job2.run();
+ assert.equal(again.existing, 1);
+ const filled = await later.ctx.repos.getRow('openvibers', 'openvibe.live');
+ assert.equal(filled.mirror_meta_state, 'ready');
+ assert.equal(filled.description, 'Live');
+ assert.deepEqual(typeof filled.topics === 'string' ? JSON.parse(filled.topics) : filled.topics, ['live', 'video']);
+ } finally { await later.close(); }
+ } finally { await t.close(); }
+ }
+
+ // ── At most a few clones at once. ──
+ {
+ const many = Array.from({ length: 6 }, (_, i) => ({
+ full_name: `OpenVibers/OpenVibe.Many${i}`, name: `OpenVibe.Many${i}`, clone_url: `https://github.com/OpenVibers/OpenVibe.Many${i}.git`,
+ }));
+ const fetchImpl = async (url) => {
+ const target = new URL(url);
+ if (target.hostname === 'github.com') return publicRefs('main');
+ if (target.pathname.startsWith('/orgs/')) return status(403);
+ if (target.pathname.startsWith('/repos/')) return status(403);
+ throw new Error(`unexpected request ${url}`);
+ };
+ const t = await boot({ env: { GIT_DRILL: '' }, fetchImpl, lookup });
+ t.config.official.concurrency = 2;
+ let active = 0;
+ let peak = 0;
+ t.ctx.git.cloneMirror = async () => {
+ active += 1;
+ peak = Math.max(peak, active);
+ await new Promise((resolve) => setTimeout(resolve, 15));
+ active -= 1;
+ return true;
+ };
+ t.ctx.git.languages = async () => ({});
+ const job = createOfficialMirrors({
+ store: t.store, repos: t.ctx.repos, config: t.config, github: t.ctx.github,
+ log: { log() {}, warn() {}, error() {} }, contractList: () => many,
+ });
+ try {
+ t.config.official.org = 'OpenVibers';
+ const summary = await job.run();
+ assert.equal(summary.created, 6);
+ assert.ok(peak <= 2, `at most two clones at once (peak ${peak})`);
+ } finally { await t.close(); }
+ }
+
+ // ── The poll reads refs first: nothing changed means no fetch, and REST is never called. ──
+ {
+ const restCalls = [];
+ const t = await boot({ fetchImpl: async (url) => { restCalls.push(String(url)); throw new Error('REST is not allowed in the poll'); }, lookup });
+ try {
+ await t.ctx.repos.create({ actor: t.asUser('sam'), name: 'polled', starter: 'empty' });
+ await t.store.execute("UPDATE git_repos SET mirror_url = 'https://github.com/OpenVibers/OpenVibe.Live.git' WHERE name = 'polled'");
+ let changed = 0;
+ let fetched = 0;
+ t.ctx.git.mirrorChanged = async () => { changed += 1; return false; };
+ t.ctx.git.fetchMirror = async () => { fetched += 1; return true; };
+ t.ctx.git.languages = async () => ({});
+ const mirrors = createMirrors({ store: t.store, repos: t.ctx.repos, config: t.config, git: t.ctx.git, github: t.ctx.github, log: { warn() {} } });
+
+ await mirrors.poll();
+ assert.equal(changed, 1);
+ assert.equal(fetched, 0, 'an unchanged mirror is not fetched');
+ const after = await t.ctx.repos.getRow('sam', 'polled');
+ assert.equal(after.mirror_state, 'ready');
+ assert.ok(after.mirror_last_synced_at);
+ assert.equal(restCalls.length, 0, 'the poll spent no REST call');
+
+ t.ctx.git.mirrorChanged = async () => true;
+ await mirrors.poll();
+ assert.equal(fetched, 1, 'a changed mirror is fetched');
+ assert.equal(restCalls.length, 0);
+ } finally { await t.close(); }
+ }
+
+ console.log('mirrors-quota: 27 checks passed');
+})().catch((err) => { console.error(err); process.exit(1); });
@@ -13,6 +13,13 @@ const response = (body, link = null) => ({
@@ -13,6 +13,13 @@ const response = (body, link = null) => ({
json: async () => body,
json: async () => body,
});
});
+// git's protocol v2 answer to ls-refs for HEAD: one pkt-line naming the default branch, then a flush.
+const pkt = (text) => (Buffer.byteLength(text) + 4).toString(16).padStart(4, '0') + text;
+const publicRefs = (branch = 'main') => ({
+ ok: true, status: 200, headers: { get: () => null }, json: async () => ({}),
+ text: async () => pkt(`${'a'.repeat(40)} HEAD symref-target:refs/heads/${branch}\n`) + '0000',
+});
+
const LISTING = {
const LISTING = {
1: [
1: [
{ name: 'OpenVibe.Live', full_name: 'OpenVibers/OpenVibe.Live', fork: false, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Live.git' },
{ name: 'OpenVibe.Live', full_name: 'OpenVibers/OpenVibe.Live', fork: false, archived: false, private: false, clone_url: 'https://github.com/OpenVibers/OpenVibe.Live.git' },
@@ -28,9 +35,15 @@ const LISTING = {
@@ -28,9 +35,15 @@ const LISTING = {
(async () => {
(async () => {
let nowMs = Date.now();
let nowMs = Date.now();
const calls = [];
const calls = [];
+ const probes = [];
const fetchImpl = async (url, opts = {}) => {
const fetchImpl = async (url, opts = {}) => {
calls.push({ url: String(url), authorization: opts.headers && opts.headers.authorization });
calls.push({ url: String(url), authorization: opts.headers && opts.headers.authorization });
const target = new URL(url);
const target = new URL(url);
+ // git's own ls-refs: no REST quota, and the answer names the default branch.
+ if (target.hostname === 'github.com' && target.pathname.endsWith('/git-upload-pack')) {
+ probes.push(target.pathname);
+ return publicRefs('main');
+ }
if (target.pathname === '/orgs/OpenVibers/repos') {
if (target.pathname === '/orgs/OpenVibers/repos') {
const page = Number(target.searchParams.get('page') || '1');
const page = Number(target.searchParams.get('page') || '1');
const link = page === 1 ? '<https://api.github.com/orgs/OpenVibers/repos?type=public&per_page=100&page=2>; rel="next"' : null;
const link = page === 1 ? '<https://api.github.com/orgs/OpenVibers/repos?type=public&per_page=100&page=2>; rel="next"' : null;
@@ -49,6 +62,7 @@ const LISTING = {
@@ -49,6 +62,7 @@ const LISTING = {
const t = await boot({
const t = await boot({
env: { GIT_DRILL: '', GIT_OFFICIAL_ORG: '', GIT_GITHUB_TOKEN: 'test-token' },
env: { GIT_DRILL: '', GIT_OFFICIAL_ORG: '', GIT_GITHUB_TOKEN: 'test-token' },
fetchImpl,
fetchImpl,
+ lookup: (host, options, callback) => callback(null, [{ address: '140.82.112.3', family: 4 }]),
now: () => nowMs,
now: () => nowMs,
});
});
const clones = [];
const clones = [];
@@ -72,10 +86,15 @@ const LISTING = {
@@ -72,10 +86,15 @@ const LISTING = {
result = await job.tick();
result = await job.tick();
assert.equal(result.ran, true);
assert.equal(result.ran, true);
assert.equal(result.listed, 5);
assert.equal(result.listed, 5);
+ assert.equal(result.source, 'github');
assert.equal(result.created, 2);
assert.equal(result.created, 2);
assert.equal(result.existing, 0);
assert.equal(result.existing, 0);
- assert.equal(result.failed, 1);
+ assert.equal(result.failed, 0);
+ assert.equal(result.skipped.invalid, 1);
+ assert.equal(result.skipped.fork, 1);
+ assert.equal(result.skipped.archived, 1);
assert.deepEqual(clones.sort(), ['openvibers/openvibe.contracts', 'openvibers/openvibe.live']);
assert.deepEqual(clones.sort(), ['openvibers/openvibe.contracts', 'openvibers/openvibe.live']);
+ assert.deepEqual(probes.sort(), ['/OpenVibers/OpenVibe.Contracts.git/git-upload-pack', '/OpenVibers/OpenVibe.Live.git/git-upload-pack']);
assert.ok(calls.some((c) => c.url.includes('page=2')), 'page 2 was read');
assert.ok(calls.some((c) => c.url.includes('page=2')), 'page 2 was read');
assert.ok(calls.filter((c) => c.url.includes('/orgs/')).every((c) => c.authorization === 'Bearer test-token'));
assert.ok(calls.filter((c) => c.url.includes('/orgs/')).every((c) => c.authorization === 'Bearer test-token'));
@@ -87,9 +106,9 @@ const LISTING = {
@@ -87,9 +106,9 @@ const LISTING = {
assert.ok(rows.every((r) => r.mirror_state === 'ready'));
assert.ok(rows.every((r) => r.mirror_state === 'ready'));
assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE name IN ('openvibe.fork', 'openvibe.archived')")), 0);
assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE name IN ('openvibe.fork', 'openvibe.archived')")), 0);
- const summary = logs.filter((l) => l.includes('official mirrors: created'));
+ const summary = logs.filter((l) => l.includes('official mirrors: source'));
assert.equal(summary.length, 1);
assert.equal(summary.length, 1);
- assert.match(summary[0], /created 2, existing 0, failed 1/);
+ assert.match(summary[0], /source github, created 2, existing 0, skipped 3 \(private 0, missing 0, invalid 1, fork 1, archived 1\), failed 0/);
// Not due right away: no second listing and no second clone.
// Not due right away: no second listing and no second clone.
calls.length = 0;
calls.length = 0;
@@ -104,7 +123,8 @@ const LISTING = {
@@ -104,7 +123,8 @@ const LISTING = {
assert.equal(result.ran, true);
assert.equal(result.ran, true);
assert.equal(result.created, 0);
assert.equal(result.created, 0);
assert.equal(result.existing, 2);
assert.equal(result.existing, 2);
- assert.equal(result.failed, 1);
+ assert.equal(result.failed, 0);
+ assert.equal(result.skipped.invalid, 1);
assert.equal(clones.length, 2);
assert.equal(clones.length, 2);
assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE owner_username = 'openvibers'")), 2);
assert.equal(Number(await t.store.value("SELECT count(*)::int FROM git_repos WHERE owner_username = 'openvibers'")), 2);
const state = await job.lastRun();
const state = await job.lastRun();