openvibers/openvibe.shared Commits

CI: the security scan retries its downloads and falls back to the pinned image (#345)

OP OpenVibers committed 5 days ago · e80c595ff3 · 3 files, +136 −21

The gitleaks release asset can answer 504 for hours, which turned every
repository's security job red and skipped the scan. The install step now
caches the checksum-verified binary (keyed by version and SHA-256), downloads
with retries, and when the asset stays unavailable runs the same gitleaks
release from ghcr.io/gitleaks/gitleaks pinned by digest. The scan step calls
one command in both cases, so a scan is never skipped.

scripts/pin-bump.js now moves each repository's security.yml@<sha> to the
newest OpenVibe.Shared main commit that changed .github/workflows, carrying
it in the same PR as a package pin (or alone on a ci-shared-workflows-<sha>
branch when only the workflow pin is stale).

Changes

ddiff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
iindex f10e20a..1bfc736 100644
---- a/.github/workflows/security.yml
++++ b/.github/workflows/security.yml
@@@ -7,7 +7,9 @@ name: security
# security:
# uses: OpenVibers/OpenVibe.Shared/.github/workflows/security.yml@<sha>
#
--# secrets gitleaks (pinned binary, checksum-verified). A push scans the whole git history,
++# secrets gitleaks (pinned binary, checksum-verified, cached by version and checksum). When the
++# GitHub release asset is unavailable, the same release runs from its pinned container
++# image, so the scan never silently passes. A push scans the whole git history,
# a pull request scans only its own commits. Findings are redacted in the log: the
# repositories and their CI logs are public. Proven false positives (test fixtures,
# doc placeholders) go in a narrow [[allowlists]] entry in the repo's .gitleaks.toml
@@@ -42,17 +44,60 @@ jobs:
env:
GITLEAKS_VERSION: 8.30.1
GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb # gitleaks_8.30.1_linux_x64.tar.gz
++ GITLEAKS_IMAGE: ghcr.io/gitleaks/gitleaks:v8.30.1@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
++ - name: cache gitleaks
++ uses: actions/cache@v4
++ with:
++ path: |
++ ${{ runner.temp }}/gitleaks-bin/gitleaks
++ ${{ runner.temp }}/gitleaks-bin/gitleaks.sha256
++ key: gitleaks-${{ env.GITLEAKS_VERSION }}-${{ env.GITLEAKS_SHA256 }}-${{ runner.os }}-${{ runner.arch }}
- name: install gitleaks
run: |
-- cd "$RUNNER_TEMP"
-- curl -fsSLo gitleaks.tar.gz "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
-- echo "${GITLEAKS_SHA256} gitleaks.tar.gz" | sha256sum -c -
-- tar xzf gitleaks.tar.gz gitleaks
-- ./gitleaks version
++ set -euo pipefail
++ bin_dir="$RUNNER_TEMP/gitleaks-bin"
++ bin="$bin_dir/gitleaks"
++ mkdir -p "$bin_dir"
++ # A cached binary is trusted only while the checksum written when it was extracted still matches.
++ if [ -f "$bin" ] && [ -f "$bin_dir/gitleaks.sha256" ] && (cd "$bin_dir" && sha256sum -c --status gitleaks.sha256); then
++ chmod +x "$bin"
++ echo "gitleaks ${GITLEAKS_VERSION}: restored from cache"
++ else
++ rm -f "$bin" "$bin_dir/gitleaks.sha256"
++ if curl --fail --location --retry 4 --retry-delay 8 --retry-all-errors --connect-timeout 20 \
++ --output "$RUNNER_TEMP/gitleaks.tar.gz" \
++ "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" \
++ && echo "${GITLEAKS_SHA256} $RUNNER_TEMP/gitleaks.tar.gz" | sha256sum -c - \
++ && tar xzf "$RUNNER_TEMP/gitleaks.tar.gz" -C "$bin_dir" gitleaks \
++ && (cd "$bin_dir" && sha256sum gitleaks > gitleaks.sha256) \
++ && "$bin" version | grep -q "$GITLEAKS_VERSION"; then
++ echo "gitleaks ${GITLEAKS_VERSION}: downloaded and checksum-verified"
++ else
++ rm -f "$bin" "$bin_dir/gitleaks.sha256"
++ echo "::warning::the gitleaks release asset is unavailable; scanning with the pinned container image"
++ fi
++ fi
++ # The scan step calls one command either way: the verified binary, or the same release as a
++ # container. The workspace, git dir and RUNNER_TEMP keep their host paths so every argument
++ # (--report-path included) resolves inside the container too.
++ cat > "$RUNNER_TEMP/gitleaks" <<WRAPPER
++ #!/usr/bin/env bash
++ set -euo pipefail
++ if [ -x "$bin" ]; then exec "$bin" "\$@"; fi
++ mounts=(-v "\$PWD:\$PWD" -v "\$RUNNER_TEMP:\$RUNNER_TEMP")
++ if [ -f "\$PWD/.git" ]; then
++ gitdir=\$(git -C "\$PWD" rev-parse --absolute-git-dir)
++ common=\$(git -C "\$PWD" rev-parse --path-format=absolute --git-common-dir)
++ mounts+=(-v "\$gitdir:\$gitdir" -v "\$common:\$common")
++ fi
++ exec docker run --rm --user "\$(id -u):\$(id -g)" "\${mounts[@]}" -w "\$PWD" "$GITLEAKS_IMAGE" "\$@"
++ WRAPPER
++ chmod +x "$RUNNER_TEMP/gitleaks"
++ "$RUNNER_TEMP/gitleaks" version
- name: scan
env:
EVENT: ${{ github.event_name }}
ddiff --git a/scripts/pin-bump.js b/scripts/pin-bump.js
iindex c9d3d88..ab452f1 100644
---- a/scripts/pin-bump.js
++++ b/scripts/pin-bump.js
@@@ -3,8 +3,11 @@
/**
* Pin bump (plan T1: "a bot bumps pins on each Contracts release"): after a library release, open one pull request
* per OpenVibers repository whose package.json files pin that library below the new tag. Each PR moves every pin of
-- * the library in the repository to the tag, refreshes the lockfile that goes with it (npm or pnpm), and updates the
-- * version the repository's STATUS.json and README.md name for the library. Nothing is tested here: each PR's CI runs
++ * the library in the repository to the tag, refreshes the lockfile that goes with it (npm or pnpm), updates the
++ * version the repository's STATUS.json and README.md name for the library, and moves the repository's
++ * security.yml@<sha> pin of the reusable security workflow to the newest OpenVibe.Shared commit that changed
++ * .github/workflows (also opened alone, on a ci-shared-workflows-<sha> branch, when only that pin is stale).
++ * Nothing is tested here: each PR's CI runs
* the full suite and pin drift (scripts/pin-drift.js), and the PR is merged only when it is green.
*
* node scripts/pin-bump.js <library> <tag> [--repos A,B] [--apply] [--workdir DIR]
@@@ -26,6 +29,10 @@ const cp = require('child_process');
const LIBS = { 'openvibe-contracts': 'OpenVibe.Contracts', 'openvibe-shared': 'OpenVibe.Shared', 'openvibe-sdk': 'OpenVibe.SDK', 'openvibe-publishing': 'OpenVibe.Publishing' };
const ORG = 'OpenVibers';
const AUTHOR = ['-c', 'user.name=OpenVibers', '-c', '[email protected]'];
++// Every repository pins the reusable security workflow by commit (the workflow cannot pin itself by release).
++const SHARED = 'OpenVibe.Shared';
++const SECURITY_REF_RE = new RegExp(`OpenVibers/${SHARED.replace('.', '\\.')}/\\.github/workflows/security\\.yml@([0-9a-f]{40})`, 'g');
++const securityRef = (sha) => `OpenVibers/${SHARED}/.github/workflows/security.yml@${sha}`;
const parse = (t) => { const m = /^v?(\d+)\.(\d+)\.(\d+)$/.exec(t); return m ? m.slice(1).map(Number) : null; };
const cmp = (a, b) => a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
@@@ -66,6 +73,37 @@ function stalePins(dir, library, tag) {
return out;
}
++/**
++ * The newest OpenVibe.Shared main commit that changed .github/workflows. The reusable security workflow is pinned
++ * by commit, not by release, so this is the commit every repository's security.yml@<sha> should name.
++ */
++function latestWorkflowSha() {
++ const sha = run('gh', ['api', `repos/${ORG}/${SHARED}/commits?path=.github/workflows&sha=main&per_page=1`, '--jq', '.[0].sha']);
++ if (!/^[0-9a-f]{40}$/.test(sha)) throw new Error(`could not resolve the newest ${SHARED} workflow commit (got "${sha}")`);
++ return sha;
++}
++
++/** Tracked workflow files that pin the Shared security workflow at a commit other than `sha`, with the commits they name. */
++function staleSecurityPins(dir, sha) {
++ const files = run('git', ['-C', dir, 'ls-files', '.github/workflows']).split('\n').filter(Boolean);
++ const out = [];
++ for (const f of files) {
++ const found = [...fs.readFileSync(path.join(dir, f), 'utf8').matchAll(SECURITY_REF_RE)].map((m) => m[1]);
++ const stale = [...new Set(found)].filter((v) => v !== sha);
++ if (stale.length) out.push({ file: f, from: stale });
++ }
++ return out;
++}
++
++/** Move every tracked pin of the Shared security workflow to `sha`; the rewritten files are returned so they join the commit. */
++function updateSecurityPins(dir, sha) {
++ return staleSecurityPins(dir, sha).map(({ file }) => {
++ const p = path.join(dir, file);
++ fs.writeFileSync(p, fs.readFileSync(p, 'utf8').replace(SECURITY_REF_RE, securityRef(sha)));
++ return file;
++ });
++}
++
/** Refresh the lockfile that belongs to a bumped package.json: pnpm at the root if the repository uses it, else npm beside it. */
function refreshLock(dir, file) {
if (fs.existsSync(path.join(dir, 'pnpm-lock.yaml'))) {
@@@ -116,6 +154,12 @@ function main() {
if (!tags.split('\n').some((l) => l.endsWith(`refs/tags/${tag}`))) throw new Error(`${tag} is not published in ${ORG}/${LIBS[library]}`);
const work = opts.workdir || fs.mkdtempSync(path.join(os.tmpdir(), 'pin-bump-'));
const branch = `pin-bump/${library}-${tag}`;
++ let sharedSha = null;
++ try {
++ sharedSha = latestWorkflowSha();
++ } catch (err) {
++ console.error(`pin-bump: ${err.message}; the Shared security workflow pins are left alone`);
++ }
let failed = 0;
const summary = [];
for (const repo of repositories(opts.repos)) {
@@@ -125,10 +169,14 @@ function main() {
// gh clones with the same authentication it opens the PRs with, so private repositories work too.
run('gh', ['repo', 'clone', `${ORG}/${repo}`, dir, '--', '-q', '--depth', '1']);
const stale = stalePins(dir, library, tag);
-- if (!stale.length) { summary.push(`${repo}: current`); continue; }
++ const staleSecurity = sharedSha ? staleSecurityPins(dir, sharedSha) : [];
++ if (!stale.length && !staleSecurity.length) { summary.push(`${repo}: current`); continue; }
++ // A repository with only a stale workflow pin gets its own small CI branch, so it never collides with a pin PR.
++ const target = stale.length ? branch : `ci-shared-workflows-${sharedSha.slice(0, 12)}`;
if (opts.apply) {
-- const open = run('gh', ['pr', 'list', '-R', `${ORG}/${repo}`, '--head', branch, '--state', 'open', '--json', 'number']);
-- if (JSON.parse(open).length) { summary.push(`${repo}: an open PR from ${branch} already exists`); continue; }
++ const open = JSON.parse(run('gh', ['pr', 'list', '-R', `${ORG}/${repo}`, '--state', 'open', '--json', 'number,headRefName']));
++ const existing = open.find((p) => p.headRefName === target || (!stale.length && p.headRefName.startsWith('ci-shared-workflows-')));
++ if (existing) { summary.push(`${repo}: an open PR (#${existing.number}) already carries this`); continue; }
}
const changed = new Set();
for (const { file } of stale) {
@@@ -138,21 +186,25 @@ function main() {
const lock = refreshLock(dir, file);
if (lock) changed.add(lock);
}
++ if (staleSecurity.length) updateSecurityPins(dir, sharedSha).forEach((f) => changed.add(f));
const from = [...new Set(stale.flatMap((s) => s.from))].sort((a, b) => cmp(parse(a), parse(b)));
-- updateVersionLines(dir, library, from, tag).forEach((f) => changed.add(f));
-- const title = `${library} ${tag} (was ${from.join(', ')})`;
++ if (stale.length) updateVersionLines(dir, library, from, tag).forEach((f) => changed.add(f));
++ const title = stale.length
++ ? `${library} ${tag} (was ${from.join(', ')})`
++ : `CI: pin the OpenVibe.Shared security workflow at ${sharedSha.slice(0, 12)}`;
if (!opts.apply) { summary.push(`${repo}: would open "${title}" changing ${[...changed].join(', ')}`); continue; }
-- run('git', ['-C', dir, 'checkout', '-q', '-b', branch]);
++ run('git', ['-C', dir, 'checkout', '-q', '-b', target]);
run('git', ['-C', dir, 'add', '--', ...changed]);
run('git', ['-C', dir, ...AUTHOR, 'commit', '-q', '-m', `${title}\n\nOpened by scripts/pin-bump.js in OpenVibers/OpenVibe.Shared after the ${tag} release.`]);
-- run('git', ['-C', dir, 'push', '-q', 'origin', branch]);
++ run('git', ['-C', dir, 'push', '-q', 'origin', target]);
const body = [
-- `Moves every \`${library}\` pin in this repository to ${tag} (from ${from.join(', ')}), refreshes the lockfile and the`,
-- 'version lines in STATUS.json / README.md. CI runs the full suite and pin drift; merge when it is green.',
++ ...(stale.length ? [`Moves every \`${library}\` pin in this repository to ${tag} (from ${from.join(', ')}), refreshes the lockfile and the`, 'version lines in STATUS.json / README.md.'] : []),
++ ...(staleSecurity.length ? [`Moves the OpenVibe.Shared security workflow pin to ${sharedSha}, the newest Shared main commit that changed \`.github/workflows\`.`] : []),
++ 'CI runs the full suite and pin drift; merge when it is green.',
'', 'Files: ' + [...changed].map((f) => `\`${f}\``).join(', '),
'', `Opened by \`scripts/pin-bump.js\` (OpenVibers/OpenVibe.Shared) after the ${LIBS[library]} ${tag} release.`,
].join('\n');
-- const url = run('gh', ['pr', 'create', '-R', `${ORG}/${repo}`, '--head', branch, '--title', title, '--body', body]);
++ const url = run('gh', ['pr', 'create', '-R', `${ORG}/${repo}`, '--head', target, '--title', title, '--body', body]);
summary.push(`${repo}: ${url}`);
} catch (err) {
failed++;
@@@ -169,4 +221,4 @@ if (require.main === module) {
try { main(); } catch (err) { console.error(`pin-bump: ${err.message}`); process.exitCode = 1; }
}
--module.exports = { stalePins, updateVersionLines, pinUrl, parseArgs };
++module.exports = { stalePins, staleSecurityPins, updateSecurityPins, updateVersionLines, pinUrl, parseArgs };
ddiff --git a/test/pin-bump.test.js b/test/pin-bump.test.js
iindex 0bd7a12..22cc542 100644
---- a/test/pin-bump.test.js
++++ b/test/pin-bump.test.js
@@@ -6,7 +6,7 @@ const fs = require('fs');
const os = require('os');
const path = require('path');
const cp = require('child_process');
--const { stalePins, updateVersionLines, pinUrl, parseArgs } = require('../scripts/pin-bump');
++const { stalePins, staleSecurityPins, updateSecurityPins, updateVersionLines, pinUrl, parseArgs } = require('../scripts/pin-bump');
const url = (repo, v) => `https://codeload.github.com/OpenVibers/${repo}/tar.gz/refs/tags/${v}`;
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pin-bump-'));
@@@ -17,8 +17,17 @@ write('apps/b/package.json', { dependencies: { 'openvibe-contracts': url('OpenVi
write('node_modules/x/package.json', { dependencies: { 'openvibe-contracts': url('OpenVibe.Contracts', 'v0.1.0') } });
write('STATUS.json', { repository: 'OpenVibers/Example', contracts: 'openvibe-contracts v0.110.0', sdk: 'openvibe-sdk v0.26.0', updated: '2026-01-01' });
write('README.md', '- `openvibe-contracts` v0.110.0, `openvibe-sdk` v0.26.0\n\nThe manifest was first released in openvibe-contracts v0.8.0.\n');
++const oldSecurity = 'b'.repeat(40);
++const targetSecurity = 'c'.repeat(40);
++write('.github/workflows/ci.yml', [
++ 'jobs:',
++ ' test:',
++ ` uses: OpenVibers/OpenVibe.Shared/.github/workflows/test.yml@${'a'.repeat(40)}`,
++ ' security:',
++ ` uses: OpenVibers/OpenVibe.Shared/.github/workflows/security.yml@${oldSecurity}`,
++].join('\n') + '\n');
cp.execFileSync('git', ['init', '-q', dir]);
--cp.execFileSync('git', ['-C', dir, 'add', 'package.json', 'apps', 'STATUS.json', 'README.md']);
++cp.execFileSync('git', ['-C', dir, 'add', 'package.json', 'apps', 'STATUS.json', 'README.md', '.github']);
// Only tracked package.json files count, and only pins below the tag; a current pin and a range are left alone.
const stale = stalePins(dir, 'openvibe-contracts', 'v0.112.0').map((s) => s.file).sort();
@@@ -27,6 +36,15 @@ assert.deepStrictEqual(stalePins(dir, 'openvibe-contracts', 'v0.112.0').find((x)
assert.deepStrictEqual(stalePins(dir, 'openvibe-sdk', 'v0.26.0'), [], 'already at the tag');
assert.deepStrictEqual(stalePins(dir, 'openvibe-shared', 'v2.13.0'), [], 'a peer range is not a pin');
++// The reusable security workflow is pinned by commit: a stale pin is found and moved, a current one is left alone,
++// and another Shared workflow's pin is not this bot's business.
++assert.deepStrictEqual(staleSecurityPins(dir, targetSecurity), [{ file: '.github/workflows/ci.yml', from: [oldSecurity] }]);
++assert.deepStrictEqual(staleSecurityPins(dir, oldSecurity), [], 'already at the target');
++assert.deepStrictEqual(updateSecurityPins(dir, targetSecurity), ['.github/workflows/ci.yml']);
++const ci = fs.readFileSync(path.join(dir, '.github/workflows/ci.yml'), 'utf8');
++assert.ok(ci.includes(`security.yml@${targetSecurity}`), 'the security pin moved');
++assert.ok(ci.includes(`test.yml@${'a'.repeat(40)}`), 'the test workflow pin stays as it was');
++
// STATUS.json follows the pins once package.json moves.
const pkg = path.join(dir, 'package.json');
fs.writeFileSync(pkg, fs.readFileSync(pkg, 'utf8').replace(url('OpenVibe.Contracts', 'v0.110.0'), pinUrl('openvibe-contracts', 'v0.112.0')));