openvibers/openvibe.blog Commits
changelog: every site's updates log — commit authors, a page cursor (before), a one-time import of each site's last CHANGELOG_HISTORY commits (never pending, never in a post), the recent patch notes posts and a per-site summary in the feed; polls every 5 minutes
Changes
@@@ -149,6 +149,39 @@ Other tables in the same database:| `/auth/*` | sign-in; the same session layer as OpenVibe.Community || `/api/health`, `/api/ready`, `/release.json`, `/metrics` | health, readiness and metrics; `/metrics` is loopback only |++### The network changelog and patch notes++++`server/changelog.js` keeps what shipped on every OpenVibe site. Every 5 minutes it reads each service's++repository and current release from OpenVibe.Network's registry and takes the new commits from GitHub's++compare API. The first time it sees a site it also imports that site's last `CHANGELOG_HISTORY` commits++(60 by default). Those history entries are shown in the feed but never go into a post.++++A **Patch notes** post on the official blog (series "Patch notes", imported authorship, grouped by++site) is published when either of these happens:++++- `CHANGELOG_BATCH_SIZE` changes have gathered.++- A large release (`CHANGELOG_MAJOR_LINES`, a version bump or `[major]`) has been quiet for 30 minutes.++++There is always at least a 6-hour gap between posts. An AI-written dev-blog version is filed as a draft++for review.++++`GET /api/v1/changelog?service=&limit=&before=` is public (CORS `*`, cached for 60 s). It returns:++++| Field | Contents |++|---|---|++| `entries` | Each entry has its site, sha, subject, author, time and commit link |++| `next` | The cursor for the next page |++| `latest_post` | The newest patch notes post |++| `posts` | The recent patch notes posts |++| `sites` | Only without `service`: each site with its count and latest time |++++OpenVibe.Network proxies this feed at `openvibe.network/api/v1/changelog`. Every site shows it through++`openvibe-shared/shipped.js`:++++- the "shipped X ago" line in the shared footer;++- the home pill and the recent list;++- the `/updates` log.++### Discoverability (roadmap §32)- Every page gets its robots meta, canonical and `X-Robots-Tag` from the@@@ -63,7 +63,25 @@ function ensureSchema(db) {ai_draft_id TEXT,created_at INTEGER NOT NULL);++ CREATE TABLE IF NOT EXISTS changelog_history (++ service TEXT PRIMARY KEY,++ imported INTEGER NOT NULL,++ imported_at INTEGER NOT NULL++ );++ CREATE INDEX IF NOT EXISTS idx_changelog_service ON changelog_entries(service, deployed_at);`);++ // Added after the first release: who wrote each change (the commit's author name).++ const cols = db.prepare('PRAGMA table_info(changelog_entries)').all().map((c) => c.name);++ if (!cols.includes('author')) db.exec('ALTER TABLE changelog_entries ADD COLUMN author TEXT');++}++++/** The feed's page cursor: "<deployed_at>|<id>" of the last entry shown. */++function encodeCursor(e) { return Buffer.from(`${e.deployed_at}|${e.id}`).toString('base64url'); }++function decodeCursor(s) {++ try {++ const [at, id] = Buffer.from(String(s || ''), 'base64url').toString('utf8').split('|');++ return at && !Number.isNaN(Date.parse(at)) && /^\d+$/.test(id) ? { at, id: Number(id) } : null;++ } catch { return null; }}/** The first line of a commit message, cut at a word near `n` characters. */@@@ -87,8 +105,13 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchImstate: db.prepare('SELECT * FROM changelog_state WHERE service = ?'),putState: db.prepare(`INSERT INTO changelog_state (service, release, head, checked_at) VALUES (@service, @release, @head, @at)ON CONFLICT(service) DO UPDATE SET release = excluded.release, head = excluded.head, checked_at = excluded.checked_at`),-- insert: db.prepare(`INSERT OR IGNORE INTO changelog_entries (service, repo, sha, subject, committed_at, deployed_at, release, release_lines, major)-- VALUES (@service, @repo, @sha, @subject, @committed_at, @deployed_at, @release, @release_lines, @major)`),++ insert: db.prepare(`INSERT OR IGNORE INTO changelog_entries (service, repo, sha, subject, author, committed_at, deployed_at, release, release_lines, major, post_id)++ VALUES (@service, @repo, @sha, @subject, @author, @committed_at, @deployed_at, @release, @release_lines, @major, @post_id)`),++ fillAuthor: db.prepare('UPDATE changelog_entries SET author = ? WHERE service = ? AND sha = ? AND author IS NULL'),++ history: db.prepare('SELECT * FROM changelog_history WHERE service = ?'),++ putHistory: db.prepare('INSERT OR REPLACE INTO changelog_history (service, imported, imported_at) VALUES (?, ?, ?)'),++ recentPosts: db.prepare('SELECT * FROM changelog_posts ORDER BY created_at DESC LIMIT ?'),++ sites: db.prepare(`SELECT service, COUNT(*) AS entries, MAX(deployed_at) AS latest_at FROM changelog_entries GROUP BY service ORDER BY latest_at DESC`),pending: db.prepare('SELECT * FROM changelog_entries WHERE post_id IS NULL ORDER BY deployed_at, committed_at, id'),mark: db.prepare('UPDATE changelog_entries SET post_id = ? WHERE id = ? AND post_id IS NULL'),lastPost: db.prepare('SELECT * FROM changelog_posts ORDER BY created_at DESC LIMIT 1'),@@@ -122,21 +145,46 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchImreturn out;}-- function addEntries(svc, commits, { releaseLines = 0, major = false } = {}) {++ const authorOf = (cm) => clean((cm.commit && cm.commit.author && cm.commit.author.name) || (cm.author && cm.author.login) || '', 80) || null;++++ // history: the commits a site shipped before the changelog watched it, so its updates page is not++ // empty on day one. They carry post_id 'history' (never pending, never in a patch notes post) and++ // their commit time stands in for the deploy time nobody recorded.++ function addEntries(svc, commits, { releaseLines = 0, major = false, history = false } = {}) {const deployedAt = svc.releasedAt || new Date(nowMs()).toISOString();let n = 0;for (const cm of commits) {const subject = subjectOf(cm.commit && cm.commit.message);if (!subject || isNoise(subject)) continue;const flagged = major || isVersionBump(subject) || /\[major\]/i.test((cm.commit && cm.commit.message) || '');-- n += q.insert.run({-- service: svc.id, repo: svc.repo, sha: cm.sha, subject, committed_at: (cm.commit && cm.commit.author && cm.commit.author.date) || null,-- deployed_at: deployedAt, release: svc.release, release_lines: releaseLines, major: flagged ? 1 : 0,++ const committedAt = (cm.commit && cm.commit.author && cm.commit.author.date) || null;++ if (history && !committedAt) continue;++ const author = authorOf(cm);++ const added = q.insert.run({++ service: svc.id, repo: svc.repo, sha: cm.sha, subject, author, committed_at: committedAt,++ deployed_at: history ? new Date(committedAt).toISOString() : deployedAt, release: svc.release, release_lines: releaseLines, major: flagged ? 1 : 0,++ post_id: history ? 'history' : null,}).changes;++ if (!added && author) q.fillAuthor.run(author, svc.id, cm.sha);++ n += added;}return n;}++ /** Once per service: its last c.history commits on the deployed release. */++ async function importHistory(svc) {++ if (!(c.history > 0) || q.history.get(svc.id)) return 0;++ let list;++ try { list = await gh(`/repos/${svc.repo}/commits?sha=${svc.release}&per_page=${Math.min(c.history, 100)}`); } catch (err) {++ // A repository GitHub will not list (private, empty, gone) is not asked again; anything else is.++ if ([404, 409, 422].includes(err.status)) { q.putHistory.run(svc.id, 0, nowMs()); return 0; }++ throw err;++ }++ const n = addEntries(svc, Array.isArray(list) ? list : [], { history: true });++ q.putHistory.run(svc.id, n, nowMs());++ return n;++ }++async function collectOne(svc) {const st = q.state.get(svc.id);if (st && st.release === svc.release) return 0;@@@ -160,8 +208,13 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchImasync function collect() {let added = 0;++ let histories = 0;for (const svc of await registry()) {-- try { added += await collectOne(svc); } catch (err) {++ try {++ added += await collectOne(svc);++ // A few history imports per run keep an unauthenticated GitHub budget intact.++ if (histories < 6 && !q.history.get(svc.id)) { histories++; await importHistory(svc); }++ } catch (err) {stats.lastError = `${svc.id}: ${err.message}`;if (err.status === 403 || err.status === 429) break; // GitHub's rate limit: try again next run}@@@ -271,23 +324,35 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchIm}function stop() { if (timer) clearInterval(timer); timer = null; }-- /** The public feed: newest first. */-- function entries({ service = null, limit = 20 } = {}) {++ /** The public feed, newest first: { entries, next } (next = the cursor for the page after, or null). */++ function page({ service = null, limit = 20, before = null } = {}) {const n = Math.min(Math.max(parseInt(limit, 10) || 20, 1), 100);-- const rows = service-- ? db.prepare('SELECT * FROM changelog_entries WHERE service = ? ORDER BY deployed_at DESC, id DESC LIMIT ?').all(String(service), n)-- : db.prepare('SELECT * FROM changelog_entries ORDER BY deployed_at DESC, id DESC LIMIT ?').all(n);-- return rows.map((e) => ({-- service: e.service, sha: e.sha, short: e.sha.slice(0, 7), subject: e.subject, committed_at: e.committed_at, deployed_at: e.deployed_at,-- major: Boolean(e.major), url: `https://github.com/${e.repo}/commit/${e.sha}`, post_id: e.post_id || null,-- }));++ const cur = decodeCursor(before);++ const where = [];++ const args = [];++ if (service) { where.push('service = ?'); args.push(String(service)); }++ if (cur) { where.push('(deployed_at < ? OR (deployed_at = ? AND id < ?))'); args.push(cur.at, cur.at, cur.id); }++ const rows = db.prepare(`SELECT * FROM changelog_entries ${where.length ? `WHERE ${where.join(' AND ')}` : ''} ORDER BY deployed_at DESC, id DESC LIMIT ?`).all(...args, n + 1);++ const more = rows.length > n;++ const shown = rows.slice(0, n);++ return {++ entries: shown.map((e) => ({++ service: e.service, sha: e.sha, short: e.sha.slice(0, 7), subject: e.subject, author: e.author || null, committed_at: e.committed_at, deployed_at: e.deployed_at,++ major: Boolean(e.major), url: `https://github.com/${e.repo}/commit/${e.sha}`, post_id: e.post_id && e.post_id !== 'history' ? e.post_id : null,++ })),++ next: more && shown.length ? encodeCursor(shown[shown.length - 1]) : null,++ };}++ function entries(opts = {}) { return page(opts).entries; }function latestPost() { return q.lastPost.get() || null; }++ function recentPosts(limit = 5) { return q.recentPosts.all(Math.min(Math.max(parseInt(limit, 10) || 5, 1), 20)); }++ /** Every site with entries: [{ service, entries, latest_at }], most recently shipped first. */++ function sites() { return q.sites.all(); }-- return { tick, collect, due, render, publish, start, stop, entries, latestPost, stats: () => ({ enabled: c.enabled, ...stats, pending: q.pending.all().length }) };++ return { tick, collect, due, render, publish, start, stop, entries, page, latestPost, recentPosts, sites, importHistory, stats: () => ({ enabled: c.enabled, ...stats, pending: q.pending.all().length }) };}/** A stable id for a batch (tests). */function batchKey(list) { return crypto.createHash('sha256').update(list.map((e) => `${e.service}:${e.sha}`).join(',')).digest('hex').slice(0, 16); }--module.exports = { createChangelog, subjectOf, ensureSchema, batchKey };++module.exports = { createChangelog, subjectOf, ensureSchema, batchKey, encodeCursor, decodeCursor };@@@ -45,7 +45,8 @@ function load(env = process.env) {quietMs: Math.max(0, parseInt(env.CHANGELOG_QUIET_MS, 10) || 30 * 60 * 1000),maxAgeMs: Math.max(0, parseInt(env.CHANGELOG_MAX_AGE_MS, 10) || 7 * 24 * 3600 * 1000),minGapMs: Math.max(0, parseInt(env.CHANGELOG_MIN_GAP_MS, 10) || 6 * 3600 * 1000),-- intervalMs: Math.max(60 * 1000, parseInt(env.CHANGELOG_INTERVAL_MS, 10) || 10 * 60 * 1000),++ intervalMs: Math.max(60 * 1000, parseInt(env.CHANGELOG_INTERVAL_MS, 10) || 5 * 60 * 1000),++ history: Math.max(0, Math.min(100, parseInt(env.CHANGELOG_HISTORY ?? '60', 10) || 0)),firstDelayMs: 60 * 1000,since: env.CHANGELOG_SINCE && !Number.isNaN(Date.parse(env.CHANGELOG_SINCE)) ? new Date(env.CHANGELOG_SINCE).toISOString() : null,aiDraft: env.CHANGELOG_AI_DRAFT !== 'off',@@@ -188,14 +188,18 @@ function createApi(ctx) {res.set('Access-Control-Allow-Origin', '*');res.set('Cache-Control', 'public, max-age=60');const service = /^[a-z][a-z0-9-]{1,39}$/.test(String(req.query.service || '')) ? String(req.query.service) : null;-- const latest = ctx.changelog.latestPost();-- let post = null;-- if (latest) {-- const p = posts.get(latest.post_id);++ const postOf = (row) => {++ const p = row && posts.get(row.post_id);const blog = p ? blogs.get(p.blog_id) : null;-- if (p && blog && p.state === 'published') post = { id: p.id, title: (posts.head(p) || { fields: {} }).fields.title || null, url: publication.abs(publication.postPath(blog, p)), published_at: p.published_at || null, entries: latest.entries };-- }-- return { service, entries: ctx.changelog.entries({ service, limit: req.query.limit }), latest_post: post };++ if (!p || !blog || p.state !== 'published') return null;++ return { id: p.id, title: (posts.head(p) || { fields: {} }).fields.title || null, url: publication.abs(publication.postPath(blog, p)), published_at: p.published_at || null, entries: row.entries };++ };++ const page = ctx.changelog.page({ service, limit: req.query.limit, before: req.query.before });++ const recent = ctx.changelog.recentPosts(5).map(postOf).filter(Boolean);++ return {++ service, entries: page.entries, next: page.next, latest_post: recent[0] || null, posts: recent,++ ...(service ? {} : { sites: ctx.changelog.sites() }),++ };}));// Draft with AI: OpenVibe.AI's blog.draft_post writes a draft (AI-authored, noindex until reviewed).@@@ -14,6 +14,7 @@ const H = (n) => n.toString(16).padStart(40, 'a');const releases = { live: 'aaaaaaa00001', tools: 'bbbbbbb00001' };const compares = {};const ghCalls = [];++ const history = {};const fetchImpl = async (url) => {const u = new URL(url);const json = (body, status = 200) => ({ ok: status < 300, status, json: async () => body });@@@ -27,11 +28,13 @@ const H = (n) => n.toString(16).padStart(40, 'a');ghCalls.push(u.pathname);const m = u.pathname.match(/^\/repos\/(.+)\/compare\/(.+)\.\.\.(.+)$/);if (m && compares[m[3]]) return json(compares[m[3]]);++ const h = u.pathname.match(/^\/repos\/(.+)\/commits$/);++ if (h && history[h[1]]) return json(history[h[1]]);return json({ message: 'Not Found' }, 404);}return json({}, 404);};-- const config = { ...t.ctx.config, changelog: { ...t.ctx.config.changelog, enabled: true, batchSize: 5, majorLines: 500, quietMs: 30 * 60 * 1000, maxAgeMs: 7 * 24 * 3600 * 1000, minGapMs: 6 * 3600 * 1000, since: null, aiDraft: false, blogHandle: 'openvibe' } };++ const config = { ...t.ctx.config, changelog: { ...t.ctx.config.changelog, enabled: true, batchSize: 5, majorLines: 500, quietMs: 30 * 60 * 1000, maxAgeMs: 7 * 24 * 3600 * 1000, minGapMs: 6 * 3600 * 1000, since: null, aiDraft: false, blogHandle: 'openvibe', history: 0 } };const cl = createChangelog({ config, store: t.ctx.store, blogs: t.ctx.blogs, posts: t.ctx.posts, aiDrafts: null, fetchImpl, log: { log() {}, warn() {} } });const commit = (i, msg) => ({ sha: H(i), commit: { message: msg, author: { date: new Date(t.clock.now()).toISOString() } } });@@@ -100,6 +103,32 @@ const H = (n) => n.toString(16).padStart(40, 'a');assert.strictEqual(cl.entries({ service: 'tools' })[0].major, true);});++ await check('history: each site\'s earlier commits once, never pending, with authors; the feed pages by cursor', async () => {++ history['OpenVibers/OpenVibe.Live'] = [20, 21, 22].map((i) => ({ sha: H(i), commit: { message: `Earlier change ${i}\n\nbody`, author: { name: 'OpenVibers', date: new Date(t.clock.now() - (30 - i) * 86400000).toISOString() } } }))++ .concat([{ sha: H(5), commit: { message: 'Small change 5', author: { name: 'OpenVibers', date: new Date(t.clock.now()).toISOString() } } }]);++ const hc = createChangelog({ config: { ...config, changelog: { ...config.changelog, history: 50 } }, store: t.ctx.store, blogs: t.ctx.blogs, posts: t.ctx.posts, fetchImpl, log: { log() {}, warn() {} } });++ const before = hc.stats().pending;++ await hc.collect();++ assert.strictEqual(hc.stats().pending, before, 'history never waits for a patch notes post');++ const all = hc.entries({ service: 'live', limit: 100 });++ assert.ok(all.some((e) => e.subject === 'Earlier change 20' && e.author === 'OpenVibers' && e.post_id === null));++ assert.strictEqual(all.find((e) => e.sha === H(5)).author, 'OpenVibers', 'a known commit gains its author');++ const calls = ghCalls.length;++ await hc.collect();++ assert.strictEqual(ghCalls.length, calls, 'imported once (a repository GitHub will not list is not asked again either)');++ const p1 = hc.page({ service: 'live', limit: 2 });++ assert.strictEqual(p1.entries.length, 2);++ assert.ok(p1.next);++ const p2 = hc.page({ service: 'live', limit: 2, before: p1.next });++ assert.ok(p2.entries.length >= 1 && !p2.entries.some((e) => p1.entries.some((x) => x.sha === e.sha)), 'the next page continues without repeats');++ const r = (await t.get(`/api/v1/changelog?limit=2&before=${encodeURIComponent(p1.next)}`)).json();++ assert.ok(Array.isArray(r.sites) && r.sites.some((x) => x.service === 'live'), 'network-wide answers list the sites');++ assert.ok(Array.isArray(r.posts) && r.posts.length >= 1);++ const one = (await t.get('/api/v1/changelog?service=live&limit=1')).json();++ assert.strictEqual(one.sites, undefined);++ assert.ok(one.next);++ });++await t.close();done();})();
@@ -149,6 +149,39 @@ Other tables in the same database:
@@ -149,6 +149,39 @@ Other tables in the same database:
| `/auth/*` | sign-in; the same session layer as OpenVibe.Community |
| `/auth/*` | sign-in; the same session layer as OpenVibe.Community |
| `/api/health`, `/api/ready`, `/release.json`, `/metrics` | health, readiness and metrics; `/metrics` is loopback only |
| `/api/health`, `/api/ready`, `/release.json`, `/metrics` | health, readiness and metrics; `/metrics` is loopback only |
+### The network changelog and patch notes
+
+`server/changelog.js` keeps what shipped on every OpenVibe site. Every 5 minutes it reads each service's
+repository and current release from OpenVibe.Network's registry and takes the new commits from GitHub's
+compare API. The first time it sees a site it also imports that site's last `CHANGELOG_HISTORY` commits
+(60 by default). Those history entries are shown in the feed but never go into a post.
+
+A **Patch notes** post on the official blog (series "Patch notes", imported authorship, grouped by
+site) is published when either of these happens:
+
+- `CHANGELOG_BATCH_SIZE` changes have gathered.
+- A large release (`CHANGELOG_MAJOR_LINES`, a version bump or `[major]`) has been quiet for 30 minutes.
+
+There is always at least a 6-hour gap between posts. An AI-written dev-blog version is filed as a draft
+for review.
+
+`GET /api/v1/changelog?service=&limit=&before=` is public (CORS `*`, cached for 60 s). It returns:
+
+| Field | Contents |
+|---|---|
+| `entries` | Each entry has its site, sha, subject, author, time and commit link |
+| `next` | The cursor for the next page |
+| `latest_post` | The newest patch notes post |
+| `posts` | The recent patch notes posts |
+| `sites` | Only without `service`: each site with its count and latest time |
+
+OpenVibe.Network proxies this feed at `openvibe.network/api/v1/changelog`. Every site shows it through
+`openvibe-shared/shipped.js`:
+
+- the "shipped X ago" line in the shared footer;
+- the home pill and the recent list;
+- the `/updates` log.
+
### Discoverability (roadmap §32)
### Discoverability (roadmap §32)
- Every page gets its robots meta, canonical and `X-Robots-Tag` from the
- Every page gets its robots meta, canonical and `X-Robots-Tag` from the
@@ -63,7 +63,25 @@ function ensureSchema(db) {
@@ -63,7 +63,25 @@ function ensureSchema(db) {
ai_draft_id TEXT,
ai_draft_id TEXT,
created_at INTEGER NOT NULL
created_at INTEGER NOT NULL
);
);
+ CREATE TABLE IF NOT EXISTS changelog_history (
+ service TEXT PRIMARY KEY,
+ imported INTEGER NOT NULL,
+ imported_at INTEGER NOT NULL
+ );
+ CREATE INDEX IF NOT EXISTS idx_changelog_service ON changelog_entries(service, deployed_at);
`);
`);
+ // Added after the first release: who wrote each change (the commit's author name).
+ const cols = db.prepare('PRAGMA table_info(changelog_entries)').all().map((c) => c.name);
+ if (!cols.includes('author')) db.exec('ALTER TABLE changelog_entries ADD COLUMN author TEXT');
+}
+
+/** The feed's page cursor: "<deployed_at>|<id>" of the last entry shown. */
+function encodeCursor(e) { return Buffer.from(`${e.deployed_at}|${e.id}`).toString('base64url'); }
+function decodeCursor(s) {
+ try {
+ const [at, id] = Buffer.from(String(s || ''), 'base64url').toString('utf8').split('|');
+ return at && !Number.isNaN(Date.parse(at)) && /^\d+$/.test(id) ? { at, id: Number(id) } : null;
+ } catch { return null; }
}
}
/** The first line of a commit message, cut at a word near `n` characters. */
/** The first line of a commit message, cut at a word near `n` characters. */
@@ -87,8 +105,13 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchIm
@@ -87,8 +105,13 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchIm
state: db.prepare('SELECT * FROM changelog_state WHERE service = ?'),
state: db.prepare('SELECT * FROM changelog_state WHERE service = ?'),
putState: db.prepare(`INSERT INTO changelog_state (service, release, head, checked_at) VALUES (@service, @release, @head, @at)
putState: db.prepare(`INSERT INTO changelog_state (service, release, head, checked_at) VALUES (@service, @release, @head, @at)
ON CONFLICT(service) DO UPDATE SET release = excluded.release, head = excluded.head, checked_at = excluded.checked_at`),
ON CONFLICT(service) DO UPDATE SET release = excluded.release, head = excluded.head, checked_at = excluded.checked_at`),
- insert: db.prepare(`INSERT OR IGNORE INTO changelog_entries (service, repo, sha, subject, committed_at, deployed_at, release, release_lines, major)
- VALUES (@service, @repo, @sha, @subject, @committed_at, @deployed_at, @release, @release_lines, @major)`),
+ insert: db.prepare(`INSERT OR IGNORE INTO changelog_entries (service, repo, sha, subject, author, committed_at, deployed_at, release, release_lines, major, post_id)
+ VALUES (@service, @repo, @sha, @subject, @author, @committed_at, @deployed_at, @release, @release_lines, @major, @post_id)`),
+ fillAuthor: db.prepare('UPDATE changelog_entries SET author = ? WHERE service = ? AND sha = ? AND author IS NULL'),
+ history: db.prepare('SELECT * FROM changelog_history WHERE service = ?'),
+ putHistory: db.prepare('INSERT OR REPLACE INTO changelog_history (service, imported, imported_at) VALUES (?, ?, ?)'),
+ recentPosts: db.prepare('SELECT * FROM changelog_posts ORDER BY created_at DESC LIMIT ?'),
+ sites: db.prepare(`SELECT service, COUNT(*) AS entries, MAX(deployed_at) AS latest_at FROM changelog_entries GROUP BY service ORDER BY latest_at DESC`),
pending: db.prepare('SELECT * FROM changelog_entries WHERE post_id IS NULL ORDER BY deployed_at, committed_at, id'),
pending: db.prepare('SELECT * FROM changelog_entries WHERE post_id IS NULL ORDER BY deployed_at, committed_at, id'),
mark: db.prepare('UPDATE changelog_entries SET post_id = ? WHERE id = ? AND post_id IS NULL'),
mark: db.prepare('UPDATE changelog_entries SET post_id = ? WHERE id = ? AND post_id IS NULL'),
lastPost: db.prepare('SELECT * FROM changelog_posts ORDER BY created_at DESC LIMIT 1'),
lastPost: db.prepare('SELECT * FROM changelog_posts ORDER BY created_at DESC LIMIT 1'),
@@ -122,21 +145,46 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchIm
@@ -122,21 +145,46 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchIm
return out;
return out;
}
}
- function addEntries(svc, commits, { releaseLines = 0, major = false } = {}) {
+ const authorOf = (cm) => clean((cm.commit && cm.commit.author && cm.commit.author.name) || (cm.author && cm.author.login) || '', 80) || null;
+
+ // history: the commits a site shipped before the changelog watched it, so its updates page is not
+ // empty on day one. They carry post_id 'history' (never pending, never in a patch notes post) and
+ // their commit time stands in for the deploy time nobody recorded.
+ function addEntries(svc, commits, { releaseLines = 0, major = false, history = false } = {}) {
const deployedAt = svc.releasedAt || new Date(nowMs()).toISOString();
const deployedAt = svc.releasedAt || new Date(nowMs()).toISOString();
let n = 0;
let n = 0;
for (const cm of commits) {
for (const cm of commits) {
const subject = subjectOf(cm.commit && cm.commit.message);
const subject = subjectOf(cm.commit && cm.commit.message);
if (!subject || isNoise(subject)) continue;
if (!subject || isNoise(subject)) continue;
const flagged = major || isVersionBump(subject) || /\[major\]/i.test((cm.commit && cm.commit.message) || '');
const flagged = major || isVersionBump(subject) || /\[major\]/i.test((cm.commit && cm.commit.message) || '');
- n += q.insert.run({
- service: svc.id, repo: svc.repo, sha: cm.sha, subject, committed_at: (cm.commit && cm.commit.author && cm.commit.author.date) || null,
- deployed_at: deployedAt, release: svc.release, release_lines: releaseLines, major: flagged ? 1 : 0,
+ const committedAt = (cm.commit && cm.commit.author && cm.commit.author.date) || null;
+ if (history && !committedAt) continue;
+ const author = authorOf(cm);
+ const added = q.insert.run({
+ service: svc.id, repo: svc.repo, sha: cm.sha, subject, author, committed_at: committedAt,
+ deployed_at: history ? new Date(committedAt).toISOString() : deployedAt, release: svc.release, release_lines: releaseLines, major: flagged ? 1 : 0,
+ post_id: history ? 'history' : null,
}).changes;
}).changes;
+ if (!added && author) q.fillAuthor.run(author, svc.id, cm.sha);
+ n += added;
}
}
return n;
return n;
}
}
+ /** Once per service: its last c.history commits on the deployed release. */
+ async function importHistory(svc) {
+ if (!(c.history > 0) || q.history.get(svc.id)) return 0;
+ let list;
+ try { list = await gh(`/repos/${svc.repo}/commits?sha=${svc.release}&per_page=${Math.min(c.history, 100)}`); } catch (err) {
+ // A repository GitHub will not list (private, empty, gone) is not asked again; anything else is.
+ if ([404, 409, 422].includes(err.status)) { q.putHistory.run(svc.id, 0, nowMs()); return 0; }
+ throw err;
+ }
+ const n = addEntries(svc, Array.isArray(list) ? list : [], { history: true });
+ q.putHistory.run(svc.id, n, nowMs());
+ return n;
+ }
+
async function collectOne(svc) {
async function collectOne(svc) {
const st = q.state.get(svc.id);
const st = q.state.get(svc.id);
if (st && st.release === svc.release) return 0;
if (st && st.release === svc.release) return 0;
@@ -160,8 +208,13 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchIm
@@ -160,8 +208,13 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchIm
async function collect() {
async function collect() {
let added = 0;
let added = 0;
+ let histories = 0;
for (const svc of await registry()) {
for (const svc of await registry()) {
- try { added += await collectOne(svc); } catch (err) {
+ try {
+ added += await collectOne(svc);
+ // A few history imports per run keep an unauthenticated GitHub budget intact.
+ if (histories < 6 && !q.history.get(svc.id)) { histories++; await importHistory(svc); }
+ } catch (err) {
stats.lastError = `${svc.id}: ${err.message}`;
stats.lastError = `${svc.id}: ${err.message}`;
if (err.status === 403 || err.status === 429) break; // GitHub's rate limit: try again next run
if (err.status === 403 || err.status === 429) break; // GitHub's rate limit: try again next run
}
}
@@ -271,23 +324,35 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchIm
@@ -271,23 +324,35 @@ function createChangelog({ config, store, blogs, posts, aiDrafts = null, fetchIm
}
}
function stop() { if (timer) clearInterval(timer); timer = null; }
function stop() { if (timer) clearInterval(timer); timer = null; }
- /** The public feed: newest first. */
- function entries({ service = null, limit = 20 } = {}) {
+ /** The public feed, newest first: { entries, next } (next = the cursor for the page after, or null). */
+ function page({ service = null, limit = 20, before = null } = {}) {
const n = Math.min(Math.max(parseInt(limit, 10) || 20, 1), 100);
const n = Math.min(Math.max(parseInt(limit, 10) || 20, 1), 100);
- const rows = service
- ? db.prepare('SELECT * FROM changelog_entries WHERE service = ? ORDER BY deployed_at DESC, id DESC LIMIT ?').all(String(service), n)
- : db.prepare('SELECT * FROM changelog_entries ORDER BY deployed_at DESC, id DESC LIMIT ?').all(n);
- return rows.map((e) => ({
- service: e.service, sha: e.sha, short: e.sha.slice(0, 7), subject: e.subject, committed_at: e.committed_at, deployed_at: e.deployed_at,
- major: Boolean(e.major), url: `https://github.com/${e.repo}/commit/${e.sha}`, post_id: e.post_id || null,
- }));
+ const cur = decodeCursor(before);
+ const where = [];
+ const args = [];
+ if (service) { where.push('service = ?'); args.push(String(service)); }
+ if (cur) { where.push('(deployed_at < ? OR (deployed_at = ? AND id < ?))'); args.push(cur.at, cur.at, cur.id); }
+ const rows = db.prepare(`SELECT * FROM changelog_entries ${where.length ? `WHERE ${where.join(' AND ')}` : ''} ORDER BY deployed_at DESC, id DESC LIMIT ?`).all(...args, n + 1);
+ const more = rows.length > n;
+ const shown = rows.slice(0, n);
+ return {
+ entries: shown.map((e) => ({
+ service: e.service, sha: e.sha, short: e.sha.slice(0, 7), subject: e.subject, author: e.author || null, committed_at: e.committed_at, deployed_at: e.deployed_at,
+ major: Boolean(e.major), url: `https://github.com/${e.repo}/commit/${e.sha}`, post_id: e.post_id && e.post_id !== 'history' ? e.post_id : null,
+ })),
+ next: more && shown.length ? encodeCursor(shown[shown.length - 1]) : null,
+ };
}
}
+ function entries(opts = {}) { return page(opts).entries; }
function latestPost() { return q.lastPost.get() || null; }
function latestPost() { return q.lastPost.get() || null; }
+ function recentPosts(limit = 5) { return q.recentPosts.all(Math.min(Math.max(parseInt(limit, 10) || 5, 1), 20)); }
+ /** Every site with entries: [{ service, entries, latest_at }], most recently shipped first. */
+ function sites() { return q.sites.all(); }
- return { tick, collect, due, render, publish, start, stop, entries, latestPost, stats: () => ({ enabled: c.enabled, ...stats, pending: q.pending.all().length }) };
+ return { tick, collect, due, render, publish, start, stop, entries, page, latestPost, recentPosts, sites, importHistory, stats: () => ({ enabled: c.enabled, ...stats, pending: q.pending.all().length }) };
}
}
/** A stable id for a batch (tests). */
/** A stable id for a batch (tests). */
function batchKey(list) { return crypto.createHash('sha256').update(list.map((e) => `${e.service}:${e.sha}`).join(',')).digest('hex').slice(0, 16); }
function batchKey(list) { return crypto.createHash('sha256').update(list.map((e) => `${e.service}:${e.sha}`).join(',')).digest('hex').slice(0, 16); }
-module.exports = { createChangelog, subjectOf, ensureSchema, batchKey };
+module.exports = { createChangelog, subjectOf, ensureSchema, batchKey, encodeCursor, decodeCursor };
@@ -45,7 +45,8 @@ function load(env = process.env) {
@@ -45,7 +45,8 @@ function load(env = process.env) {
quietMs: Math.max(0, parseInt(env.CHANGELOG_QUIET_MS, 10) || 30 * 60 * 1000),
quietMs: Math.max(0, parseInt(env.CHANGELOG_QUIET_MS, 10) || 30 * 60 * 1000),
maxAgeMs: Math.max(0, parseInt(env.CHANGELOG_MAX_AGE_MS, 10) || 7 * 24 * 3600 * 1000),
maxAgeMs: Math.max(0, parseInt(env.CHANGELOG_MAX_AGE_MS, 10) || 7 * 24 * 3600 * 1000),
minGapMs: Math.max(0, parseInt(env.CHANGELOG_MIN_GAP_MS, 10) || 6 * 3600 * 1000),
minGapMs: Math.max(0, parseInt(env.CHANGELOG_MIN_GAP_MS, 10) || 6 * 3600 * 1000),
- intervalMs: Math.max(60 * 1000, parseInt(env.CHANGELOG_INTERVAL_MS, 10) || 10 * 60 * 1000),
+ intervalMs: Math.max(60 * 1000, parseInt(env.CHANGELOG_INTERVAL_MS, 10) || 5 * 60 * 1000),
+ history: Math.max(0, Math.min(100, parseInt(env.CHANGELOG_HISTORY ?? '60', 10) || 0)),
firstDelayMs: 60 * 1000,
firstDelayMs: 60 * 1000,
since: env.CHANGELOG_SINCE && !Number.isNaN(Date.parse(env.CHANGELOG_SINCE)) ? new Date(env.CHANGELOG_SINCE).toISOString() : null,
since: env.CHANGELOG_SINCE && !Number.isNaN(Date.parse(env.CHANGELOG_SINCE)) ? new Date(env.CHANGELOG_SINCE).toISOString() : null,
aiDraft: env.CHANGELOG_AI_DRAFT !== 'off',
aiDraft: env.CHANGELOG_AI_DRAFT !== 'off',
@@ -188,14 +188,18 @@ function createApi(ctx) {
@@ -188,14 +188,18 @@ function createApi(ctx) {
res.set('Access-Control-Allow-Origin', '*');
res.set('Access-Control-Allow-Origin', '*');
res.set('Cache-Control', 'public, max-age=60');
res.set('Cache-Control', 'public, max-age=60');
const service = /^[a-z][a-z0-9-]{1,39}$/.test(String(req.query.service || '')) ? String(req.query.service) : null;
const service = /^[a-z][a-z0-9-]{1,39}$/.test(String(req.query.service || '')) ? String(req.query.service) : null;
- const latest = ctx.changelog.latestPost();
- let post = null;
- if (latest) {
- const p = posts.get(latest.post_id);
+ const postOf = (row) => {
+ const p = row && posts.get(row.post_id);
const blog = p ? blogs.get(p.blog_id) : null;
const blog = p ? blogs.get(p.blog_id) : null;
- if (p && blog && p.state === 'published') post = { id: p.id, title: (posts.head(p) || { fields: {} }).fields.title || null, url: publication.abs(publication.postPath(blog, p)), published_at: p.published_at || null, entries: latest.entries };
- }
- return { service, entries: ctx.changelog.entries({ service, limit: req.query.limit }), latest_post: post };
+ if (!p || !blog || p.state !== 'published') return null;
+ return { id: p.id, title: (posts.head(p) || { fields: {} }).fields.title || null, url: publication.abs(publication.postPath(blog, p)), published_at: p.published_at || null, entries: row.entries };
+ };
+ const page = ctx.changelog.page({ service, limit: req.query.limit, before: req.query.before });
+ const recent = ctx.changelog.recentPosts(5).map(postOf).filter(Boolean);
+ return {
+ service, entries: page.entries, next: page.next, latest_post: recent[0] || null, posts: recent,
+ ...(service ? {} : { sites: ctx.changelog.sites() }),
+ };
}));
}));
// Draft with AI: OpenVibe.AI's blog.draft_post writes a draft (AI-authored, noindex until reviewed).
// Draft with AI: OpenVibe.AI's blog.draft_post writes a draft (AI-authored, noindex until reviewed).
@@ -14,6 +14,7 @@ const H = (n) => n.toString(16).padStart(40, 'a');
@@ -14,6 +14,7 @@ const H = (n) => n.toString(16).padStart(40, 'a');
const releases = { live: 'aaaaaaa00001', tools: 'bbbbbbb00001' };
const releases = { live: 'aaaaaaa00001', tools: 'bbbbbbb00001' };
const compares = {};
const compares = {};
const ghCalls = [];
const ghCalls = [];
+ const history = {};
const fetchImpl = async (url) => {
const fetchImpl = async (url) => {
const u = new URL(url);
const u = new URL(url);
const json = (body, status = 200) => ({ ok: status < 300, status, json: async () => body });
const json = (body, status = 200) => ({ ok: status < 300, status, json: async () => body });
@@ -27,11 +28,13 @@ const H = (n) => n.toString(16).padStart(40, 'a');
@@ -27,11 +28,13 @@ const H = (n) => n.toString(16).padStart(40, 'a');
ghCalls.push(u.pathname);
ghCalls.push(u.pathname);
const m = u.pathname.match(/^\/repos\/(.+)\/compare\/(.+)\.\.\.(.+)$/);
const m = u.pathname.match(/^\/repos\/(.+)\/compare\/(.+)\.\.\.(.+)$/);
if (m && compares[m[3]]) return json(compares[m[3]]);
if (m && compares[m[3]]) return json(compares[m[3]]);
+ const h = u.pathname.match(/^\/repos\/(.+)\/commits$/);
+ if (h && history[h[1]]) return json(history[h[1]]);
return json({ message: 'Not Found' }, 404);
return json({ message: 'Not Found' }, 404);
}
}
return json({}, 404);
return json({}, 404);
};
};
- const config = { ...t.ctx.config, changelog: { ...t.ctx.config.changelog, enabled: true, batchSize: 5, majorLines: 500, quietMs: 30 * 60 * 1000, maxAgeMs: 7 * 24 * 3600 * 1000, minGapMs: 6 * 3600 * 1000, since: null, aiDraft: false, blogHandle: 'openvibe' } };
+ const config = { ...t.ctx.config, changelog: { ...t.ctx.config.changelog, enabled: true, batchSize: 5, majorLines: 500, quietMs: 30 * 60 * 1000, maxAgeMs: 7 * 24 * 3600 * 1000, minGapMs: 6 * 3600 * 1000, since: null, aiDraft: false, blogHandle: 'openvibe', history: 0 } };
const cl = createChangelog({ config, store: t.ctx.store, blogs: t.ctx.blogs, posts: t.ctx.posts, aiDrafts: null, fetchImpl, log: { log() {}, warn() {} } });
const cl = createChangelog({ config, store: t.ctx.store, blogs: t.ctx.blogs, posts: t.ctx.posts, aiDrafts: null, fetchImpl, log: { log() {}, warn() {} } });
const commit = (i, msg) => ({ sha: H(i), commit: { message: msg, author: { date: new Date(t.clock.now()).toISOString() } } });
const commit = (i, msg) => ({ sha: H(i), commit: { message: msg, author: { date: new Date(t.clock.now()).toISOString() } } });
@@ -100,6 +103,32 @@ const H = (n) => n.toString(16).padStart(40, 'a');
@@ -100,6 +103,32 @@ const H = (n) => n.toString(16).padStart(40, 'a');
assert.strictEqual(cl.entries({ service: 'tools' })[0].major, true);
assert.strictEqual(cl.entries({ service: 'tools' })[0].major, true);
});
});
+ await check('history: each site\'s earlier commits once, never pending, with authors; the feed pages by cursor', async () => {
+ history['OpenVibers/OpenVibe.Live'] = [20, 21, 22].map((i) => ({ sha: H(i), commit: { message: `Earlier change ${i}\n\nbody`, author: { name: 'OpenVibers', date: new Date(t.clock.now() - (30 - i) * 86400000).toISOString() } } }))
+ .concat([{ sha: H(5), commit: { message: 'Small change 5', author: { name: 'OpenVibers', date: new Date(t.clock.now()).toISOString() } } }]);
+ const hc = createChangelog({ config: { ...config, changelog: { ...config.changelog, history: 50 } }, store: t.ctx.store, blogs: t.ctx.blogs, posts: t.ctx.posts, fetchImpl, log: { log() {}, warn() {} } });
+ const before = hc.stats().pending;
+ await hc.collect();
+ assert.strictEqual(hc.stats().pending, before, 'history never waits for a patch notes post');
+ const all = hc.entries({ service: 'live', limit: 100 });
+ assert.ok(all.some((e) => e.subject === 'Earlier change 20' && e.author === 'OpenVibers' && e.post_id === null));
+ assert.strictEqual(all.find((e) => e.sha === H(5)).author, 'OpenVibers', 'a known commit gains its author');
+ const calls = ghCalls.length;
+ await hc.collect();
+ assert.strictEqual(ghCalls.length, calls, 'imported once (a repository GitHub will not list is not asked again either)');
+ const p1 = hc.page({ service: 'live', limit: 2 });
+ assert.strictEqual(p1.entries.length, 2);
+ assert.ok(p1.next);
+ const p2 = hc.page({ service: 'live', limit: 2, before: p1.next });
+ assert.ok(p2.entries.length >= 1 && !p2.entries.some((e) => p1.entries.some((x) => x.sha === e.sha)), 'the next page continues without repeats');
+ const r = (await t.get(`/api/v1/changelog?limit=2&before=${encodeURIComponent(p1.next)}`)).json();
+ assert.ok(Array.isArray(r.sites) && r.sites.some((x) => x.service === 'live'), 'network-wide answers list the sites');
+ assert.ok(Array.isArray(r.posts) && r.posts.length >= 1);
+ const one = (await t.get('/api/v1/changelog?service=live&limit=1')).json();
+ assert.strictEqual(one.sites, undefined);
+ assert.ok(one.next);
+ });
+
await t.close();
await t.close();
done();
done();
})();
})();