openvibers/openvibe.sdk Commits

Migration tools and lessons from the Community pass (G3).

OP OpenVibers committed 28 Sept 2026 · fc2a299c81 · 5 files, +118 −3

asyncify.js: a namespace require (const store = require('./store')) and a factory instance (const svc =
createPasteService(…)) are that module's API in the file where they appear; require('./x').fn() calls are awaited;
fileApis in the config for holders that differ by file; a promise passed to a local helper that awaits it
(rejects(p, …)) stays a promise; never a second await in front of one. sqlfix.py: an immediately invoked
db.transaction(fn)() is rewritten before the named-function rule (which had wrapped it in an arrow the codemod then
awaited over and over); camelCase aliases quoted; explicit ids into GENERATED ALWAYS identities get OVERRIDING SYSTEM
VALUE; inline INSERT … .lastInsertRowid gets RETURNING id; SQLite constraint-message regexes in tests become
PostgreSQL's. sqlite-schema-to-pg.js: an INTEGER PRIMARY KEY that references another table is a plain key.
typecheck.js lists columns whose values do not match their declared type in a SQLite file. §7 gains the Community
lessons.

Changes

ddiff --git a/docs/migrating-to-postgresql.md b/docs/migrating-to-postgresql.md
iindex 1189ec2..ee91edc 100644
---- a/docs/migrating-to-postgresql.md
++++ b/docs/migrating-to-postgresql.md
@@@ -165,3 +165,15 @@ It changes nothing in the SQLite file.
- **No DDL at boot.** A module that ran `CREATE TABLE IF NOT EXISTS` on the serving handle fails on production roles (`permission denied for schema`) while PGlite allows it. The table goes in the migration; `sqlfix.py` reports `db.exec(`.
- **Test boots that call `start()` directly** fall back to the development PGlite directory and share rows across runs. Pass every boot a `createTestDb` handle.
- **Redaction.** PostgreSQL has no `secure_delete`: an overwritten row version stays until vacuum. Lower the table's `autovacuum_vacuum_scale_factor` and say so in the README.
++- **Community (2026-09-28)** added these:
++ - **A schema changed by `migrate()` after `SCHEMA`.** `gen-migration.js --open "require('./server/db').openDb(':memory:')"` takes the schema the SQLite release's own boot leaves (every ALTER applied, tables in dependency order, boot seeds as INSERTs). Run it before converting any code.
++ - **SQLite text timestamps.** `DATETIME` columns hold `'YYYY-MM-DD HH:MM:SS'` text. Keep it: the migration gets `SQLITE_DATE_FUNCTIONS` (`ov_now()`, `datetime(t, modifier)`, `julianday(t)`), and `sqlfix.py` rewrites `CURRENT_TIMESTAMP` to `ov_now()` (PostgreSQL's would be a `timestamptz`, cast to different text).
++ - **Type affinity.** SQLite stores text in an `INTEGER` column without complaint (a Chat room id); PostgreSQL refuses. Check the production copy (`tools/asyncify/typecheck.js <file>` lists columns holding another storage class) and read the code that writes each id column.
++ - **`INTEGER PRIMARY KEY REFERENCES …`** is the other row's key, not a counter: `bigint PRIMARY KEY` (the converter does this now).
++ - **camelCase aliases** fold to lower case (`AS totalViews` comes back as `totalviews`): quote them (`sqlfix.py` does).
++ - **Explicit ids** into a `GENERATED ALWAYS` identity need `OVERRIDING SYSTEM VALUE` (`sqlfix.py` adds it where the migration says ALWAYS); `.lastInsertRowid` needs `RETURNING id`.
++ - **`IMMEDIATE` transactions.** A read-then-write that relied on SQLite's write lock (a vote recount) locks its row first: `SELECT … FOR UPDATE`.
++ - **Work kicked from inside a transaction** (`setImmediate(drain)`) runs before the commit now and misses the row: `db.afterCommit`, and a short delay when changes made together should go out as one pass.
++ - **Event handlers that became async** (a WebSocket dispatch) can overtake each other: queue them in arrival order.
++ - **Fire-and-forget helpers** (`tell(fn) { try { fn(x) } catch {} }`) catch nothing once `fn` is async: await it.
++ - **Tests**: `const x = createFoo(…)` instances, `require('./x').fn()` calls and a test's own `rejects(promise)` helper are handled by the codemod now; a boot helper that re-requires the server gets its database passed in (`createApp({ db })`); SQLite-only upgrade tests (a `migrate()` rebuilding an old table) go, the migration ledger replaces them.
ddiff --git a/tools/asyncify/asyncify.js b/tools/asyncify/asyncify.js
iindex 32394a1..6006cba 100644
---- a/tools/asyncify/asyncify.js
++++ b/tools/asyncify/asyncify.js
@@@ -59,9 +59,50 @@ function analyse(file, src, cfg, asyncNames) {
const isFn = (n) => n.type === 'FunctionDeclaration' || n.type === 'FunctionExpression' || n.type === 'ArrowFunctionExpression';
const stores = new Set(cfg.stores || []);
const syncStoreMethods = new Set(cfg.syncStoreMethods || []);
-- const apis = cfg.apis || {};
++ const apis = { ...(cfg.apis || {}) };
const syncNames = new Set(cfg.syncNames || []);
const localAsync = new Set(asyncNames.local.get(file) || []);
++ // const store = require('./store'): in this file, `store` is that module's API (the same name can mean another
++ // module elsewhere, so a namespace require wins over the config's global apis entry).
++ walk.full(parse(src), (n) => {
++ if (n.type !== 'VariableDeclarator' || n.id.type !== 'Identifier' || !n.init || n.init.type !== 'CallExpression') return;
++ const c = n.init;
++ if (c.callee.type !== 'Identifier' || c.callee.name !== 'require' || !c.arguments[0] || typeof c.arguments[0].value !== 'string' || !c.arguments[0].value.startsWith('.')) return;
++ const base = path.relative(process.cwd(), path.resolve(path.dirname(file), c.arguments[0].value));
++ const target = base.endsWith('.js') ? base : fs.existsSync(`${base}.js`) ? `${base}.js` : path.join(base, 'index.js');
++ if (fs.existsSync(target)) apis[n.id.name] = target;
++ });
++ // const svc = createPasteService(…) with createPasteService imported from a local file (destructured): `svc` is that
++ // file's API here (a factory's instance; its methods are the file's inner functions).
++ {
++ const imported = new Map();
++ const tree = parse(src);
++ walk.full(tree, (n) => {
++ if (n.type !== 'VariableDeclarator' || n.id.type !== 'ObjectPattern' || !n.init || n.init.type !== 'CallExpression') return;
++ const c = n.init;
++ if (c.callee.type !== 'Identifier' || c.callee.name !== 'require' || !c.arguments[0] || typeof c.arguments[0].value !== 'string' || !c.arguments[0].value.startsWith('.')) return;
++ const base = path.relative(process.cwd(), path.resolve(path.dirname(file), c.arguments[0].value));
++ const target = base.endsWith('.js') ? base : fs.existsSync(`${base}.js`) ? `${base}.js` : path.join(base, 'index.js');
++ for (const prop of n.id.properties) {
++ if (prop.type === 'Property' && prop.value && prop.value.type === 'Identifier') imported.set(prop.value.name, target);
++ }
++ });
++ const factoryOf = (init) => {
++ let x = init;
++ if (x && x.type === 'AwaitExpression') x = x.argument;
++ if (x && x.type === 'LogicalExpression') x = x.right; // opts.relay || createDiscordRelay(…)
++ if (x && x.type === 'AwaitExpression') x = x.argument;
++ return x && x.type === 'CallExpression' && x.callee.type === 'Identifier' && /^create[A-Z]/.test(x.callee.name) ? imported.get(x.callee.name) : null;
++ };
++ walk.full(tree, (n) => {
++ const pairs = n.type === 'VariableDeclarator' && n.id.type === 'Identifier' ? [[n.id.name, n.init]]
++ : n.type === 'AssignmentExpression' && n.left.type === 'Identifier' ? [[n.left.name, n.right]] : [];
++ for (const [name, init] of pairs) { const f = factoryOf(init); if (f && fs.existsSync(f)) apis[name] = f; }
++ });
++ }
++ // fileApis: { "server/comments/api.js": { "service": "server/comments/service.js" } } for a holder whose module
++ // differs by file (a factory's instance passed in under a generic name).
++ Object.assign(apis, (cfg.fileApis || {})[path.relative(process.cwd(), file)] || {});
// const { seed } = require('./workflows/seed'): a function imported from a converted file is async when it is there.
walk.full(ast, (n) => {
if (n.type !== 'VariableDeclarator' || n.id.type !== 'ObjectPattern' || !n.init || n.init.type !== 'CallExpression') return;
@@@ -112,6 +153,15 @@ function analyse(file, src, cfg, asyncNames) {
});
});
++ // Local functions that await a parameter themselves: rejects(p, …) → 'rejects:0'.
++ const promiseTakers = new Set();
++ for (const [name, f] of localFns) {
++ const params = f.params.map((q) => (q.type === 'Identifier' ? q.name : null));
++ walk.full(f.body, (x) => {
++ if (x.type === 'AwaitExpression' && x.argument.type === 'Identifier' && params.includes(x.argument.name)) promiseTakers.add(`${name}:${params.indexOf(x.argument.name)}`);
++ });
++ }
++
function asyncCall(call) {
const c = call.callee;
if (c.type === 'MemberExpression' && !c.computed) {
@@@ -124,6 +174,12 @@ function analyse(file, src, cfg, asyncNames) {
if (p.length && stmtNames.has(p[p.length - 1])) return true;
}
if ((m === 'tx' || m === 'transaction') && p.length && ['store', 'db', 'this'].includes(p[p.length - 1])) return m === 'tx';
++ // require('./events').pasteCreated(: a module's function reached through an inline require.
++ if (obj.type === 'CallExpression' && obj.callee.type === 'Identifier' && obj.callee.name === 'require' && obj.arguments[0] && typeof obj.arguments[0].value === 'string' && obj.arguments[0].value.startsWith('.')) {
++ const base = path.relative(process.cwd(), path.resolve(path.dirname(file), obj.arguments[0].value));
++ const set = asyncNames.api.get(base.endsWith('.js') ? base : `${base}.js`) || asyncNames.api.get(path.join(base, 'index.js'));
++ if (set && set.has(m) && !syncNames.has(m)) return true;
++ }
// <x>.<store>.<method>(: a store reached through an object; a bare identifier only when listed in bareStores.
if (p.length >= 2 && stores.has(p[p.length - 1]) && !syncStoreMethods.has(m)) return true;
if (p.length === 1 && (cfg.bareStores || []).includes(p[0]) && !syncStoreMethods.has(m)) return true;
@@@ -226,6 +282,8 @@ function analyse(file, src, cfg, asyncNames) {
if (parent && parent.type === 'VariableDeclarator' && parent.init === node && parent.id.type === 'Identifier' && awaitedLater.has(parent.id.name)) return;
// assert.rejects(promise) / doesNotReject(promise) take the promise itself.
if (parent && parent.type === 'CallExpression' && parent.arguments[0] === node && parent.callee.type === 'MemberExpression' && ['rejects', 'doesNotReject'].includes(parent.callee.property.name)) return;
++ // So does a local helper that awaits that parameter itself (a test's rejects(p, status)).
++ if (parent && parent.type === 'CallExpression' && parent.callee.type === 'Identifier' && promiseTakers.has(`${parent.callee.name}:${parent.arguments.indexOf(node)}`)) return;
// Returned directly from an arrow body or a return statement of an async fn is fine too, but awaiting is harmless.
const fnIdx = (() => { for (let i = parents.length - 1; i >= 0; i--) if (isFn(parents[i])) return i; return -1; })();
const fn = fnIdx >= 0 ? parents[fnIdx] : null;
@@@ -252,6 +310,8 @@ function analyse(file, src, cfg, asyncNames) {
makeAsync(fn);
}
const wrap = parent && parent.type === 'MemberExpression' && parent.object === node;
++ // Never a second await in front of one already there (a call the parser saw inside another's callee).
++ if (!wrap && /\bawait\s+$/.test(src.slice(Math.max(0, node.start - 12), node.start))) return;
edits.push({ at: node.start, text: wrap ? '(await ' : 'await ' });
if (wrap) edits.push({ at: node.end, text: ')' });
});
ddiff --git a/tools/asyncify/sqlfix.py b/tools/asyncify/sqlfix.py
iindex 75acf05..ddef00a 100644
---- a/tools/asyncify/sqlfix.py
++++ b/tools/asyncify/sqlfix.py
@@@ -42,6 +42,7 @@ changed = []
# Table columns from the service's migrations, so upsert right-hand sides can be qualified even for columns the INSERT
# does not list (a counter bumped in place, a timestamp cleared).
TABLE_COLS = {}
++ALWAYS_IDENTITY = {} # table → its GENERATED ALWAYS identity column
import glob as _glob
for mf in sorted(_glob.glob('migrations/*.sql')):
ddl = open(mf).read()
@@@ -50,6 +51,9 @@ for mf in sorted(_glob.glob('migrations/*.sql')):
TABLE_COLS.setdefault(tm.group(1), set()).update(cols)
for am in re.finditer(r'ALTER TABLE (\w+) ADD COLUMN (\w+)', ddl):
TABLE_COLS.setdefault(am.group(1), set()).add(am.group(2))
++ for tm in re.finditer(r'CREATE TABLE (?:IF NOT EXISTS )?(\w+) \(([\s\S]*?)\n\);', ddl):
++ for im in re.finditer(r'^\s*"?(\w+)"?\s+bigint GENERATED ALWAYS AS IDENTITY', tm.group(2), re.M):
++ ALWAYS_IDENTITY.setdefault(tm.group(1), im.group(1))
# A migration carrying SQLITE_DATE_FUNCTIONS (text timestamps kept as SQLite wrote them): CURRENT_TIMESTAMP in code
# becomes ov_now() (PostgreSQL's would be a timestamptz, cast to other text), strftime's ISO 'now' ov_now_iso().
SQLITE_DATES = any('CREATE FUNCTION ov_now()' in open(mf).read() for mf in _glob.glob('migrations/*.sql'))
@@@ -74,8 +78,6 @@ for p in files:
s = re.sub(r"assert\.throws\(\(\) => ((?:t\.)?[\w.]*db\.prepare\((?:[^()]|\([^()]*\))*\)\.(?:run|get|all)\((?:[^()]|\([^()]*\))*\)), ", r"await assert.rejects(\1, ", s)
# A jsonb column (the SDK outbox's envelope) comes back as an object.
s = re.sub(r"JSON\.parse\((\w+)\.envelope\)", r"(typeof \1.envelope === 'string' ? JSON.parse(\1.envelope) : \1.envelope)", s)
-- # const f = db.transaction((a, b) => { … }) (called later as f(a, b)) → const f = (a, b) => db.tx(async () => { … })
-- s = re.sub(r'const (\w+) = (\w+(?:\.\w+)?)\.transaction\(\(([^()]*)\) => \{', r'const \1 = (\3) => \2.tx(async () => {', s)
# db.transaction(fn)() → db.tx(fn) (balanced parentheses; the codemod then awaits it and makes fn async).
out, i = [], 0
while True:
@@@ -89,6 +91,17 @@ for p in files:
else:
out.append(s[i:k]); i = k
s = ''.join(out)
++ # const f = db.transaction((a, b) => { … }) (called later as f(a, b); an immediately invoked one was rewritten just
++ # above) → const f = (a, b) => db.tx(async () => { … })
++ s = re.sub(r'const (\w+) = (\w+(?:\.\w+)?)\.transaction\(\(([^()]*)\) => \{', r'const \1 = (\3) => \2.tx(async () => {', s)
++ # An unquoted alias is folded to lower case in PostgreSQL (AS totalViews comes back as totalviews): quote
++ # camelCase aliases inside SQL string literals.
++ def quote_aliases(lit):
++ body = lit.group(0)
++ if not re.search(r'\bSELECT\b', body): return body
++ q = '\\"' if body[0] == '"' else '"'
++ return re.sub(r'\bAS ([a-z]+[A-Z]\w*)\b', lambda m: f'AS {q}{m.group(1)}{q}', body)
++ s = re.sub(r'`[^`]*`|\'(?:[^\'\\\n]|\\.)*\'|"(?:[^"\\\n]|\\.)*"', quote_aliases, s)
# Column names PostgreSQL reserves (SQLite does not): quoted inside SQL string literals ("window", "user").
def quote_reserved(lit):
body = lit.group(0)
@@@ -114,6 +127,20 @@ for p in files:
parts = re.split(r'(,\s*(?=\w+\s*=))', sets)
return f'INSERT INTO {table} ({m.group(2)}){head}' + ''.join(fix_rhs(x) if not re.match(r',\s*$', x) else x for x in parts)
s = re.sub(r'INSERT INTO (\w+) \(([^)]*)\)((?:(?!INSERT INTO)[^`\'"])*?DO UPDATE SET[^`\'"]*)', qualify, s)
++ # INSERT INTO t (id, …) VALUES with an explicit id into a GENERATED ALWAYS identity (tests, fixtures, imports):
++ # PostgreSQL refuses it without OVERRIDING SYSTEM VALUE.
++ def override(m):
++ table, cols = m.group(1), [c.strip().strip('"') for c in m.group(2).split(',')]
++ return m.group(0) + ' OVERRIDING SYSTEM VALUE' if ALWAYS_IDENTITY.get(table) in cols else m.group(0)
++ s = re.sub(r'INSERT INTO (\w+) \(([^)]*)\)(?! OVERRIDING)(?=\s*(?:VALUES|SELECT))', override, s)
++ # (await db.prepare('INSERT …').run(…)).lastInsertRowid: the id comes back through RETURNING.
++ s = re.sub(r"(prepare\()(`|'|\")(INSERT INTO (\w+)(?:(?!\2).)*?)(\2\)\s*\.run\((?:[^()]|\([^()]*\))*\)\)\s*\.lastInsertRowid)",
++ lambda m: m.group(0) if 'RETURNING' in m.group(3) else m.group(1) + m.group(2) + m.group(3) + ' RETURNING id' + m.group(5), s)
++ # Tests matching SQLite's constraint messages: PostgreSQL's say "duplicate key", "violates … constraint".
++ s = re.sub(r'/UNIQUE constraint failed[^/\n]*/|/UNIQUE/', '/duplicate key/', s)
++ s = re.sub(r'/CHECK constraint failed[^/\n]*/', '/violates check constraint/', s)
++ s = re.sub(r'/NOT NULL constraint failed[^/\n]*/', '/violates not-null constraint/', s)
++ s = re.sub(r'/FOREIGN KEY constraint failed[^/\n]*/', '/violates foreign key constraint/', s)
# openvibe-publishing 1.0: the index sequencer stamps through a handle (the ambient transaction joins through store.db).
handle = 'store.db' if 'store.db' in s else 'db'
s = re.sub(r'((?:store\.)?sequencer\.stamp\()(?!store\.db|db,|t,)', lambda m: m.group(1) + handle + ', ', s)
ddiff --git a/tools/asyncify/sqlite-schema-to-pg.js b/tools/asyncify/sqlite-schema-to-pg.js
iindex cb7a293..d7aa1c3 100644
---- a/tools/asyncify/sqlite-schema-to-pg.js
++++ b/tools/asyncify/sqlite-schema-to-pg.js
@@@ -36,6 +36,8 @@ function toPg(sql) {
sql = sql.replace(/^(\s*)(window|user)(\s+[A-Z])/gm, '$1"$2"$3').replace(/([(,]\s*)(window|user)(\s*[,)])/g, '$1"$2"$3');
return sql
.replace(/\bINTEGER PRIMARY KEY AUTOINCREMENT\b/g, 'bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY')
++ // An INTEGER PRIMARY KEY that references another table is that row's key, not a counter.
++ .replace(/\bINTEGER PRIMARY KEY\b(?! AUTOINCREMENT)(?=[^,\n]*\bREFERENCES\b)/g, 'bigint PRIMARY KEY')
// A bare INTEGER PRIMARY KEY is SQLite's rowid: code may insert without it or with an explicit value.
.replace(/\bINTEGER PRIMARY KEY\b(?! AUTOINCREMENT)/g, 'bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY')
.replace(/CREATE (UNIQUE )?INDEX IF NOT EXISTS/g, (m, u) => `CREATE ${u || ''}INDEX`)
ddiff --git a/tools/asyncify/typecheck.js b/tools/asyncify/typecheck.js
nnew file mode 100644
iindex 0000000..bf2e9f3
---- /dev/null
++++ b/tools/asyncify/typecheck.js
@@@ -0,0 +1,14 @@
++#!/usr/bin/env node
++'use strict';
++// Every column declared INTEGER/REAL holding a value of another storage class (SQLite's affinity lets text in).
++const Database = require(require.resolve('better-sqlite3', { paths: [process.cwd()] }));
++const db = new Database(process.argv[2], { readonly: true });
++for (const { name } of db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all()) {
++ for (const c of db.prepare(`PRAGMA table_info("${name}")`).all()) {
++ const t = String(c.type).toUpperCase();
++ const want = /INT/.test(t) ? 'integer' : /REAL|FLOA|DOUB/.test(t) ? 'real' : null;
++ if (!want) continue;
++ const rows = db.prepare(`SELECT typeof("${c.name}") AS k, COUNT(*) AS n, MIN("${c.name}") AS ex FROM "${name}" WHERE "${c.name}" IS NOT NULL AND typeof("${c.name}") NOT IN ('${want}'${want === 'real' ? ", 'integer'" : ''}) GROUP BY 1`).all();
++ for (const r of rows) console.log(`${name}.${c.name} (${t}): ${r.n} ${r.k} value(s), e.g. ${String(r.ex).slice(0, 40)}`);
++ }
++}