openvibers/openvibe.sdk Commits
Migration tools and lessons from the Community pass (G3).
Changes
@@@ -165,3 +165,15 @@ It changes nothing in the SQLite file.- **No DDL at boot.** A module that ran `CREATE TABLE IF NOT EXISTS` on the serving handle fails on production roles (`permission denied for schema`) while PGlite allows it. The table goes in the migration; `sqlfix.py` reports `db.exec(`.- **Test boots that call `start()` directly** fall back to the development PGlite directory and share rows across runs. Pass every boot a `createTestDb` handle.- **Redaction.** PostgreSQL has no `secure_delete`: an overwritten row version stays until vacuum. Lower the table's `autovacuum_vacuum_scale_factor` and say so in the README.++- **Community (2026-09-28)** added these:++ - **A schema changed by `migrate()` after `SCHEMA`.** `gen-migration.js --open "require('./server/db').openDb(':memory:')"` takes the schema the SQLite release's own boot leaves (every ALTER applied, tables in dependency order, boot seeds as INSERTs). Run it before converting any code.++ - **SQLite text timestamps.** `DATETIME` columns hold `'YYYY-MM-DD HH:MM:SS'` text. Keep it: the migration gets `SQLITE_DATE_FUNCTIONS` (`ov_now()`, `datetime(t, modifier)`, `julianday(t)`), and `sqlfix.py` rewrites `CURRENT_TIMESTAMP` to `ov_now()` (PostgreSQL's would be a `timestamptz`, cast to different text).++ - **Type affinity.** SQLite stores text in an `INTEGER` column without complaint (a Chat room id); PostgreSQL refuses. Check the production copy (`tools/asyncify/typecheck.js <file>` lists columns holding another storage class) and read the code that writes each id column.++ - **`INTEGER PRIMARY KEY REFERENCES …`** is the other row's key, not a counter: `bigint PRIMARY KEY` (the converter does this now).++ - **camelCase aliases** fold to lower case (`AS totalViews` comes back as `totalviews`): quote them (`sqlfix.py` does).++ - **Explicit ids** into a `GENERATED ALWAYS` identity need `OVERRIDING SYSTEM VALUE` (`sqlfix.py` adds it where the migration says ALWAYS); `.lastInsertRowid` needs `RETURNING id`.++ - **`IMMEDIATE` transactions.** A read-then-write that relied on SQLite's write lock (a vote recount) locks its row first: `SELECT … FOR UPDATE`.++ - **Work kicked from inside a transaction** (`setImmediate(drain)`) runs before the commit now and misses the row: `db.afterCommit`, and a short delay when changes made together should go out as one pass.++ - **Event handlers that became async** (a WebSocket dispatch) can overtake each other: queue them in arrival order.++ - **Fire-and-forget helpers** (`tell(fn) { try { fn(x) } catch {} }`) catch nothing once `fn` is async: await it.++ - **Tests**: `const x = createFoo(…)` instances, `require('./x').fn()` calls and a test's own `rejects(promise)` helper are handled by the codemod now; a boot helper that re-requires the server gets its database passed in (`createApp({ db })`); SQLite-only upgrade tests (a `migrate()` rebuilding an old table) go, the migration ledger replaces them.@@@ -59,9 +59,50 @@ function analyse(file, src, cfg, asyncNames) {const isFn = (n) => n.type === 'FunctionDeclaration' || n.type === 'FunctionExpression' || n.type === 'ArrowFunctionExpression';const stores = new Set(cfg.stores || []);const syncStoreMethods = new Set(cfg.syncStoreMethods || []);-- const apis = cfg.apis || {};++ const apis = { ...(cfg.apis || {}) };const syncNames = new Set(cfg.syncNames || []);const localAsync = new Set(asyncNames.local.get(file) || []);++ // const store = require('./store'): in this file, `store` is that module's API (the same name can mean another++ // module elsewhere, so a namespace require wins over the config's global apis entry).++ walk.full(parse(src), (n) => {++ if (n.type !== 'VariableDeclarator' || n.id.type !== 'Identifier' || !n.init || n.init.type !== 'CallExpression') return;++ const c = n.init;++ if (c.callee.type !== 'Identifier' || c.callee.name !== 'require' || !c.arguments[0] || typeof c.arguments[0].value !== 'string' || !c.arguments[0].value.startsWith('.')) return;++ const base = path.relative(process.cwd(), path.resolve(path.dirname(file), c.arguments[0].value));++ const target = base.endsWith('.js') ? base : fs.existsSync(`${base}.js`) ? `${base}.js` : path.join(base, 'index.js');++ if (fs.existsSync(target)) apis[n.id.name] = target;++ });++ // const svc = createPasteService(…) with createPasteService imported from a local file (destructured): `svc` is that++ // file's API here (a factory's instance; its methods are the file's inner functions).++ {++ const imported = new Map();++ const tree = parse(src);++ walk.full(tree, (n) => {++ if (n.type !== 'VariableDeclarator' || n.id.type !== 'ObjectPattern' || !n.init || n.init.type !== 'CallExpression') return;++ const c = n.init;++ if (c.callee.type !== 'Identifier' || c.callee.name !== 'require' || !c.arguments[0] || typeof c.arguments[0].value !== 'string' || !c.arguments[0].value.startsWith('.')) return;++ const base = path.relative(process.cwd(), path.resolve(path.dirname(file), c.arguments[0].value));++ const target = base.endsWith('.js') ? base : fs.existsSync(`${base}.js`) ? `${base}.js` : path.join(base, 'index.js');++ for (const prop of n.id.properties) {++ if (prop.type === 'Property' && prop.value && prop.value.type === 'Identifier') imported.set(prop.value.name, target);++ }++ });++ const factoryOf = (init) => {++ let x = init;++ if (x && x.type === 'AwaitExpression') x = x.argument;++ if (x && x.type === 'LogicalExpression') x = x.right; // opts.relay || createDiscordRelay(…)++ if (x && x.type === 'AwaitExpression') x = x.argument;++ return x && x.type === 'CallExpression' && x.callee.type === 'Identifier' && /^create[A-Z]/.test(x.callee.name) ? imported.get(x.callee.name) : null;++ };++ walk.full(tree, (n) => {++ const pairs = n.type === 'VariableDeclarator' && n.id.type === 'Identifier' ? [[n.id.name, n.init]]++ : n.type === 'AssignmentExpression' && n.left.type === 'Identifier' ? [[n.left.name, n.right]] : [];++ for (const [name, init] of pairs) { const f = factoryOf(init); if (f && fs.existsSync(f)) apis[name] = f; }++ });++ }++ // fileApis: { "server/comments/api.js": { "service": "server/comments/service.js" } } for a holder whose module++ // differs by file (a factory's instance passed in under a generic name).++ Object.assign(apis, (cfg.fileApis || {})[path.relative(process.cwd(), file)] || {});// const { seed } = require('./workflows/seed'): a function imported from a converted file is async when it is there.walk.full(ast, (n) => {if (n.type !== 'VariableDeclarator' || n.id.type !== 'ObjectPattern' || !n.init || n.init.type !== 'CallExpression') return;@@@ -112,6 +153,15 @@ function analyse(file, src, cfg, asyncNames) {});});++ // Local functions that await a parameter themselves: rejects(p, …) → 'rejects:0'.++ const promiseTakers = new Set();++ for (const [name, f] of localFns) {++ const params = f.params.map((q) => (q.type === 'Identifier' ? q.name : null));++ walk.full(f.body, (x) => {++ if (x.type === 'AwaitExpression' && x.argument.type === 'Identifier' && params.includes(x.argument.name)) promiseTakers.add(`${name}:${params.indexOf(x.argument.name)}`);++ });++ }++function asyncCall(call) {const c = call.callee;if (c.type === 'MemberExpression' && !c.computed) {@@@ -124,6 +174,12 @@ function analyse(file, src, cfg, asyncNames) {if (p.length && stmtNames.has(p[p.length - 1])) return true;}if ((m === 'tx' || m === 'transaction') && p.length && ['store', 'db', 'this'].includes(p[p.length - 1])) return m === 'tx';++ // require('./events').pasteCreated(: a module's function reached through an inline require.++ if (obj.type === 'CallExpression' && obj.callee.type === 'Identifier' && obj.callee.name === 'require' && obj.arguments[0] && typeof obj.arguments[0].value === 'string' && obj.arguments[0].value.startsWith('.')) {++ const base = path.relative(process.cwd(), path.resolve(path.dirname(file), obj.arguments[0].value));++ const set = asyncNames.api.get(base.endsWith('.js') ? base : `${base}.js`) || asyncNames.api.get(path.join(base, 'index.js'));++ if (set && set.has(m) && !syncNames.has(m)) return true;++ }// <x>.<store>.<method>(: a store reached through an object; a bare identifier only when listed in bareStores.if (p.length >= 2 && stores.has(p[p.length - 1]) && !syncStoreMethods.has(m)) return true;if (p.length === 1 && (cfg.bareStores || []).includes(p[0]) && !syncStoreMethods.has(m)) return true;@@@ -226,6 +282,8 @@ function analyse(file, src, cfg, asyncNames) {if (parent && parent.type === 'VariableDeclarator' && parent.init === node && parent.id.type === 'Identifier' && awaitedLater.has(parent.id.name)) return;// assert.rejects(promise) / doesNotReject(promise) take the promise itself.if (parent && parent.type === 'CallExpression' && parent.arguments[0] === node && parent.callee.type === 'MemberExpression' && ['rejects', 'doesNotReject'].includes(parent.callee.property.name)) return;++ // So does a local helper that awaits that parameter itself (a test's rejects(p, status)).++ if (parent && parent.type === 'CallExpression' && parent.callee.type === 'Identifier' && promiseTakers.has(`${parent.callee.name}:${parent.arguments.indexOf(node)}`)) return;// Returned directly from an arrow body or a return statement of an async fn is fine too, but awaiting is harmless.const fnIdx = (() => { for (let i = parents.length - 1; i >= 0; i--) if (isFn(parents[i])) return i; return -1; })();const fn = fnIdx >= 0 ? parents[fnIdx] : null;@@@ -252,6 +310,8 @@ function analyse(file, src, cfg, asyncNames) {makeAsync(fn);}const wrap = parent && parent.type === 'MemberExpression' && parent.object === node;++ // Never a second await in front of one already there (a call the parser saw inside another's callee).++ if (!wrap && /\bawait\s+$/.test(src.slice(Math.max(0, node.start - 12), node.start))) return;edits.push({ at: node.start, text: wrap ? '(await ' : 'await ' });if (wrap) edits.push({ at: node.end, text: ')' });});@@@ -42,6 +42,7 @@ changed = []# Table columns from the service's migrations, so upsert right-hand sides can be qualified even for columns the INSERT# does not list (a counter bumped in place, a timestamp cleared).TABLE_COLS = {}++ALWAYS_IDENTITY = {} # table → its GENERATED ALWAYS identity columnimport glob as _globfor mf in sorted(_glob.glob('migrations/*.sql')):ddl = open(mf).read()@@@ -50,6 +51,9 @@ for mf in sorted(_glob.glob('migrations/*.sql')):TABLE_COLS.setdefault(tm.group(1), set()).update(cols)for am in re.finditer(r'ALTER TABLE (\w+) ADD COLUMN (\w+)', ddl):TABLE_COLS.setdefault(am.group(1), set()).add(am.group(2))++ for tm in re.finditer(r'CREATE TABLE (?:IF NOT EXISTS )?(\w+) \(([\s\S]*?)\n\);', ddl):++ for im in re.finditer(r'^\s*"?(\w+)"?\s+bigint GENERATED ALWAYS AS IDENTITY', tm.group(2), re.M):++ ALWAYS_IDENTITY.setdefault(tm.group(1), im.group(1))# A migration carrying SQLITE_DATE_FUNCTIONS (text timestamps kept as SQLite wrote them): CURRENT_TIMESTAMP in code# becomes ov_now() (PostgreSQL's would be a timestamptz, cast to other text), strftime's ISO 'now' ov_now_iso().SQLITE_DATES = any('CREATE FUNCTION ov_now()' in open(mf).read() for mf in _glob.glob('migrations/*.sql'))@@@ -74,8 +78,6 @@ for p in files:s = re.sub(r"assert\.throws\(\(\) => ((?:t\.)?[\w.]*db\.prepare\((?:[^()]|\([^()]*\))*\)\.(?:run|get|all)\((?:[^()]|\([^()]*\))*\)), ", r"await assert.rejects(\1, ", s)# A jsonb column (the SDK outbox's envelope) comes back as an object.s = re.sub(r"JSON\.parse\((\w+)\.envelope\)", r"(typeof \1.envelope === 'string' ? JSON.parse(\1.envelope) : \1.envelope)", s)-- # const f = db.transaction((a, b) => { … }) (called later as f(a, b)) → const f = (a, b) => db.tx(async () => { … })-- s = re.sub(r'const (\w+) = (\w+(?:\.\w+)?)\.transaction\(\(([^()]*)\) => \{', r'const \1 = (\3) => \2.tx(async () => {', s)# db.transaction(fn)() → db.tx(fn) (balanced parentheses; the codemod then awaits it and makes fn async).out, i = [], 0while True:@@@ -89,6 +91,17 @@ for p in files:else:out.append(s[i:k]); i = ks = ''.join(out)++ # const f = db.transaction((a, b) => { … }) (called later as f(a, b); an immediately invoked one was rewritten just++ # above) → const f = (a, b) => db.tx(async () => { … })++ s = re.sub(r'const (\w+) = (\w+(?:\.\w+)?)\.transaction\(\(([^()]*)\) => \{', r'const \1 = (\3) => \2.tx(async () => {', s)++ # An unquoted alias is folded to lower case in PostgreSQL (AS totalViews comes back as totalviews): quote++ # camelCase aliases inside SQL string literals.++ def quote_aliases(lit):++ body = lit.group(0)++ if not re.search(r'\bSELECT\b', body): return body++ q = '\\"' if body[0] == '"' else '"'++ return re.sub(r'\bAS ([a-z]+[A-Z]\w*)\b', lambda m: f'AS {q}{m.group(1)}{q}', body)++ s = re.sub(r'`[^`]*`|\'(?:[^\'\\\n]|\\.)*\'|"(?:[^"\\\n]|\\.)*"', quote_aliases, s)# Column names PostgreSQL reserves (SQLite does not): quoted inside SQL string literals ("window", "user").def quote_reserved(lit):body = lit.group(0)@@@ -114,6 +127,20 @@ for p in files:parts = re.split(r'(,\s*(?=\w+\s*=))', sets)return f'INSERT INTO {table} ({m.group(2)}){head}' + ''.join(fix_rhs(x) if not re.match(r',\s*$', x) else x for x in parts)s = re.sub(r'INSERT INTO (\w+) \(([^)]*)\)((?:(?!INSERT INTO)[^`\'"])*?DO UPDATE SET[^`\'"]*)', qualify, s)++ # INSERT INTO t (id, …) VALUES with an explicit id into a GENERATED ALWAYS identity (tests, fixtures, imports):++ # PostgreSQL refuses it without OVERRIDING SYSTEM VALUE.++ def override(m):++ table, cols = m.group(1), [c.strip().strip('"') for c in m.group(2).split(',')]++ return m.group(0) + ' OVERRIDING SYSTEM VALUE' if ALWAYS_IDENTITY.get(table) in cols else m.group(0)++ s = re.sub(r'INSERT INTO (\w+) \(([^)]*)\)(?! OVERRIDING)(?=\s*(?:VALUES|SELECT))', override, s)++ # (await db.prepare('INSERT …').run(…)).lastInsertRowid: the id comes back through RETURNING.++ s = re.sub(r"(prepare\()(`|'|\")(INSERT INTO (\w+)(?:(?!\2).)*?)(\2\)\s*\.run\((?:[^()]|\([^()]*\))*\)\)\s*\.lastInsertRowid)",++ lambda m: m.group(0) if 'RETURNING' in m.group(3) else m.group(1) + m.group(2) + m.group(3) + ' RETURNING id' + m.group(5), s)++ # Tests matching SQLite's constraint messages: PostgreSQL's say "duplicate key", "violates … constraint".++ s = re.sub(r'/UNIQUE constraint failed[^/\n]*/|/UNIQUE/', '/duplicate key/', s)++ s = re.sub(r'/CHECK constraint failed[^/\n]*/', '/violates check constraint/', s)++ s = re.sub(r'/NOT NULL constraint failed[^/\n]*/', '/violates not-null constraint/', s)++ s = re.sub(r'/FOREIGN KEY constraint failed[^/\n]*/', '/violates foreign key constraint/', s)# openvibe-publishing 1.0: the index sequencer stamps through a handle (the ambient transaction joins through store.db).handle = 'store.db' if 'store.db' in s else 'db's = re.sub(r'((?:store\.)?sequencer\.stamp\()(?!store\.db|db,|t,)', lambda m: m.group(1) + handle + ', ', s)@@@ -36,6 +36,8 @@ function toPg(sql) {sql = sql.replace(/^(\s*)(window|user)(\s+[A-Z])/gm, '$1"$2"$3').replace(/([(,]\s*)(window|user)(\s*[,)])/g, '$1"$2"$3');return sql.replace(/\bINTEGER PRIMARY KEY AUTOINCREMENT\b/g, 'bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY')++ // An INTEGER PRIMARY KEY that references another table is that row's key, not a counter.++ .replace(/\bINTEGER PRIMARY KEY\b(?! AUTOINCREMENT)(?=[^,\n]*\bREFERENCES\b)/g, 'bigint PRIMARY KEY')// A bare INTEGER PRIMARY KEY is SQLite's rowid: code may insert without it or with an explicit value..replace(/\bINTEGER PRIMARY KEY\b(?! AUTOINCREMENT)/g, 'bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY').replace(/CREATE (UNIQUE )?INDEX IF NOT EXISTS/g, (m, u) => `CREATE ${u || ''}INDEX`)nnew file mode 100644@@@ -0,0 +1,14 @@++#!/usr/bin/env node++'use strict';++// Every column declared INTEGER/REAL holding a value of another storage class (SQLite's affinity lets text in).++const Database = require(require.resolve('better-sqlite3', { paths: [process.cwd()] }));++const db = new Database(process.argv[2], { readonly: true });++for (const { name } of db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all()) {++ for (const c of db.prepare(`PRAGMA table_info("${name}")`).all()) {++ const t = String(c.type).toUpperCase();++ const want = /INT/.test(t) ? 'integer' : /REAL|FLOA|DOUB/.test(t) ? 'real' : null;++ if (!want) continue;++ const rows = db.prepare(`SELECT typeof("${c.name}") AS k, COUNT(*) AS n, MIN("${c.name}") AS ex FROM "${name}" WHERE "${c.name}" IS NOT NULL AND typeof("${c.name}") NOT IN ('${want}'${want === 'real' ? ", 'integer'" : ''}) GROUP BY 1`).all();++ for (const r of rows) console.log(`${name}.${c.name} (${t}): ${r.n} ${r.k} value(s), e.g. ${String(r.ex).slice(0, 40)}`);++ }++}
@@ -165,3 +165,15 @@ It changes nothing in the SQLite file.
@@ -165,3 +165,15 @@ It changes nothing in the SQLite file.
- **No DDL at boot.** A module that ran `CREATE TABLE IF NOT EXISTS` on the serving handle fails on production roles (`permission denied for schema`) while PGlite allows it. The table goes in the migration; `sqlfix.py` reports `db.exec(`.
- **No DDL at boot.** A module that ran `CREATE TABLE IF NOT EXISTS` on the serving handle fails on production roles (`permission denied for schema`) while PGlite allows it. The table goes in the migration; `sqlfix.py` reports `db.exec(`.
- **Test boots that call `start()` directly** fall back to the development PGlite directory and share rows across runs. Pass every boot a `createTestDb` handle.
- **Test boots that call `start()` directly** fall back to the development PGlite directory and share rows across runs. Pass every boot a `createTestDb` handle.
- **Redaction.** PostgreSQL has no `secure_delete`: an overwritten row version stays until vacuum. Lower the table's `autovacuum_vacuum_scale_factor` and say so in the README.
- **Redaction.** PostgreSQL has no `secure_delete`: an overwritten row version stays until vacuum. Lower the table's `autovacuum_vacuum_scale_factor` and say so in the README.
+- **Community (2026-09-28)** added these:
+ - **A schema changed by `migrate()` after `SCHEMA`.** `gen-migration.js --open "require('./server/db').openDb(':memory:')"` takes the schema the SQLite release's own boot leaves (every ALTER applied, tables in dependency order, boot seeds as INSERTs). Run it before converting any code.
+ - **SQLite text timestamps.** `DATETIME` columns hold `'YYYY-MM-DD HH:MM:SS'` text. Keep it: the migration gets `SQLITE_DATE_FUNCTIONS` (`ov_now()`, `datetime(t, modifier)`, `julianday(t)`), and `sqlfix.py` rewrites `CURRENT_TIMESTAMP` to `ov_now()` (PostgreSQL's would be a `timestamptz`, cast to different text).
+ - **Type affinity.** SQLite stores text in an `INTEGER` column without complaint (a Chat room id); PostgreSQL refuses. Check the production copy (`tools/asyncify/typecheck.js <file>` lists columns holding another storage class) and read the code that writes each id column.
+ - **`INTEGER PRIMARY KEY REFERENCES …`** is the other row's key, not a counter: `bigint PRIMARY KEY` (the converter does this now).
+ - **camelCase aliases** fold to lower case (`AS totalViews` comes back as `totalviews`): quote them (`sqlfix.py` does).
+ - **Explicit ids** into a `GENERATED ALWAYS` identity need `OVERRIDING SYSTEM VALUE` (`sqlfix.py` adds it where the migration says ALWAYS); `.lastInsertRowid` needs `RETURNING id`.
+ - **`IMMEDIATE` transactions.** A read-then-write that relied on SQLite's write lock (a vote recount) locks its row first: `SELECT … FOR UPDATE`.
+ - **Work kicked from inside a transaction** (`setImmediate(drain)`) runs before the commit now and misses the row: `db.afterCommit`, and a short delay when changes made together should go out as one pass.
+ - **Event handlers that became async** (a WebSocket dispatch) can overtake each other: queue them in arrival order.
+ - **Fire-and-forget helpers** (`tell(fn) { try { fn(x) } catch {} }`) catch nothing once `fn` is async: await it.
+ - **Tests**: `const x = createFoo(…)` instances, `require('./x').fn()` calls and a test's own `rejects(promise)` helper are handled by the codemod now; a boot helper that re-requires the server gets its database passed in (`createApp({ db })`); SQLite-only upgrade tests (a `migrate()` rebuilding an old table) go, the migration ledger replaces them.
@@ -59,9 +59,50 @@ function analyse(file, src, cfg, asyncNames) {
@@ -59,9 +59,50 @@ function analyse(file, src, cfg, asyncNames) {
const isFn = (n) => n.type === 'FunctionDeclaration' || n.type === 'FunctionExpression' || n.type === 'ArrowFunctionExpression';
const isFn = (n) => n.type === 'FunctionDeclaration' || n.type === 'FunctionExpression' || n.type === 'ArrowFunctionExpression';
const stores = new Set(cfg.stores || []);
const stores = new Set(cfg.stores || []);
const syncStoreMethods = new Set(cfg.syncStoreMethods || []);
const syncStoreMethods = new Set(cfg.syncStoreMethods || []);
- const apis = cfg.apis || {};
+ const apis = { ...(cfg.apis || {}) };
const syncNames = new Set(cfg.syncNames || []);
const syncNames = new Set(cfg.syncNames || []);
const localAsync = new Set(asyncNames.local.get(file) || []);
const localAsync = new Set(asyncNames.local.get(file) || []);
+ // const store = require('./store'): in this file, `store` is that module's API (the same name can mean another
+ // module elsewhere, so a namespace require wins over the config's global apis entry).
+ walk.full(parse(src), (n) => {
+ if (n.type !== 'VariableDeclarator' || n.id.type !== 'Identifier' || !n.init || n.init.type !== 'CallExpression') return;
+ const c = n.init;
+ if (c.callee.type !== 'Identifier' || c.callee.name !== 'require' || !c.arguments[0] || typeof c.arguments[0].value !== 'string' || !c.arguments[0].value.startsWith('.')) return;
+ const base = path.relative(process.cwd(), path.resolve(path.dirname(file), c.arguments[0].value));
+ const target = base.endsWith('.js') ? base : fs.existsSync(`${base}.js`) ? `${base}.js` : path.join(base, 'index.js');
+ if (fs.existsSync(target)) apis[n.id.name] = target;
+ });
+ // const svc = createPasteService(…) with createPasteService imported from a local file (destructured): `svc` is that
+ // file's API here (a factory's instance; its methods are the file's inner functions).
+ {
+ const imported = new Map();
+ const tree = parse(src);
+ walk.full(tree, (n) => {
+ if (n.type !== 'VariableDeclarator' || n.id.type !== 'ObjectPattern' || !n.init || n.init.type !== 'CallExpression') return;
+ const c = n.init;
+ if (c.callee.type !== 'Identifier' || c.callee.name !== 'require' || !c.arguments[0] || typeof c.arguments[0].value !== 'string' || !c.arguments[0].value.startsWith('.')) return;
+ const base = path.relative(process.cwd(), path.resolve(path.dirname(file), c.arguments[0].value));
+ const target = base.endsWith('.js') ? base : fs.existsSync(`${base}.js`) ? `${base}.js` : path.join(base, 'index.js');
+ for (const prop of n.id.properties) {
+ if (prop.type === 'Property' && prop.value && prop.value.type === 'Identifier') imported.set(prop.value.name, target);
+ }
+ });
+ const factoryOf = (init) => {
+ let x = init;
+ if (x && x.type === 'AwaitExpression') x = x.argument;
+ if (x && x.type === 'LogicalExpression') x = x.right; // opts.relay || createDiscordRelay(…)
+ if (x && x.type === 'AwaitExpression') x = x.argument;
+ return x && x.type === 'CallExpression' && x.callee.type === 'Identifier' && /^create[A-Z]/.test(x.callee.name) ? imported.get(x.callee.name) : null;
+ };
+ walk.full(tree, (n) => {
+ const pairs = n.type === 'VariableDeclarator' && n.id.type === 'Identifier' ? [[n.id.name, n.init]]
+ : n.type === 'AssignmentExpression' && n.left.type === 'Identifier' ? [[n.left.name, n.right]] : [];
+ for (const [name, init] of pairs) { const f = factoryOf(init); if (f && fs.existsSync(f)) apis[name] = f; }
+ });
+ }
+ // fileApis: { "server/comments/api.js": { "service": "server/comments/service.js" } } for a holder whose module
+ // differs by file (a factory's instance passed in under a generic name).
+ Object.assign(apis, (cfg.fileApis || {})[path.relative(process.cwd(), file)] || {});
// const { seed } = require('./workflows/seed'): a function imported from a converted file is async when it is there.
// const { seed } = require('./workflows/seed'): a function imported from a converted file is async when it is there.
walk.full(ast, (n) => {
walk.full(ast, (n) => {
if (n.type !== 'VariableDeclarator' || n.id.type !== 'ObjectPattern' || !n.init || n.init.type !== 'CallExpression') return;
if (n.type !== 'VariableDeclarator' || n.id.type !== 'ObjectPattern' || !n.init || n.init.type !== 'CallExpression') return;
@@ -112,6 +153,15 @@ function analyse(file, src, cfg, asyncNames) {
@@ -112,6 +153,15 @@ function analyse(file, src, cfg, asyncNames) {
});
});
});
});
+ // Local functions that await a parameter themselves: rejects(p, …) → 'rejects:0'.
+ const promiseTakers = new Set();
+ for (const [name, f] of localFns) {
+ const params = f.params.map((q) => (q.type === 'Identifier' ? q.name : null));
+ walk.full(f.body, (x) => {
+ if (x.type === 'AwaitExpression' && x.argument.type === 'Identifier' && params.includes(x.argument.name)) promiseTakers.add(`${name}:${params.indexOf(x.argument.name)}`);
+ });
+ }
+
function asyncCall(call) {
function asyncCall(call) {
const c = call.callee;
const c = call.callee;
if (c.type === 'MemberExpression' && !c.computed) {
if (c.type === 'MemberExpression' && !c.computed) {
@@ -124,6 +174,12 @@ function analyse(file, src, cfg, asyncNames) {
@@ -124,6 +174,12 @@ function analyse(file, src, cfg, asyncNames) {
if (p.length && stmtNames.has(p[p.length - 1])) return true;
if (p.length && stmtNames.has(p[p.length - 1])) return true;
}
}
if ((m === 'tx' || m === 'transaction') && p.length && ['store', 'db', 'this'].includes(p[p.length - 1])) return m === 'tx';
if ((m === 'tx' || m === 'transaction') && p.length && ['store', 'db', 'this'].includes(p[p.length - 1])) return m === 'tx';
+ // require('./events').pasteCreated(: a module's function reached through an inline require.
+ if (obj.type === 'CallExpression' && obj.callee.type === 'Identifier' && obj.callee.name === 'require' && obj.arguments[0] && typeof obj.arguments[0].value === 'string' && obj.arguments[0].value.startsWith('.')) {
+ const base = path.relative(process.cwd(), path.resolve(path.dirname(file), obj.arguments[0].value));
+ const set = asyncNames.api.get(base.endsWith('.js') ? base : `${base}.js`) || asyncNames.api.get(path.join(base, 'index.js'));
+ if (set && set.has(m) && !syncNames.has(m)) return true;
+ }
// <x>.<store>.<method>(: a store reached through an object; a bare identifier only when listed in bareStores.
// <x>.<store>.<method>(: a store reached through an object; a bare identifier only when listed in bareStores.
if (p.length >= 2 && stores.has(p[p.length - 1]) && !syncStoreMethods.has(m)) return true;
if (p.length >= 2 && stores.has(p[p.length - 1]) && !syncStoreMethods.has(m)) return true;
if (p.length === 1 && (cfg.bareStores || []).includes(p[0]) && !syncStoreMethods.has(m)) return true;
if (p.length === 1 && (cfg.bareStores || []).includes(p[0]) && !syncStoreMethods.has(m)) return true;
@@ -226,6 +282,8 @@ function analyse(file, src, cfg, asyncNames) {
@@ -226,6 +282,8 @@ function analyse(file, src, cfg, asyncNames) {
if (parent && parent.type === 'VariableDeclarator' && parent.init === node && parent.id.type === 'Identifier' && awaitedLater.has(parent.id.name)) return;
if (parent && parent.type === 'VariableDeclarator' && parent.init === node && parent.id.type === 'Identifier' && awaitedLater.has(parent.id.name)) return;
// assert.rejects(promise) / doesNotReject(promise) take the promise itself.
// assert.rejects(promise) / doesNotReject(promise) take the promise itself.
if (parent && parent.type === 'CallExpression' && parent.arguments[0] === node && parent.callee.type === 'MemberExpression' && ['rejects', 'doesNotReject'].includes(parent.callee.property.name)) return;
if (parent && parent.type === 'CallExpression' && parent.arguments[0] === node && parent.callee.type === 'MemberExpression' && ['rejects', 'doesNotReject'].includes(parent.callee.property.name)) return;
+ // So does a local helper that awaits that parameter itself (a test's rejects(p, status)).
+ if (parent && parent.type === 'CallExpression' && parent.callee.type === 'Identifier' && promiseTakers.has(`${parent.callee.name}:${parent.arguments.indexOf(node)}`)) return;
// Returned directly from an arrow body or a return statement of an async fn is fine too, but awaiting is harmless.
// Returned directly from an arrow body or a return statement of an async fn is fine too, but awaiting is harmless.
const fnIdx = (() => { for (let i = parents.length - 1; i >= 0; i--) if (isFn(parents[i])) return i; return -1; })();
const fnIdx = (() => { for (let i = parents.length - 1; i >= 0; i--) if (isFn(parents[i])) return i; return -1; })();
const fn = fnIdx >= 0 ? parents[fnIdx] : null;
const fn = fnIdx >= 0 ? parents[fnIdx] : null;
@@ -252,6 +310,8 @@ function analyse(file, src, cfg, asyncNames) {
@@ -252,6 +310,8 @@ function analyse(file, src, cfg, asyncNames) {
makeAsync(fn);
makeAsync(fn);
}
}
const wrap = parent && parent.type === 'MemberExpression' && parent.object === node;
const wrap = parent && parent.type === 'MemberExpression' && parent.object === node;
+ // Never a second await in front of one already there (a call the parser saw inside another's callee).
+ if (!wrap && /\bawait\s+$/.test(src.slice(Math.max(0, node.start - 12), node.start))) return;
edits.push({ at: node.start, text: wrap ? '(await ' : 'await ' });
edits.push({ at: node.start, text: wrap ? '(await ' : 'await ' });
if (wrap) edits.push({ at: node.end, text: ')' });
if (wrap) edits.push({ at: node.end, text: ')' });
});
});
@@ -42,6 +42,7 @@ changed = []
@@ -42,6 +42,7 @@ changed = []
# Table columns from the service's migrations, so upsert right-hand sides can be qualified even for columns the INSERT
# Table columns from the service's migrations, so upsert right-hand sides can be qualified even for columns the INSERT
# does not list (a counter bumped in place, a timestamp cleared).
# does not list (a counter bumped in place, a timestamp cleared).
TABLE_COLS = {}
TABLE_COLS = {}
+ALWAYS_IDENTITY = {} # table → its GENERATED ALWAYS identity column
import glob as _glob
import glob as _glob
for mf in sorted(_glob.glob('migrations/*.sql')):
for mf in sorted(_glob.glob('migrations/*.sql')):
ddl = open(mf).read()
ddl = open(mf).read()
@@ -50,6 +51,9 @@ for mf in sorted(_glob.glob('migrations/*.sql')):
@@ -50,6 +51,9 @@ for mf in sorted(_glob.glob('migrations/*.sql')):
TABLE_COLS.setdefault(tm.group(1), set()).update(cols)
TABLE_COLS.setdefault(tm.group(1), set()).update(cols)
for am in re.finditer(r'ALTER TABLE (\w+) ADD COLUMN (\w+)', ddl):
for am in re.finditer(r'ALTER TABLE (\w+) ADD COLUMN (\w+)', ddl):
TABLE_COLS.setdefault(am.group(1), set()).add(am.group(2))
TABLE_COLS.setdefault(am.group(1), set()).add(am.group(2))
+ for tm in re.finditer(r'CREATE TABLE (?:IF NOT EXISTS )?(\w+) \(([\s\S]*?)\n\);', ddl):
+ for im in re.finditer(r'^\s*"?(\w+)"?\s+bigint GENERATED ALWAYS AS IDENTITY', tm.group(2), re.M):
+ ALWAYS_IDENTITY.setdefault(tm.group(1), im.group(1))
# A migration carrying SQLITE_DATE_FUNCTIONS (text timestamps kept as SQLite wrote them): CURRENT_TIMESTAMP in code
# A migration carrying SQLITE_DATE_FUNCTIONS (text timestamps kept as SQLite wrote them): CURRENT_TIMESTAMP in code
# becomes ov_now() (PostgreSQL's would be a timestamptz, cast to other text), strftime's ISO 'now' ov_now_iso().
# becomes ov_now() (PostgreSQL's would be a timestamptz, cast to other text), strftime's ISO 'now' ov_now_iso().
SQLITE_DATES = any('CREATE FUNCTION ov_now()' in open(mf).read() for mf in _glob.glob('migrations/*.sql'))
SQLITE_DATES = any('CREATE FUNCTION ov_now()' in open(mf).read() for mf in _glob.glob('migrations/*.sql'))
@@ -74,8 +78,6 @@ for p in files:
@@ -74,8 +78,6 @@ for p in files:
s = re.sub(r"assert\.throws\(\(\) => ((?:t\.)?[\w.]*db\.prepare\((?:[^()]|\([^()]*\))*\)\.(?:run|get|all)\((?:[^()]|\([^()]*\))*\)), ", r"await assert.rejects(\1, ", s)
s = re.sub(r"assert\.throws\(\(\) => ((?:t\.)?[\w.]*db\.prepare\((?:[^()]|\([^()]*\))*\)\.(?:run|get|all)\((?:[^()]|\([^()]*\))*\)), ", r"await assert.rejects(\1, ", s)
# A jsonb column (the SDK outbox's envelope) comes back as an object.
# A jsonb column (the SDK outbox's envelope) comes back as an object.
s = re.sub(r"JSON\.parse\((\w+)\.envelope\)", r"(typeof \1.envelope === 'string' ? JSON.parse(\1.envelope) : \1.envelope)", s)
s = re.sub(r"JSON\.parse\((\w+)\.envelope\)", r"(typeof \1.envelope === 'string' ? JSON.parse(\1.envelope) : \1.envelope)", s)
- # const f = db.transaction((a, b) => { … }) (called later as f(a, b)) → const f = (a, b) => db.tx(async () => { … })
- s = re.sub(r'const (\w+) = (\w+(?:\.\w+)?)\.transaction\(\(([^()]*)\) => \{', r'const \1 = (\3) => \2.tx(async () => {', s)
# db.transaction(fn)() → db.tx(fn) (balanced parentheses; the codemod then awaits it and makes fn async).
# db.transaction(fn)() → db.tx(fn) (balanced parentheses; the codemod then awaits it and makes fn async).
out, i = [], 0
out, i = [], 0
while True:
while True:
@@ -89,6 +91,17 @@ for p in files:
@@ -89,6 +91,17 @@ for p in files:
else:
else:
out.append(s[i:k]); i = k
out.append(s[i:k]); i = k
s = ''.join(out)
s = ''.join(out)
+ # const f = db.transaction((a, b) => { … }) (called later as f(a, b); an immediately invoked one was rewritten just
+ # above) → const f = (a, b) => db.tx(async () => { … })
+ s = re.sub(r'const (\w+) = (\w+(?:\.\w+)?)\.transaction\(\(([^()]*)\) => \{', r'const \1 = (\3) => \2.tx(async () => {', s)
+ # An unquoted alias is folded to lower case in PostgreSQL (AS totalViews comes back as totalviews): quote
+ # camelCase aliases inside SQL string literals.
+ def quote_aliases(lit):
+ body = lit.group(0)
+ if not re.search(r'\bSELECT\b', body): return body
+ q = '\\"' if body[0] == '"' else '"'
+ return re.sub(r'\bAS ([a-z]+[A-Z]\w*)\b', lambda m: f'AS {q}{m.group(1)}{q}', body)
+ s = re.sub(r'`[^`]*`|\'(?:[^\'\\\n]|\\.)*\'|"(?:[^"\\\n]|\\.)*"', quote_aliases, s)
# Column names PostgreSQL reserves (SQLite does not): quoted inside SQL string literals ("window", "user").
# Column names PostgreSQL reserves (SQLite does not): quoted inside SQL string literals ("window", "user").
def quote_reserved(lit):
def quote_reserved(lit):
body = lit.group(0)
body = lit.group(0)
@@ -114,6 +127,20 @@ for p in files:
@@ -114,6 +127,20 @@ for p in files:
parts = re.split(r'(,\s*(?=\w+\s*=))', sets)
parts = re.split(r'(,\s*(?=\w+\s*=))', sets)
return f'INSERT INTO {table} ({m.group(2)}){head}' + ''.join(fix_rhs(x) if not re.match(r',\s*$', x) else x for x in parts)
return f'INSERT INTO {table} ({m.group(2)}){head}' + ''.join(fix_rhs(x) if not re.match(r',\s*$', x) else x for x in parts)
s = re.sub(r'INSERT INTO (\w+) \(([^)]*)\)((?:(?!INSERT INTO)[^`\'"])*?DO UPDATE SET[^`\'"]*)', qualify, s)
s = re.sub(r'INSERT INTO (\w+) \(([^)]*)\)((?:(?!INSERT INTO)[^`\'"])*?DO UPDATE SET[^`\'"]*)', qualify, s)
+ # INSERT INTO t (id, …) VALUES with an explicit id into a GENERATED ALWAYS identity (tests, fixtures, imports):
+ # PostgreSQL refuses it without OVERRIDING SYSTEM VALUE.
+ def override(m):
+ table, cols = m.group(1), [c.strip().strip('"') for c in m.group(2).split(',')]
+ return m.group(0) + ' OVERRIDING SYSTEM VALUE' if ALWAYS_IDENTITY.get(table) in cols else m.group(0)
+ s = re.sub(r'INSERT INTO (\w+) \(([^)]*)\)(?! OVERRIDING)(?=\s*(?:VALUES|SELECT))', override, s)
+ # (await db.prepare('INSERT …').run(…)).lastInsertRowid: the id comes back through RETURNING.
+ s = re.sub(r"(prepare\()(`|'|\")(INSERT INTO (\w+)(?:(?!\2).)*?)(\2\)\s*\.run\((?:[^()]|\([^()]*\))*\)\)\s*\.lastInsertRowid)",
+ lambda m: m.group(0) if 'RETURNING' in m.group(3) else m.group(1) + m.group(2) + m.group(3) + ' RETURNING id' + m.group(5), s)
+ # Tests matching SQLite's constraint messages: PostgreSQL's say "duplicate key", "violates … constraint".
+ s = re.sub(r'/UNIQUE constraint failed[^/\n]*/|/UNIQUE/', '/duplicate key/', s)
+ s = re.sub(r'/CHECK constraint failed[^/\n]*/', '/violates check constraint/', s)
+ s = re.sub(r'/NOT NULL constraint failed[^/\n]*/', '/violates not-null constraint/', s)
+ s = re.sub(r'/FOREIGN KEY constraint failed[^/\n]*/', '/violates foreign key constraint/', s)
# openvibe-publishing 1.0: the index sequencer stamps through a handle (the ambient transaction joins through store.db).
# openvibe-publishing 1.0: the index sequencer stamps through a handle (the ambient transaction joins through store.db).
handle = 'store.db' if 'store.db' in s else 'db'
handle = 'store.db' if 'store.db' in s else 'db'
s = re.sub(r'((?:store\.)?sequencer\.stamp\()(?!store\.db|db,|t,)', lambda m: m.group(1) + handle + ', ', s)
s = re.sub(r'((?:store\.)?sequencer\.stamp\()(?!store\.db|db,|t,)', lambda m: m.group(1) + handle + ', ', s)
@@ -36,6 +36,8 @@ function toPg(sql) {
@@ -36,6 +36,8 @@ function toPg(sql) {
sql = sql.replace(/^(\s*)(window|user)(\s+[A-Z])/gm, '$1"$2"$3').replace(/([(,]\s*)(window|user)(\s*[,)])/g, '$1"$2"$3');
sql = sql.replace(/^(\s*)(window|user)(\s+[A-Z])/gm, '$1"$2"$3').replace(/([(,]\s*)(window|user)(\s*[,)])/g, '$1"$2"$3');
return sql
return sql
.replace(/\bINTEGER PRIMARY KEY AUTOINCREMENT\b/g, 'bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY')
.replace(/\bINTEGER PRIMARY KEY AUTOINCREMENT\b/g, 'bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY')
+ // An INTEGER PRIMARY KEY that references another table is that row's key, not a counter.
+ .replace(/\bINTEGER PRIMARY KEY\b(?! AUTOINCREMENT)(?=[^,\n]*\bREFERENCES\b)/g, 'bigint PRIMARY KEY')
// A bare INTEGER PRIMARY KEY is SQLite's rowid: code may insert without it or with an explicit value.
// A bare INTEGER PRIMARY KEY is SQLite's rowid: code may insert without it or with an explicit value.
.replace(/\bINTEGER PRIMARY KEY\b(?! AUTOINCREMENT)/g, 'bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY')
.replace(/\bINTEGER PRIMARY KEY\b(?! AUTOINCREMENT)/g, 'bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY')
.replace(/CREATE (UNIQUE )?INDEX IF NOT EXISTS/g, (m, u) => `CREATE ${u || ''}INDEX`)
.replace(/CREATE (UNIQUE )?INDEX IF NOT EXISTS/g, (m, u) => `CREATE ${u || ''}INDEX`)
new file mode 100644
new file mode 100644
@@ -0,0 +1,14 @@
@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+'use strict';
+// Every column declared INTEGER/REAL holding a value of another storage class (SQLite's affinity lets text in).
+const Database = require(require.resolve('better-sqlite3', { paths: [process.cwd()] }));
+const db = new Database(process.argv[2], { readonly: true });
+for (const { name } of db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all()) {
+ for (const c of db.prepare(`PRAGMA table_info("${name}")`).all()) {
+ const t = String(c.type).toUpperCase();
+ const want = /INT/.test(t) ? 'integer' : /REAL|FLOA|DOUB/.test(t) ? 'real' : null;
+ if (!want) continue;
+ const rows = db.prepare(`SELECT typeof("${c.name}") AS k, COUNT(*) AS n, MIN("${c.name}") AS ex FROM "${name}" WHERE "${c.name}" IS NOT NULL AND typeof("${c.name}") NOT IN ('${want}'${want === 'real' ? ", 'integer'" : ''}) GROUP BY 1`).all();
+ for (const r of rows) console.log(`${name}.${c.name} (${t}): ${r.n} ${r.k} value(s), e.g. ${String(r.ex).slice(0, 40)}`);
+ }
+}