openvibers/openvibe.sdk Commits
v0.20.2: importSqlite refuses a source table with no target table instead of skipping it silently (a table a module created at runtime and the migration forgot would have been dropped by a switch; found converting AI). SQLite's own tables and FTS5 virtual tables with their shadows are still skipped; anything else not imported on purpose goes in the new skipSource option (runSqliteMigration too). An audit of the eleven switched services found nothing lost. tools/asyncify: the codemod follows aliases of async module methods, reports computed calls on module APIs, orders nested insertions explicitly (no more garbled Promise.all wraps); the schema converter handles WHEN on immutability triggers, quoted reserved column names (window, user) and NOCASE on expressions; sqlfix quotes reserved names in SQL strings, rewrites JSON.parse of jsonb envelopes and assert.throws over statements, and lists the order sqlfix → asyncify. npm test 41/41 with the containers.
Changes
@@@ -8,6 +8,10 @@ release may change an API and says so here.++## 0.20.2 (2026-09-28)++++- **`importSqlite` refuses a source table that has no target table.** Before, such a table was skipped without a word. A table that a module created at runtime and the migration forgot would have been dropped by the switch (found converting OpenVibe.AI: `subject_credentials`). SQLite's own tables and FTS5 virtual tables with their shadow tables are still skipped, because an index is rebuilt, not copied. Anything else not imported on purpose goes in the new `skipSource` option, also on `runSqliteMigration`. An audit of the eleven services already switched found nothing lost. `test/db.test.js`.++## 0.20.1 (2026-09-28)- **`openvibe-sdk/db`: one query at a time per transaction connection.** `await Promise.all(xs.map((x) => db.….get(x)))` inside a transaction used to put several queries on the transaction's client at once. node-postgres queues them, but warns that pg@9 will refuse. Such calls are now chained per connection, in call order. Outside a transaction, each call still takes its own pool connection and runs concurrently. `test/db-ambient.test.js` checks this on PGlite and through PgBouncer.@@@ -3,7 +3,7 @@"stage": "alpha","code": true,"wave": "W2",-- "version": "0.20.1",++ "version": "0.20.2","plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §3.2; roadmap §30","deployed": false,"deployment": "none: a package released as git tags; services and apps pin a tag",@@@ -1,4 +1,4 @@--// Generated by scripts/browser-bundle.js from browser.js (openvibe-sdk 0.20.1, MIT). Do not edit.++// Generated by scripts/browser-bundle.js from browser.js (openvibe-sdk 0.20.2, MIT). Do not edit.// One self-contained ES module: core at the top level plus the browser-safe subpaths as namespaces// (auth, registry, modules, realtime, media, community, search, chat, ai, jobs, tools, projects, geo). No server-only code: no service tokens, JWT verification, events, identity./* eslint-disable */@@@ -41,7 +41,7 @@ const { paginate, offsetPager } = __load(7);const semver = __load(6);/** The package version (test/version.test.js keeps it equal to package.json). */--const SDK_VERSION = '0.20.1';++const SDK_VERSION = '0.20.2';module.exports = {createClient,@@@ -1,12 +1,12 @@{"name": "openvibe-sdk",-- "version": "0.20.1",++ "version": "0.20.2","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "openvibe-sdk",-- "version": "0.20.1",++ "version": "0.20.2","license": "MIT","devDependencies": {"@electric-sql/pglite": "^0.5.8",@@@ -1,6 +1,6 @@{"name": "openvibe-sdk",-- "version": "0.20.1",++ "version": "0.20.2","description": "Supported browser and server clients for the OpenVibe platform: auth (incl. developer apps), registry discovery, identity, user modules, events, realtime, media, community, search, chat, AI runs, Tools (registry, run API and jobs), VIP memberships and developer projects, the async PostgreSQL data layer (db) and the Valkey cache, limits store, queue and pub/sub, with deadlines, safe retries, problem errors and trace propagation.","main": "index.js","types": "types/index.d.ts",@@@ -8,7 +8,7 @@ const { paginate, offsetPager } = require('./paginate');const semver = require('./semver');/** The package version (test/version.test.js keeps it equal to package.json). */--const SDK_VERSION = '0.20.1';++const SDK_VERSION = '0.20.2';module.exports = {createClient,@@@ -65,7 +65,7 @@ async function cleaningOptions(db, tables = {}) {return { tables: out, cleaned: () => [...cleaned].map(([column, values]) => ({ column, values: values.size })) };}--async function runSqliteMigration({ service, sqlite, directUrl, migrations, tables = {}, argv = process.argv.slice(2), out = console.log } = {}) {++async function runSqliteMigration({ service, sqlite, directUrl, migrations, tables = {}, skipSource = [], argv = process.argv.slice(2), out = console.log } = {}) {const { createDb, importSqlite } = require('./index');const opt = (name) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : null; };const flag = (name) => argv.includes(`--${name}`);@@@ -82,7 +82,7 @@ async function runSqliteMigration({ service, sqlite, directUrl, migrations, tablawait owner.migrate({ dir: migrations, log: quiet });const t0 = Date.now();const options = await cleaningOptions(owner, tables);-- const report = await importSqlite({ sqlite: file, db: owner, truncate: true, tables: options.tables, log: quiet });++ const report = await importSqlite({ sqlite: file, db: owner, truncate: true, tables: options.tables, skipSource, log: quiet });const cleaned = options.cleaned();if (flag('json')) out(JSON.stringify({ sqlite: file, into: owner.store, ...report, cleaned }, null, 2));else {@@@ -130,9 +130,10 @@ async function schemaOf(db) {* @param {string[]} [o.only] import only these target tables* @param {boolean} [o.truncate] empty the target tables first (rehearsals)* @param {boolean} [o.verify] compare counts and checksums (default true)++ * @param {string[]} [o.skipSource] source tables deliberately not imported (every other source table must have a target)* → { ok, tables: [{ table, source, rows, ms, checksum }], problems: [] }*/--async function importSqlite({ sqlite, db, tables: opts = {}, only = null, truncate = false, verify = true, log = console } = {}) {++async function importSqlite({ sqlite, db, tables: opts = {}, only = null, truncate = false, verify = true, skipSource = [], log = console } = {}) {let Database;const src = typeof sqlite === 'string' ? (() => { try { Database = require('better-sqlite3'); } catch { throw new Error('importSqlite: install better-sqlite3 to read the SQLite file'); } return new Database(sqlite, { readonly: true, fileMustExist: true }); })() : sqlite;const report = { ok: true, tables: [], problems: [] };@@@ -140,6 +141,18 @@ async function importSqlite({ sqlite, db, tables: opts = {}, only = null, truncatry {const schema = (await schemaOf(db)).filter((t) => !only || only.includes(t.name));const srcTables = new Set(src.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map((r) => r.name));++ // Every source table must land somewhere: a table the target lacks would be dropped without a word (a table a++ // module created at runtime and the migration forgot). Skipped: SQLite's own, FTS5 virtual tables and their++ // shadow tables (an index is rebuilt, not copied), and skipSource.++ if (!only) {++ const virtuals = src.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'").all().map((r) => r.name);++ const targets = new Set(schema.map((t) => (opts[t.name] && opts[t.name].from) || t.name));++ for (const name of srcTables) {++ if (targets.has(name) || name.startsWith('sqlite_') || skipSource.includes(name)) continue;++ if (virtuals.some((v) => name === v || name.startsWith(`${v}_`))) continue;++ problem({ table: name, problem: 'a source table with no target table (add it to the migrations, or list it in skipSource)' });++ }++ }const plan = [];for (const t of schema) {const o = opts[t.name] || {};@@@ -138,7 +138,9 @@ const tests = [const s = new Database(file);s.exec(`CREATE TABLE people (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL, is_admin INTEGER DEFAULT 0, settings TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, legacy_note TEXT);CREATE TABLE notes (id INTEGER PRIMARY KEY, person_id INTEGER REFERENCES people(id), body TEXT, pinned INTEGER, posted INTEGER, tags TEXT);-- CREATE TABLE loose (k TEXT, v TEXT);`);++ CREATE TABLE loose (k TEXT, v TEXT);++ CREATE TABLE forgotten (id INTEGER PRIMARY KEY, secret TEXT);++ CREATE VIRTUAL TABLE docs_fts USING fts5(title);`);const ins = s.prepare('INSERT INTO people (id, username, is_admin, settings, created_at, legacy_note) VALUES (?, ?, ?, ?, ?, ?)');ins.run(3, 'ann', 1, '{"theme":"blue","n":[1,2]}', '2026-09-01 10:00:00', 'drop me');ins.run(7, 'bob', 0, null, '2026-09-02T11:30:00.000Z', null);@@@ -159,8 +161,13 @@ CREATE TABLE loose (k text, v text);` }), log: { log() {} } });// A source column with no target column is a problem until it is dropped on purpose.let r = await importSqlite({ sqlite: file, db, log: quiet });assert.equal(r.ok, false);-- assert.match(r.problems[0].problem, /legacy_note/);++ assert.ok(r.problems.some((p) => /legacy_note/.test(p.problem)), JSON.stringify(r.problems));r = await importSqlite({ sqlite: file, db, log: quiet, tables: { people: { dropColumns: ['legacy_note'] } } });++ // A source table the target lacks would be dropped without a word: refused until skipped on purpose++ // (an FTS5 index and its shadow tables are skipped: an index is rebuilt, not copied).++ assert.equal(r.ok, false);++ assert.deepEqual(r.problems.map((p) => p.table), ['forgotten']);++ r = await importSqlite({ sqlite: file, db, log: quiet, truncate: true, skipSource: ['forgotten'], tables: { people: { dropColumns: ['legacy_note'] } } });assert.equal(r.ok, true, JSON.stringify(r.problems));assert.deepEqual(r.tables.map((t) => [t.table, t.rows]), [['loose', 2], ['people', 3], ['notes', 2]], 'parents before children');const ann = await db.one(sql`SELECT * FROM people WHERE id = 3`);@@@ -171,7 +178,7 @@ CREATE TABLE loose (k text, v text);` }), log: { log() {} } });const next = await db.one(sql`INSERT INTO people (username) VALUES ('new') RETURNING id`);assert.equal(next.id, 13, 'the identity continues after the imported maximum');// A second import onto the same rows fails loudly; truncate makes a rehearsal repeatable.-- r = await importSqlite({ sqlite: file, db, log: quiet, truncate: true, tables: { people: { dropColumns: ['legacy_note'] } } });++ r = await importSqlite({ sqlite: file, db, log: quiet, truncate: true, skipSource: ['forgotten'], tables: { people: { dropColumns: ['legacy_note'] } } });assert.equal(r.ok, true);// A transformation that differs between copy and check shows up as a failed verification.let flip = 0;@@@ -45,8 +45,8 @@ python3 g1pass.py <svc> <Name> <ENV> # package pins, config, outbox, linpm installnode gen-migration.js <Name> --publishing … # migrations/0001_initial.sql from the SQLite SCHEMApython3 g1db.py <svc> <Name> # server/db.js: openDb / createStore / openStore--node asyncify.js --config <svc>.json … # await everything; wraps .map(async …) in Promise.all--python3 sqlfix.py # the SQL rewrites, and CHECK lines for a person++python3 sqlfix.py # the SQL rewrites (db.transaction(fn)() → db.tx(fn) too), and CHECK lines++node asyncify.js --config <svc>.json … # then await everything (db.tx included); wraps .map(async …) in Promise.allnpm test && npm run test:pg # then rehearse: scripts/migrate-to-postgres.js --sqlite <copy> --pglitepython3 g1docs.py <svc> <Name> <ENV> # README and STATUS.json```@@@ -61,7 +61,14 @@ function analyse(file, src, cfg, asyncNames) {const syncStoreMethods = new Set(cfg.syncStoreMethods || []);const apis = cfg.apis || {};const syncNames = new Set(cfg.syncNames || []);-- const localAsync = asyncNames.local.get(file) || new Set();++ const localAsync = new Set(asyncNames.local.get(file) || []);++ // const view = ops.providerView: an alias of an async module method is async too.++ walk.full(ast, (n) => {++ if (n.type !== 'VariableDeclarator' || n.id.type !== 'Identifier' || !n.init || n.init.type !== 'MemberExpression' || n.init.computed) return;++ const holder = n.init.object.type === 'Identifier' ? n.init.object.name : null;++ const set = holder && apis[holder] ? asyncNames.api.get(apis[holder]) : null;++ if (set && set.has(n.init.property.name) && !syncNames.has(n.init.property.name)) localAsync.add(n.id.name);++ });// Names that hold a prepared statement: const x = db.prepare(...), or { x: db.prepare(...) }.const stmtNames = new Set();@@@ -141,10 +148,16 @@ function analyse(file, src, cfg, asyncNames) {if (call.callee.property.name !== 'map') { manual.push(`${file}:${call.loc.start.line}: a .${call.callee.property.name}() callback becomes async`); return; }if (inPromiseArg(call) || wrappedMaps.has(call)) return;wrappedMaps.add(call);-- edits.push({ at: call.start, text: '(await Promise.all(' });-- edits.push({ at: call.end, text: '))' });++ edits.push({ at: call.start, text: '(await Promise.all(', depth: 1 });++ edits.push({ at: call.end, text: '))', depth: 1 });makeAsync([...anc].reverse().find((x) => x !== fn && isFnN(x)));}++ // registry[k.list](): a computed call on a module API cannot be resolved here; a person awaits it if needed.++ walk.full(ast, (n) => {++ if (n.type !== 'CallExpression' || n.callee.type !== 'MemberExpression' || !n.callee.computed || n.callee.object.type !== 'Identifier' || !apis[n.callee.object.name]) return;++ const p = ancestors.get(n) || []; const up = p[p.length - 1];++ if (!(up && up.type === 'AwaitExpression')) manual.push(`${file}:${n.loc.start.line}: computed call ${n.callee.object.name}[…]() on a module API: await it if the method is async`);++ });// An async callback already there whose map result is not awaited as a whole is almost always a bug.walk.full(ast, (n) => {if (n.type === 'CallExpression' && n.callee.type === 'MemberExpression' && n.callee.property.name === 'map' && isFnN(n.arguments[0]) && n.arguments[0].async && !inPromiseArg(n)) {@@@ -172,8 +185,8 @@ function analyse(file, src, cfg, asyncNames) {const outer = [...p].reverse().find(isFnNode);if (n.callee.property.name === 'map' && outer && !wrappedMaps.has(n)) {wrappedMaps.add(n);-- edits.push({ at: n.start, text: '(await Promise.all(' });-- edits.push({ at: n.end, text: '))' });++ edits.push({ at: n.start, text: '(await Promise.all(', depth: 1 });++ edits.push({ at: n.end, text: '))', depth: 1 });makeAsync(outer);return;}@@@ -212,8 +225,8 @@ function analyse(file, src, cfg, asyncNames) {if (fnParent.callee.property.name === 'map' && outer) {if (!wrappedMaps.has(fnParent)) {wrappedMaps.add(fnParent);-- edits.push({ at: fnParent.start, text: '(await Promise.all(' });-- edits.push({ at: fnParent.end, text: '))' });++ edits.push({ at: fnParent.start, text: '(await Promise.all(', depth: 1 });++ edits.push({ at: fnParent.end, text: '))', depth: 1 });makeAsync(outer);}} else {@@@ -244,7 +257,13 @@ function analyse(file, src, cfg, asyncNames) {function apply(src, edits) {// At one position, closing text goes in first so it ends up after what opens there.-- const sorted = edits.slice().sort((a, b) => b.at - a.at || ((a.text[0] === ')' ? 0 : 1) - (b.text[0] === ')' ? 0 : 1)));++ // At one position: closers go in before openers (so an opener ends up left of a closer), an outer closer before an++ // inner one (it ends up right of it), an inner opener before an outer one (the outer ends up leftmost). A++ // (await Promise.all( wrap is outer (depth 1) to the await of the call it starts with (depth 0).++ const closer = (e) => e.text[0] === ')';++ const sorted = edits.slice().sort((a, b) => b.at - a.at++ || (closer(a) ? 0 : 1) - (closer(b) ? 0 : 1)++ || (closer(a) ? (b.depth || 0) - (a.depth || 0) : (a.depth || 0) - (b.depth || 0)));let out = src;for (const e of sorted) out = out.slice(0, e.at) + e.text + out.slice(e.at);return out;@@@ -53,11 +53,14 @@ if m:s = s.replace('function createApp(opts = {}) {', 'async function createApp(opts = {}) {', 1)open('server/app.js', 'w').write(s); print('app store')# limits + valkey--edit('server/http/actor-limits.js', [-- ("const { createActorLimiter, defaultActor } = require('openvibe-sdk/limits');", "const { createActorLimiter, createValkeyLimitStore, defaultActor } = require('openvibe-sdk/limits');"),-- ("function createActorLimits({ config, now = () => Date.now(), registry = null, log = console }) {", "function createActorLimits({ config, now = () => Date.now(), registry = null, log = console, valkey = null }) {"),-- (" now,\n", " now,\n // Shared across processes on Valkey (ADR-035) when VALKEY_URL is set; in-process otherwise.\n ...(valkey ? { store: createValkeyLimitStore(valkey) } : {}),\n"),--])++if os.path.exists('server/http/actor-limits.js'):++ s = open('server/http/actor-limits.js').read(); o = s++ if 'createValkeyLimitStore' not in s:++ s = re.sub(r"const \{ createActorLimiter(, defaultActor)? \} = require\('openvibe-sdk/limits'\);",++ lambda m: "const { createActorLimiter, createValkeyLimitStore" + (m.group(1) or '') + " } = require('openvibe-sdk/limits');", s, 1)++ s = re.sub(r"(function createActorLimits\(\{[^)]*?)( \}\) \{)", lambda m: m.group(1) + ', valkey = null' + m.group(2), s, 1)++ s = s.replace(" now,\n", " now,\n // Shared across processes on Valkey (ADR-035) when VALKEY_URL is set; in-process otherwise.\n ...(valkey ? { store: createValkeyLimitStore(valkey) } : {}),\n", 1)++ if s != o: open('server/http/actor-limits.js', 'w').write(s); print('edited server/http/actor-limits.js')s = open('server/app.js').read()m = re.search(r" ctx\.(limits|actorLimits) = createActorLimits\(\{ config, now: opts\.limitsNow \|\| \(\(\) => Date\.now\(\)\), registry: metrics\.registry, log \}\);", s)if m:@@@ -154,7 +157,7 @@ const {{ MIGRATIONS }} = require('../server/db');const TABLES = {{}};if (require.main === module) {{-- const config = configLib.load();++ const config = (configLib.load || configLib.loadConfig)();runSqliteMigration({{ service: '{svc}', sqlite: config.dbPath, directUrl: config.db.directUrl, migrations: MIGRATIONS, tables: TABLES }}).then((code) => process.exit(code), (err) => {{ console.error(`migrate-to-postgres failed: ${{err.message}}`); process.exit(1); }});}}@@@ -43,10 +43,15 @@ for p in files:i = s.index('INSERT OR IGNORE INTO'); q = s[i - 1] if s[i - 1] in '\'"`' else '`'j = s.index(q, i)s = s[:i] + 'INSERT INTO' + s[i + len('INSERT OR IGNORE INTO'):j] + ' ON CONFLICT DO NOTHING' + s[j:]++ s = re.sub(r'(\w+\((?:[^()]|\([^()]*\))*\)) COLLATE NOCASE', r'lower(\1)', s) # an expression: COALESCE(a, b) COLLATE NOCASEs = re.sub(r'([\w.]+) COLLATE NOCASE', r'lower(\1)', s)s = re.sub(r'(?<![IN\w])LIKE (?=[@?:$\'(]|[a-z])', 'ILIKE ', s)s = s.replace('json_each(?)', 'jsonb_array_elements_text(?::jsonb)')s = re.sub(r'\bIFNULL\(', 'COALESCE(', s)++ # assert.throws(() => db.prepare(…).run(…), re): the statement rejects now.++ s = re.sub(r"assert\.throws\(\(\) => ((?:t\.)?[\w.]*db\.prepare\((?:[^()]|\([^()]*\))*\)\.(?:run|get|all)\((?:[^()]|\([^()]*\))*\)), ", r"await assert.rejects(\1, ", s)++ # A jsonb column (the SDK outbox's envelope) comes back as an object.++ s = re.sub(r"JSON\.parse\((\w+)\.envelope\)", r"(typeof \1.envelope === 'string' ? JSON.parse(\1.envelope) : \1.envelope)", s)# db.transaction(fn)() → db.tx(fn) (balanced parentheses; the codemod then awaits it and makes fn async).out, i = [], 0while True:@@@ -60,6 +65,13 @@ for p in files:else:out.append(s[i:k]); i = ks = ''.join(out)++ # Column names PostgreSQL reserves (SQLite does not): quoted inside SQL string literals ("window", "user").++ def quote_reserved(lit):++ body = lit.group(0)++ if not re.search(r'\b(SELECT|INSERT|UPDATE|DELETE|CREATE)\b', body): return body++ q = '\\"' if body[0] == '"' else '"' # inside a double-quoted JS string the quotes are escaped++ return re.sub(r"(?<![.\w\"$\\'])(window|user)(?![\w\"(\\'])", lambda m: q + m.group(1) + q, body)++ s = re.sub(r'`[^`]*`|\'(?:[^\'\\\n]|\\.)*\'|"(?:[^"\\\n]|\\.)*"', quote_reserved, s)# better-sqlite3's db.inTransaction was a property; openvibe-sdk/db's is a function.s = re.sub(r'\b(db|this\.db|store\.db)\.inTransaction\b(?!\s*\()', r'\1.inTransaction()', s)# ON CONFLICT … DO UPDATE SET x = x + 1 / COALESCE(excluded.x, x): a bare column on the right is ambiguous in@@@ -15,10 +15,13 @@*/function toPg(sql) {// The one trigger shape services use: a row that must never change (BEFORE UPDATE/DELETE … RAISE(ABORT, msg)).-- sql = sql.replace(/CREATE TRIGGER(?: IF NOT EXISTS)? (\w+) BEFORE (UPDATE(?: OF [\w\s,]+?)?|DELETE) ON (\w+)\s+BEGIN SELECT RAISE\(ABORT, '([^']*)'\); END;/g,-- (m, name, op, table, msg) => `CREATE FUNCTION ${name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION '${msg}'; END $$;\n`-- + `CREATE TRIGGER ${name} BEFORE ${op.replace(/\s+/g, ' ')} ON ${table} FOR EACH ROW EXECUTE FUNCTION ${name}();`);-- if (sql.split(';').some((st) => /CREATE\s+TRIGGER/i.test(st) && !/FOR EACH ROW EXECUTE FUNCTION/.test(st))) {++ // With an optional WHEN condition (x IS NOT y between OLD and NEW becomes IS DISTINCT FROM).++ sql = sql.replace(/CREATE TRIGGER(?: IF NOT EXISTS)? (\w+) BEFORE (UPDATE(?: OF [\w\s,]+?)?|DELETE) ON (\w+)(?:\s+WHEN ([\s\S]+?))?\s+BEGIN SELECT RAISE\(ABORT, '([^']*)'\); END;/g,++ (m, name, op, table, when, msg) => `CREATE FUNCTION ${name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION '${msg}'; END $$;\n`++ + `CREATE TRIGGER ${name} BEFORE ${op.replace(/\s+/g, ' ')} ON ${table} FOR EACH ROW`++ + (when ? ` WHEN (${when.replace(/\s+/g, ' ').replace(/ IS NOT (OLD|NEW)\./g, ' IS DISTINCT FROM $1.')})` : '')++ + ` EXECUTE FUNCTION ${name}();`);++ if (sql.split(';').some((st) => /CREATE\s+TRIGGER/i.test(st) && !/FOR EACH ROW[\s\S]*EXECUTE FUNCTION/.test(st))) {throw new Error('sqlite-schema-to-pg: a trigger other than the immutable-row shape needs writing as PL/pgSQL by hand');}// COLLATE NOCASE: in an index, the index is on lower(col) (queries compare lower(col) too: sqlfix.py); on a column@@@ -26,6 +29,8 @@ function toPg(sql) {sql = sql.replace(/(CREATE (?:UNIQUE )?INDEX[^;]*?\()([^;]*?)\)/g, (m, head, cols) => head + cols.replace(/(\w+) COLLATE NOCASE/g, 'lower($1)') + ')');if (/^\s*\w+\s+TEXT\b[^\n]*COLLATE NOCASE[^\n]*UNIQUE|UNIQUE[^\n]*COLLATE NOCASE/m.test(sql)) throw new Error('sqlite-schema-to-pg: a case-insensitive UNIQUE column needs a unique index on lower(col), by hand');sql = sql.replace(/^(\s*\w+\s+TEXT\b[^\n]*?) COLLATE NOCASE/gm, '$1');++ // Column names PostgreSQL reserves (SQLite does not) are quoted: a column definition, and inside index/unique lists.++ sql = sql.replace(/^(\s*)(window|user)(\s+[A-Z])/gm, '$1"$2"$3').replace(/([(,]\s*)(window|user)(\s*[,)])/g, '$1"$2"$3');return sql.replace(/\bINTEGER PRIMARY KEY AUTOINCREMENT\b/g, 'bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY')// A bare INTEGER PRIMARY KEY is SQLite's rowid: code may insert without it or with an explicit value.@@@ -34,10 +39,10 @@ function toPg(sql) {.replace(/CREATE TABLE IF NOT EXISTS/g, 'CREATE TABLE').replace(/\)\s*WITHOUT ROWID\s*;/g, ');').replace(/CREATE VIEW IF NOT EXISTS/g, 'CREATE VIEW')-- .replace(/^(\s*\w+\s+)TEXT\b/gm, '$1text COLLATE "C"')-- .replace(/^(\s*\w+\s+)INTEGER\b/gm, '$1bigint')-- .replace(/^(\s*\w+\s+)REAL\b/gm, '$1double precision')-- .replace(/^(\s*\w+\s+)BLOB\b/gm, '$1bytea');++ .replace(/^(\s*"?\w+"?\s+)TEXT\b/gm, '$1text COLLATE "C"')++ .replace(/^(\s*"?\w+"?\s+)INTEGER\b/gm, '$1bigint')++ .replace(/^(\s*"?\w+"?\s+)REAL\b/gm, '$1double precision')++ .replace(/^(\s*"?\w+"?\s+)BLOB\b/gm, '$1bytea');}module.exports = { toPg };
@@ -8,6 +8,10 @@ release may change an API and says so here.
@@ -8,6 +8,10 @@ release may change an API and says so here.
+## 0.20.2 (2026-09-28)
+
+- **`importSqlite` refuses a source table that has no target table.** Before, such a table was skipped without a word. A table that a module created at runtime and the migration forgot would have been dropped by the switch (found converting OpenVibe.AI: `subject_credentials`). SQLite's own tables and FTS5 virtual tables with their shadow tables are still skipped, because an index is rebuilt, not copied. Anything else not imported on purpose goes in the new `skipSource` option, also on `runSqliteMigration`. An audit of the eleven services already switched found nothing lost. `test/db.test.js`.
+
## 0.20.1 (2026-09-28)
## 0.20.1 (2026-09-28)
- **`openvibe-sdk/db`: one query at a time per transaction connection.** `await Promise.all(xs.map((x) => db.….get(x)))` inside a transaction used to put several queries on the transaction's client at once. node-postgres queues them, but warns that pg@9 will refuse. Such calls are now chained per connection, in call order. Outside a transaction, each call still takes its own pool connection and runs concurrently. `test/db-ambient.test.js` checks this on PGlite and through PgBouncer.
- **`openvibe-sdk/db`: one query at a time per transaction connection.** `await Promise.all(xs.map((x) => db.….get(x)))` inside a transaction used to put several queries on the transaction's client at once. node-postgres queues them, but warns that pg@9 will refuse. Such calls are now chained per connection, in call order. Outside a transaction, each call still takes its own pool connection and runs concurrently. `test/db-ambient.test.js` checks this on PGlite and through PgBouncer.
@@ -3,7 +3,7 @@
@@ -3,7 +3,7 @@
"stage": "alpha",
"stage": "alpha",
"code": true,
"code": true,
"wave": "W2",
"wave": "W2",
- "version": "0.20.1",
+ "version": "0.20.2",
"plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §3.2; roadmap §30",
"plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §3.2; roadmap §30",
"deployed": false,
"deployed": false,
"deployment": "none: a package released as git tags; services and apps pin a tag",
"deployment": "none: a package released as git tags; services and apps pin a tag",
@@ -1,4 +1,4 @@
@@ -1,4 +1,4 @@
-// Generated by scripts/browser-bundle.js from browser.js (openvibe-sdk 0.20.1, MIT). Do not edit.
+// Generated by scripts/browser-bundle.js from browser.js (openvibe-sdk 0.20.2, MIT). Do not edit.
// One self-contained ES module: core at the top level plus the browser-safe subpaths as namespaces
// One self-contained ES module: core at the top level plus the browser-safe subpaths as namespaces
// (auth, registry, modules, realtime, media, community, search, chat, ai, jobs, tools, projects, geo). No server-only code: no service tokens, JWT verification, events, identity.
// (auth, registry, modules, realtime, media, community, search, chat, ai, jobs, tools, projects, geo). No server-only code: no service tokens, JWT verification, events, identity.
/* eslint-disable */
/* eslint-disable */
@@ -41,7 +41,7 @@ const { paginate, offsetPager } = __load(7);
@@ -41,7 +41,7 @@ const { paginate, offsetPager } = __load(7);
const semver = __load(6);
const semver = __load(6);
/** The package version (test/version.test.js keeps it equal to package.json). */
/** The package version (test/version.test.js keeps it equal to package.json). */
-const SDK_VERSION = '0.20.1';
+const SDK_VERSION = '0.20.2';
module.exports = {
module.exports = {
createClient,
createClient,
@@ -1,12 +1,12 @@
@@ -1,12 +1,12 @@
{
{
"name": "openvibe-sdk",
"name": "openvibe-sdk",
- "version": "0.20.1",
+ "version": "0.20.2",
"lockfileVersion": 3,
"lockfileVersion": 3,
"requires": true,
"requires": true,
"packages": {
"packages": {
"": {
"": {
"name": "openvibe-sdk",
"name": "openvibe-sdk",
- "version": "0.20.1",
+ "version": "0.20.2",
"license": "MIT",
"license": "MIT",
"devDependencies": {
"devDependencies": {
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",
@@ -1,6 +1,6 @@
@@ -1,6 +1,6 @@
{
{
"name": "openvibe-sdk",
"name": "openvibe-sdk",
- "version": "0.20.1",
+ "version": "0.20.2",
"description": "Supported browser and server clients for the OpenVibe platform: auth (incl. developer apps), registry discovery, identity, user modules, events, realtime, media, community, search, chat, AI runs, Tools (registry, run API and jobs), VIP memberships and developer projects, the async PostgreSQL data layer (db) and the Valkey cache, limits store, queue and pub/sub, with deadlines, safe retries, problem errors and trace propagation.",
"description": "Supported browser and server clients for the OpenVibe platform: auth (incl. developer apps), registry discovery, identity, user modules, events, realtime, media, community, search, chat, AI runs, Tools (registry, run API and jobs), VIP memberships and developer projects, the async PostgreSQL data layer (db) and the Valkey cache, limits store, queue and pub/sub, with deadlines, safe retries, problem errors and trace propagation.",
"main": "index.js",
"main": "index.js",
"types": "types/index.d.ts",
"types": "types/index.d.ts",
@@ -8,7 +8,7 @@ const { paginate, offsetPager } = require('./paginate');
@@ -8,7 +8,7 @@ const { paginate, offsetPager } = require('./paginate');
const semver = require('./semver');
const semver = require('./semver');
/** The package version (test/version.test.js keeps it equal to package.json). */
/** The package version (test/version.test.js keeps it equal to package.json). */
-const SDK_VERSION = '0.20.1';
+const SDK_VERSION = '0.20.2';
module.exports = {
module.exports = {
createClient,
createClient,
@@ -65,7 +65,7 @@ async function cleaningOptions(db, tables = {}) {
@@ -65,7 +65,7 @@ async function cleaningOptions(db, tables = {}) {
return { tables: out, cleaned: () => [...cleaned].map(([column, values]) => ({ column, values: values.size })) };
return { tables: out, cleaned: () => [...cleaned].map(([column, values]) => ({ column, values: values.size })) };
}
}
-async function runSqliteMigration({ service, sqlite, directUrl, migrations, tables = {}, argv = process.argv.slice(2), out = console.log } = {}) {
+async function runSqliteMigration({ service, sqlite, directUrl, migrations, tables = {}, skipSource = [], argv = process.argv.slice(2), out = console.log } = {}) {
const { createDb, importSqlite } = require('./index');
const { createDb, importSqlite } = require('./index');
const opt = (name) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : null; };
const opt = (name) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : null; };
const flag = (name) => argv.includes(`--${name}`);
const flag = (name) => argv.includes(`--${name}`);
@@ -82,7 +82,7 @@ async function runSqliteMigration({ service, sqlite, directUrl, migrations, tabl
@@ -82,7 +82,7 @@ async function runSqliteMigration({ service, sqlite, directUrl, migrations, tabl
await owner.migrate({ dir: migrations, log: quiet });
await owner.migrate({ dir: migrations, log: quiet });
const t0 = Date.now();
const t0 = Date.now();
const options = await cleaningOptions(owner, tables);
const options = await cleaningOptions(owner, tables);
- const report = await importSqlite({ sqlite: file, db: owner, truncate: true, tables: options.tables, log: quiet });
+ const report = await importSqlite({ sqlite: file, db: owner, truncate: true, tables: options.tables, skipSource, log: quiet });
const cleaned = options.cleaned();
const cleaned = options.cleaned();
if (flag('json')) out(JSON.stringify({ sqlite: file, into: owner.store, ...report, cleaned }, null, 2));
if (flag('json')) out(JSON.stringify({ sqlite: file, into: owner.store, ...report, cleaned }, null, 2));
else {
else {
@@ -130,9 +130,10 @@ async function schemaOf(db) {
@@ -130,9 +130,10 @@ async function schemaOf(db) {
* @param {string[]} [o.only] import only these target tables
* @param {string[]} [o.only] import only these target tables
* @param {boolean} [o.truncate] empty the target tables first (rehearsals)
* @param {boolean} [o.truncate] empty the target tables first (rehearsals)
* @param {boolean} [o.verify] compare counts and checksums (default true)
* @param {boolean} [o.verify] compare counts and checksums (default true)
+ * @param {string[]} [o.skipSource] source tables deliberately not imported (every other source table must have a target)
* → { ok, tables: [{ table, source, rows, ms, checksum }], problems: [] }
* → { ok, tables: [{ table, source, rows, ms, checksum }], problems: [] }
*/
*/
-async function importSqlite({ sqlite, db, tables: opts = {}, only = null, truncate = false, verify = true, log = console } = {}) {
+async function importSqlite({ sqlite, db, tables: opts = {}, only = null, truncate = false, verify = true, skipSource = [], log = console } = {}) {
let Database;
let Database;
const src = typeof sqlite === 'string' ? (() => { try { Database = require('better-sqlite3'); } catch { throw new Error('importSqlite: install better-sqlite3 to read the SQLite file'); } return new Database(sqlite, { readonly: true, fileMustExist: true }); })() : sqlite;
const src = typeof sqlite === 'string' ? (() => { try { Database = require('better-sqlite3'); } catch { throw new Error('importSqlite: install better-sqlite3 to read the SQLite file'); } return new Database(sqlite, { readonly: true, fileMustExist: true }); })() : sqlite;
const report = { ok: true, tables: [], problems: [] };
const report = { ok: true, tables: [], problems: [] };
@@ -140,6 +141,18 @@ async function importSqlite({ sqlite, db, tables: opts = {}, only = null, trunca
@@ -140,6 +141,18 @@ async function importSqlite({ sqlite, db, tables: opts = {}, only = null, trunca
try {
try {
const schema = (await schemaOf(db)).filter((t) => !only || only.includes(t.name));
const schema = (await schemaOf(db)).filter((t) => !only || only.includes(t.name));
const srcTables = new Set(src.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map((r) => r.name));
const srcTables = new Set(src.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map((r) => r.name));
+ // Every source table must land somewhere: a table the target lacks would be dropped without a word (a table a
+ // module created at runtime and the migration forgot). Skipped: SQLite's own, FTS5 virtual tables and their
+ // shadow tables (an index is rebuilt, not copied), and skipSource.
+ if (!only) {
+ const virtuals = src.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'").all().map((r) => r.name);
+ const targets = new Set(schema.map((t) => (opts[t.name] && opts[t.name].from) || t.name));
+ for (const name of srcTables) {
+ if (targets.has(name) || name.startsWith('sqlite_') || skipSource.includes(name)) continue;
+ if (virtuals.some((v) => name === v || name.startsWith(`${v}_`))) continue;
+ problem({ table: name, problem: 'a source table with no target table (add it to the migrations, or list it in skipSource)' });
+ }
+ }
const plan = [];
const plan = [];
for (const t of schema) {
for (const t of schema) {
const o = opts[t.name] || {};
const o = opts[t.name] || {};
@@ -138,7 +138,9 @@ const tests = [
@@ -138,7 +138,9 @@ const tests = [
const s = new Database(file);
const s = new Database(file);
s.exec(`CREATE TABLE people (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL, is_admin INTEGER DEFAULT 0, settings TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, legacy_note TEXT);
s.exec(`CREATE TABLE people (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL, is_admin INTEGER DEFAULT 0, settings TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, legacy_note TEXT);
CREATE TABLE notes (id INTEGER PRIMARY KEY, person_id INTEGER REFERENCES people(id), body TEXT, pinned INTEGER, posted INTEGER, tags TEXT);
CREATE TABLE notes (id INTEGER PRIMARY KEY, person_id INTEGER REFERENCES people(id), body TEXT, pinned INTEGER, posted INTEGER, tags TEXT);
- CREATE TABLE loose (k TEXT, v TEXT);`);
+ CREATE TABLE loose (k TEXT, v TEXT);
+ CREATE TABLE forgotten (id INTEGER PRIMARY KEY, secret TEXT);
+ CREATE VIRTUAL TABLE docs_fts USING fts5(title);`);
const ins = s.prepare('INSERT INTO people (id, username, is_admin, settings, created_at, legacy_note) VALUES (?, ?, ?, ?, ?, ?)');
const ins = s.prepare('INSERT INTO people (id, username, is_admin, settings, created_at, legacy_note) VALUES (?, ?, ?, ?, ?, ?)');
ins.run(3, 'ann', 1, '{"theme":"blue","n":[1,2]}', '2026-09-01 10:00:00', 'drop me');
ins.run(3, 'ann', 1, '{"theme":"blue","n":[1,2]}', '2026-09-01 10:00:00', 'drop me');
ins.run(7, 'bob', 0, null, '2026-09-02T11:30:00.000Z', null);
ins.run(7, 'bob', 0, null, '2026-09-02T11:30:00.000Z', null);
@@ -159,8 +161,13 @@ CREATE TABLE loose (k text, v text);` }), log: { log() {} } });
@@ -159,8 +161,13 @@ CREATE TABLE loose (k text, v text);` }), log: { log() {} } });
// A source column with no target column is a problem until it is dropped on purpose.
// A source column with no target column is a problem until it is dropped on purpose.
let r = await importSqlite({ sqlite: file, db, log: quiet });
let r = await importSqlite({ sqlite: file, db, log: quiet });
assert.equal(r.ok, false);
assert.equal(r.ok, false);
- assert.match(r.problems[0].problem, /legacy_note/);
+ assert.ok(r.problems.some((p) => /legacy_note/.test(p.problem)), JSON.stringify(r.problems));
r = await importSqlite({ sqlite: file, db, log: quiet, tables: { people: { dropColumns: ['legacy_note'] } } });
r = await importSqlite({ sqlite: file, db, log: quiet, tables: { people: { dropColumns: ['legacy_note'] } } });
+ // A source table the target lacks would be dropped without a word: refused until skipped on purpose
+ // (an FTS5 index and its shadow tables are skipped: an index is rebuilt, not copied).
+ assert.equal(r.ok, false);
+ assert.deepEqual(r.problems.map((p) => p.table), ['forgotten']);
+ r = await importSqlite({ sqlite: file, db, log: quiet, truncate: true, skipSource: ['forgotten'], tables: { people: { dropColumns: ['legacy_note'] } } });
assert.equal(r.ok, true, JSON.stringify(r.problems));
assert.equal(r.ok, true, JSON.stringify(r.problems));
assert.deepEqual(r.tables.map((t) => [t.table, t.rows]), [['loose', 2], ['people', 3], ['notes', 2]], 'parents before children');
assert.deepEqual(r.tables.map((t) => [t.table, t.rows]), [['loose', 2], ['people', 3], ['notes', 2]], 'parents before children');
const ann = await db.one(sql`SELECT * FROM people WHERE id = 3`);
const ann = await db.one(sql`SELECT * FROM people WHERE id = 3`);
@@ -171,7 +178,7 @@ CREATE TABLE loose (k text, v text);` }), log: { log() {} } });
@@ -171,7 +178,7 @@ CREATE TABLE loose (k text, v text);` }), log: { log() {} } });
const next = await db.one(sql`INSERT INTO people (username) VALUES ('new') RETURNING id`);
const next = await db.one(sql`INSERT INTO people (username) VALUES ('new') RETURNING id`);
assert.equal(next.id, 13, 'the identity continues after the imported maximum');
assert.equal(next.id, 13, 'the identity continues after the imported maximum');
// A second import onto the same rows fails loudly; truncate makes a rehearsal repeatable.
// A second import onto the same rows fails loudly; truncate makes a rehearsal repeatable.
- r = await importSqlite({ sqlite: file, db, log: quiet, truncate: true, tables: { people: { dropColumns: ['legacy_note'] } } });
+ r = await importSqlite({ sqlite: file, db, log: quiet, truncate: true, skipSource: ['forgotten'], tables: { people: { dropColumns: ['legacy_note'] } } });
assert.equal(r.ok, true);
assert.equal(r.ok, true);
// A transformation that differs between copy and check shows up as a failed verification.
// A transformation that differs between copy and check shows up as a failed verification.
let flip = 0;
let flip = 0;
@@ -45,8 +45,8 @@ python3 g1pass.py <svc> <Name> <ENV> # package pins, config, outbox, li
@@ -45,8 +45,8 @@ python3 g1pass.py <svc> <Name> <ENV> # package pins, config, outbox, li
npm install
npm install
node gen-migration.js <Name> --publishing … # migrations/0001_initial.sql from the SQLite SCHEMA
node gen-migration.js <Name> --publishing … # migrations/0001_initial.sql from the SQLite SCHEMA
python3 g1db.py <svc> <Name> # server/db.js: openDb / createStore / openStore
python3 g1db.py <svc> <Name> # server/db.js: openDb / createStore / openStore
-node asyncify.js --config <svc>.json … # await everything; wraps .map(async …) in Promise.all
-python3 sqlfix.py # the SQL rewrites, and CHECK lines for a person
+python3 sqlfix.py # the SQL rewrites (db.transaction(fn)() → db.tx(fn) too), and CHECK lines
+node asyncify.js --config <svc>.json … # then await everything (db.tx included); wraps .map(async …) in Promise.all
npm test && npm run test:pg # then rehearse: scripts/migrate-to-postgres.js --sqlite <copy> --pglite
npm test && npm run test:pg # then rehearse: scripts/migrate-to-postgres.js --sqlite <copy> --pglite
python3 g1docs.py <svc> <Name> <ENV> # README and STATUS.json
python3 g1docs.py <svc> <Name> <ENV> # README and STATUS.json
```
```
@@ -61,7 +61,14 @@ function analyse(file, src, cfg, asyncNames) {
@@ -61,7 +61,14 @@ function analyse(file, src, cfg, asyncNames) {
const syncStoreMethods = new Set(cfg.syncStoreMethods || []);
const syncStoreMethods = new Set(cfg.syncStoreMethods || []);
const apis = cfg.apis || {};
const apis = cfg.apis || {};
const syncNames = new Set(cfg.syncNames || []);
const syncNames = new Set(cfg.syncNames || []);
- const localAsync = asyncNames.local.get(file) || new Set();
+ const localAsync = new Set(asyncNames.local.get(file) || []);
+ // const view = ops.providerView: an alias of an async module method is async too.
+ walk.full(ast, (n) => {
+ if (n.type !== 'VariableDeclarator' || n.id.type !== 'Identifier' || !n.init || n.init.type !== 'MemberExpression' || n.init.computed) return;
+ const holder = n.init.object.type === 'Identifier' ? n.init.object.name : null;
+ const set = holder && apis[holder] ? asyncNames.api.get(apis[holder]) : null;
+ if (set && set.has(n.init.property.name) && !syncNames.has(n.init.property.name)) localAsync.add(n.id.name);
+ });
// Names that hold a prepared statement: const x = db.prepare(...), or { x: db.prepare(...) }.
// Names that hold a prepared statement: const x = db.prepare(...), or { x: db.prepare(...) }.
const stmtNames = new Set();
const stmtNames = new Set();
@@ -141,10 +148,16 @@ function analyse(file, src, cfg, asyncNames) {
@@ -141,10 +148,16 @@ function analyse(file, src, cfg, asyncNames) {
if (call.callee.property.name !== 'map') { manual.push(`${file}:${call.loc.start.line}: a .${call.callee.property.name}() callback becomes async`); return; }
if (call.callee.property.name !== 'map') { manual.push(`${file}:${call.loc.start.line}: a .${call.callee.property.name}() callback becomes async`); return; }
if (inPromiseArg(call) || wrappedMaps.has(call)) return;
if (inPromiseArg(call) || wrappedMaps.has(call)) return;
wrappedMaps.add(call);
wrappedMaps.add(call);
- edits.push({ at: call.start, text: '(await Promise.all(' });
- edits.push({ at: call.end, text: '))' });
+ edits.push({ at: call.start, text: '(await Promise.all(', depth: 1 });
+ edits.push({ at: call.end, text: '))', depth: 1 });
makeAsync([...anc].reverse().find((x) => x !== fn && isFnN(x)));
makeAsync([...anc].reverse().find((x) => x !== fn && isFnN(x)));
}
}
+ // registry[k.list](): a computed call on a module API cannot be resolved here; a person awaits it if needed.
+ walk.full(ast, (n) => {
+ if (n.type !== 'CallExpression' || n.callee.type !== 'MemberExpression' || !n.callee.computed || n.callee.object.type !== 'Identifier' || !apis[n.callee.object.name]) return;
+ const p = ancestors.get(n) || []; const up = p[p.length - 1];
+ if (!(up && up.type === 'AwaitExpression')) manual.push(`${file}:${n.loc.start.line}: computed call ${n.callee.object.name}[…]() on a module API: await it if the method is async`);
+ });
// An async callback already there whose map result is not awaited as a whole is almost always a bug.
// An async callback already there whose map result is not awaited as a whole is almost always a bug.
walk.full(ast, (n) => {
walk.full(ast, (n) => {
if (n.type === 'CallExpression' && n.callee.type === 'MemberExpression' && n.callee.property.name === 'map' && isFnN(n.arguments[0]) && n.arguments[0].async && !inPromiseArg(n)) {
if (n.type === 'CallExpression' && n.callee.type === 'MemberExpression' && n.callee.property.name === 'map' && isFnN(n.arguments[0]) && n.arguments[0].async && !inPromiseArg(n)) {
@@ -172,8 +185,8 @@ function analyse(file, src, cfg, asyncNames) {
@@ -172,8 +185,8 @@ function analyse(file, src, cfg, asyncNames) {
const outer = [...p].reverse().find(isFnNode);
const outer = [...p].reverse().find(isFnNode);
if (n.callee.property.name === 'map' && outer && !wrappedMaps.has(n)) {
if (n.callee.property.name === 'map' && outer && !wrappedMaps.has(n)) {
wrappedMaps.add(n);
wrappedMaps.add(n);
- edits.push({ at: n.start, text: '(await Promise.all(' });
- edits.push({ at: n.end, text: '))' });
+ edits.push({ at: n.start, text: '(await Promise.all(', depth: 1 });
+ edits.push({ at: n.end, text: '))', depth: 1 });
makeAsync(outer);
makeAsync(outer);
return;
return;
}
}
@@ -212,8 +225,8 @@ function analyse(file, src, cfg, asyncNames) {
@@ -212,8 +225,8 @@ function analyse(file, src, cfg, asyncNames) {
if (fnParent.callee.property.name === 'map' && outer) {
if (fnParent.callee.property.name === 'map' && outer) {
if (!wrappedMaps.has(fnParent)) {
if (!wrappedMaps.has(fnParent)) {
wrappedMaps.add(fnParent);
wrappedMaps.add(fnParent);
- edits.push({ at: fnParent.start, text: '(await Promise.all(' });
- edits.push({ at: fnParent.end, text: '))' });
+ edits.push({ at: fnParent.start, text: '(await Promise.all(', depth: 1 });
+ edits.push({ at: fnParent.end, text: '))', depth: 1 });
makeAsync(outer);
makeAsync(outer);
}
}
} else {
} else {
@@ -244,7 +257,13 @@ function analyse(file, src, cfg, asyncNames) {
@@ -244,7 +257,13 @@ function analyse(file, src, cfg, asyncNames) {
function apply(src, edits) {
function apply(src, edits) {
// At one position, closing text goes in first so it ends up after what opens there.
// At one position, closing text goes in first so it ends up after what opens there.
- const sorted = edits.slice().sort((a, b) => b.at - a.at || ((a.text[0] === ')' ? 0 : 1) - (b.text[0] === ')' ? 0 : 1)));
+ // At one position: closers go in before openers (so an opener ends up left of a closer), an outer closer before an
+ // inner one (it ends up right of it), an inner opener before an outer one (the outer ends up leftmost). A
+ // (await Promise.all( wrap is outer (depth 1) to the await of the call it starts with (depth 0).
+ const closer = (e) => e.text[0] === ')';
+ const sorted = edits.slice().sort((a, b) => b.at - a.at
+ || (closer(a) ? 0 : 1) - (closer(b) ? 0 : 1)
+ || (closer(a) ? (b.depth || 0) - (a.depth || 0) : (a.depth || 0) - (b.depth || 0)));
let out = src;
let out = src;
for (const e of sorted) out = out.slice(0, e.at) + e.text + out.slice(e.at);
for (const e of sorted) out = out.slice(0, e.at) + e.text + out.slice(e.at);
return out;
return out;
@@ -53,11 +53,14 @@ if m:
@@ -53,11 +53,14 @@ if m:
s = s.replace('function createApp(opts = {}) {', 'async function createApp(opts = {}) {', 1)
s = s.replace('function createApp(opts = {}) {', 'async function createApp(opts = {}) {', 1)
open('server/app.js', 'w').write(s); print('app store')
open('server/app.js', 'w').write(s); print('app store')
# limits + valkey
# limits + valkey
-edit('server/http/actor-limits.js', [
- ("const { createActorLimiter, defaultActor } = require('openvibe-sdk/limits');", "const { createActorLimiter, createValkeyLimitStore, defaultActor } = require('openvibe-sdk/limits');"),
- ("function createActorLimits({ config, now = () => Date.now(), registry = null, log = console }) {", "function createActorLimits({ config, now = () => Date.now(), registry = null, log = console, valkey = null }) {"),
- (" now,\n", " now,\n // Shared across processes on Valkey (ADR-035) when VALKEY_URL is set; in-process otherwise.\n ...(valkey ? { store: createValkeyLimitStore(valkey) } : {}),\n"),
-])
+if os.path.exists('server/http/actor-limits.js'):
+ s = open('server/http/actor-limits.js').read(); o = s
+ if 'createValkeyLimitStore' not in s:
+ s = re.sub(r"const \{ createActorLimiter(, defaultActor)? \} = require\('openvibe-sdk/limits'\);",
+ lambda m: "const { createActorLimiter, createValkeyLimitStore" + (m.group(1) or '') + " } = require('openvibe-sdk/limits');", s, 1)
+ s = re.sub(r"(function createActorLimits\(\{[^)]*?)( \}\) \{)", lambda m: m.group(1) + ', valkey = null' + m.group(2), s, 1)
+ s = s.replace(" now,\n", " now,\n // Shared across processes on Valkey (ADR-035) when VALKEY_URL is set; in-process otherwise.\n ...(valkey ? { store: createValkeyLimitStore(valkey) } : {}),\n", 1)
+ if s != o: open('server/http/actor-limits.js', 'w').write(s); print('edited server/http/actor-limits.js')
s = open('server/app.js').read()
s = open('server/app.js').read()
m = re.search(r" ctx\.(limits|actorLimits) = createActorLimits\(\{ config, now: opts\.limitsNow \|\| \(\(\) => Date\.now\(\)\), registry: metrics\.registry, log \}\);", s)
m = re.search(r" ctx\.(limits|actorLimits) = createActorLimits\(\{ config, now: opts\.limitsNow \|\| \(\(\) => Date\.now\(\)\), registry: metrics\.registry, log \}\);", s)
if m:
if m:
@@ -154,7 +157,7 @@ const {{ MIGRATIONS }} = require('../server/db');
@@ -154,7 +157,7 @@ const {{ MIGRATIONS }} = require('../server/db');
const TABLES = {{}};
const TABLES = {{}};
if (require.main === module) {{
if (require.main === module) {{
- const config = configLib.load();
+ const config = (configLib.load || configLib.loadConfig)();
runSqliteMigration({{ service: '{svc}', sqlite: config.dbPath, directUrl: config.db.directUrl, migrations: MIGRATIONS, tables: TABLES }})
runSqliteMigration({{ service: '{svc}', sqlite: config.dbPath, directUrl: config.db.directUrl, migrations: MIGRATIONS, tables: TABLES }})
.then((code) => process.exit(code), (err) => {{ console.error(`migrate-to-postgres failed: ${{err.message}}`); process.exit(1); }});
.then((code) => process.exit(code), (err) => {{ console.error(`migrate-to-postgres failed: ${{err.message}}`); process.exit(1); }});
}}
}}
@@ -43,10 +43,15 @@ for p in files:
@@ -43,10 +43,15 @@ for p in files:
i = s.index('INSERT OR IGNORE INTO'); q = s[i - 1] if s[i - 1] in '\'"`' else '`'
i = s.index('INSERT OR IGNORE INTO'); q = s[i - 1] if s[i - 1] in '\'"`' else '`'
j = s.index(q, i)
j = s.index(q, i)
s = s[:i] + 'INSERT INTO' + s[i + len('INSERT OR IGNORE INTO'):j] + ' ON CONFLICT DO NOTHING' + s[j:]
s = s[:i] + 'INSERT INTO' + s[i + len('INSERT OR IGNORE INTO'):j] + ' ON CONFLICT DO NOTHING' + s[j:]
+ s = re.sub(r'(\w+\((?:[^()]|\([^()]*\))*\)) COLLATE NOCASE', r'lower(\1)', s) # an expression: COALESCE(a, b) COLLATE NOCASE
s = re.sub(r'([\w.]+) COLLATE NOCASE', r'lower(\1)', s)
s = re.sub(r'([\w.]+) COLLATE NOCASE', r'lower(\1)', s)
s = re.sub(r'(?<![IN\w])LIKE (?=[@?:$\'(]|[a-z])', 'ILIKE ', s)
s = re.sub(r'(?<![IN\w])LIKE (?=[@?:$\'(]|[a-z])', 'ILIKE ', s)
s = s.replace('json_each(?)', 'jsonb_array_elements_text(?::jsonb)')
s = s.replace('json_each(?)', 'jsonb_array_elements_text(?::jsonb)')
s = re.sub(r'\bIFNULL\(', 'COALESCE(', s)
s = re.sub(r'\bIFNULL\(', 'COALESCE(', s)
+ # assert.throws(() => db.prepare(…).run(…), re): the statement rejects now.
+ s = re.sub(r"assert\.throws\(\(\) => ((?:t\.)?[\w.]*db\.prepare\((?:[^()]|\([^()]*\))*\)\.(?:run|get|all)\((?:[^()]|\([^()]*\))*\)), ", r"await assert.rejects(\1, ", s)
+ # A jsonb column (the SDK outbox's envelope) comes back as an object.
+ s = re.sub(r"JSON\.parse\((\w+)\.envelope\)", r"(typeof \1.envelope === 'string' ? JSON.parse(\1.envelope) : \1.envelope)", s)
# db.transaction(fn)() → db.tx(fn) (balanced parentheses; the codemod then awaits it and makes fn async).
# db.transaction(fn)() → db.tx(fn) (balanced parentheses; the codemod then awaits it and makes fn async).
out, i = [], 0
out, i = [], 0
while True:
while True:
@@ -60,6 +65,13 @@ for p in files:
@@ -60,6 +65,13 @@ for p in files:
else:
else:
out.append(s[i:k]); i = k
out.append(s[i:k]); i = k
s = ''.join(out)
s = ''.join(out)
+ # Column names PostgreSQL reserves (SQLite does not): quoted inside SQL string literals ("window", "user").
+ def quote_reserved(lit):
+ body = lit.group(0)
+ if not re.search(r'\b(SELECT|INSERT|UPDATE|DELETE|CREATE)\b', body): return body
+ q = '\\"' if body[0] == '"' else '"' # inside a double-quoted JS string the quotes are escaped
+ return re.sub(r"(?<![.\w\"$\\'])(window|user)(?![\w\"(\\'])", lambda m: q + m.group(1) + q, body)
+ s = re.sub(r'`[^`]*`|\'(?:[^\'\\\n]|\\.)*\'|"(?:[^"\\\n]|\\.)*"', quote_reserved, s)
# better-sqlite3's db.inTransaction was a property; openvibe-sdk/db's is a function.
# better-sqlite3's db.inTransaction was a property; openvibe-sdk/db's is a function.
s = re.sub(r'\b(db|this\.db|store\.db)\.inTransaction\b(?!\s*\()', r'\1.inTransaction()', s)
s = re.sub(r'\b(db|this\.db|store\.db)\.inTransaction\b(?!\s*\()', r'\1.inTransaction()', s)
# ON CONFLICT … DO UPDATE SET x = x + 1 / COALESCE(excluded.x, x): a bare column on the right is ambiguous in
# ON CONFLICT … DO UPDATE SET x = x + 1 / COALESCE(excluded.x, x): a bare column on the right is ambiguous in
@@ -15,10 +15,13 @@
@@ -15,10 +15,13 @@
*/
*/
function toPg(sql) {
function toPg(sql) {
// The one trigger shape services use: a row that must never change (BEFORE UPDATE/DELETE … RAISE(ABORT, msg)).
// The one trigger shape services use: a row that must never change (BEFORE UPDATE/DELETE … RAISE(ABORT, msg)).
- sql = sql.replace(/CREATE TRIGGER(?: IF NOT EXISTS)? (\w+) BEFORE (UPDATE(?: OF [\w\s,]+?)?|DELETE) ON (\w+)\s+BEGIN SELECT RAISE\(ABORT, '([^']*)'\); END;/g,
- (m, name, op, table, msg) => `CREATE FUNCTION ${name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION '${msg}'; END $$;\n`
- + `CREATE TRIGGER ${name} BEFORE ${op.replace(/\s+/g, ' ')} ON ${table} FOR EACH ROW EXECUTE FUNCTION ${name}();`);
- if (sql.split(';').some((st) => /CREATE\s+TRIGGER/i.test(st) && !/FOR EACH ROW EXECUTE FUNCTION/.test(st))) {
+ // With an optional WHEN condition (x IS NOT y between OLD and NEW becomes IS DISTINCT FROM).
+ sql = sql.replace(/CREATE TRIGGER(?: IF NOT EXISTS)? (\w+) BEFORE (UPDATE(?: OF [\w\s,]+?)?|DELETE) ON (\w+)(?:\s+WHEN ([\s\S]+?))?\s+BEGIN SELECT RAISE\(ABORT, '([^']*)'\); END;/g,
+ (m, name, op, table, when, msg) => `CREATE FUNCTION ${name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION '${msg}'; END $$;\n`
+ + `CREATE TRIGGER ${name} BEFORE ${op.replace(/\s+/g, ' ')} ON ${table} FOR EACH ROW`
+ + (when ? ` WHEN (${when.replace(/\s+/g, ' ').replace(/ IS NOT (OLD|NEW)\./g, ' IS DISTINCT FROM $1.')})` : '')
+ + ` EXECUTE FUNCTION ${name}();`);
+ if (sql.split(';').some((st) => /CREATE\s+TRIGGER/i.test(st) && !/FOR EACH ROW[\s\S]*EXECUTE FUNCTION/.test(st))) {
throw new Error('sqlite-schema-to-pg: a trigger other than the immutable-row shape needs writing as PL/pgSQL by hand');
throw new Error('sqlite-schema-to-pg: a trigger other than the immutable-row shape needs writing as PL/pgSQL by hand');
}
}
// COLLATE NOCASE: in an index, the index is on lower(col) (queries compare lower(col) too: sqlfix.py); on a column
// COLLATE NOCASE: in an index, the index is on lower(col) (queries compare lower(col) too: sqlfix.py); on a column
@@ -26,6 +29,8 @@ function toPg(sql) {
@@ -26,6 +29,8 @@ function toPg(sql) {
sql = sql.replace(/(CREATE (?:UNIQUE )?INDEX[^;]*?\()([^;]*?)\)/g, (m, head, cols) => head + cols.replace(/(\w+) COLLATE NOCASE/g, 'lower($1)') + ')');
sql = sql.replace(/(CREATE (?:UNIQUE )?INDEX[^;]*?\()([^;]*?)\)/g, (m, head, cols) => head + cols.replace(/(\w+) COLLATE NOCASE/g, 'lower($1)') + ')');
if (/^\s*\w+\s+TEXT\b[^\n]*COLLATE NOCASE[^\n]*UNIQUE|UNIQUE[^\n]*COLLATE NOCASE/m.test(sql)) throw new Error('sqlite-schema-to-pg: a case-insensitive UNIQUE column needs a unique index on lower(col), by hand');
if (/^\s*\w+\s+TEXT\b[^\n]*COLLATE NOCASE[^\n]*UNIQUE|UNIQUE[^\n]*COLLATE NOCASE/m.test(sql)) throw new Error('sqlite-schema-to-pg: a case-insensitive UNIQUE column needs a unique index on lower(col), by hand');
sql = sql.replace(/^(\s*\w+\s+TEXT\b[^\n]*?) COLLATE NOCASE/gm, '$1');
sql = sql.replace(/^(\s*\w+\s+TEXT\b[^\n]*?) COLLATE NOCASE/gm, '$1');
+ // Column names PostgreSQL reserves (SQLite does not) are quoted: a column definition, and inside index/unique lists.
+ sql = sql.replace(/^(\s*)(window|user)(\s+[A-Z])/gm, '$1"$2"$3').replace(/([(,]\s*)(window|user)(\s*[,)])/g, '$1"$2"$3');
return sql
return sql
.replace(/\bINTEGER PRIMARY KEY AUTOINCREMENT\b/g, 'bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY')
.replace(/\bINTEGER PRIMARY KEY AUTOINCREMENT\b/g, 'bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY')
// A bare INTEGER PRIMARY KEY is SQLite's rowid: code may insert without it or with an explicit value.
// A bare INTEGER PRIMARY KEY is SQLite's rowid: code may insert without it or with an explicit value.
@@ -34,10 +39,10 @@ function toPg(sql) {
@@ -34,10 +39,10 @@ function toPg(sql) {
.replace(/CREATE TABLE IF NOT EXISTS/g, 'CREATE TABLE')
.replace(/CREATE TABLE IF NOT EXISTS/g, 'CREATE TABLE')
.replace(/\)\s*WITHOUT ROWID\s*;/g, ');')
.replace(/\)\s*WITHOUT ROWID\s*;/g, ');')
.replace(/CREATE VIEW IF NOT EXISTS/g, 'CREATE VIEW')
.replace(/CREATE VIEW IF NOT EXISTS/g, 'CREATE VIEW')
- .replace(/^(\s*\w+\s+)TEXT\b/gm, '$1text COLLATE "C"')
- .replace(/^(\s*\w+\s+)INTEGER\b/gm, '$1bigint')
- .replace(/^(\s*\w+\s+)REAL\b/gm, '$1double precision')
- .replace(/^(\s*\w+\s+)BLOB\b/gm, '$1bytea');
+ .replace(/^(\s*"?\w+"?\s+)TEXT\b/gm, '$1text COLLATE "C"')
+ .replace(/^(\s*"?\w+"?\s+)INTEGER\b/gm, '$1bigint')
+ .replace(/^(\s*"?\w+"?\s+)REAL\b/gm, '$1double precision')
+ .replace(/^(\s*"?\w+"?\s+)BLOB\b/gm, '$1bytea');
}
}
module.exports = { toPg };
module.exports = { toPg };