openvibers/openvibe.sdk Commits
0.19.0: runSqliteMigration in openvibe-sdk/db, a service's scripts/migrate-to-postgres.js in one call. It migrates as the owner (or into PGlite with --pglite for a rehearsal) and imports into emptied tables. It makes text PostgreSQL refuses storable: a NUL is dropped and an unpaired surrogate becomes U+FFFD, in text and inside JSON, and every column where that happened is reported. It verifies counts and checksums, prints the report (--json for all of it), and answers 0 only when everything verified. Generalised from Tips' script; Wiki's is a few lines around it. test/sqlite-cli.test.js; 41/41 test files with the containers.
Changes
@@@ -7,6 +7,17 @@ release may change an API and says so here.++++## 0.19.0 (2026-09-28)++++- **`openvibe-sdk/db` `runSqliteMigration({ service, sqlite, directUrl, migrations, tables, argv })`** is a service's `scripts/migrate-to-postgres.js` in one call. It:++ - applies the migrations as the owner, or on an in-memory PGlite with `--pglite` for a rehearsal;++ - imports with `importSqlite` into emptied tables;++ - makes text PostgreSQL refuses storable (a NUL is dropped, an unpaired surrogate becomes U+FFFD, in text and inside JSON), and reports every column where that happened;++ - verifies counts and checksums, prints the report (`--json` for all of it), and answers 0 only when everything verified.++++ It was generalised from Tips' script and first used by Wiki. `test/sqlite-cli.test.js`.++## 0.18.1 (2026-09-28)- **`db.prepare`:** a statement with one parameter takes a single array argument as that parameter's value (`WHERE id = ANY(?)`). Only statements with several parameters read a single array as the list of values (better-sqlite3's array form). Found converting OpenVibe.Wiki.@@@ -3,7 +3,7 @@"stage": "alpha","code": true,"wave": "W2",-- "version": "0.18.1",++ "version": "0.19.0","plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §3.2; roadmap §30","deployed": false,"deployment": "none: a package released as git tags; services and apps pin a tag",@@@ -1,4 +1,4 @@--// Generated by scripts/browser-bundle.js from browser.js (openvibe-sdk 0.18.1, MIT). Do not edit.++// Generated by scripts/browser-bundle.js from browser.js (openvibe-sdk 0.19.0, MIT). Do not edit.// One self-contained ES module: core at the top level plus the browser-safe subpaths as namespaces// (auth, registry, modules, realtime, media, community, search, chat, ai, jobs, tools, projects, geo). No server-only code: no service tokens, JWT verification, events, identity./* eslint-disable */@@@ -41,7 +41,7 @@ const { paginate, offsetPager } = __load(7);const semver = __load(6);/** The package version (test/version.test.js keeps it equal to package.json). */--const SDK_VERSION = '0.18.1';++const SDK_VERSION = '0.19.0';module.exports = {createClient,@@@ -105,7 +105,7 @@ Anything held in a process that must be shared once there are two processes:## 6. The one-time import, and the production switch--Add `scripts/migrate-to-postgres.js`. It:++Add `scripts/migrate-to-postgres.js`, a few lines around `runSqliteMigration` from `openvibe-sdk/db` (SDK ≥ 0.19; Wiki's is the example). It:1. runs the migrations with `DATABASE_DIRECT_URL`;2. runs `importSqlite({ sqlite: <the service's DB path>, db: owner, truncate: true, tables: { … dropColumns … } })`;3. prints the report and exits 1 unless `report.ok`.@@@ -2,6 +2,7 @@import cjs from '../src/db/index.js';export const {-- createDb, sql, DbError, importSqlite, PARSERS, ISOLATION,++ createDb, sql, DbError, importSqlite, runSqliteMigration, PARSERS,++ ISOLATION,} = cjs;export default cjs;@@@ -1,12 +1,12 @@{"name": "openvibe-sdk",-- "version": "0.18.1",++ "version": "0.19.0","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "openvibe-sdk",-- "version": "0.18.1",++ "version": "0.19.0","license": "MIT","devDependencies": {"@electric-sql/pglite": "^0.5.8",@@@ -1,6 +1,6 @@{"name": "openvibe-sdk",-- "version": "0.18.1",++ "version": "0.19.0","description": "Supported browser and server clients for the OpenVibe platform: auth (incl. developer apps), registry discovery, identity, user modules, events, realtime, media, community, search, chat, AI runs, Tools (registry, run API and jobs), VIP memberships and developer projects, the async PostgreSQL data layer (db) and the Valkey cache, limits store, queue and pub/sub, with deadlines, safe retries, problem errors and trace propagation.","main": "index.js","types": "types/index.d.ts",@@@ -8,7 +8,7 @@ const { paginate, offsetPager } = require('./paginate');const semver = require('./semver');/** The package version (test/version.test.js keeps it equal to package.json). */--const SDK_VERSION = '0.18.1';++const SDK_VERSION = '0.19.0';module.exports = {createClient,@@@ -290,5 +290,7 @@ function pgliteAdapter(opt) {/** The one-time SQLite → PostgreSQL move (loaded on use: it needs better-sqlite3). */function importSqlite(opts) { return require('./sqlite-import').importSqlite(opts); }++/** A service's scripts/migrate-to-postgres.js in one call (./sqlite-cli.js). */++function runSqliteMigration(opts) { return require('./sqlite-cli').runSqliteMigration(opts); }--module.exports = { createDb, sql, DbError, importSqlite, PARSERS, ISOLATION };++module.exports = { createDb, sql, DbError, importSqlite, runSqliteMigration, PARSERS, ISOLATION };nnew file mode 100644@@@ -0,0 +1,100 @@++'use strict';++/**++ * The one-time move of a service's SQLite database into its PostgreSQL schema (ADR-035; the procedure is++ * docs/migrating-to-postgresql.md, section 6), shared by every service's scripts/migrate-to-postgres.js:++ *++ * const { runSqliteMigration } = require('openvibe-sdk/db');++ * runSqliteMigration({ service: 'wiki', sqlite: config.dbPath, directUrl: config.db.directUrl,++ * migrations: MIGRATIONS, tables: { new_name: { from: 'old_name' } } }).then((code) => process.exit(code));++ *++ * node scripts/migrate-to-postgres.js [--sqlite <file>] [--pglite] [--json]++ *++ * 1. applies the migrations as the owner (directUrl, a direct connection); --pglite: an in-memory PostgreSQL++ * instead (a rehearsal with nothing to set up; the report is all it leaves);++ * 2. copies every table with importSqlite (batched, parents first, identities kept and sequences advanced)++ * into emptied tables (truncate: a rehearsal can be repeated);++ * 3. makes text PostgreSQL refuses storable, and reports it: a NUL character is dropped and an unpaired++ * surrogate becomes U+FFFD, in text and inside JSON values alike (SQLite took both);++ * 4. verifies each table's row count and a checksum of every row on both sides, prints the report, and++ * answers 1 unless report.ok. The SQLite file is opened read-only: nothing in it changes.++ * `tables` is importSqlite's per-table options (from, dropColumns, where, map), merged with the cleaning.++ */++const path = require('path');++++const cleanText = (v) => v.toWellFormed().replace(/\u0000/g, '');++function cleanJson(v) {++ if (typeof v === 'string') return cleanText(v);++ if (Array.isArray(v)) return v.map(cleanJson);++ if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [cleanText(k), cleanJson(x)]));++ return v;++}++++/** importSqlite's `tables` for the target schema: `tables`, plus a map per table making text storable. */++async function cleaningOptions(db, tables = {}) {++ const cols = await db.many(`SELECT table_name, column_name, udt_name FROM information_schema.columns++ WHERE table_schema = current_schema() AND table_name <> 'ov_migrations' AND udt_name IN ('text', 'varchar', 'json', 'jsonb')`);++ const byTable = new Map();++ for (const c of cols) { if (!byTable.has(c.table_name)) byTable.set(c.table_name, []); byTable.get(c.table_name).push(c); }++ const cleaned = new Map();++ const note = (t, c, v) => { const k = `${t}.${c}`; if (!cleaned.has(k)) cleaned.set(k, new Set()); cleaned.get(k).add(v); };++ const out = { ...tables };++ for (const [table, list] of byTable) {++ const base = tables[table] || {};++ out[table] = {++ ...base,++ map(row) {++ const r = base.map ? base.map(row) : row;++ if (!r) return r;++ for (const { column_name: c, udt_name: udt } of list) {++ const v = r[c];++ if (typeof v !== 'string') continue;++ if (udt === 'json' || udt === 'jsonb') {++ let parsed;++ try { parsed = JSON.parse(v); } catch { continue; } // left for the importer to report++ const fixed = JSON.stringify(cleanJson(parsed));++ if (fixed !== JSON.stringify(parsed)) { note(table, c, v); r[c] = fixed; }++ } else {++ const fixed = cleanText(v);++ if (fixed !== v) { note(table, c, v); r[c] = fixed; }++ }++ }++ return r;++ },++ };++ }++ return { tables: out, cleaned: () => [...cleaned].map(([column, values]) => ({ column, values: values.size })) };++}++++async function runSqliteMigration({ service, sqlite, directUrl, migrations, tables = {}, argv = process.argv.slice(2), out = console.log } = {}) {++ const { createDb, importSqlite } = require('./index');++ const opt = (name) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : null; };++ const flag = (name) => argv.includes(`--${name}`);++ const file = path.resolve(opt('sqlite') || sqlite || '');++ const quiet = { log() {}, warn: console.warn, error: console.error };++ let owner;++ if (flag('pglite')) {++ owner = createDb({ pglite: true, service: `${service}-import`, log: quiet });++ } else {++ if (!directUrl) throw new Error('DATABASE_DIRECT_URL is not set: the import runs as the owner on a direct connection');++ owner = createDb({ url: directUrl, service: `${service}-import`, max: 2, log: quiet });++ }++ try {++ await owner.migrate({ dir: migrations, log: quiet });++ const t0 = Date.now();++ const options = await cleaningOptions(owner, tables);++ const report = await importSqlite({ sqlite: file, db: owner, truncate: true, tables: options.tables, log: quiet });++ const cleaned = options.cleaned();++ if (flag('json')) out(JSON.stringify({ sqlite: file, into: owner.store, ...report, cleaned }, null, 2));++ else {++ out(`import ${file} → ${owner.store} (${Date.now() - t0} ms): ${report.ok ? 'OK' : 'PROBLEMS'}`);++ for (const t of report.tables) out(` ${t.table.padEnd(30)} ${String(t.rows).padStart(8)} rows ${t.checksum || '-'}${t.source !== t.table ? ` (from ${t.source})` : ''}`);++ for (const c of cleaned) out(` cleaned: ${c.column}: ${c.values} value(s) with a NUL or an unpaired surrogate`);++ for (const p of report.problems) out(` problem: ${p.table}: ${p.problem}`);++ }++ return report.ok ? 0 : 1;++ } finally {++ await owner.close();++ }++}++++module.exports = { runSqliteMigration, cleaningOptions, cleanText, cleanJson };nnew file mode 100644@@@ -0,0 +1,58 @@++'use strict';++/** runSqliteMigration: migrate, import, clean what PostgreSQL refuses (NUL, unpaired surrogates) and report it,++ * verify, answer 0; a column the schema lacks is a problem and answers 1; --json prints the report. */++const assert = require('node:assert/strict');++const fs = require('fs');++const os = require('os');++const path = require('path');++const { run } = require('./helpers');++const { runSqliteMigration } = require('../src/db');++++let Database;++try { Database = require('better-sqlite3'); } catch { Database = null; }++++const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sdk-sqlitecli-'));++const migrations = path.join(dir, 'migrations');++fs.mkdirSync(migrations);++fs.writeFileSync(path.join(migrations, '0001_initial.sql'), `-- phase: expand++CREATE TABLE notes (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, body text NOT NULL, meta jsonb, created_at bigint NOT NULL);++CREATE TABLE renamed (id text PRIMARY KEY, n integer NOT NULL);++`);++++function sqliteFile(extraColumn = false) {++ const file = path.join(dir, `src-${extraColumn ? 'x' : 'ok'}.db`);++ const db = new Database(file);++ db.exec(`CREATE TABLE notes (id INTEGER PRIMARY KEY AUTOINCREMENT, body TEXT NOT NULL, meta TEXT, created_at INTEGER NOT NULL${extraColumn ? ', stray TEXT' : ''});++ CREATE TABLE old_name (id TEXT PRIMARY KEY, n INTEGER NOT NULL);`);++ const ins = db.prepare('INSERT INTO notes (body, meta, created_at) VALUES (?, ?, ?)');++ ins.run('plain', '{"a":1}', 1);++ ins.run('with\u0000nul', '{"k":"bad \\ud800 surrogate"}', 2);++ ins.run('none', null, 3);++ db.prepare('INSERT INTO old_name (id, n) VALUES (?, ?)').run('x', 7);++ db.close();++ return file;++}++++const tests = [];++if (Database) {++ tests.push(['imports, cleans and reports, verifies and answers 0', async () => {++ const lines = [];++ const code = await runSqliteMigration({ service: 'sdk', sqlite: sqliteFile(), migrations, tables: { renamed: { from: 'old_name' } }, argv: ['--pglite'], out: (l) => lines.push(l) });++ assert.equal(code, 0, lines.join('\n'));++ assert.match(lines[0], /→ pglite .*: OK/);++ assert.ok(lines.some((l) => /notes .* 3 rows/.test(l)));++ assert.ok(lines.some((l) => /renamed .* 1 rows .*\(from old_name\)/.test(l)));++ assert.ok(lines.some((l) => /cleaned: notes\.body: 1 value/.test(l)), lines.join('\n'));++ assert.ok(lines.some((l) => /cleaned: notes\.meta: 1 value/.test(l)), lines.join('\n'));++ }]);++ tests.push(['a column the schema lacks is a problem: answers 1; --json prints the report', async () => {++ const lines = [];++ const code = await runSqliteMigration({ service: 'sdk', sqlite: sqliteFile(true), migrations, tables: { renamed: { from: 'old_name' } }, argv: ['--pglite', '--json'], out: (l) => lines.push(l) });++ assert.equal(code, 1);++ const report = JSON.parse(lines.join('\n'));++ assert.equal(report.ok, false);++ assert.ok(report.problems.some((p) => p.table === 'notes' && /stray/.test(p.problem)), JSON.stringify(report.problems));++ }]);++} else console.log('runSqliteMigration: skipped (better-sqlite3 not installed)');++++run(tests);@@@ -82,3 +82,11 @@ export interface ImportReport { ok: boolean; tables: { table: string; source: stexport function importSqlite(o: { sqlite: string | object; db: Db; tables?: Record<string, ImportTableOptions>; only?: string[]; truncate?: boolean; verify?: boolean; log?: { log(msg: string): void } }): Promise<ImportReport>;export const PARSERS: Record<number, (v: string) => unknown>;export const ISOLATION: Record<string, string>;++/**++ * A service's scripts/migrate-to-postgres.js in one call: migrate (or --pglite), import into emptied tables, clean++ * text PostgreSQL refuses (reported), verify counts and checksums; resolves 0 when everything verified, else 1.++ */++export function runSqliteMigration(opts: {++ service: string; sqlite: string; directUrl?: string; migrations: string;++ tables?: Record<string, ImportTableOptions>; argv?: string[]; out?: (line: string) => void;++}): Promise<0 | 1>;
@@ -7,6 +7,17 @@ release may change an API and says so here.
@@ -7,6 +7,17 @@ release may change an API and says so here.
+
+## 0.19.0 (2026-09-28)
+
+- **`openvibe-sdk/db` `runSqliteMigration({ service, sqlite, directUrl, migrations, tables, argv })`** is a service's `scripts/migrate-to-postgres.js` in one call. It:
+ - applies the migrations as the owner, or on an in-memory PGlite with `--pglite` for a rehearsal;
+ - imports with `importSqlite` into emptied tables;
+ - makes text PostgreSQL refuses storable (a NUL is dropped, an unpaired surrogate becomes U+FFFD, in text and inside JSON), and reports every column where that happened;
+ - verifies counts and checksums, prints the report (`--json` for all of it), and answers 0 only when everything verified.
+
+ It was generalised from Tips' script and first used by Wiki. `test/sqlite-cli.test.js`.
+
## 0.18.1 (2026-09-28)
## 0.18.1 (2026-09-28)
- **`db.prepare`:** a statement with one parameter takes a single array argument as that parameter's value (`WHERE id = ANY(?)`). Only statements with several parameters read a single array as the list of values (better-sqlite3's array form). Found converting OpenVibe.Wiki.
- **`db.prepare`:** a statement with one parameter takes a single array argument as that parameter's value (`WHERE id = ANY(?)`). Only statements with several parameters read a single array as the list of values (better-sqlite3's array form). Found converting OpenVibe.Wiki.
@@ -3,7 +3,7 @@
@@ -3,7 +3,7 @@
"stage": "alpha",
"stage": "alpha",
"code": true,
"code": true,
"wave": "W2",
"wave": "W2",
- "version": "0.18.1",
+ "version": "0.19.0",
"plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §3.2; roadmap §30",
"plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §3.2; roadmap §30",
"deployed": false,
"deployed": false,
"deployment": "none: a package released as git tags; services and apps pin a tag",
"deployment": "none: a package released as git tags; services and apps pin a tag",
@@ -1,4 +1,4 @@
@@ -1,4 +1,4 @@
-// Generated by scripts/browser-bundle.js from browser.js (openvibe-sdk 0.18.1, MIT). Do not edit.
+// Generated by scripts/browser-bundle.js from browser.js (openvibe-sdk 0.19.0, MIT). Do not edit.
// One self-contained ES module: core at the top level plus the browser-safe subpaths as namespaces
// One self-contained ES module: core at the top level plus the browser-safe subpaths as namespaces
// (auth, registry, modules, realtime, media, community, search, chat, ai, jobs, tools, projects, geo). No server-only code: no service tokens, JWT verification, events, identity.
// (auth, registry, modules, realtime, media, community, search, chat, ai, jobs, tools, projects, geo). No server-only code: no service tokens, JWT verification, events, identity.
/* eslint-disable */
/* eslint-disable */
@@ -41,7 +41,7 @@ const { paginate, offsetPager } = __load(7);
@@ -41,7 +41,7 @@ const { paginate, offsetPager } = __load(7);
const semver = __load(6);
const semver = __load(6);
/** The package version (test/version.test.js keeps it equal to package.json). */
/** The package version (test/version.test.js keeps it equal to package.json). */
-const SDK_VERSION = '0.18.1';
+const SDK_VERSION = '0.19.0';
module.exports = {
module.exports = {
createClient,
createClient,
@@ -105,7 +105,7 @@ Anything held in a process that must be shared once there are two processes:
@@ -105,7 +105,7 @@ Anything held in a process that must be shared once there are two processes:
## 6. The one-time import, and the production switch
## 6. The one-time import, and the production switch
-Add `scripts/migrate-to-postgres.js`. It:
+Add `scripts/migrate-to-postgres.js`, a few lines around `runSqliteMigration` from `openvibe-sdk/db` (SDK ≥ 0.19; Wiki's is the example). It:
1. runs the migrations with `DATABASE_DIRECT_URL`;
1. runs the migrations with `DATABASE_DIRECT_URL`;
2. runs `importSqlite({ sqlite: <the service's DB path>, db: owner, truncate: true, tables: { … dropColumns … } })`;
2. runs `importSqlite({ sqlite: <the service's DB path>, db: owner, truncate: true, tables: { … dropColumns … } })`;
3. prints the report and exits 1 unless `report.ok`.
3. prints the report and exits 1 unless `report.ok`.
@@ -2,6 +2,7 @@
@@ -2,6 +2,7 @@
import cjs from '../src/db/index.js';
import cjs from '../src/db/index.js';
export const {
export const {
- createDb, sql, DbError, importSqlite, PARSERS, ISOLATION,
+ createDb, sql, DbError, importSqlite, runSqliteMigration, PARSERS,
+ ISOLATION,
} = cjs;
} = cjs;
export default cjs;
export default cjs;
@@ -1,12 +1,12 @@
@@ -1,12 +1,12 @@
{
{
"name": "openvibe-sdk",
"name": "openvibe-sdk",
- "version": "0.18.1",
+ "version": "0.19.0",
"lockfileVersion": 3,
"lockfileVersion": 3,
"requires": true,
"requires": true,
"packages": {
"packages": {
"": {
"": {
"name": "openvibe-sdk",
"name": "openvibe-sdk",
- "version": "0.18.1",
+ "version": "0.19.0",
"license": "MIT",
"license": "MIT",
"devDependencies": {
"devDependencies": {
"@electric-sql/pglite": "^0.5.8",
"@electric-sql/pglite": "^0.5.8",
@@ -1,6 +1,6 @@
@@ -1,6 +1,6 @@
{
{
"name": "openvibe-sdk",
"name": "openvibe-sdk",
- "version": "0.18.1",
+ "version": "0.19.0",
"description": "Supported browser and server clients for the OpenVibe platform: auth (incl. developer apps), registry discovery, identity, user modules, events, realtime, media, community, search, chat, AI runs, Tools (registry, run API and jobs), VIP memberships and developer projects, the async PostgreSQL data layer (db) and the Valkey cache, limits store, queue and pub/sub, with deadlines, safe retries, problem errors and trace propagation.",
"description": "Supported browser and server clients for the OpenVibe platform: auth (incl. developer apps), registry discovery, identity, user modules, events, realtime, media, community, search, chat, AI runs, Tools (registry, run API and jobs), VIP memberships and developer projects, the async PostgreSQL data layer (db) and the Valkey cache, limits store, queue and pub/sub, with deadlines, safe retries, problem errors and trace propagation.",
"main": "index.js",
"main": "index.js",
"types": "types/index.d.ts",
"types": "types/index.d.ts",
@@ -8,7 +8,7 @@ const { paginate, offsetPager } = require('./paginate');
@@ -8,7 +8,7 @@ const { paginate, offsetPager } = require('./paginate');
const semver = require('./semver');
const semver = require('./semver');
/** The package version (test/version.test.js keeps it equal to package.json). */
/** The package version (test/version.test.js keeps it equal to package.json). */
-const SDK_VERSION = '0.18.1';
+const SDK_VERSION = '0.19.0';
module.exports = {
module.exports = {
createClient,
createClient,
@@ -290,5 +290,7 @@ function pgliteAdapter(opt) {
@@ -290,5 +290,7 @@ function pgliteAdapter(opt) {
/** The one-time SQLite → PostgreSQL move (loaded on use: it needs better-sqlite3). */
/** The one-time SQLite → PostgreSQL move (loaded on use: it needs better-sqlite3). */
function importSqlite(opts) { return require('./sqlite-import').importSqlite(opts); }
function importSqlite(opts) { return require('./sqlite-import').importSqlite(opts); }
+/** A service's scripts/migrate-to-postgres.js in one call (./sqlite-cli.js). */
+function runSqliteMigration(opts) { return require('./sqlite-cli').runSqliteMigration(opts); }
-module.exports = { createDb, sql, DbError, importSqlite, PARSERS, ISOLATION };
+module.exports = { createDb, sql, DbError, importSqlite, runSqliteMigration, PARSERS, ISOLATION };
new file mode 100644
new file mode 100644
@@ -0,0 +1,100 @@
@@ -0,0 +1,100 @@
+'use strict';
+/**
+ * The one-time move of a service's SQLite database into its PostgreSQL schema (ADR-035; the procedure is
+ * docs/migrating-to-postgresql.md, section 6), shared by every service's scripts/migrate-to-postgres.js:
+ *
+ * const { runSqliteMigration } = require('openvibe-sdk/db');
+ * runSqliteMigration({ service: 'wiki', sqlite: config.dbPath, directUrl: config.db.directUrl,
+ * migrations: MIGRATIONS, tables: { new_name: { from: 'old_name' } } }).then((code) => process.exit(code));
+ *
+ * node scripts/migrate-to-postgres.js [--sqlite <file>] [--pglite] [--json]
+ *
+ * 1. applies the migrations as the owner (directUrl, a direct connection); --pglite: an in-memory PostgreSQL
+ * instead (a rehearsal with nothing to set up; the report is all it leaves);
+ * 2. copies every table with importSqlite (batched, parents first, identities kept and sequences advanced)
+ * into emptied tables (truncate: a rehearsal can be repeated);
+ * 3. makes text PostgreSQL refuses storable, and reports it: a NUL character is dropped and an unpaired
+ * surrogate becomes U+FFFD, in text and inside JSON values alike (SQLite took both);
+ * 4. verifies each table's row count and a checksum of every row on both sides, prints the report, and
+ * answers 1 unless report.ok. The SQLite file is opened read-only: nothing in it changes.
+ * `tables` is importSqlite's per-table options (from, dropColumns, where, map), merged with the cleaning.
+ */
+const path = require('path');
+
+const cleanText = (v) => v.toWellFormed().replace(/\u0000/g, '');
+function cleanJson(v) {
+ if (typeof v === 'string') return cleanText(v);
+ if (Array.isArray(v)) return v.map(cleanJson);
+ if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [cleanText(k), cleanJson(x)]));
+ return v;
+}
+
+/** importSqlite's `tables` for the target schema: `tables`, plus a map per table making text storable. */
+async function cleaningOptions(db, tables = {}) {
+ const cols = await db.many(`SELECT table_name, column_name, udt_name FROM information_schema.columns
+ WHERE table_schema = current_schema() AND table_name <> 'ov_migrations' AND udt_name IN ('text', 'varchar', 'json', 'jsonb')`);
+ const byTable = new Map();
+ for (const c of cols) { if (!byTable.has(c.table_name)) byTable.set(c.table_name, []); byTable.get(c.table_name).push(c); }
+ const cleaned = new Map();
+ const note = (t, c, v) => { const k = `${t}.${c}`; if (!cleaned.has(k)) cleaned.set(k, new Set()); cleaned.get(k).add(v); };
+ const out = { ...tables };
+ for (const [table, list] of byTable) {
+ const base = tables[table] || {};
+ out[table] = {
+ ...base,
+ map(row) {
+ const r = base.map ? base.map(row) : row;
+ if (!r) return r;
+ for (const { column_name: c, udt_name: udt } of list) {
+ const v = r[c];
+ if (typeof v !== 'string') continue;
+ if (udt === 'json' || udt === 'jsonb') {
+ let parsed;
+ try { parsed = JSON.parse(v); } catch { continue; } // left for the importer to report
+ const fixed = JSON.stringify(cleanJson(parsed));
+ if (fixed !== JSON.stringify(parsed)) { note(table, c, v); r[c] = fixed; }
+ } else {
+ const fixed = cleanText(v);
+ if (fixed !== v) { note(table, c, v); r[c] = fixed; }
+ }
+ }
+ return r;
+ },
+ };
+ }
+ return { tables: out, cleaned: () => [...cleaned].map(([column, values]) => ({ column, values: values.size })) };
+}
+
+async function runSqliteMigration({ service, sqlite, directUrl, migrations, tables = {}, argv = process.argv.slice(2), out = console.log } = {}) {
+ const { createDb, importSqlite } = require('./index');
+ const opt = (name) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : null; };
+ const flag = (name) => argv.includes(`--${name}`);
+ const file = path.resolve(opt('sqlite') || sqlite || '');
+ const quiet = { log() {}, warn: console.warn, error: console.error };
+ let owner;
+ if (flag('pglite')) {
+ owner = createDb({ pglite: true, service: `${service}-import`, log: quiet });
+ } else {
+ if (!directUrl) throw new Error('DATABASE_DIRECT_URL is not set: the import runs as the owner on a direct connection');
+ owner = createDb({ url: directUrl, service: `${service}-import`, max: 2, log: quiet });
+ }
+ try {
+ await owner.migrate({ dir: migrations, log: quiet });
+ const t0 = Date.now();
+ const options = await cleaningOptions(owner, tables);
+ const report = await importSqlite({ sqlite: file, db: owner, truncate: true, tables: options.tables, log: quiet });
+ const cleaned = options.cleaned();
+ if (flag('json')) out(JSON.stringify({ sqlite: file, into: owner.store, ...report, cleaned }, null, 2));
+ else {
+ out(`import ${file} → ${owner.store} (${Date.now() - t0} ms): ${report.ok ? 'OK' : 'PROBLEMS'}`);
+ for (const t of report.tables) out(` ${t.table.padEnd(30)} ${String(t.rows).padStart(8)} rows ${t.checksum || '-'}${t.source !== t.table ? ` (from ${t.source})` : ''}`);
+ for (const c of cleaned) out(` cleaned: ${c.column}: ${c.values} value(s) with a NUL or an unpaired surrogate`);
+ for (const p of report.problems) out(` problem: ${p.table}: ${p.problem}`);
+ }
+ return report.ok ? 0 : 1;
+ } finally {
+ await owner.close();
+ }
+}
+
+module.exports = { runSqliteMigration, cleaningOptions, cleanText, cleanJson };
new file mode 100644
new file mode 100644
@@ -0,0 +1,58 @@
@@ -0,0 +1,58 @@
+'use strict';
+/** runSqliteMigration: migrate, import, clean what PostgreSQL refuses (NUL, unpaired surrogates) and report it,
+ * verify, answer 0; a column the schema lacks is a problem and answers 1; --json prints the report. */
+const assert = require('node:assert/strict');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { run } = require('./helpers');
+const { runSqliteMigration } = require('../src/db');
+
+let Database;
+try { Database = require('better-sqlite3'); } catch { Database = null; }
+
+const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sdk-sqlitecli-'));
+const migrations = path.join(dir, 'migrations');
+fs.mkdirSync(migrations);
+fs.writeFileSync(path.join(migrations, '0001_initial.sql'), `-- phase: expand
+CREATE TABLE notes (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, body text NOT NULL, meta jsonb, created_at bigint NOT NULL);
+CREATE TABLE renamed (id text PRIMARY KEY, n integer NOT NULL);
+`);
+
+function sqliteFile(extraColumn = false) {
+ const file = path.join(dir, `src-${extraColumn ? 'x' : 'ok'}.db`);
+ const db = new Database(file);
+ db.exec(`CREATE TABLE notes (id INTEGER PRIMARY KEY AUTOINCREMENT, body TEXT NOT NULL, meta TEXT, created_at INTEGER NOT NULL${extraColumn ? ', stray TEXT' : ''});
+ CREATE TABLE old_name (id TEXT PRIMARY KEY, n INTEGER NOT NULL);`);
+ const ins = db.prepare('INSERT INTO notes (body, meta, created_at) VALUES (?, ?, ?)');
+ ins.run('plain', '{"a":1}', 1);
+ ins.run('with\u0000nul', '{"k":"bad \\ud800 surrogate"}', 2);
+ ins.run('none', null, 3);
+ db.prepare('INSERT INTO old_name (id, n) VALUES (?, ?)').run('x', 7);
+ db.close();
+ return file;
+}
+
+const tests = [];
+if (Database) {
+ tests.push(['imports, cleans and reports, verifies and answers 0', async () => {
+ const lines = [];
+ const code = await runSqliteMigration({ service: 'sdk', sqlite: sqliteFile(), migrations, tables: { renamed: { from: 'old_name' } }, argv: ['--pglite'], out: (l) => lines.push(l) });
+ assert.equal(code, 0, lines.join('\n'));
+ assert.match(lines[0], /→ pglite .*: OK/);
+ assert.ok(lines.some((l) => /notes .* 3 rows/.test(l)));
+ assert.ok(lines.some((l) => /renamed .* 1 rows .*\(from old_name\)/.test(l)));
+ assert.ok(lines.some((l) => /cleaned: notes\.body: 1 value/.test(l)), lines.join('\n'));
+ assert.ok(lines.some((l) => /cleaned: notes\.meta: 1 value/.test(l)), lines.join('\n'));
+ }]);
+ tests.push(['a column the schema lacks is a problem: answers 1; --json prints the report', async () => {
+ const lines = [];
+ const code = await runSqliteMigration({ service: 'sdk', sqlite: sqliteFile(true), migrations, tables: { renamed: { from: 'old_name' } }, argv: ['--pglite', '--json'], out: (l) => lines.push(l) });
+ assert.equal(code, 1);
+ const report = JSON.parse(lines.join('\n'));
+ assert.equal(report.ok, false);
+ assert.ok(report.problems.some((p) => p.table === 'notes' && /stray/.test(p.problem)), JSON.stringify(report.problems));
+ }]);
+} else console.log('runSqliteMigration: skipped (better-sqlite3 not installed)');
+
+run(tests);
@@ -82,3 +82,11 @@ export interface ImportReport { ok: boolean; tables: { table: string; source: st
@@ -82,3 +82,11 @@ export interface ImportReport { ok: boolean; tables: { table: string; source: st
export function importSqlite(o: { sqlite: string | object; db: Db; tables?: Record<string, ImportTableOptions>; only?: string[]; truncate?: boolean; verify?: boolean; log?: { log(msg: string): void } }): Promise<ImportReport>;
export function importSqlite(o: { sqlite: string | object; db: Db; tables?: Record<string, ImportTableOptions>; only?: string[]; truncate?: boolean; verify?: boolean; log?: { log(msg: string): void } }): Promise<ImportReport>;
export const PARSERS: Record<number, (v: string) => unknown>;
export const PARSERS: Record<number, (v: string) => unknown>;
export const ISOLATION: Record<string, string>;
export const ISOLATION: Record<string, string>;
+/**
+ * A service's scripts/migrate-to-postgres.js in one call: migrate (or --pglite), import into emptied tables, clean
+ * text PostgreSQL refuses (reported), verify counts and checksums; resolves 0 when everything verified, else 1.
+ */
+export function runSqliteMigration(opts: {
+ service: string; sqlite: string; directUrl?: string; migrations: string;
+ tables?: Record<string, ImportTableOptions>; argv?: string[]; out?: (line: string) => void;
+}): Promise<0 | 1>;