openvibers/openvibe.sdk Commits
tools/asyncify: the codemod that converted Wiki and Blog from better-sqlite3 to openvibe-sdk/db (ADR-035, WS-X2). It works on syntax (acorn). It awaits prepared statements, Publishing store calls and module API methods that became async, makes the enclosing functions async, and repeats across files to a fixpoint. It never awaits a promise used as a value, and it reports calls inside array callbacks for a hand fix instead of breaking them. There is also the earlier regex pass for tests and an example config. Not part of the published package.
Changes
nnew file mode 100644@@@ -0,0 +1,40 @@++# asyncify: converting a service from better-sqlite3 to openvibe-sdk/db++++This directory holds the tools used to convert Wiki and Blog (2026-09-28). They are not part of the published package.++++```++cd tools/asyncify && npm install++node asyncify.js --config my-service.json $(git -C <service> ls-files 'server/*.js' 'server/**/*.js') <service>/test/*.test.js++```++++## asyncify.js++++It parses every file with acorn and does the following:++++1. It adds `await` to calls that now return a promise:++ - prepared statements (`db.prepare(…).get/all/run`, `q.<name>.…`, and any name assigned from `.prepare(…)`);++ - Publishing store methods reached as `<x>.<store>.<method>(`;++ - `store.tx(`;++ - `asyncMethods` and `asyncGlobals` from the config;++ - methods of module APIs (`apis`: a variable name → the file whose functions became async).++2. It makes the enclosing functions `async`.++3. It repeats across all the files until nothing changes.++++It leaves some calls unawaited:++- A promise used as a value (`.then`, `.catch`, `.finally`, `Promise.all/race([...])`, `assert.rejects(p)`) stays a promise.++- A call inside an array callback (`map`, `filter`, `forEach`, …) is **reported** (`MANUAL …`), never changed, because an async callback there silently breaks the result. The same goes for an async function passed by name to an array method.++++Fix those by hand, with a `for … of` loop, `await Promise.all(xs.map(async …))`, or better a batch read (`revisions.getMany`, `citations.forRevisions`, `reviews.latestMany`, or `= ANY(?)`).++++It also rewrites `assert.throws(async …)` as `await assert.rejects(async …)`. An edit that would break the syntax is refused and reported.++++The config has these keys: `stores`, `bareStores`, `syncStoreMethods`, `apis`, `syncNames`, `asyncGlobals` and `asyncMethods`. See `example-blog.json`.++++What stays for a person:++- the SQL dialect: `INSERT OR IGNORE/REPLACE` → `ON CONFLICT`, `COLLATE NOCASE` → `lower()`, `LIKE` → `ILIKE`, `IS ?` → `IS NOT DISTINCT FROM`, `DISTINCT` + `ORDER BY`, `PRAGMA` and `sqlite_master` → `information_schema`;++- `RETURNING` in place of `lastInsertRowid`;++- the boot (open, migrate, `createApp` async) and the test helpers (`openvibe-sdk/testing` `createTestDb`).++++## awaitify-tests.py++++This is the earlier regex pass for test files (Wiki). `asyncify.js` over the server and the tests together supersedes it.nnew file mode 100644@@@ -0,0 +1,212 @@++#!/usr/bin/env node++'use strict';++/**++ * asyncify: the mechanical half of moving a service from better-sqlite3 to openvibe-sdk/db (ADR-035).++ *++ * node asyncify.js --config asyncify.json file.js ...++ *++ * config: {++ * "stores": ["revisions", ...], // publishing store property names reached as <x>.<store>.<method>(++ * "syncStoreMethods": ["..."], // methods of those stores that stay synchronous++ * "apis": { "blogs": "server/domain/blogs.js", ... }, // variable/property names that hold a module's API object++ * "syncNames": ["validHandle", ...], // functions that stay synchronous even when their module changes++ * "asyncGlobals": ["tx"] // extra callee names that are async (e.g. a local tx helper)++ * }++ *++ * Async calls: <stmt>.get/.all/.run(...) where <stmt> is db.prepare(...)/<x>.prepare(...)/q.<name>/<x>.q.<name>;++ * <x>.<store>.<method>(...); store.tx(...) / db.tx(...); calls to functions of these files that became async++ * (fixpoint, across files through `apis`). Each gets `await` (parenthesised when a member access follows), its++ * enclosing function becomes async. A function passed to map/filter/forEach/some/every/find/reduce/sort that would++ * become async is NOT changed: it is reported for a hand fix (an async callback there silently breaks semantics).++ */++const fs = require('fs');++const path = require('path');++const acorn = require('acorn');++const walk = require('acorn-walk');++++const ARRAY_CB = new Set(['map', 'filter', 'forEach', 'some', 'every', 'find', 'findIndex', 'reduce', 'sort', 'flatMap']);++++function parse(src) {++ return acorn.parse(src, { ecmaVersion: 'latest', sourceType: 'script', allowHashBang: true, allowReturnOutsideFunction: true, locations: true });++}++++function memberPath(node) {++ const parts = [];++ let n = node;++ while (n) {++ if (n.type === 'MemberExpression' && !n.computed && n.property.type === 'Identifier') { parts.unshift(n.property.name); n = n.object; continue; }++ if (n.type === 'Identifier') { parts.unshift(n.name); break; }++ if (n.type === 'ThisExpression') { parts.unshift('this'); break; }++ if (n.type === 'CallExpression') { parts.unshift('()'); n = n.callee; continue; }++ return parts.length ? ['?', ...parts] : null;++ }++ return parts;++}++++function functionName(fn, parent) {++ if (fn.id) return fn.id.name;++ if (parent && parent.type === 'VariableDeclarator' && parent.id.type === 'Identifier') return parent.id.name;++ if (parent && parent.type === 'Property' && parent.key) return parent.key.name || parent.key.value;++ if (parent && parent.type === 'AssignmentExpression' && parent.left.type === 'MemberExpression') return parent.left.property.name;++ return null;++}++++function analyse(file, src, cfg, asyncNames) {++ const ast = parse(src);++ const ancestors = new Map();++ // parent links++ walk.fullAncestor(ast, (node, _state, anc) => { ancestors.set(node, anc.slice(0, -1)); });++ const isFn = (n) => n.type === 'FunctionDeclaration' || n.type === 'FunctionExpression' || n.type === 'ArrowFunctionExpression';++ const stores = new Set(cfg.stores || []);++ const syncStoreMethods = new Set(cfg.syncStoreMethods || []);++ const apis = cfg.apis || {};++ const syncNames = new Set(cfg.syncNames || []);++ const localAsync = asyncNames.local.get(file) || new Set();++++ // Names that hold a prepared statement: const x = db.prepare(...), or { x: db.prepare(...) }.++ const stmtNames = new Set();++ const isPrepare = (n) => n && n.type === 'CallExpression' && n.callee.type === 'MemberExpression' && n.callee.property.name === 'prepare';++ walk.full(ast, (n) => {++ if (n.type === 'VariableDeclarator' && n.id.type === 'Identifier' && isPrepare(n.init)) stmtNames.add(n.id.name);++ if (n.type === 'Property' && n.key && isPrepare(n.value)) stmtNames.add(n.key.name || n.key.value);++ });++++ function asyncCall(call) {++ const c = call.callee;++ if (c.type === 'MemberExpression' && !c.computed) {++ const m = c.property.name;++ const obj = c.object;++ const p = memberPath(obj) || [];++ if (['get', 'all', 'run'].includes(m)) {++ if (obj.type === 'CallExpression' && obj.callee.type === 'MemberExpression' && ['prepare', 'pluck'].includes(obj.callee.property.name)) return true;++ if (p.length >= 2 && (p[p.length - 2] === 'q' || p[p.length - 2] === 'stmts' || p[p.length - 2] === 'Q')) return true;++ if (p.length && stmtNames.has(p[p.length - 1])) return true;++ }++ if ((m === 'tx' || m === 'transaction') && p.length && ['store', 'db', 'this'].includes(p[p.length - 1])) return m === 'tx';++ // <x>.<store>.<method>(: a store reached through an object; a bare identifier only when listed in bareStores.++ if (p.length >= 2 && stores.has(p[p.length - 1]) && !syncStoreMethods.has(m)) return true;++ if (p.length === 1 && (cfg.bareStores || []).includes(p[0]) && !syncStoreMethods.has(m)) return true;++ if ((cfg.asyncMethods || []).includes(m)) return true;++ // <api>.<method>( across modules++ const holder = p[p.length - 1];++ if (apis[holder]) {++ const set = asyncNames.api.get(apis[holder]);++ if (set && set.has(m) && !syncNames.has(m)) return true;++ }++ if (p.length && (p[p.length - 1] === 'api' || p[p.length - 1] === 'svc' || p[p.length - 1] === 'self') && localAsync.has(m)) return true;++ }++ if (c.type === 'Identifier') {++ if ((cfg.asyncGlobals || []).includes(c.name)) return true;++ if (localAsync.has(c.name) && !syncNames.has(c.name)) return true;++ }++ return false;++ }++++ const edits = []; // { at, text } insertions++ const manual = [];++ const becameAsync = new Set();++ const newlyAsyncNames = new Set();++ const fnsToAsync = new Set();++++ // An async function passed by name to an array method (xs.map(asyncFn)) needs a hand fix too.++ walk.full(ast, (n) => {++ if (n.type === 'CallExpression' && n.callee.type === 'MemberExpression' && ARRAY_CB.has(n.callee.property.name)++ && n.arguments[0] && n.arguments[0].type === 'Identifier' && localAsync.has(n.arguments[0].name)) {++ const gp = null;++ manual.push(`${file}:${n.loc.start.line}: async function ${n.arguments[0].name} passed to .${n.callee.property.name}()`);++ }++ });++ walk.fullAncestor(ast, (node, _s, anc) => {++ if (node.type !== 'CallExpression' || !asyncCall(node)) return;++ const parents = anc.slice(0, -1);++ const parent = parents[parents.length - 1];++ if (parent && parent.type === 'AwaitExpression') return;++ // A promise used as a value stays a promise: p.then/catch/finally, or an element of Promise.all/race/allSettled/any.++ if (parent && parent.type === 'MemberExpression' && parent.object === node && ['then', 'catch', 'finally'].includes(parent.property.name)) return;++ if (parent && parent.type === 'ArrayExpression') {++ const gp = parents[parents.length - 2];++ if (gp && gp.type === 'CallExpression' && gp.callee.type === 'MemberExpression' && gp.callee.object.name === 'Promise') return;++ }++ if (parent && parent.type === 'CallExpression' && parent.callee.type === 'MemberExpression' && parent.callee.object.name === 'Promise') return;++ // assert.rejects(promise) / doesNotReject(promise) take the promise itself.++ if (parent && parent.type === 'CallExpression' && parent.arguments[0] === node && parent.callee.type === 'MemberExpression' && ['rejects', 'doesNotReject'].includes(parent.callee.property.name)) return;++ // Returned directly from an arrow body or a return statement of an async fn is fine too, but awaiting is harmless.++ const fnIdx = (() => { for (let i = parents.length - 1; i >= 0; i--) if (isFn(parents[i])) return i; return -1; })();++ const fn = fnIdx >= 0 ? parents[fnIdx] : null;++ if (fn) {++ const fnParent = parents[fnIdx - 1];++ if (fnParent && fnParent.type === 'CallExpression' && fnParent.arguments.includes(fn) && fnParent.callee.type === 'MemberExpression' && ARRAY_CB.has(fnParent.callee.property.name)) {++ manual.push(`${file}:${node.loc.start.line}: async call inside a .${fnParent.callee.property.name}() callback`);++ return;++ }++ fnsToAsync.add(fn);++ }++ const wrap = parent && parent.type === 'MemberExpression' && parent.object === node;++ edits.push({ at: node.start, text: wrap ? '(await ' : 'await ' });++ if (wrap) edits.push({ at: node.end, text: ')' });++ });++++ for (const fn of fnsToAsync) {++ if (fn.async) continue;++ const anc = ancestors.get(fn) || [];++ const parent = anc[anc.length - 1];++ becameAsync.add(fn);++ const name = functionName(fn, parent);++ if (name) newlyAsyncNames.add(name);++ if (parent && parent.type === 'Property' && (parent.method || parent.kind !== 'init')) edits.push({ at: parent.start, text: 'async ' });++ else if (parent && parent.type === 'MethodDefinition') edits.push({ at: parent.key.start, text: 'async ' });++ else edits.push({ at: fn.start, text: 'async ' });++ }++ return { edits, manual, newlyAsyncNames };++}++++function apply(src, edits) {++ const sorted = edits.slice().sort((a, b) => b.at - a.at || (a.text === ')' ? -1 : 1));++ let out = src;++ for (const e of sorted) out = out.slice(0, e.at) + e.text + out.slice(e.at);++ return out;++}++++function main() {++ const args = process.argv.slice(2);++ const ci = args.indexOf('--config');++ const cfg = ci >= 0 ? JSON.parse(fs.readFileSync(args[ci + 1], 'utf8')) : {};++ const files = args.filter((a, i) => a.endsWith('.js') && i !== ci + 1).map((f) => path.relative(process.cwd(), path.resolve(f)));++ const asyncNames = { local: new Map(), api: new Map() };++ let manual = [];++ for (let round = 0; round < 12; round++) {++ let changed = false;++ manual = [];++ for (const f of files) {++ const src = fs.readFileSync(f, 'utf8');++ let r;++ try { r = analyse(f, src, cfg, asyncNames); } catch (e) { console.error(`${f}: parse error ${e.message}`); process.exitCode = 1; continue; }++ manual.push(...r.manual);++ if (r.edits.length) {++ let next = apply(src, r.edits);++ // An assertion about a function that is async now: assert.throws cannot see a rejection.++ next = next.replace(/(^|[^.\w])assert\.throws\(async /g, '$1await assert.rejects(async ').replace(/(^|[^.\w])assert\.doesNotThrow\(async /g, '$1await assert.doesNotReject(async ');++ try { parse(next); } catch (e) { console.error(`${f}: the edits would break the syntax (${e.message}); left unchanged`); manual.push(`${f}: edits refused (syntax)`); continue; }++ fs.writeFileSync(f, next); changed = true;++ }++ // Every async function of the file (new or already) counts for callers.++ const all = new Set([...(asyncNames.local.get(f) || []), ...r.newlyAsyncNames]);++ walk.fullAncestor(parse(fs.readFileSync(f, 'utf8')), (node, _s, anc) => {++ if ((node.type === 'FunctionDeclaration' || node.type === 'FunctionExpression' || node.type === 'ArrowFunctionExpression') && node.async) {++ const parent = anc[anc.length - 2];++ const name = functionName(node, parent);++ if (name) all.add(name);++ }++ });++ const before = (asyncNames.local.get(f) || new Set()).size;++ asyncNames.local.set(f, all);++ asyncNames.api.set(f, all);++ if (all.size !== before) changed = true;++ }++ if (!changed) break;++ }++ for (const m of [...new Set(manual)]) console.log(`MANUAL ${m}`);++ for (const f of files) console.log(`async ${f}: ${[...(asyncNames.local.get(f) || [])].sort().join(', ')}`);++}++++main();nnew file mode 100644@@@ -0,0 +1,115 @@++"""Add `await` to async service/db calls in test files converted from better-sqlite3 (openvibe-sdk/db ≥ 0.18).++Usage: python3 awaitify.py <sync-member,...> file... (sync members: svc members that stay synchronous)"""++import re, sys++++def skip_string(s, j):++ q = s[j]; j += 1++ while j < len(s):++ c = s[j]++ if c == '\\': j += 2; continue++ if q == '`' and c == '$' and j + 1 < len(s) and s[j+1] == '{':++ j = skip_braces(s, j + 1); continue++ if c == q: return j + 1++ j += 1++ return j++++def skip_braces(s, j): # s[j] == '{'++ depth = 0++ while j < len(s):++ c = s[j]++ if c in '\'"`': j = skip_string(s, j); continue++ if c == '{': depth += 1++ elif c == '}':++ depth -= 1++ if depth == 0: return j + 1++ j += 1++ return j++++def call_end(s, i): # s[i] == '(' -> index after matching ')'++ depth = 0; j = i++ while j < len(s):++ c = s[j]++ if c in '\'"`': j = skip_string(s, j); continue++ if c == '/' and s[j+1:j+2] == '/': j = s.find('\n', j); j = len(s) if j < 0 else j; continue++ if c in '([{': depth += 1++ elif c in ')]}':++ depth -= 1++ if depth == 0: return j + 1++ j += 1++ return j++++def transform(src, sync_members):++ out = []; i = 0; changed = 0++ pat = re.compile(r'((?:[A-Za-z_$][\w$]*\.)*(?:svc|db)\.(\w+)\(|(?:\bH\.)?\boutbox\(\s*h\w*\s*\)|(?:[A-Za-z_$][\w$]*\.)*(?:stores\.\w+|platform\.outbox)\.(\w+)\(|(?<![\w.$])(?:summarize|seedFn|seed)\()')++ while True:++ m = pat.search(src, i)++ if not m: out.append(src[i:]); break++ start = m.start()++ # not part of a longer member chain (e.g. foo.bar.svc handled by regex prefix); skip when preceded by '.'++ if start > 0 and src[start-1] in '.$':++ out.append(src[i:m.end()]); i = m.end(); continue++ before = src[:start].rstrip()++ text = m.group(0)++ if text.startswith('outbox') or text.startswith('H.outbox'):++ end = m.end()++ elif m.group(3) is not None or re.match(r'(summarize|seedFn|seed)\($', text):++ if m.group(3) in ('start', 'stop', 'counts'):++ out.append(src[i:m.end()]); i = m.end(); continue++ end = call_end(src, m.end() - 1)++ else:++ member = m.group(2)++ is_db = '.db.' in text or text.startswith('db.')++ if (not is_db and member in sync_members) or (is_db and member not in ('prepare', 'value', 'many', 'maybe', 'one', 'exec', 'query', 'tx')):++ out.append(src[i:m.end()]); i = m.end(); continue++ end = call_end(src, m.end() - 1)++ if is_db and member == 'prepare':++ # .pluck(...)? then .get/.all/.run(...)++ mm = re.match(r'\s*\.pluck\(', src[end:])++ if mm: end = call_end(src, end + mm.end() - 1)++ mm = re.match(r'\s*\.(get|all|run)\(', src[end:])++ if not mm:++ out.append(src[i:m.end()]); i = m.end(); continue++ end = call_end(src, end + mm.end() - 1)++ if before.endswith('await') or before.endswith('await ('):++ out.append(src[i:end]); i = end; continue++ head = src[start:m.end()]++ inner, _ = transform(src[m.end():end], sync_members) if end > m.end() else ('', 0)++ expr = head + inner++ nxt = src[end:end+1]++ rep = f'(await {expr})' if nxt in ('.', '[') else f'await {expr}'++ out.append(src[i:start]); out.append(rep); i = end; changed += 1++ return ''.join(out), changed++++if __name__ == '__main__':++ sync = set(sys.argv[1].split(','))++ for f in sys.argv[2:]:++ s = open(f).read()++ t, n = transform(s, sync)++ # arrows/functions that now contain await on the same line: make them async++ t = re.sub(r'(const \w+ = )\(([^)]*)\) => (?=.*\bawait\b)', r'\1async (\2) => ', t)++ if n: open(f, 'w').write(t); print(f, n)++++def await_helpers(src):++ names = set(re.findall(r'(?:const|let)\s+(\w+)\s*=\s*async\b', src)) | set(re.findall(r'async function\s+(\w+)\s*\(', src))++ names -= {'main', 'run'}++ if not names: return src, 0++ pat = re.compile(r'(?<![\w.$])(' + '|'.join(sorted(names, key=len, reverse=True)) + r')\(')++ out = []; i = 0; n = 0++ while True:++ m = pat.search(src, i)++ if not m: out.append(src[i:]); break++ start = m.start()++ before = src[:start].rstrip()++ # a definition, or already awaited++ if re.search(r'(function|const|let)\s*$', before) or re.search(r'(const|let)\s+$', src[:start]) or before.endswith('await') or re.search(r'function\s+$', src[:start]):++ out.append(src[i:m.end()]); i = m.end(); continue++ end = call_end(src, m.end() - 1)++ expr = src[start:end]; nxt = src[end:end+1]++ rep = f'(await {expr})' if nxt in ('.', '[') else f'await {expr}'++ out.append(src[i:start]); out.append(rep); i = end; n += 1++ return ''.join(out), n++++if __name__ == '__main__' and len(sys.argv) > 1 and sys.argv[1] == '--helpers':++ for f in sys.argv[2:]:++ s = open(f).read(); t, n = await_helpers(s)++ if n: open(f, 'w').write(t); print(f, n)nnew file mode 100644@@@ -0,0 +1,58 @@++{++ "stores": [++ "revisions",++ "citations",++ "taxonomy",++ "scheduler",++ "attachments",++ "discussion",++ "reviews",++ "redirects",++ "sequencer",++ "outbox"++ ],++ "syncStoreMethods": [++ "start",++ "stop",++ "slugify",++ "enabled"++ ],++ "apis": {++ "blogs": "server/domain/blogs.js",++ "posts": "server/domain/posts.js",++ "publication": "server/domain/publication.js",++ "reading": "server/domain/reading.js",++ "access": "server/domain/access.js",++ "aiDrafts": "server/domain/ai-drafts.js",++ "changelog": "server/changelog.js",++ "effects": "server/domain/effects.js",++ "people": "server/clients/network.js",++ "worker": "server/worker.js",++ "entitlements": "server/domain/access.js",++ "ctx": "server/app.js",++ "cl": "server/changelog.js"++ },++ "syncNames": [++ "validHandle",++ "themeVars",++ "categoryVocabulary",++ "actorRef",++ "listOf",++ "truthy",++ "encodeCursor",++ "decodeCursor",++ "batchKey",++ "subjectOf",++ "start",++ "stop"++ ],++ "asyncGlobals": [++ "seedOfficialPost"++ ],++ "bareStores": [++ "outbox"++ ],++ "asyncMethods": [++ "events"++ ]++}\\ No newline at end of filennew file mode 100644@@@ -0,0 +1,10 @@++{++ "name": "openvibe-asyncify",++ "private": true,++ "description": "The mechanical half of moving a service from better-sqlite3 to openvibe-sdk/db (ADR-035). Not part of the published SDK package.",++ "license": "MIT",++ "dependencies": {++ "acorn": "^8.15.0",++ "acorn-walk": "^8.3.4"++ }++}
new file mode 100644
new file mode 100644
@@ -0,0 +1,40 @@
@@ -0,0 +1,40 @@
+# asyncify: converting a service from better-sqlite3 to openvibe-sdk/db
+
+This directory holds the tools used to convert Wiki and Blog (2026-09-28). They are not part of the published package.
+
+```
+cd tools/asyncify && npm install
+node asyncify.js --config my-service.json $(git -C <service> ls-files 'server/*.js' 'server/**/*.js') <service>/test/*.test.js
+```
+
+## asyncify.js
+
+It parses every file with acorn and does the following:
+
+1. It adds `await` to calls that now return a promise:
+ - prepared statements (`db.prepare(…).get/all/run`, `q.<name>.…`, and any name assigned from `.prepare(…)`);
+ - Publishing store methods reached as `<x>.<store>.<method>(`;
+ - `store.tx(`;
+ - `asyncMethods` and `asyncGlobals` from the config;
+ - methods of module APIs (`apis`: a variable name → the file whose functions became async).
+2. It makes the enclosing functions `async`.
+3. It repeats across all the files until nothing changes.
+
+It leaves some calls unawaited:
+- A promise used as a value (`.then`, `.catch`, `.finally`, `Promise.all/race([...])`, `assert.rejects(p)`) stays a promise.
+- A call inside an array callback (`map`, `filter`, `forEach`, …) is **reported** (`MANUAL …`), never changed, because an async callback there silently breaks the result. The same goes for an async function passed by name to an array method.
+
+Fix those by hand, with a `for … of` loop, `await Promise.all(xs.map(async …))`, or better a batch read (`revisions.getMany`, `citations.forRevisions`, `reviews.latestMany`, or `= ANY(?)`).
+
+It also rewrites `assert.throws(async …)` as `await assert.rejects(async …)`. An edit that would break the syntax is refused and reported.
+
+The config has these keys: `stores`, `bareStores`, `syncStoreMethods`, `apis`, `syncNames`, `asyncGlobals` and `asyncMethods`. See `example-blog.json`.
+
+What stays for a person:
+- the SQL dialect: `INSERT OR IGNORE/REPLACE` → `ON CONFLICT`, `COLLATE NOCASE` → `lower()`, `LIKE` → `ILIKE`, `IS ?` → `IS NOT DISTINCT FROM`, `DISTINCT` + `ORDER BY`, `PRAGMA` and `sqlite_master` → `information_schema`;
+- `RETURNING` in place of `lastInsertRowid`;
+- the boot (open, migrate, `createApp` async) and the test helpers (`openvibe-sdk/testing` `createTestDb`).
+
+## awaitify-tests.py
+
+This is the earlier regex pass for test files (Wiki). `asyncify.js` over the server and the tests together supersedes it.
new file mode 100644
new file mode 100644
@@ -0,0 +1,212 @@
@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * asyncify: the mechanical half of moving a service from better-sqlite3 to openvibe-sdk/db (ADR-035).
+ *
+ * node asyncify.js --config asyncify.json file.js ...
+ *
+ * config: {
+ * "stores": ["revisions", ...], // publishing store property names reached as <x>.<store>.<method>(
+ * "syncStoreMethods": ["..."], // methods of those stores that stay synchronous
+ * "apis": { "blogs": "server/domain/blogs.js", ... }, // variable/property names that hold a module's API object
+ * "syncNames": ["validHandle", ...], // functions that stay synchronous even when their module changes
+ * "asyncGlobals": ["tx"] // extra callee names that are async (e.g. a local tx helper)
+ * }
+ *
+ * Async calls: <stmt>.get/.all/.run(...) where <stmt> is db.prepare(...)/<x>.prepare(...)/q.<name>/<x>.q.<name>;
+ * <x>.<store>.<method>(...); store.tx(...) / db.tx(...); calls to functions of these files that became async
+ * (fixpoint, across files through `apis`). Each gets `await` (parenthesised when a member access follows), its
+ * enclosing function becomes async. A function passed to map/filter/forEach/some/every/find/reduce/sort that would
+ * become async is NOT changed: it is reported for a hand fix (an async callback there silently breaks semantics).
+ */
+const fs = require('fs');
+const path = require('path');
+const acorn = require('acorn');
+const walk = require('acorn-walk');
+
+const ARRAY_CB = new Set(['map', 'filter', 'forEach', 'some', 'every', 'find', 'findIndex', 'reduce', 'sort', 'flatMap']);
+
+function parse(src) {
+ return acorn.parse(src, { ecmaVersion: 'latest', sourceType: 'script', allowHashBang: true, allowReturnOutsideFunction: true, locations: true });
+}
+
+function memberPath(node) {
+ const parts = [];
+ let n = node;
+ while (n) {
+ if (n.type === 'MemberExpression' && !n.computed && n.property.type === 'Identifier') { parts.unshift(n.property.name); n = n.object; continue; }
+ if (n.type === 'Identifier') { parts.unshift(n.name); break; }
+ if (n.type === 'ThisExpression') { parts.unshift('this'); break; }
+ if (n.type === 'CallExpression') { parts.unshift('()'); n = n.callee; continue; }
+ return parts.length ? ['?', ...parts] : null;
+ }
+ return parts;
+}
+
+function functionName(fn, parent) {
+ if (fn.id) return fn.id.name;
+ if (parent && parent.type === 'VariableDeclarator' && parent.id.type === 'Identifier') return parent.id.name;
+ if (parent && parent.type === 'Property' && parent.key) return parent.key.name || parent.key.value;
+ if (parent && parent.type === 'AssignmentExpression' && parent.left.type === 'MemberExpression') return parent.left.property.name;
+ return null;
+}
+
+function analyse(file, src, cfg, asyncNames) {
+ const ast = parse(src);
+ const ancestors = new Map();
+ // parent links
+ walk.fullAncestor(ast, (node, _state, anc) => { ancestors.set(node, anc.slice(0, -1)); });
+ const isFn = (n) => n.type === 'FunctionDeclaration' || n.type === 'FunctionExpression' || n.type === 'ArrowFunctionExpression';
+ const stores = new Set(cfg.stores || []);
+ const syncStoreMethods = new Set(cfg.syncStoreMethods || []);
+ const apis = cfg.apis || {};
+ const syncNames = new Set(cfg.syncNames || []);
+ const localAsync = asyncNames.local.get(file) || new Set();
+
+ // Names that hold a prepared statement: const x = db.prepare(...), or { x: db.prepare(...) }.
+ const stmtNames = new Set();
+ const isPrepare = (n) => n && n.type === 'CallExpression' && n.callee.type === 'MemberExpression' && n.callee.property.name === 'prepare';
+ walk.full(ast, (n) => {
+ if (n.type === 'VariableDeclarator' && n.id.type === 'Identifier' && isPrepare(n.init)) stmtNames.add(n.id.name);
+ if (n.type === 'Property' && n.key && isPrepare(n.value)) stmtNames.add(n.key.name || n.key.value);
+ });
+
+ function asyncCall(call) {
+ const c = call.callee;
+ if (c.type === 'MemberExpression' && !c.computed) {
+ const m = c.property.name;
+ const obj = c.object;
+ const p = memberPath(obj) || [];
+ if (['get', 'all', 'run'].includes(m)) {
+ if (obj.type === 'CallExpression' && obj.callee.type === 'MemberExpression' && ['prepare', 'pluck'].includes(obj.callee.property.name)) return true;
+ if (p.length >= 2 && (p[p.length - 2] === 'q' || p[p.length - 2] === 'stmts' || p[p.length - 2] === 'Q')) return true;
+ if (p.length && stmtNames.has(p[p.length - 1])) return true;
+ }
+ if ((m === 'tx' || m === 'transaction') && p.length && ['store', 'db', 'this'].includes(p[p.length - 1])) return m === 'tx';
+ // <x>.<store>.<method>(: a store reached through an object; a bare identifier only when listed in bareStores.
+ if (p.length >= 2 && stores.has(p[p.length - 1]) && !syncStoreMethods.has(m)) return true;
+ if (p.length === 1 && (cfg.bareStores || []).includes(p[0]) && !syncStoreMethods.has(m)) return true;
+ if ((cfg.asyncMethods || []).includes(m)) return true;
+ // <api>.<method>( across modules
+ const holder = p[p.length - 1];
+ if (apis[holder]) {
+ const set = asyncNames.api.get(apis[holder]);
+ if (set && set.has(m) && !syncNames.has(m)) return true;
+ }
+ if (p.length && (p[p.length - 1] === 'api' || p[p.length - 1] === 'svc' || p[p.length - 1] === 'self') && localAsync.has(m)) return true;
+ }
+ if (c.type === 'Identifier') {
+ if ((cfg.asyncGlobals || []).includes(c.name)) return true;
+ if (localAsync.has(c.name) && !syncNames.has(c.name)) return true;
+ }
+ return false;
+ }
+
+ const edits = []; // { at, text } insertions
+ const manual = [];
+ const becameAsync = new Set();
+ const newlyAsyncNames = new Set();
+ const fnsToAsync = new Set();
+
+ // An async function passed by name to an array method (xs.map(asyncFn)) needs a hand fix too.
+ walk.full(ast, (n) => {
+ if (n.type === 'CallExpression' && n.callee.type === 'MemberExpression' && ARRAY_CB.has(n.callee.property.name)
+ && n.arguments[0] && n.arguments[0].type === 'Identifier' && localAsync.has(n.arguments[0].name)) {
+ const gp = null;
+ manual.push(`${file}:${n.loc.start.line}: async function ${n.arguments[0].name} passed to .${n.callee.property.name}()`);
+ }
+ });
+ walk.fullAncestor(ast, (node, _s, anc) => {
+ if (node.type !== 'CallExpression' || !asyncCall(node)) return;
+ const parents = anc.slice(0, -1);
+ const parent = parents[parents.length - 1];
+ if (parent && parent.type === 'AwaitExpression') return;
+ // A promise used as a value stays a promise: p.then/catch/finally, or an element of Promise.all/race/allSettled/any.
+ if (parent && parent.type === 'MemberExpression' && parent.object === node && ['then', 'catch', 'finally'].includes(parent.property.name)) return;
+ if (parent && parent.type === 'ArrayExpression') {
+ const gp = parents[parents.length - 2];
+ if (gp && gp.type === 'CallExpression' && gp.callee.type === 'MemberExpression' && gp.callee.object.name === 'Promise') return;
+ }
+ if (parent && parent.type === 'CallExpression' && parent.callee.type === 'MemberExpression' && parent.callee.object.name === 'Promise') return;
+ // assert.rejects(promise) / doesNotReject(promise) take the promise itself.
+ if (parent && parent.type === 'CallExpression' && parent.arguments[0] === node && parent.callee.type === 'MemberExpression' && ['rejects', 'doesNotReject'].includes(parent.callee.property.name)) return;
+ // Returned directly from an arrow body or a return statement of an async fn is fine too, but awaiting is harmless.
+ const fnIdx = (() => { for (let i = parents.length - 1; i >= 0; i--) if (isFn(parents[i])) return i; return -1; })();
+ const fn = fnIdx >= 0 ? parents[fnIdx] : null;
+ if (fn) {
+ const fnParent = parents[fnIdx - 1];
+ if (fnParent && fnParent.type === 'CallExpression' && fnParent.arguments.includes(fn) && fnParent.callee.type === 'MemberExpression' && ARRAY_CB.has(fnParent.callee.property.name)) {
+ manual.push(`${file}:${node.loc.start.line}: async call inside a .${fnParent.callee.property.name}() callback`);
+ return;
+ }
+ fnsToAsync.add(fn);
+ }
+ const wrap = parent && parent.type === 'MemberExpression' && parent.object === node;
+ edits.push({ at: node.start, text: wrap ? '(await ' : 'await ' });
+ if (wrap) edits.push({ at: node.end, text: ')' });
+ });
+
+ for (const fn of fnsToAsync) {
+ if (fn.async) continue;
+ const anc = ancestors.get(fn) || [];
+ const parent = anc[anc.length - 1];
+ becameAsync.add(fn);
+ const name = functionName(fn, parent);
+ if (name) newlyAsyncNames.add(name);
+ if (parent && parent.type === 'Property' && (parent.method || parent.kind !== 'init')) edits.push({ at: parent.start, text: 'async ' });
+ else if (parent && parent.type === 'MethodDefinition') edits.push({ at: parent.key.start, text: 'async ' });
+ else edits.push({ at: fn.start, text: 'async ' });
+ }
+ return { edits, manual, newlyAsyncNames };
+}
+
+function apply(src, edits) {
+ const sorted = edits.slice().sort((a, b) => b.at - a.at || (a.text === ')' ? -1 : 1));
+ let out = src;
+ for (const e of sorted) out = out.slice(0, e.at) + e.text + out.slice(e.at);
+ return out;
+}
+
+function main() {
+ const args = process.argv.slice(2);
+ const ci = args.indexOf('--config');
+ const cfg = ci >= 0 ? JSON.parse(fs.readFileSync(args[ci + 1], 'utf8')) : {};
+ const files = args.filter((a, i) => a.endsWith('.js') && i !== ci + 1).map((f) => path.relative(process.cwd(), path.resolve(f)));
+ const asyncNames = { local: new Map(), api: new Map() };
+ let manual = [];
+ for (let round = 0; round < 12; round++) {
+ let changed = false;
+ manual = [];
+ for (const f of files) {
+ const src = fs.readFileSync(f, 'utf8');
+ let r;
+ try { r = analyse(f, src, cfg, asyncNames); } catch (e) { console.error(`${f}: parse error ${e.message}`); process.exitCode = 1; continue; }
+ manual.push(...r.manual);
+ if (r.edits.length) {
+ let next = apply(src, r.edits);
+ // An assertion about a function that is async now: assert.throws cannot see a rejection.
+ next = next.replace(/(^|[^.\w])assert\.throws\(async /g, '$1await assert.rejects(async ').replace(/(^|[^.\w])assert\.doesNotThrow\(async /g, '$1await assert.doesNotReject(async ');
+ try { parse(next); } catch (e) { console.error(`${f}: the edits would break the syntax (${e.message}); left unchanged`); manual.push(`${f}: edits refused (syntax)`); continue; }
+ fs.writeFileSync(f, next); changed = true;
+ }
+ // Every async function of the file (new or already) counts for callers.
+ const all = new Set([...(asyncNames.local.get(f) || []), ...r.newlyAsyncNames]);
+ walk.fullAncestor(parse(fs.readFileSync(f, 'utf8')), (node, _s, anc) => {
+ if ((node.type === 'FunctionDeclaration' || node.type === 'FunctionExpression' || node.type === 'ArrowFunctionExpression') && node.async) {
+ const parent = anc[anc.length - 2];
+ const name = functionName(node, parent);
+ if (name) all.add(name);
+ }
+ });
+ const before = (asyncNames.local.get(f) || new Set()).size;
+ asyncNames.local.set(f, all);
+ asyncNames.api.set(f, all);
+ if (all.size !== before) changed = true;
+ }
+ if (!changed) break;
+ }
+ for (const m of [...new Set(manual)]) console.log(`MANUAL ${m}`);
+ for (const f of files) console.log(`async ${f}: ${[...(asyncNames.local.get(f) || [])].sort().join(', ')}`);
+}
+
+main();
new file mode 100644
new file mode 100644
@@ -0,0 +1,115 @@
@@ -0,0 +1,115 @@
+"""Add `await` to async service/db calls in test files converted from better-sqlite3 (openvibe-sdk/db ≥ 0.18).
+Usage: python3 awaitify.py <sync-member,...> file... (sync members: svc members that stay synchronous)"""
+import re, sys
+
+def skip_string(s, j):
+ q = s[j]; j += 1
+ while j < len(s):
+ c = s[j]
+ if c == '\\': j += 2; continue
+ if q == '`' and c == '$' and j + 1 < len(s) and s[j+1] == '{':
+ j = skip_braces(s, j + 1); continue
+ if c == q: return j + 1
+ j += 1
+ return j
+
+def skip_braces(s, j): # s[j] == '{'
+ depth = 0
+ while j < len(s):
+ c = s[j]
+ if c in '\'"`': j = skip_string(s, j); continue
+ if c == '{': depth += 1
+ elif c == '}':
+ depth -= 1
+ if depth == 0: return j + 1
+ j += 1
+ return j
+
+def call_end(s, i): # s[i] == '(' -> index after matching ')'
+ depth = 0; j = i
+ while j < len(s):
+ c = s[j]
+ if c in '\'"`': j = skip_string(s, j); continue
+ if c == '/' and s[j+1:j+2] == '/': j = s.find('\n', j); j = len(s) if j < 0 else j; continue
+ if c in '([{': depth += 1
+ elif c in ')]}':
+ depth -= 1
+ if depth == 0: return j + 1
+ j += 1
+ return j
+
+def transform(src, sync_members):
+ out = []; i = 0; changed = 0
+ pat = re.compile(r'((?:[A-Za-z_$][\w$]*\.)*(?:svc|db)\.(\w+)\(|(?:\bH\.)?\boutbox\(\s*h\w*\s*\)|(?:[A-Za-z_$][\w$]*\.)*(?:stores\.\w+|platform\.outbox)\.(\w+)\(|(?<![\w.$])(?:summarize|seedFn|seed)\()')
+ while True:
+ m = pat.search(src, i)
+ if not m: out.append(src[i:]); break
+ start = m.start()
+ # not part of a longer member chain (e.g. foo.bar.svc handled by regex prefix); skip when preceded by '.'
+ if start > 0 and src[start-1] in '.$':
+ out.append(src[i:m.end()]); i = m.end(); continue
+ before = src[:start].rstrip()
+ text = m.group(0)
+ if text.startswith('outbox') or text.startswith('H.outbox'):
+ end = m.end()
+ elif m.group(3) is not None or re.match(r'(summarize|seedFn|seed)\($', text):
+ if m.group(3) in ('start', 'stop', 'counts'):
+ out.append(src[i:m.end()]); i = m.end(); continue
+ end = call_end(src, m.end() - 1)
+ else:
+ member = m.group(2)
+ is_db = '.db.' in text or text.startswith('db.')
+ if (not is_db and member in sync_members) or (is_db and member not in ('prepare', 'value', 'many', 'maybe', 'one', 'exec', 'query', 'tx')):
+ out.append(src[i:m.end()]); i = m.end(); continue
+ end = call_end(src, m.end() - 1)
+ if is_db and member == 'prepare':
+ # .pluck(...)? then .get/.all/.run(...)
+ mm = re.match(r'\s*\.pluck\(', src[end:])
+ if mm: end = call_end(src, end + mm.end() - 1)
+ mm = re.match(r'\s*\.(get|all|run)\(', src[end:])
+ if not mm:
+ out.append(src[i:m.end()]); i = m.end(); continue
+ end = call_end(src, end + mm.end() - 1)
+ if before.endswith('await') or before.endswith('await ('):
+ out.append(src[i:end]); i = end; continue
+ head = src[start:m.end()]
+ inner, _ = transform(src[m.end():end], sync_members) if end > m.end() else ('', 0)
+ expr = head + inner
+ nxt = src[end:end+1]
+ rep = f'(await {expr})' if nxt in ('.', '[') else f'await {expr}'
+ out.append(src[i:start]); out.append(rep); i = end; changed += 1
+ return ''.join(out), changed
+
+if __name__ == '__main__':
+ sync = set(sys.argv[1].split(','))
+ for f in sys.argv[2:]:
+ s = open(f).read()
+ t, n = transform(s, sync)
+ # arrows/functions that now contain await on the same line: make them async
+ t = re.sub(r'(const \w+ = )\(([^)]*)\) => (?=.*\bawait\b)', r'\1async (\2) => ', t)
+ if n: open(f, 'w').write(t); print(f, n)
+
+def await_helpers(src):
+ names = set(re.findall(r'(?:const|let)\s+(\w+)\s*=\s*async\b', src)) | set(re.findall(r'async function\s+(\w+)\s*\(', src))
+ names -= {'main', 'run'}
+ if not names: return src, 0
+ pat = re.compile(r'(?<![\w.$])(' + '|'.join(sorted(names, key=len, reverse=True)) + r')\(')
+ out = []; i = 0; n = 0
+ while True:
+ m = pat.search(src, i)
+ if not m: out.append(src[i:]); break
+ start = m.start()
+ before = src[:start].rstrip()
+ # a definition, or already awaited
+ if re.search(r'(function|const|let)\s*$', before) or re.search(r'(const|let)\s+$', src[:start]) or before.endswith('await') or re.search(r'function\s+$', src[:start]):
+ out.append(src[i:m.end()]); i = m.end(); continue
+ end = call_end(src, m.end() - 1)
+ expr = src[start:end]; nxt = src[end:end+1]
+ rep = f'(await {expr})' if nxt in ('.', '[') else f'await {expr}'
+ out.append(src[i:start]); out.append(rep); i = end; n += 1
+ return ''.join(out), n
+
+if __name__ == '__main__' and len(sys.argv) > 1 and sys.argv[1] == '--helpers':
+ for f in sys.argv[2:]:
+ s = open(f).read(); t, n = await_helpers(s)
+ if n: open(f, 'w').write(t); print(f, n)
new file mode 100644
new file mode 100644
@@ -0,0 +1,58 @@
@@ -0,0 +1,58 @@
+{
+ "stores": [
+ "revisions",
+ "citations",
+ "taxonomy",
+ "scheduler",
+ "attachments",
+ "discussion",
+ "reviews",
+ "redirects",
+ "sequencer",
+ "outbox"
+ ],
+ "syncStoreMethods": [
+ "start",
+ "stop",
+ "slugify",
+ "enabled"
+ ],
+ "apis": {
+ "blogs": "server/domain/blogs.js",
+ "posts": "server/domain/posts.js",
+ "publication": "server/domain/publication.js",
+ "reading": "server/domain/reading.js",
+ "access": "server/domain/access.js",
+ "aiDrafts": "server/domain/ai-drafts.js",
+ "changelog": "server/changelog.js",
+ "effects": "server/domain/effects.js",
+ "people": "server/clients/network.js",
+ "worker": "server/worker.js",
+ "entitlements": "server/domain/access.js",
+ "ctx": "server/app.js",
+ "cl": "server/changelog.js"
+ },
+ "syncNames": [
+ "validHandle",
+ "themeVars",
+ "categoryVocabulary",
+ "actorRef",
+ "listOf",
+ "truthy",
+ "encodeCursor",
+ "decodeCursor",
+ "batchKey",
+ "subjectOf",
+ "start",
+ "stop"
+ ],
+ "asyncGlobals": [
+ "seedOfficialPost"
+ ],
+ "bareStores": [
+ "outbox"
+ ],
+ "asyncMethods": [
+ "events"
+ ]
+}
\ No newline at end of file
\ No newline at end of file
new file mode 100644
new file mode 100644
@@ -0,0 +1,10 @@
@@ -0,0 +1,10 @@
+{
+ "name": "openvibe-asyncify",
+ "private": true,
+ "description": "The mechanical half of moving a service from better-sqlite3 to openvibe-sdk/db (ADR-035). Not part of the published SDK package.",
+ "license": "MIT",
+ "dependencies": {
+ "acorn": "^8.15.0",
+ "acorn-walk": "^8.3.4"
+ }
+}