openvibers/openvibe.news Commits
News moves to PostgreSQL and Valkey (ADR-035, roadmap WS-X2, repository-pass group G1), async from end to end.
Changes
@@@ -19,6 +19,14 @@ TRUST_PROXY=2#NEWS_LIMITS_HOUR=3000# SQLite database (production unit: /var/lib/openvibe-news/news.db)++# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh news: the runtime role through PgBouncer,++# the owner on a direct connection (migrations and the one-time import), and Valkey (shared limit counters).++# Without DATABASE_URL, development uses an embedded PGlite database in data/pglite.++DATABASE_URL=++DATABASE_DIRECT_URL=++VALKEY_URL=++VALKEY_PREFIX=++# The SQLite file of releases before PostgreSQL: read once by scripts/migrate-to-postgres.js.NEWS_DB_PATH=./data/news.db# OpenVibe.Network — SSO (OAuth2), JWKS, client-credentials tokens@@@ -9,6 +9,9 @@ jobs:uses: OpenVibers/OpenVibe.Shared/.github/workflows/test.yml@423fe519b5fe9594cc81b670e5ba5cb9f21f6176with:contracts-service: news++ # ADR-035: PostgreSQL 18 + PgBouncer (transaction mode) + Valkey 9 containers (openvibe-sdk++ # scripts/test-services.sh). npm test runs every file on PGlite; test:pg runs them all through PgBouncer.++ test: 'eval "$(node_modules/openvibe-sdk/scripts/test-services.sh up)" && npm test && npm run test:pg'# Secret scan (gitleaks) and dependency audit, shared by every OpenVibers repo (Track Q).security:uses: OpenVibers/OpenVibe.Shared/.github/workflows/security.yml@423fe519b5fe9594cc81b670e5ba5cb9f21f6176@@@ -27,7 +27,7 @@ News publishes nothing until an editor does: the only seed is the topic list.## Owns--The nine charter tables live in News' own SQLite (`NEWS_DB_PATH`):++The nine charter tables live in News' own PostgreSQL database (`ov_news` on the host's data role, ADR-035; schema in [migrations/](migrations/)):| Charter table | What it is ||---|---|@@@ -222,11 +222,13 @@ under [Grants the Network must hold](#grants-the-network-must-hold).## Depends on--- **Packages** (pinned by release tarball): `openvibe-publishing` v0.4.0 (revisions, citations,-- authorship, seo, index-hooks, ssr, taxonomy slugify), `openvibe-contracts` v0.49.0,-- `openvibe-shared` v1.25.0 (chrome, app icon, footer, legal, release, metrics, ready, seo),-- `openvibe-sdk` v0.12.0 (events outbox and inbox, webhook signatures v2, service tokens, per-actor-- limits).++- **PostgreSQL 18 and Valkey 9** (OpenVibe.Host `roles/data/`, ADR-035): every read and write is async through++ `openvibe-sdk/db`; Valkey holds the per-actor limit counters (optional: without `VALKEY_URL` they count per process).++- **Packages** (pinned by release tarball): `openvibe-publishing` v1.0.0 (async PostgreSQL stores: revisions,++ citations, authorship, seo, index-hooks, ssr, taxonomy slugify), `openvibe-contracts` v0.76.0,++ `openvibe-shared` v1.25.0 (Frame, app icon, footer, legal, release, metrics, ready, seo),++ `openvibe-sdk` v0.20.0 (db with after-commit hooks, PostgreSQL events outbox and inbox, webhook signatures v2,++ service tokens, per-actor limits, testing).- **OpenVibe.Sources** (4720): `sources.item.read`; optionally `sources.source.read` (outletnames and source health; without it the outlet is the URL's host).- **OpenVibe.Events** (4300): `events.event.publish`; `events.subscription.manage` to create the@@@ -374,10 +376,13 @@ fnm exec --using=22.22.1 npm run dev # http://localhost:4820 (set OV_OAUTHProduction deploys with `sudo ovhost deploy news` on the host (strategy `git-checkout`: fetch,fast-forward `/opt/openvibe.news`, install on a lockfile change, restart, wait for `/api/ready`).--The unit is `openvibe-news.service` on `127.0.0.1:4820`, the env file `/etc/openvibe/news.env`.++The unit is `openvibe-news.service` on `127.0.0.1:4820`, the env file `/etc/openvibe/news.env`. The database is++`ov_news` on the host's data role (`sudo /opt/openvibe.host/roles/data/add-service.sh news` writes its settings); the++release migrates it at boot. The one-time move from SQLite is `scripts/migrate-to-postgres.js` (openvibe-sdk++`runSqliteMigration`, with a `--pglite` rehearsal mode), run while the service is stopped; the old++`/var/lib/openvibe-news/news.db` stays read-only for 7 days as the rollback.Rollback: ovhost puts the previous sha back by itself when `/api/ready` does not answer 2xx after the--restart; afterwards `sudo ovhost rollback news --to <sha>`. Nothing blocks a rollback: the schema--code only adds tables and columns.++restart; afterwards `sudo ovhost rollback news --to <sha>`. Migrations only add tables and columns.First install (done once; kept for a rebuild):@@@ -4,11 +4,16 @@"code": true,"wave": "W17","plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026)",-- "runtime": "node 22, express 4, better-sqlite3 11",++ "runtime": "node 22, express 4, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9","port": 4820,"domain": "openvibe.news",-- "contracts": "openvibe-contracts v0.49.0",-- "packages": ["openvibe-publishing v0.4.0", "openvibe-contracts v0.49.0", "openvibe-shared v1.25.0", "openvibe-sdk v0.12.0"],++ "contracts": "openvibe-contracts v0.76.0",++ "packages": [++ "openvibe-publishing v1.0.0",++ "openvibe-contracts v0.76.0",++ "openvibe-shared v1.25.0",++ "openvibe-sdk v0.20.0"++ ],"capabilities": ["news.story.create","news.story.read",@@@ -3,8 +3,8 @@ Description=OpenVibe.News — source-backed stories from OpenVibe.Sources items# Network (sign-in, JWKS, service tokens), Sources (the items, over loopback) and Events (the outbox# relay and the webhook) should be up first, but News degrades gracefully without them: pages and# feeds serve; sign-in, ingestion and events wait and are reported by /api/ready.--After=network.target--Wants=openvibe-network.service++After=network.target postgresql.service pgbouncer.service valkey-server.service++Wants=openvibe-network.service postgresql.service pgbouncer.service valkey-server.service[Service]Type=simplennew file mode 100644@@@ -0,0 +1,360 @@++-- phase: expand++-- OpenVibe.News on PostgreSQL (ADR-035, roadmap WS-X2): the tables as they were on SQLite (converted by openvibe-sdk++-- tools/asyncify/sqlite-schema-to-pg: text COLLATE "C" compares like SQLite, integers are bigint, identities keep their ids),++-- then the openvibe-publishing stores and the openvibe-sdk outbox. Generated once on 2026-09-28; never edited after it runs.++++CREATE TABLE news_topics (++ id text COLLATE "C" PRIMARY KEY, -- top_<ULID>++ slug text COLLATE "C" NOT NULL UNIQUE,++ name text COLLATE "C" NOT NULL,++ description text COLLATE "C",++ status text COLLATE "C" NOT NULL DEFAULT 'active' CHECK (status IN ('active','archived')),++ created_at bigint NOT NULL,++ updated_at bigint NOT NULL++);++++CREATE TABLE news_story_clusters (++ id text COLLATE "C" PRIMARY KEY, -- clu_<ULID>++ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)++ label text COLLATE "C" NOT NULL, -- derived from the key terms, never invented++ status text COLLATE "C" NOT NULL DEFAULT 'open' CHECK (status IN ('open','merged','dissolved')),++ merged_into text COLLATE "C" REFERENCES news_story_clusters(id),++ split_from text COLLATE "C" REFERENCES news_story_clusters(id),++ terms text COLLATE "C" NOT NULL DEFAULT '{}', -- { term: count } over its members++ entities text COLLATE "C" NOT NULL DEFAULT '{}', -- { entity: count } over its members++ window_start bigint, -- earliest member time (published_at, else first seen)++ window_end bigint, -- latest member time++ created_by text COLLATE "C" NOT NULL, -- 'svc:news' (clustering) or usr_… (a split)++ created_at bigint NOT NULL,++ updated_at bigint NOT NULL++);++CREATE INDEX news_story_clusters_open ON news_story_clusters (status, window_end);++++CREATE TABLE news_source_items (++ id text COLLATE "C" PRIMARY KEY, -- nsi_<ULID>++ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)++ sources_item_id text COLLATE "C" NOT NULL UNIQUE, -- itm_… in OpenVibe.Sources (a typed reference)++ sources_revision bigint NOT NULL, -- the Sources revision these fields came from++ source_key text COLLATE "C" NOT NULL,++ canonical_url text COLLATE "C",++ url_key text COLLATE "C", -- normalised URL for dedupe++ headline text COLLATE "C" NOT NULL,++ outlet text COLLATE "C" NOT NULL,++ authors text COLLATE "C" NOT NULL DEFAULT '[]',++ published_at text COLLATE "C", -- as the source states it; NULL when it does not++ summary text COLLATE "C", -- a short summary ONLY when the terms allow it++ summary_basis text COLLATE "C", -- why a summary is (not) stored++ license_note text COLLATE "C",++ terms_note text COLLATE "C",++ content_hash text COLLATE "C", -- Sources' hash of the parsed fields++ title_key text COLLATE "C" NOT NULL, -- normalised headline for near-duplicate checks++ status text COLLATE "C" NOT NULL DEFAULT 'active' CHECK (status IN ('active','duplicate','removed')),++ duplicate_of text COLLATE "C" REFERENCES news_source_items(id),++ dedupe text COLLATE "C", -- { rule, detail } when a duplicate++ cluster_id text COLLATE "C" REFERENCES news_story_clusters(id),++ cluster_reason text COLLATE "C", -- { rule, shared_entities, shared_terms, score } (explanation)++ retrieved_at text COLLATE "C",++ first_seen_at bigint NOT NULL,++ upstream_updated_at bigint, -- the last time Sources revised it after first ingest++ removed_at bigint,++ removed_reason text COLLATE "C",++ updated_at bigint NOT NULL++);++CREATE INDEX news_source_items_url ON news_source_items (url_key);++CREATE INDEX news_source_items_hash ON news_source_items (content_hash);++CREATE INDEX news_source_items_cluster ON news_source_items (cluster_id);++CREATE INDEX news_source_items_seen ON news_source_items (first_seen_at);++++CREATE TABLE news_stories (++ id text COLLATE "C" PRIMARY KEY, -- sty_<ULID>++ slug text COLLATE "C" NOT NULL UNIQUE,++ working_headline text COLLATE "C" NOT NULL, -- the editors' latest headline (readers see the revision's)++ cluster_id text COLLATE "C" REFERENCES news_story_clusters(id),++ topic_id text COLLATE "C" REFERENCES news_topics(id),++ state text COLLATE "C" NOT NULL DEFAULT 'draft' CHECK (state IN ('draft','published','unpublished','retracted')),++ published_revision bigint, -- which revision readers see (NULL: never published)++ first_published_at bigint,++ published_at bigint, -- the latest publication of published_revision++ retracted_at bigint,++ noindex bigint NOT NULL DEFAULT 0, -- an editor asked search engines not to index it++ created_by text COLLATE "C" NOT NULL, -- usr_… (the editor who opened the story)++ created_at bigint NOT NULL,++ updated_at bigint NOT NULL++);++CREATE INDEX news_stories_listing ON news_stories (state, published_at);++CREATE INDEX news_stories_topic ON news_stories (topic_id, state, published_at);++CREATE INDEX news_stories_cluster ON news_stories (cluster_id);++++CREATE TABLE news_perspectives (++ id text COLLATE "C" PRIMARY KEY, -- per_<ULID>++ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)++ story_id text COLLATE "C" NOT NULL REFERENCES news_stories(id),++ label text COLLATE "C" NOT NULL, -- written by an editor; never generated++ description text COLLATE "C",++ position bigint NOT NULL DEFAULT 0,++ created_by text COLLATE "C" NOT NULL,++ created_at bigint NOT NULL,++ updated_at bigint NOT NULL,++ removed_at bigint++);++CREATE INDEX news_perspectives_story ON news_perspectives (story_id, position);++++CREATE TABLE news_story_sources (++ story_id text COLLATE "C" NOT NULL REFERENCES news_stories(id),++ source_item_id text COLLATE "C" NOT NULL REFERENCES news_source_items(id),++ position bigint NOT NULL, -- the [n] readers and editors cite++ perspective_id text COLLATE "C" REFERENCES news_perspectives(id),++ added_by text COLLATE "C" NOT NULL,++ added_at bigint NOT NULL,++ detached_at bigint,++ PRIMARY KEY (story_id, source_item_id)++);++CREATE INDEX news_story_sources_item ON news_story_sources (source_item_id);++++CREATE TABLE news_timeline_entries (++ id text COLLATE "C" PRIMARY KEY, -- tle_<ULID>++ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)++ story_id text COLLATE "C" NOT NULL REFERENCES news_stories(id),++ occurred_on text COLLATE "C" NOT NULL, -- YYYY-MM-DD or an ISO instant, as the source states it++ text text COLLATE "C" NOT NULL,++ source_item_id text COLLATE "C" NOT NULL REFERENCES news_source_items(id),++ created_by text COLLATE "C" NOT NULL,++ created_at bigint NOT NULL,++ removed_at bigint++);++CREATE INDEX news_timeline_story ON news_timeline_entries (story_id, occurred_on);++++CREATE TABLE news_editorial_flags (++ id text COLLATE "C" PRIMARY KEY, -- flg_<ULID>++ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)++ story_id text COLLATE "C" NOT NULL REFERENCES news_stories(id),++ kind text COLLATE "C" NOT NULL CHECK (kind IN ('correction','update','retraction','source_updated','source_removed')),++ status text COLLATE "C" NOT NULL DEFAULT 'open' CHECK (status IN ('open','published','resolved')),++ note text COLLATE "C", -- public text for correction/update/retraction++ source_item_id text COLLATE "C" REFERENCES news_source_items(id),++ pending_revision bigint, -- the revision the system prepared for an upstream change++ revision bigint, -- the published revision it belongs to++ created_by text COLLATE "C" NOT NULL,++ created_at bigint NOT NULL,++ resolved_by text COLLATE "C",++ resolved_at bigint++);++CREATE INDEX news_flags_story ON news_editorial_flags (story_id, status, created_at);++++-- Every merge, split and reversal, with exactly which items moved (so each can be undone).++CREATE TABLE news_cluster_audit (++ id text COLLATE "C" PRIMARY KEY, -- cla_<ULID>++ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)++ action text COLLATE "C" NOT NULL CHECK (action IN ('merge','split','reverse_merge','reverse_split')),++ cluster_id text COLLATE "C" NOT NULL, -- merge: the surviving cluster; split: the source cluster++ other_id text COLLATE "C" NOT NULL, -- merge: the absorbed cluster; split: the new cluster++ item_ids text COLLATE "C" NOT NULL, -- JSON array of news_source_items ids that moved++ reason text COLLATE "C",++ actor text COLLATE "C" NOT NULL,++ reverses text COLLATE "C" REFERENCES news_cluster_audit(id),++ reversed_by text COLLATE "C" REFERENCES news_cluster_audit(id),++ created_at bigint NOT NULL++);++CREATE INDEX news_cluster_audit_cluster ON news_cluster_audit (cluster_id, created_at);++++-- What ingestion did: webhook deliveries, cursor pulls and upstream fetch failures.++CREATE TABLE news_ingest_runs (++ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,++ origin text COLLATE "C" NOT NULL CHECK (origin IN ('webhook','pull','sources')),++ state text COLLATE "C" NOT NULL, -- ok | failed | ignored | upstream state (http_error, timeout, …)++ source_key text COLLATE "C",++ sources_item_id text COLLATE "C",++ error_code text COLLATE "C",++ detail text COLLATE "C",++ counts text COLLATE "C", -- { created, updated, removed, duplicates, unchanged }++ cursor_before bigint,++ cursor_after bigint,++ at bigint NOT NULL++);++CREATE INDEX news_ingest_runs_at ON news_ingest_runs (at);++++CREATE TABLE news_state (++ key text COLLATE "C" PRIMARY KEY,++ value text COLLATE "C" NOT NULL,++ updated_at bigint NOT NULL++);++++-- Display cache of OpenVibe.Sources' registry (outlet names) and health; never authority.++CREATE TABLE news_source_status (++ source_key text COLLATE "C" PRIMARY KEY,++ name text COLLATE "C",++ homepage_url text COLLATE "C",++ status text COLLATE "C",++ stale bigint,++ last_success_at text COLLATE "C",++ refreshed_at bigint NOT NULL++);++++-- Display cache of Network names for subjects (from sign-in claims or identity.subject.resolve).++CREATE TABLE subject_projections (++ subject text COLLATE "C" PRIMARY KEY,++ username text COLLATE "C",++ display_name text COLLATE "C",++ avatar_url text COLLATE "C",++ refreshed_at bigint NOT NULL++);++CREATE INDEX subject_projections_username ON subject_projections (username);++++CREATE INDEX news_stories_published ON news_stories (published_at DESC, id) WHERE state = 'published';++-- openvibe-publishing/revisions (prefix news_story)++CREATE TABLE IF NOT EXISTS news_story_revisions (++ id text PRIMARY KEY,++ entity_id text COLLATE "C" NOT NULL,++ number integer NOT NULL CHECK (number >= 1),++ parent_id text,++ parent_number integer,++ kind text NOT NULL CHECK (kind IN ('edit','revert','import')),++ reverted_to integer,++ content text NOT NULL,++ fields jsonb NOT NULL DEFAULT '{}',++ meta jsonb NOT NULL DEFAULT '{}',++ content_hash text NOT NULL,++ author text,++ message text,++ created_at bigint NOT NULL++);++CREATE UNIQUE INDEX IF NOT EXISTS news_story_revisions_entity_num ON news_story_revisions (entity_id, number);++CREATE TABLE IF NOT EXISTS news_story_drafts (++ entity_id text COLLATE "C" NOT NULL,++ owner text COLLATE "C" NOT NULL,++ base_revision integer NOT NULL,++ content text NOT NULL,++ fields jsonb NOT NULL DEFAULT '{}',++ meta jsonb NOT NULL DEFAULT '{}',++ created_at bigint NOT NULL,++ updated_at bigint NOT NULL,++ PRIMARY KEY (entity_id, owner)++);++CREATE INDEX IF NOT EXISTS news_story_drafts_updated ON news_story_drafts (entity_id, updated_at DESC, owner);++CREATE TABLE IF NOT EXISTS news_story_revision_purges (++ entity_id text COLLATE "C" PRIMARY KEY,++ reason text NOT NULL,++ purged_by text,++ purged_at bigint NOT NULL++);++CREATE OR REPLACE FUNCTION news_story_revisions_guard() RETURNS trigger LANGUAGE plpgsql AS $$++BEGIN++ IF TG_OP = 'UPDATE' THEN++ RAISE EXCEPTION 'news_story_revisions rows are immutable' USING ERRCODE = 'restrict_violation';++ END IF;++ IF NOT EXISTS (SELECT 1 FROM news_story_revision_purges WHERE entity_id = OLD.entity_id) THEN++ RAISE EXCEPTION 'news_story_revisions rows are never deleted outside a recorded purge' USING ERRCODE = 'restrict_violation';++ END IF;++ RETURN OLD;++END++$$;++CREATE OR REPLACE TRIGGER news_story_revisions_no_update BEFORE UPDATE ON news_story_revisions FOR EACH ROW EXECUTE FUNCTION news_story_revisions_guard();++CREATE OR REPLACE TRIGGER news_story_revisions_no_delete BEFORE DELETE ON news_story_revisions FOR EACH ROW EXECUTE FUNCTION news_story_revisions_guard();++++-- openvibe-publishing/citations (prefix news_story)++CREATE TABLE IF NOT EXISTS news_story_citations (++ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,++ entity_id text COLLATE "C" NOT NULL,++ revision integer NOT NULL CHECK (revision >= 1),++ anchor text,++ source_item_id text COLLATE "C",++ url text,++ title text,++ retrieved_at timestamptz,++ quote_text text,++ quote_start integer,++ quote_end integer,++ license_note text,++ carried_from bigint REFERENCES news_story_citations(id),++ attached_by text,++ attached_at bigint NOT NULL,++ CHECK (source_item_id IS NOT NULL OR url IS NOT NULL)++);++CREATE INDEX IF NOT EXISTS news_story_citations_rev ON news_story_citations (entity_id, revision, id);++CREATE INDEX IF NOT EXISTS news_story_citations_source ON news_story_citations (source_item_id, entity_id, revision, id) WHERE source_item_id IS NOT NULL;++CREATE INDEX IF NOT EXISTS news_story_citations_carried ON news_story_citations (carried_from) WHERE carried_from IS NOT NULL;++CREATE TABLE IF NOT EXISTS news_story_citation_purges (++ entity_id text COLLATE "C" PRIMARY KEY,++ reason text NOT NULL,++ purged_by text,++ purged_at bigint NOT NULL++);++CREATE OR REPLACE FUNCTION news_story_citations_guard() RETURNS trigger LANGUAGE plpgsql AS $$++BEGIN++ IF TG_OP = 'UPDATE' THEN++ RAISE EXCEPTION 'news_story_citations rows are immutable' USING ERRCODE = 'restrict_violation';++ END IF;++ IF NOT EXISTS (SELECT 1 FROM news_story_citation_purges WHERE entity_id = OLD.entity_id) THEN++ RAISE EXCEPTION 'news_story_citations rows are never deleted outside a recorded purge' USING ERRCODE = 'restrict_violation';++ END IF;++ RETURN OLD;++END++$$;++CREATE OR REPLACE TRIGGER news_story_citations_no_update BEFORE UPDATE ON news_story_citations FOR EACH ROW EXECUTE FUNCTION news_story_citations_guard();++CREATE OR REPLACE TRIGGER news_story_citations_no_delete BEFORE DELETE ON news_story_citations FOR EACH ROW EXECUTE FUNCTION news_story_citations_guard();++++-- openvibe-publishing/authorship (prefix news_story)++CREATE TABLE IF NOT EXISTS news_story_reviews (++ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,++ entity_id text COLLATE "C" NOT NULL,++ revision integer NOT NULL,++ reviewer text NOT NULL,++ decision text NOT NULL CHECK (decision IN ('approved','rejected')),++ note text,++ reviewed_at bigint NOT NULL++);++CREATE INDEX IF NOT EXISTS news_story_reviews_rev ON news_story_reviews (entity_id, revision, id);++CREATE INDEX IF NOT EXISTS news_story_reviews_entity ON news_story_reviews (entity_id, id);++CREATE OR REPLACE FUNCTION news_story_reviews_guard() RETURNS trigger LANGUAGE plpgsql AS $$++BEGIN++ IF TG_OP = 'UPDATE' THEN++ RAISE EXCEPTION 'news_story_reviews rows are immutable' USING ERRCODE = 'restrict_violation';++ END IF;++ RETURN OLD;++END++$$;++CREATE OR REPLACE TRIGGER news_story_reviews_no_update BEFORE UPDATE ON news_story_reviews FOR EACH ROW EXECUTE FUNCTION news_story_reviews_guard();++++-- openvibe-publishing/index-hooks (prefix news)++CREATE TABLE IF NOT EXISTS news_index_revisions (++ owner text COLLATE "C" NOT NULL,++ type text COLLATE "C" NOT NULL,++ id text COLLATE "C" NOT NULL,++ revision integer NOT NULL,++ hash text NOT NULL,++ updated_at bigint NOT NULL,++ PRIMARY KEY (owner, type, id)++);++++-- openvibe-publishing/discussion (prefix news_story)++CREATE TABLE IF NOT EXISTS news_story_discussion_refs (++ entity_id text COLLATE "C" PRIMARY KEY,++ thread_id text NOT NULL,++ ref jsonb NOT NULL,++ resolved_at bigint NOT NULL++);++++-- openvibe-sdk/events inbox: one receipt per (consumer, event) handled++CREATE TABLE IF NOT EXISTS idempotency_receipts (++ consumer text NOT NULL,++ event_id text NOT NULL,++ processed_at bigint NOT NULL,++ PRIMARY KEY (consumer, event_id)++);++++-- openvibe-sdk/events PostgreSQL outbox++CREATE TABLE IF NOT EXISTS event_outbox (++ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,++ event_id text NOT NULL UNIQUE,++ envelope jsonb NOT NULL,++ traceparent text,++ created_at bigint NOT NULL,++ attempts integer NOT NULL DEFAULT 0,++ next_attempt_at bigint NOT NULL DEFAULT 0,++ sent_at bigint,++ seq bigint,++ rejected_at bigint,++ last_error text++);++CREATE INDEX IF NOT EXISTS event_outbox_due ON event_outbox (next_attempt_at, id) WHERE sent_at IS NULL AND rejected_at IS NULL;++CREATE INDEX IF NOT EXISTS event_outbox_sent ON event_outbox (sent_at) WHERE sent_at IS NOT NULL;@@@ -15,18 +15,34 @@"express": "^4.21.2","express-rate-limit": "^7.5.0","helmet": "^8.1.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.49.0",-- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v0.4.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.12.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.25.0"++ "iovalkey": "^0.4.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.76.0",++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.20.0",++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.25.0",++ "pg": "^8.23.0"},"devDependencies": {++ "@electric-sql/pglite": "^0.5.8","jsonwebtoken": "^9.0.2"},"engines": {"node": ">=22.0.0"}},++ "node_modules/@electric-sql/pglite": {++ "version": "0.5.8",++ "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.5.8.tgz",++ "integrity": "sha512-n9tsbUOhwx2epK1V0ZG9Ar4SHWUju04dhmzZXiSBXwBoleOvIfals33NAaWgagQVAL4Rbvx/Ptsu3P+pA09f6Q==",++ "devOptional": true,++ "license": "Apache-2.0"++ },++ "node_modules/@iovalkey/commands": {++ "version": "0.1.0",++ "resolved": "https://registry.npmjs.org/@iovalkey/commands/-/commands-0.1.0.tgz",++ "integrity": "sha512-/B9W4qKSSITDii5nkBCHyPkIkAi+ealUtr1oqBJsLxjSRLka4pxun2VvMNSmcwgAMxgXtQfl0qRv7TE+udPJzg==",++ "license": "MIT"++ },"node_modules/accepts": {"version": "1.3.8","resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",@@@ -228,6 +244,15 @@"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==","license": "ISC"},++ "node_modules/cluster-key-slot": {++ "version": "1.1.2",++ "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",++ "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",++ "license": "Apache-2.0",++ "engines": {++ "node": ">=0.10.0"++ }++ },"node_modules/content-disposition": {"version": "0.5.4","resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",@@@ -310,6 +335,15 @@"node": ">=4.0.0"}},++ "node_modules/denque": {++ "version": "2.1.0",++ "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",++ "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",++ "license": "Apache-2.0",++ "engines": {++ "node": ">=0.10"++ }++ },"node_modules/depd": {"version": "2.0.0","resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",@@@ -746,6 +780,49 @@"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==","license": "ISC"},++ "node_modules/iovalkey": {++ "version": "0.4.0",++ "resolved": "https://registry.npmjs.org/iovalkey/-/iovalkey-0.4.0.tgz",++ "integrity": "sha512-OSUKxJ+s44CLdUTaicX4+pVrBN/zHKIYwr2oKhRDuczr19FwcSCXEZHzcMUYWZNh1mNSCV9bNJHW/T6cHVm9Aw==",++ "license": "MIT",++ "dependencies": {++ "@iovalkey/commands": "^0.1.0",++ "cluster-key-slot": "^1.1.0",++ "debug": "^4.3.4",++ "denque": "^2.1.0",++ "lodash.defaults": "^4.2.0",++ "lodash.isarguments": "^3.1.0",++ "redis-errors": "^1.2.0",++ "redis-parser": "^3.0.0",++ "standard-as-callback": "^2.1.0"++ },++ "engines": {++ "node": ">=18.12.0"++ }++ },++ "node_modules/iovalkey/node_modules/debug": {++ "version": "4.4.3",++ "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",++ "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",++ "license": "MIT",++ "dependencies": {++ "ms": "^2.1.3"++ },++ "engines": {++ "node": ">=6.0"++ },++ "peerDependenciesMeta": {++ "supports-color": {++ "optional": true++ }++ }++ },++ "node_modules/iovalkey/node_modules/ms": {++ "version": "2.1.3",++ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",++ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",++ "license": "MIT"++ },"node_modules/ipaddr.js": {"version": "1.9.1","resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",@@@ -810,12 +887,24 @@"safe-buffer": "^5.0.1"}},++ "node_modules/lodash.defaults": {++ "version": "4.2.0",++ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",++ "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",++ "license": "MIT"++ },"node_modules/lodash.includes": {"version": "4.3.0","resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz","integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==","license": "MIT"},++ "node_modules/lodash.isarguments": {++ "version": "3.1.0",++ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",++ "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",++ "license": "MIT"++ },"node_modules/lodash.isboolean": {"version": "3.0.3","resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",@@@ -1015,9 +1104,9 @@}},"node_modules/openvibe-contracts": {-- "version": "0.49.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.49.0",-- "integrity": "sha512-lapsLoBm3ZpndgiOMELdCBh2PJtuuyqj6SUhQsfwJkxnLdQfyhXfRFgiuLh+iQrg4kqtG9611191Mp8zAfk7fQ==",++ "version": "0.76.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.76.0",++ "integrity": "sha512-zTdx776KWrcJKm1JTnf+0+BnsvswdvXy1ldPyse1hSqhnW/SSr98sX696eRetFUhTl8qVM1xSRCykqIUbqItPw==","license": "AGPL-3.0-only","dependencies": {"ajv": "^8.17.1",@@@ -1031,15 +1120,15 @@}},"node_modules/openvibe-publishing": {-- "version": "0.4.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v0.4.0",-- "integrity": "sha512-zipRyxNSb39fTk6ZtXfCygWgxSRVj/fOVGc31hynBANM5JCwErX/dYKsE/XmdYGlId29E5O3MyZLohgfocBP4w==",++ "version": "1.0.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",++ "integrity": "sha512-g4mU0PPtI4y38NcsdppEbrXTrCYp0rPmIYWyxLxh4GpEYhn48jMWq2+cMQA2uwZCeeZlB3UiXVrQNQyQLiliwg==","license": "MIT","engines": {"node": ">=20"},"peerDependencies": {-- "better-sqlite3": ">=11",++ "openvibe-sdk": ">=0.15.0","openvibe-shared": ">=1.5.0"},"peerDependenciesMeta": {@@@ -1049,12 +1138,32 @@}},"node_modules/openvibe-sdk": {-- "version": "0.12.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.12.0",-- "integrity": "sha512-iG1h905O1f7y/PnP8xLA6GCVzXEBDfSaBj2tc1ucNreWg2sT/AJWosb8NW0WRMK5WS4sIQ6tMsprmDLj/fjjeQ==",++ "version": "0.20.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.20.0",++ "integrity": "sha512-CqZFfVuSv3ZPoyG1LuWCAdvo8TG2ZuUCeGaqTsxDKS3gsymSFV2QXPq/5QUotTd82WShTGUKKhTgcyp9ggUMfQ==","license": "MIT","engines": {"node": ">=20"++ },++ "peerDependencies": {++ "@electric-sql/pglite": "^0.5.8",++ "better-sqlite3": ">=11",++ "iovalkey": "^0.4.0",++ "pg": "^8.23.0"++ },++ "peerDependenciesMeta": {++ "@electric-sql/pglite": {++ "optional": true++ },++ "better-sqlite3": {++ "optional": true++ },++ "iovalkey": {++ "optional": true++ },++ "pg": {++ "optional": true++ }}},"node_modules/openvibe-shared": {@@@ -1081,6 +1190,134 @@"integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==","license": "MIT"},++ "node_modules/pg": {++ "version": "8.23.0",++ "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz",++ "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==",++ "license": "MIT",++ "dependencies": {++ "pg-connection-string": "^2.14.0",++ "pg-pool": "^3.14.0",++ "pg-protocol": "^1.16.0",++ "pg-types": "2.2.0",++ "pgpass": "1.0.5"++ },++ "engines": {++ "node": ">= 16.0.0"++ },++ "optionalDependencies": {++ "pg-cloudflare": "^1.4.0"++ },++ "peerDependencies": {++ "pg-native": ">=3.0.1"++ },++ "peerDependenciesMeta": {++ "pg-native": {++ "optional": true++ }++ }++ },++ "node_modules/pg-cloudflare": {++ "version": "1.4.0",++ "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz",++ "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==",++ "license": "MIT",++ "optional": true++ },++ "node_modules/pg-connection-string": {++ "version": "2.14.0",++ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz",++ "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==",++ "license": "MIT"++ },++ "node_modules/pg-int8": {++ "version": "1.0.1",++ "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",++ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",++ "license": "ISC",++ "engines": {++ "node": ">=4.0.0"++ }++ },++ "node_modules/pg-pool": {++ "version": "3.14.0",++ "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",++ "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",++ "license": "MIT",++ "peerDependencies": {++ "pg": ">=8.0"++ }++ },++ "node_modules/pg-protocol": {++ "version": "1.16.0",++ "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.0.tgz",++ "integrity": "sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg==",++ "license": "MIT"++ },++ "node_modules/pg-types": {++ "version": "2.2.0",++ "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",++ "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",++ "license": "MIT",++ "dependencies": {++ "pg-int8": "1.0.1",++ "postgres-array": "~2.0.0",++ "postgres-bytea": "~1.0.0",++ "postgres-date": "~1.0.4",++ "postgres-interval": "^1.1.0"++ },++ "engines": {++ "node": ">=4"++ }++ },++ "node_modules/pgpass": {++ "version": "1.0.5",++ "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",++ "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",++ "license": "MIT",++ "dependencies": {++ "split2": "^4.1.0"++ }++ },++ "node_modules/postgres-array": {++ "version": "2.0.0",++ "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",++ "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",++ "license": "MIT",++ "engines": {++ "node": ">=4"++ }++ },++ "node_modules/postgres-bytea": {++ "version": "1.0.1",++ "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",++ "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",++ "license": "MIT",++ "engines": {++ "node": ">=0.10.0"++ }++ },++ "node_modules/postgres-date": {++ "version": "1.0.7",++ "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",++ "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",++ "license": "MIT",++ "engines": {++ "node": ">=0.10.0"++ }++ },++ "node_modules/postgres-interval": {++ "version": "1.2.0",++ "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",++ "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",++ "license": "MIT",++ "dependencies": {++ "xtend": "^4.0.0"++ },++ "engines": {++ "node": ">=0.10.0"++ }++ },"node_modules/prebuild-install": {"version": "7.1.3","resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",@@@ -1204,6 +1441,27 @@"node": ">= 6"}},++ "node_modules/redis-errors": {++ "version": "1.2.0",++ "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",++ "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",++ "license": "MIT",++ "engines": {++ "node": ">=4"++ }++ },++ "node_modules/redis-parser": {++ "version": "3.0.0",++ "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",++ "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",++ "license": "MIT",++ "dependencies": {++ "redis-errors": "^1.0.0"++ },++ "engines": {++ "node": ">=4"++ }++ },"node_modules/require-from-string": {"version": "2.0.2","resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",@@@ -1419,6 +1677,21 @@"simple-concat": "^1.0.0"}},++ "node_modules/split2": {++ "version": "4.2.0",++ "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",++ "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",++ "license": "ISC",++ "engines": {++ "node": ">= 10.x"++ }++ },++ "node_modules/standard-as-callback": {++ "version": "2.1.0",++ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",++ "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",++ "license": "MIT"++ },"node_modules/statuses": {"version": "2.0.2","resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",@@@ -1546,6 +1819,15 @@"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz","integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==","license": "ISC"++ },++ "node_modules/xtend": {++ "version": "4.0.2",++ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",++ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",++ "license": "MIT",++ "engines": {++ "node": ">=0.4"++ }}}}@@@ -2,16 +2,23 @@"name": "openvibe-news","version": "0.1.0","private": true,-- "description": "OpenVibe.News — a source-backed publication product: source items from OpenVibe.Sources, dedupe, explainable clusters, editor-written stories whose every claim cites its sources, timelines, perspectives and correction/retraction revisions, server-rendered and useful without JavaScript.",++ "description": "OpenVibe.News \u2014 a source-backed publication product: source items from OpenVibe.Sources, dedupe, explainable clusters, editor-written stories whose every claim cites its sources, timelines, perspectives and correction/retraction revisions, server-rendered and useful without JavaScript.","main": "server/index.js","scripts": {"start": "node server/index.js","dev": "NODE_ENV=development node server/index.js","test": "node test/run.js","seed": "node scripts/seed.js",-- "subscribe": "node scripts/subscribe.js"++ "subscribe": "node scripts/subscribe.js",++ "test:pg": "NEWS_TEST_STORE=pg node test/run.js"},-- "keywords": ["openvibe", "news", "publishing", "sources", "feeds"],++ "keywords": [++ "openvibe",++ "news",++ "publishing",++ "sources",++ "feeds"++ ],"author": "OpenVibers","license": "AGPL-3.0-only","repository": "github:OpenVibers/OpenVibe.News",@@@ -22,13 +29,16 @@"express": "^4.21.2","express-rate-limit": "^7.5.0","helmet": "^8.1.0",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.49.0",-- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v0.4.0",-- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.12.0",-- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.25.0"++ "iovalkey": "^0.4.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.76.0",++ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",++ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.20.0",++ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.25.0",++ "pg": "^8.23.0"},"devDependencies": {-- "jsonwebtoken": "^9.0.2"++ "jsonwebtoken": "^9.0.2",++ "@electric-sql/pglite": "^0.5.8"},"engines": {"node": ">=22.0.0"nnew file mode 100644@@@ -0,0 +1,26 @@++#!/usr/bin/env node++'use strict';++/**++ * The one-time move of News's SQLite database (NEWS_DB_PATH) into its PostgreSQL schema (ADR-035; the procedure is++ * openvibe-sdk docs/migrating-to-postgresql.md, section 6, carried out by openvibe-sdk/db runSqliteMigration).++ *++ * node scripts/migrate-to-postgres.js [--sqlite <file>] [--pglite] [--json]++ *++ * Applies migrations/ as the owner (DATABASE_DIRECT_URL), copies every table into emptied tables, verifies row counts++ * and checksums, and exits 1 unless everything verified. The SQLite file is opened read-only. Every table keeps its++ * name and columns.++ */++require('dotenv').config();++const { runSqliteMigration } = require('openvibe-sdk/db');++const configLib = require('../server/config');++const { MIGRATIONS } = require('../server/db');++++const TABLES = {};++++if (require.main === module) {++ const config = configLib.load();++ runSqliteMigration({ service: 'news', sqlite: config.dbPath, directUrl: config.db.directUrl, migrations: MIGRATIONS, tables: TABLES })++ .then((code) => process.exit(code), (err) => { console.error(`migrate-to-postgres failed: ${err.message}`); process.exit(1); });++}++++module.exports = { TABLES };@@@ -5,7 +5,7 @@* existing ones are never renamed, re-described or re-activated. There are no seed stories,* source items or clusters: News publishes nothing until an editor does.*-- * npm run seed (uses NEWS_DB_PATH from .env or /etc/openvibe/news.env)++ * npm run seed (DATABASE_URL from .env or /etc/openvibe/news.env; without it, the development PGlite)*/require('dotenv').config();const { load } = require('../server/config');@@@ -13,7 +13,9 @@ const { openStore } = require('../server/db');const { createTopics } = require('../server/domain/topics');const config = load();--const store = openStore(config.dbPath);--const r = createTopics({ store }).seed();--console.log(`topics: ${r.created} added, ${r.existing} already present (${config.dbPath})`);--store.close();++(async () => {++ const store = await openStore(config);++ const r = await createTopics({ store }).seed();++ console.log(`topics: ${r.created} added, ${r.existing} already present (${store.db.store})`);++ await store.close();++})().catch((err) => { console.error(`[seed] ${err.message}`); process.exit(1); });@@@ -46,11 +46,12 @@ const VERSION = require('../package.json').version;/** opts: config, store | dbPath, now (clock), fetchImpl, auth (a createAuthClient-like object), log,* limitsNow (the per-actor limiter's clock, tests; default the wall clock) */--function createApp(opts = {}) {++async function createApp(opts = {}) {const config = opts.config || configLib.load();const log = opts.log || console;const fetchImpl = opts.fetchImpl || globalThis.fetch;-- const store = opts.store || openStore(opts.dbPath || config.dbPath, { now: opts.now });++ // PostgreSQL (ADR-035): opened and migrated here unless the caller (a test, a script) hands in a store.++ const store = opts.store || await openStore(config, { now: opts.now, log });const outbox = createNewsOutbox({ db: store.db, config, fetchImpl, now: store.now, log });const publication = createPublication({ store, config, outbox });@@@ -64,7 +65,7 @@ function createApp(opts = {}) {ingest.setStories(stories);const topics = createTopics({ store });// Topics are the only seed (idempotent: missing slugs are added, existing ones never changed).-- if (opts.seedTopics !== false) topics.seed();++ if (opts.seedTopics !== false) await topics.seed();const people = createPeople({ store, config, fetchImpl });const reading = createReading({ store, config, stories, publication, people, topics });const auth = opts.auth || createAuthClient(config);@@@ -82,7 +83,10 @@ function createApp(opts = {}) {app.locals.ctx = ctx;// Per-actor limits (http/actor-limits.js) on /api/v1, the desk and the comment form, counted once// each router resolved req.viewer; the per-address limits below stay.-- ctx.limits = createActorLimits({ config, now: opts.limitsNow || (() => Date.now()), registry: metrics.registry, log });++ // Valkey (ADR-035): shared, never-authoritative state (per-actor limit counters). Optional.++ const valkey = opts.valkey !== undefined ? opts.valkey : (config.valkey.url ? require('openvibe-sdk/valkey').createValkey({ url: config.valkey.url, prefix: config.valkey.prefix, log }) : null);++ ctx.valkey = valkey;++ ctx.limits = createActorLimits({ config, now: opts.limitsNow || (() => Date.now()), registry: metrics.registry, log, valkey });app.use(contracts.http.middleware());app.use(helmet({@@@ -112,7 +116,7 @@ function createApp(opts = {}) {app.get('/api/health', (_req, res) => res.json({ status: 'ok', service: 'openvibe-news', version: VERSION }));// GET /release.json (ADR-016) and POST /release-metrics: open tabs' update reports into /metrics.release.mount(app, { registry: metrics.registry });-- const readiness = createNewsReadiness({ store, auth, outbox, ingest, config, release: release.release });++ const readiness = createNewsReadiness({ store, auth, outbox, ingest, config, release: release.release, valkey: ctx.valkey });app.get('/api/ready', readiness.handler);// ── Events webhook (raw body; before any other body parser) ─@@@ -66,7 +66,7 @@ function createViewerResolver({ auth, config, people }) {const claims = await auth.verify(token);if (!claims || (typeof claims.sub === 'string' && PRINCIPAL_SUB.test(claims.sub))) return null;const subject = ids.isSubjectId('user', claims.subject_id) ? claims.subject_id : null;-- if (subject && people) { try { people.rememberClaims(subject, claims); } catch { /* display cache only */ } }++ if (subject && people) { try { await people.rememberClaims(subject, claims); } catch { /* display cache only */ } }return { kind: 'user', subject, staff: staffMap.can(claims, 'staff.editorial.manage'), origin: 'user', user: claimsToUser(claims), token };}@@@ -78,7 +78,7 @@ function createViewerResolver({ auth, config, people }) {const payload = decodeJwtPayload(token);if (payload && typeof payload.sub === 'string' && PRINCIPAL_SUB.test(payload.sub)) {if (opts.services === false) return ANONYMOUS;-- return fromServiceToken(req, token);++ return await fromServiceToken(req, token);}}const token = extractToken(req);@@@ -54,7 +54,7 @@ function createCommunity({ store, config, fetchImpl = globalThis.fetch, log = co/** The stored thread id, or resolve it once through Community (the caller checks the story may have one). */async threadFor(story, label, ctx) {-- const known = store.discussion.get(story.id);++ const known = await store.discussion.get(story.id);if (known) return known.threadId;if (!discussion) return null;const out = await store.discussion.threadFor(story.id, refFor(story, label), { client: discussion, traceparent: ctx && ctx.traceparent, requestId: ctx && ctx.requestId });@@@ -64,13 +64,13 @@ function createCommunity({ store, config, fetchImpl = globalThis.fetch, log = co/** { thread, comments, next_cursor } read as an anonymous visitor (public data only). */async readThread(threadId, { after, ctx } = {}) {const qs = after ? `?after=${encodeURIComponent(after)}` : '';-- return call('GET', `/api/v1/comments/threads/${encodeURIComponent(threadId)}${qs}`, { ctx });++ return await call('GET', `/api/v1/comments/threads/${encodeURIComponent(threadId)}${qs}`, { ctx });},/** Comment as the signed-in member. */async comment(threadId, subject, { message, parentId } = {}, ctx) {if (!writeTokens) { const e = new Error('comments are not configured'); e.status = 503; throw e; }-- return call('POST', `/api/v1/comments/threads/${encodeURIComponent(threadId)}/comments`, {++ return await call('POST', `/api/v1/comments/threads/${encodeURIComponent(threadId)}/comments`, {tokens: writeTokens, subject, body: { message, ...(parentId ? { parent_id: parentId } : {}) }, ctx,});},@@@ -80,7 +80,7 @@ function createCommunity({ store, config, fetchImpl = globalThis.fetch, log = co* had a thread. Never throws; a failure is logged.*/async setThreadVisibility(storyId, visibility, ctx) {-- const known = store.discussion.get(storyId);++ const known = await store.discussion.get(storyId);if (!known || !modTokens) return false;try {await call('PUT', `/api/v1/comments/threads/${encodeURIComponent(known.threadId)}/visibility`, { tokens: modTokens, body: { visibility }, ctx });@@@ -23,6 +23,7 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {}) : null;const q = {get: db.prepare('SELECT * FROM subject_projections WHERE subject = ?'),++ getMany: db.prepare('SELECT * FROM subject_projections WHERE subject = ANY(?)'),byUsername: db.prepare('SELECT * FROM subject_projections WHERE username = ? ORDER BY refreshed_at DESC LIMIT 1'),put: db.prepare(`INSERT INTO subject_projections (subject, username, display_name, avatar_url, refreshed_at) VALUES (?, ?, ?, ?, ?)ON CONFLICT (subject) DO UPDATE SET username = excluded.username, display_name = excluded.display_name,@@@ -30,9 +31,9 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {};const absAvatar = (u) => (!u ? null : /^https?:\/\//i.test(u) ? u : `${config.networkUrl}${u.startsWith('/') ? '' : '/'}${u}`);-- function remember(subject, p) {++ async function remember(subject, p) {if (!ids.isSubjectId('user', subject) || !p) return;-- q.put.run(subject, p.username ? String(p.username).toLowerCase() : null, p.display_name || p.username || null, absAvatar(p.avatar_url), store.now());++ await q.put.run(subject, p.username ? String(p.username).toLowerCase() : null, p.display_name || p.username || null, absAvatar(p.avatar_url), store.now());}async function resolve(subjects) {@@@ -46,7 +47,7 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {if (res.status === 401) tokens.invalidate && tokens.invalidate();const data = await res.json().catch(() => null);if (!res.ok || !data || typeof data.results !== 'object') throw new Error(`resolve-batch answered ${res.status}`);-- for (const s of subjects) if (data.results[s]) remember(s, data.results[s]);++ for (const s of subjects) if (data.results[s]) await remember(s, data.results[s]);}const shape = (row, subject) => (row@@@ -56,12 +57,14 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {return {enabled,/** From a verified sign-in: the member's own claims are a fresh projection. */-- rememberClaims(subject, claims) { remember(subject, claims); },++ async rememberClaims(subject, claims) { await remember(subject, claims); },/** Map subject → { name, username, avatarUrl, known }. Missing entries are fetched (bounded wait). */async many(subjects) {const list = [...new Set((subjects || []).filter((s) => ids.isSubjectId('user', s)))];-- const missing = list.filter((s) => { const r = q.get.get(s); return !r || store.now() - r.refreshed_at > TTL_MS; });++ const read = async () => new Map((list.length ? await q.getMany.all(list) : []).map((r) => [r.subject, r]));++ let rows = await read();++ const missing = list.filter((s) => { const r = rows.get(s); return !r || store.now() - r.refreshed_at > TTL_MS; });if (missing.length && tokens) {let timer;await Promise.race([@@@ -69,19 +72,20 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {new Promise((r) => { timer = setTimeout(r, WAIT_MS); }),]);clearTimeout(timer);++ rows = await read();}-- return new Map(list.map((s) => [s, shape(q.get.get(s), s)]));++ return new Map(list.map((s) => [s, shape(rows.get(s) || null, s)]));},async one(subject) { return (await this.many([subject])).get(subject) || shape(null, subject); },/** The subject a username belongs to, from people News has seen. */-- byUsername(username) {-- const row = q.byUsername.get(String(username || '').toLowerCase());++ async byUsername(username) {++ const row = await q.byUsername.get(String(username || '').toLowerCase());return row ? shape(row, row.subject) : null;},-- cached: (subject) => shape(q.get.get(subject), subject),++ cached: async (subject) => shape(await q.get.get(subject), subject),};}@@@ -35,6 +35,10 @@ function load(env = process.env) {hour: Math.max(1, int(env.NEWS_LIMITS_HOUR, 3000)),},++ // PostgreSQL (ADR-035): DATABASE_URL serves (PgBouncer), DATABASE_DIRECT_URL migrates (owner role).++ db: { url: env.DATABASE_URL || '', directUrl: env.DATABASE_DIRECT_URL || '' },++ valkey: { url: env.VALKEY_URL || '', prefix: env.VALKEY_PREFIX || 'ov:news:' },++ // The SQLite file of releases before the switch: read once by scripts/migrate-to-postgres.js.dbPath: env.NEWS_DB_PATH || './data/news.db',// OpenVibe.Network: SSO (OAuth2 authorization server), JWKS, client-credentials tokens.@@@ -1,7 +1,7 @@'use strict';/**-- * News' own SQLite database: created on boot, idempotently. Nothing here is shared with another++ * News' own PostgreSQL database (ADR-035, roadmap WS-X2): the schema is migrations/NNNN_*.sql, applied at boot. Nothing here is shared with another* service; the publishing packages create their tables inside this database with News' prefixes.** The nine charter tables (roadmap §15.13):@@@ -28,220 +28,45 @@*/const fs = require('fs');const path = require('path');--const Database = require('better-sqlite3');++const { createDb } = require('openvibe-sdk/db');const { createRevisionStore } = require('openvibe-publishing/revisions');const { createCitationStore } = require('openvibe-publishing/citations');const { createReviewLog } = require('openvibe-publishing/authorship');const { createIndexSequencer } = require('openvibe-publishing/index-hooks');const { createDiscussionRefs } = require('openvibe-publishing/discussion');--const SCHEMA = `--CREATE TABLE IF NOT EXISTS news_topics (-- id TEXT PRIMARY KEY, -- top_<ULID>-- slug TEXT NOT NULL UNIQUE,-- name TEXT NOT NULL,-- description TEXT,-- status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','archived')),-- created_at INTEGER NOT NULL,-- updated_at INTEGER NOT NULL--);----CREATE TABLE IF NOT EXISTS news_story_clusters (-- id TEXT PRIMARY KEY, -- clu_<ULID>-- label TEXT NOT NULL, -- derived from the key terms, never invented-- status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('open','merged','dissolved')),-- merged_into TEXT REFERENCES news_story_clusters(id),-- split_from TEXT REFERENCES news_story_clusters(id),-- terms TEXT NOT NULL DEFAULT '{}', -- { term: count } over its members-- entities TEXT NOT NULL DEFAULT '{}', -- { entity: count } over its members-- window_start INTEGER, -- earliest member time (published_at, else first seen)-- window_end INTEGER, -- latest member time-- created_by TEXT NOT NULL, -- 'svc:news' (clustering) or usr_… (a split)-- created_at INTEGER NOT NULL,-- updated_at INTEGER NOT NULL--);--CREATE INDEX IF NOT EXISTS news_story_clusters_open ON news_story_clusters (status, window_end);----CREATE TABLE IF NOT EXISTS news_source_items (-- id TEXT PRIMARY KEY, -- nsi_<ULID>-- sources_item_id TEXT NOT NULL UNIQUE, -- itm_… in OpenVibe.Sources (a typed reference)-- sources_revision INTEGER NOT NULL, -- the Sources revision these fields came from-- source_key TEXT NOT NULL,-- canonical_url TEXT,-- url_key TEXT, -- normalised URL for dedupe-- headline TEXT NOT NULL,-- outlet TEXT NOT NULL,-- authors TEXT NOT NULL DEFAULT '[]',-- published_at TEXT, -- as the source states it; NULL when it does not-- summary TEXT, -- a short summary ONLY when the terms allow it-- summary_basis TEXT, -- why a summary is (not) stored-- license_note TEXT,-- terms_note TEXT,-- content_hash TEXT, -- Sources' hash of the parsed fields-- title_key TEXT NOT NULL, -- normalised headline for near-duplicate checks-- status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','duplicate','removed')),-- duplicate_of TEXT REFERENCES news_source_items(id),-- dedupe TEXT, -- { rule, detail } when a duplicate-- cluster_id TEXT REFERENCES news_story_clusters(id),-- cluster_reason TEXT, -- { rule, shared_entities, shared_terms, score } (explanation)-- retrieved_at TEXT,-- first_seen_at INTEGER NOT NULL,-- upstream_updated_at INTEGER, -- the last time Sources revised it after first ingest-- removed_at INTEGER,-- removed_reason TEXT,-- updated_at INTEGER NOT NULL--);--CREATE INDEX IF NOT EXISTS news_source_items_url ON news_source_items (url_key);--CREATE INDEX IF NOT EXISTS news_source_items_hash ON news_source_items (content_hash);--CREATE INDEX IF NOT EXISTS news_source_items_cluster ON news_source_items (cluster_id);--CREATE INDEX IF NOT EXISTS news_source_items_seen ON news_source_items (first_seen_at);----CREATE TABLE IF NOT EXISTS news_stories (-- id TEXT PRIMARY KEY, -- sty_<ULID>-- slug TEXT NOT NULL UNIQUE,-- working_headline TEXT NOT NULL, -- the editors' latest headline (readers see the revision's)-- cluster_id TEXT REFERENCES news_story_clusters(id),-- topic_id TEXT REFERENCES news_topics(id),-- state TEXT NOT NULL DEFAULT 'draft' CHECK (state IN ('draft','published','unpublished','retracted')),-- published_revision INTEGER, -- which revision readers see (NULL: never published)-- first_published_at INTEGER,-- published_at INTEGER, -- the latest publication of published_revision-- retracted_at INTEGER,-- noindex INTEGER NOT NULL DEFAULT 0, -- an editor asked search engines not to index it-- created_by TEXT NOT NULL, -- usr_… (the editor who opened the story)-- created_at INTEGER NOT NULL,-- updated_at INTEGER NOT NULL--);--CREATE INDEX IF NOT EXISTS news_stories_listing ON news_stories (state, published_at);--CREATE INDEX IF NOT EXISTS news_stories_topic ON news_stories (topic_id, state, published_at);--CREATE INDEX IF NOT EXISTS news_stories_cluster ON news_stories (cluster_id);----CREATE TABLE IF NOT EXISTS news_perspectives (-- id TEXT PRIMARY KEY, -- per_<ULID>-- story_id TEXT NOT NULL REFERENCES news_stories(id),-- label TEXT NOT NULL, -- written by an editor; never generated-- description TEXT,-- position INTEGER NOT NULL DEFAULT 0,-- created_by TEXT NOT NULL,-- created_at INTEGER NOT NULL,-- updated_at INTEGER NOT NULL,-- removed_at INTEGER--);--CREATE INDEX IF NOT EXISTS news_perspectives_story ON news_perspectives (story_id, position);----CREATE TABLE IF NOT EXISTS news_story_sources (-- story_id TEXT NOT NULL REFERENCES news_stories(id),-- source_item_id TEXT NOT NULL REFERENCES news_source_items(id),-- position INTEGER NOT NULL, -- the [n] readers and editors cite-- perspective_id TEXT REFERENCES news_perspectives(id),-- added_by TEXT NOT NULL,-- added_at INTEGER NOT NULL,-- detached_at INTEGER,-- PRIMARY KEY (story_id, source_item_id)--);--CREATE INDEX IF NOT EXISTS news_story_sources_item ON news_story_sources (source_item_id);----CREATE TABLE IF NOT EXISTS news_timeline_entries (-- id TEXT PRIMARY KEY, -- tle_<ULID>-- story_id TEXT NOT NULL REFERENCES news_stories(id),-- occurred_on TEXT NOT NULL, -- YYYY-MM-DD or an ISO instant, as the source states it-- text TEXT NOT NULL,-- source_item_id TEXT NOT NULL REFERENCES news_source_items(id),-- created_by TEXT NOT NULL,-- created_at INTEGER NOT NULL,-- removed_at INTEGER--);--CREATE INDEX IF NOT EXISTS news_timeline_story ON news_timeline_entries (story_id, occurred_on);----CREATE TABLE IF NOT EXISTS news_editorial_flags (-- id TEXT PRIMARY KEY, -- flg_<ULID>-- story_id TEXT NOT NULL REFERENCES news_stories(id),-- kind TEXT NOT NULL CHECK (kind IN ('correction','update','retraction','source_updated','source_removed')),-- status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('open','published','resolved')),-- note TEXT, -- public text for correction/update/retraction-- source_item_id TEXT REFERENCES news_source_items(id),-- pending_revision INTEGER, -- the revision the system prepared for an upstream change-- revision INTEGER, -- the published revision it belongs to-- created_by TEXT NOT NULL,-- created_at INTEGER NOT NULL,-- resolved_by TEXT,-- resolved_at INTEGER--);--CREATE INDEX IF NOT EXISTS news_flags_story ON news_editorial_flags (story_id, status, created_at);----CREATE TABLE IF NOT EXISTS news_cluster_audit (-- id TEXT PRIMARY KEY, -- cla_<ULID>-- action TEXT NOT NULL CHECK (action IN ('merge','split','reverse_merge','reverse_split')),-- cluster_id TEXT NOT NULL, -- merge: the surviving cluster; split: the source cluster-- other_id TEXT NOT NULL, -- merge: the absorbed cluster; split: the new cluster-- item_ids TEXT NOT NULL, -- JSON array of news_source_items ids that moved-- reason TEXT,-- actor TEXT NOT NULL,-- reverses TEXT REFERENCES news_cluster_audit(id),-- reversed_by TEXT REFERENCES news_cluster_audit(id),-- created_at INTEGER NOT NULL--);--CREATE INDEX IF NOT EXISTS news_cluster_audit_cluster ON news_cluster_audit (cluster_id, created_at);----CREATE TABLE IF NOT EXISTS news_ingest_runs (-- id INTEGER PRIMARY KEY AUTOINCREMENT,-- origin TEXT NOT NULL CHECK (origin IN ('webhook','pull','sources')),-- state TEXT NOT NULL, -- ok | failed | ignored | upstream state (http_error, timeout, …)-- source_key TEXT,-- sources_item_id TEXT,-- error_code TEXT,-- detail TEXT,-- counts TEXT, -- { created, updated, removed, duplicates, unchanged }-- cursor_before INTEGER,-- cursor_after INTEGER,-- at INTEGER NOT NULL--);--CREATE INDEX IF NOT EXISTS news_ingest_runs_at ON news_ingest_runs (at);----CREATE TABLE IF NOT EXISTS news_state (-- key TEXT PRIMARY KEY,-- value TEXT NOT NULL,-- updated_at INTEGER NOT NULL--);----CREATE TABLE IF NOT EXISTS news_source_status (-- source_key TEXT PRIMARY KEY,-- name TEXT,-- homepage_url TEXT,-- status TEXT,-- stale INTEGER,-- last_success_at TEXT,-- refreshed_at INTEGER NOT NULL--);----CREATE TABLE IF NOT EXISTS subject_projections (-- subject TEXT PRIMARY KEY,-- username TEXT,-- display_name TEXT,-- avatar_url TEXT,-- refreshed_at INTEGER NOT NULL--);--CREATE INDEX IF NOT EXISTS subject_projections_username ON subject_projections (username);--`;++const MIGRATIONS = path.join(__dirname, '..', 'migrations');++const DEV_PGLITE = path.join(__dirname, '..', 'data', 'pglite');/**-- * Open (or create) the database and every store on it.-- * opts.now — injectable clock (epoch ms) shared by the stores, so tests and replays are deterministic.++ * The serving handle (ADR-035): DATABASE_URL through PgBouncer; in development without it, an embedded PGlite database++ * in data/pglite. Migrations run first, as the owner (DATABASE_DIRECT_URL), or on the embedded handle.*/--function openStore(dbPath, { now = () => Date.now() } = {}) {-- if (dbPath !== ':memory:') fs.mkdirSync(path.dirname(path.resolve(dbPath)), { recursive: true });-- const db = new Database(dbPath);-- db.pragma('journal_mode = WAL');-- db.pragma('foreign_keys = ON');-- db.pragma('busy_timeout = 5000');-- db.exec(SCHEMA);++async function openDb(config, { log = console, registry } = {}) {++ if (!config.db.url) {++ if (config.isProduction) throw new Error('DATABASE_URL is not set: production serves from PostgreSQL (OpenVibe.Host roles/data add-service.sh news)');++ log.warn(`[News] DATABASE_URL unset: embedded PGlite database in ${DEV_PGLITE} (development only, one process)`);++ fs.mkdirSync(DEV_PGLITE, { recursive: true });++ const db = createDb({ pglite: DEV_PGLITE, service: 'news', registry, log });++ await db.migrate({ dir: MIGRATIONS, log });++ return db;++ }++ if (!config.db.directUrl) throw new Error('DATABASE_DIRECT_URL is not set: migrations run with the owner role on a direct connection');++ const owner = createDb({ url: config.db.directUrl, service: 'news-migrate', max: 1, log });++ try { await owner.migrate({ dir: MIGRATIONS, log }); } finally { await owner.close(); }++ return createDb({ url: config.db.url, service: 'news', registry, log });++}++/**++ * Every store on a migrated database handle. opts.now — injectable clock (epoch ms) shared by the stores, so tests++ * and replays are deterministic. store.tx(fn) is a transaction; inside it, plain db calls join it (ambient).++ */++function createStore(db, { now = () => Date.now() } = {}) {const revisions = createRevisionStore(db, { prefix: 'news_story', now });-- const store = {++ const getStateStmt = db.prepare('SELECT value FROM news_state WHERE key = ?');++ const setStateStmt = db.prepare(`INSERT INTO news_state (key, value, updated_at) VALUES (?, ?, ?)++ ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`);++ return {db,now,revisions,@@@ -249,21 +74,24 @@ function openStore(dbPath, { now = () => Date.now() } = {}) {reviews: createReviewLog(db, { prefix: 'news_story', now }),sequencer: createIndexSequencer(db, { prefix: 'news', now }),discussion: createDiscussionRefs(db, { prefix: 'news_story', now }),-- tx: (fn) => db.transaction(fn)(),-- getState(key, def = null) {-- const r = db.prepare('SELECT value FROM news_state WHERE key = ?').get(key);++ tx: async (fn) => await db.tx(() => fn()),++ async getState(key, def = null) {++ const r = await getStateStmt.get(key);return r ? JSON.parse(r.value) : def;},-- setState(key, value) {-- db.prepare(`INSERT INTO news_state (key, value, updated_at) VALUES (?, ?, ?)-- ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`).run(key, JSON.stringify(value), now());++ async setState(key, value) {++ await setStateStmt.run(key, JSON.stringify(value), now());},close: () => db.close(),};-- return store;++}++++/** openDb + createStore. */++async function openStore(config, { now, log } = {}) {++ return createStore(await openDb(config, { log }), { now });}const CHARTER_TABLES = ['news_topics', 'news_source_items', 'news_story_clusters', 'news_stories', 'news_story_revisions','news_story_sources', 'news_perspectives', 'news_timeline_entries', 'news_editorial_flags'];--module.exports = { openStore, CHARTER_TABLES };++module.exports = { openDb, openStore, createStore, CHARTER_TABLES, MIGRATIONS };@@@ -59,32 +59,32 @@ function createClusters({ store, config, outbox }) {const { db } = store;const q = {get: db.prepare('SELECT * FROM news_story_clusters WHERE id = ?'),-- open: db.prepare(`SELECT * FROM news_story_clusters WHERE status = 'open' AND window_end >= ? AND window_start <= ? ORDER BY created_at, rowid`),++ open: db.prepare(`SELECT * FROM news_story_clusters WHERE status = 'open' AND window_end >= ? AND window_start <= ? ORDER BY created_at, seq`),insert: db.prepare(`INSERT INTO news_story_clusters (id, label, status, split_from, terms, entities, window_start, window_end, created_by, created_at, updated_at)VALUES (@id, @label, 'open', @split_from, @terms, @entities, @window_start, @window_end, @created_by, @now, @now)`),-- members: db.prepare("SELECT * FROM news_source_items WHERE cluster_id = ? ORDER BY first_seen_at, rowid"),++ members: db.prepare("SELECT * FROM news_source_items WHERE cluster_id = ? ORDER BY first_seen_at, seq"),setItem: db.prepare('UPDATE news_source_items SET cluster_id = ?, cluster_reason = ?, updated_at = ? WHERE id = ?'),audit: db.prepare('SELECT * FROM news_cluster_audit WHERE id = ?'),-- auditFor: db.prepare('SELECT * FROM news_cluster_audit WHERE cluster_id = ? OR other_id = ? ORDER BY created_at, rowid'),++ auditFor: db.prepare('SELECT * FROM news_cluster_audit WHERE cluster_id = ? OR other_id = ? ORDER BY created_at, seq'),insertAudit: db.prepare(`INSERT INTO news_cluster_audit (id, action, cluster_id, other_id, item_ids, reason, actor, reverses, created_at)VALUES (@id, @action, @cluster_id, @other_id, @item_ids, @reason, @actor, @reverses, @now)`),};-- function emit(cluster, action, extra = {}) {-- outbox.emit({++ async function emit(cluster, action, extra = {}) {++ await outbox.emit({event_type: 'news.cluster.updated', actor: { type: 'service', id: 'news' }, visibility: 'internal', priority: 'low',subject: { type: 'cluster', id: cluster.id },-- payload: { cluster_id: cluster.id, action, status: cluster.status, label: cluster.label, item_count: q.members.all(cluster.id).length, ...extra },++ payload: { cluster_id: cluster.id, action, status: cluster.status, label: cluster.label, item_count: (await q.members.all(cluster.id)).length, ...extra },});}/** Rebuild a cluster's terms, entities, window and label from its members (removed items do not count). */-- function recompute(clusterId) {++ async function recompute(clusterId) {const terms = {};const entities = {};let start = null;let end = null;-- for (const it of q.members.all(clusterId)) {++ for (const it of await q.members.all(clusterId)) {if (it.status === 'removed') continue;const sig = signature(it);for (const t of sig.terms) terms[t] = (terms[t] || 0) + 1;@@@ -93,23 +93,23 @@ function createClusters({ store, config, outbox }) {start = start == null ? at : Math.min(start, at);end = end == null ? at : Math.max(end, at);}-- db.prepare('UPDATE news_story_clusters SET terms = ?, entities = ?, window_start = COALESCE(?, window_start), window_end = COALESCE(?, window_end), label = ?, updated_at = ? WHERE id = ?')++ await db.prepare('UPDATE news_story_clusters SET terms = ?, entities = ?, window_start = COALESCE(?, window_start), window_end = COALESCE(?, window_end), label = ?, updated_at = ? WHERE id = ?').run(JSON.stringify(terms), JSON.stringify(entities), start, end, labelOf(entities, terms), store.now(), clusterId);-- return q.get.get(clusterId);++ return await q.get.get(clusterId);}/*** Place one item (inside the ingest transaction). Duplicates follow their original. Returns* { cluster, created, reason }.*/-- function assign(item) {++ async function assign(item) {if (item.status === 'duplicate' && item.duplicate_of) {-- const original = db.prepare('SELECT * FROM news_source_items WHERE id = ?').get(item.duplicate_of);++ const original = await db.prepare('SELECT * FROM news_source_items WHERE id = ?').get(item.duplicate_of);if (original && original.cluster_id) {-- const cluster = resolve(original.cluster_id);++ const cluster = await resolve(original.cluster_id);const reason = { rule: 'duplicate_of', item: original.id };-- q.setItem.run(cluster.id, JSON.stringify(reason), store.now(), item.id);-- emit(cluster, 'item_added', { item_id: item.id, rule: 'duplicate_of' });++ await q.setItem.run(cluster.id, JSON.stringify(reason), store.now(), item.id);++ await emit(cluster, 'item_added', { item_id: item.id, rule: 'duplicate_of' });return { cluster, created: false, reason };}}@@@ -117,37 +117,37 @@ function createClusters({ store, config, outbox }) {const at = itemTime(item);const w = config.clustering.windowMs;let best = null;-- for (const c of q.open.all(at - w, at + w)) {++ for (const c of await q.open.all(at - w, at + w)) {const m = compare(sig, c);if (!m.joins) continue;-- if (!best || m.score > best.m.score) best = { c, m }; // ORDER BY created_at, rowid: the oldest wins a tie++ if (!best || m.score > best.m.score) best = { c, m }; // ORDER BY created_at, seq: the oldest wins a tie}if (best) {const reason = { rule: 'shared_terms', cluster: best.c.id, shared_entities: best.m.sharedEntities, shared_terms: best.m.sharedTerms, score: best.m.score, window_hours: w / 3600000 };-- q.setItem.run(best.c.id, JSON.stringify(reason), store.now(), item.id);-- const cluster = recompute(best.c.id);-- emit(cluster, 'item_added', { item_id: item.id, rule: 'shared_terms', score: best.m.score });++ await q.setItem.run(best.c.id, JSON.stringify(reason), store.now(), item.id);++ const cluster = await recompute(best.c.id);++ await emit(cluster, 'item_added', { item_id: item.id, rule: 'shared_terms', score: best.m.score });return { cluster, created: false, reason };}const now = store.now();const id = newClusterId(now);-- q.insert.run({ id, label: 'new', split_from: null, terms: '{}', entities: '{}', window_start: at, window_end: at, created_by: 'svc:news', now });++ await q.insert.run({ id, label: 'new', split_from: null, terms: '{}', entities: '{}', window_start: at, window_end: at, created_by: 'svc:news', now });const reason = { rule: 'new_cluster', cluster: id, terms: sig.terms.slice(0, 20), entities: sig.entities.slice(0, 10) };-- q.setItem.run(id, JSON.stringify(reason), now, item.id);-- const cluster = recompute(id);-- emit(cluster, 'created', { item_id: item.id });++ await q.setItem.run(id, JSON.stringify(reason), now, item.id);++ const cluster = await recompute(id);++ await emit(cluster, 'created', { item_id: item.id });return { cluster, created: true, reason };}/** Follow merges to the cluster that holds the items now. */-- function resolve(id) {-- let c = q.get.get(id);-- for (let i = 0; c && c.status === 'merged' && c.merged_into && i < 50; i++) c = q.get.get(c.merged_into);++ async function resolve(id) {++ let c = await q.get.get(id);++ for (let i = 0; c && c.status === 'merged' && c.merged_into && i < 50; i++) c = await q.get.get(c.merged_into);return c;}-- function mustOpen(id) {-- const c = q.get.get(String(id || ''));++ async function mustOpen(id) {++ const c = await q.get.get(String(id || ''));if (!c) throw new ApiError(404, 'cluster.not_found', 'No such cluster');if (c.status !== 'open') throw new ApiError(409, 'cluster.not_open', `Cluster ${c.id} is ${c.status}`);return c;@@@ -155,11 +155,11 @@ function createClusters({ store, config, outbox }) {function actorId(viewer) { return viewer && viewer.subject ? viewer.subject : (viewer && viewer.service) || 'svc:news'; }-- function recordAudit(row) {++ async function recordAudit(row) {const now = store.now();const id = newAuditId(now);-- q.insertAudit.run({ id, reason: null, reverses: null, ...row, item_ids: JSON.stringify(row.item_ids), now });-- return q.audit.get(id);++ await q.insertAudit.run({ id, reason: null, reverses: null, ...row, item_ids: JSON.stringify(row.item_ids), now });++ return await q.audit.get(id);}function shapeAudit(a) {@@@ -169,51 +169,51 @@ function createClusters({ store, config, outbox }) {const api = {signature, compare, itemTime, labelOf, assign, resolve, recompute,-- get: (id) => q.get.get(String(id || '')) || null,-- members: (id) => q.members.all(id),-- audit: (id) => q.auditFor.all(id, id).map(shapeAudit),-- auditEntry: (id) => shapeAudit(q.audit.get(String(id || ''))),++ get: async (id) => await q.get.get(String(id || '')) || null,++ members: async (id) => await q.members.all(id),++ audit: async (id) => (await q.auditFor.all(id, id)).map(shapeAudit),++ auditEntry: async (id) => shapeAudit(await q.audit.get(String(id || ''))),/** Open clusters, newest activity first (the editor dashboard). */-- recent({ limit = 50 } = {}) {-- return db.prepare("SELECT * FROM news_story_clusters WHERE status = 'open' ORDER BY window_end DESC, id DESC LIMIT ?").all(limit);++ async recent({ limit = 50 } = {}) {++ return await db.prepare("SELECT * FROM news_story_clusters WHERE status = 'open' ORDER BY window_end DESC, id DESC LIMIT ?").all(limit);},/** Merge cluster `otherId` into `intoId`. */-- merge(viewer, intoId, otherId, { reason = null } = {}) {-- return store.tx(() => {-- const into = mustOpen(intoId);-- const other = mustOpen(otherId);++ async merge(viewer, intoId, otherId, { reason = null } = {}) {++ return await store.tx(async () => {++ const into = await mustOpen(intoId);++ const other = await mustOpen(otherId);if (into.id === other.id) throw new ApiError(422, 'cluster.same', 'A cluster cannot be merged into itself');-- const moved = q.members.all(other.id).map((i) => i.id);++ const moved = (await q.members.all(other.id)).map((i) => i.id);const now = store.now();-- for (const id of moved) q.setItem.run(into.id, JSON.stringify({ rule: 'merged', from: other.id }), now, id);-- db.prepare("UPDATE news_story_clusters SET status = 'merged', merged_into = ?, updated_at = ? WHERE id = ?").run(into.id, now, other.id);-- const audit = recordAudit({ action: 'merge', cluster_id: into.id, other_id: other.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer) });-- const c = recompute(into.id);-- emit(c, 'merged', { absorbed: other.id, audit_id: audit.id, moved: moved.length });++ for (const id of moved) await q.setItem.run(into.id, JSON.stringify({ rule: 'merged', from: other.id }), now, id);++ await db.prepare("UPDATE news_story_clusters SET status = 'merged', merged_into = ?, updated_at = ? WHERE id = ?").run(into.id, now, other.id);++ const audit = await recordAudit({ action: 'merge', cluster_id: into.id, other_id: other.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer) });++ const c = await recompute(into.id);++ await emit(c, 'merged', { absorbed: other.id, audit_id: audit.id, moved: moved.length });return { cluster: c, audit: shapeAudit(audit) };});},/** Move some items of a cluster into a new cluster. At least one item must stay. */-- split(viewer, clusterId, itemIds, { reason = null } = {}) {-- return store.tx(() => {-- const from = mustOpen(clusterId);-- const members = new Set(q.members.all(from.id).map((i) => i.id));++ async split(viewer, clusterId, itemIds, { reason = null } = {}) {++ return await store.tx(async () => {++ const from = await mustOpen(clusterId);++ const members = new Set((await q.members.all(from.id)).map((i) => i.id));const wanted = [...new Set((itemIds || []).map(String))];if (!wanted.length) throw new ApiError(422, 'cluster.split_empty', 'Choose the items to split off');for (const id of wanted) if (!members.has(id)) throw new ApiError(422, 'cluster.split_foreign', `${id} is not in cluster ${from.id}`);if (wanted.length >= members.size) throw new ApiError(422, 'cluster.split_all', 'At least one item must stay in the cluster');const now = store.now();const id = newClusterId(now);-- q.insert.run({ id, label: 'new', split_from: from.id, terms: '{}', entities: '{}', window_start: null, window_end: null, created_by: actorId(viewer), now });-- for (const it of wanted) q.setItem.run(id, JSON.stringify({ rule: 'split', from: from.id }), now, it);-- const audit = recordAudit({ action: 'split', cluster_id: from.id, other_id: id, item_ids: wanted, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer) });-- const a = recompute(from.id);-- const b = recompute(id);-- emit(a, 'split', { new_cluster: id, audit_id: audit.id, moved: wanted.length });-- emit(b, 'created', { split_from: from.id, audit_id: audit.id });++ await q.insert.run({ id, label: 'new', split_from: from.id, terms: '{}', entities: '{}', window_start: null, window_end: null, created_by: actorId(viewer), now });++ for (const it of wanted) await q.setItem.run(id, JSON.stringify({ rule: 'split', from: from.id }), now, it);++ const audit = await recordAudit({ action: 'split', cluster_id: from.id, other_id: id, item_ids: wanted, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer) });++ const a = await recompute(from.id);++ const b = await recompute(id);++ await emit(a, 'split', { new_cluster: id, audit_id: audit.id, moved: wanted.length });++ await emit(b, 'created', { split_from: from.id, audit_id: audit.id });return { cluster: a, created: b, audit: shapeAudit(audit) };});},@@@ -222,9 +222,9 @@ function createClusters({ store, config, outbox }) {* Undo a merge or a split: the items that moved (and are still where they were moved to) go* back; the absorbed cluster reopens / the split-off cluster is dissolved. Audited as well.*/-- reverse(viewer, auditId, { reason = null } = {}) {-- return store.tx(() => {-- const a = q.audit.get(String(auditId || ''));++ async reverse(viewer, auditId, { reason = null } = {}) {++ return await store.tx(async () => {++ const a = await q.audit.get(String(auditId || ''));if (!a) throw new ApiError(404, 'cluster.audit_not_found', 'No such merge or split');if (a.reversed_by) throw new ApiError(409, 'cluster.already_reversed', `Already reversed by ${a.reversed_by}`);if (a.action !== 'merge' && a.action !== 'split') throw new ApiError(422, 'cluster.not_reversible', 'Only merges and splits can be reversed');@@@ -233,32 +233,34 @@ function createClusters({ store, config, outbox }) {let moved = [];let result;if (a.action === 'merge') {-- const other = q.get.get(a.other_id);++ const other = await q.get.get(a.other_id);if (!other || other.status !== 'merged' || other.merged_into !== a.cluster_id) throw new ApiError(409, 'cluster.changed', 'The absorbed cluster has changed since the merge');-- moved = items.filter((id) => (db.prepare('SELECT cluster_id FROM news_source_items WHERE id = ?').get(id) || {}).cluster_id === a.cluster_id);-- for (const id of moved) q.setItem.run(other.id, JSON.stringify({ rule: 'merge_reversed', audit: a.id }), now, id);-- db.prepare("UPDATE news_story_clusters SET status = 'open', merged_into = NULL, updated_at = ? WHERE id = ?").run(now, other.id);-- const rev = recordAudit({ action: 'reverse_merge', cluster_id: a.cluster_id, other_id: other.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer), reverses: a.id });-- db.prepare('UPDATE news_cluster_audit SET reversed_by = ? WHERE id = ?').run(rev.id, a.id);-- const c1 = recompute(a.cluster_id);-- const c2 = recompute(other.id);-- emit(c1, 'merge_reversed', { audit_id: rev.id, restored: other.id });-- emit(c2, 'reopened', { audit_id: rev.id });-- result = { audit: shapeAudit(q.audit.get(rev.id)), clusters: [c1, c2] };++ const now1 = new Map((await db.prepare('SELECT id, cluster_id FROM news_source_items WHERE id = ANY(?)').all(items)).map((r) => [r.id, r.cluster_id]));++ moved = items.filter((id) => now1.get(id) === a.cluster_id);++ for (const id of moved) await q.setItem.run(other.id, JSON.stringify({ rule: 'merge_reversed', audit: a.id }), now, id);++ await db.prepare("UPDATE news_story_clusters SET status = 'open', merged_into = NULL, updated_at = ? WHERE id = ?").run(now, other.id);++ const rev = await recordAudit({ action: 'reverse_merge', cluster_id: a.cluster_id, other_id: other.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer), reverses: a.id });++ await db.prepare('UPDATE news_cluster_audit SET reversed_by = ? WHERE id = ?').run(rev.id, a.id);++ const c1 = await recompute(a.cluster_id);++ const c2 = await recompute(other.id);++ await emit(c1, 'merge_reversed', { audit_id: rev.id, restored: other.id });++ await emit(c2, 'reopened', { audit_id: rev.id });++ result = { audit: shapeAudit(await q.audit.get(rev.id)), clusters: [c1, c2] };} else {-- const split = q.get.get(a.other_id);-- const from = q.get.get(a.cluster_id);++ const split = await q.get.get(a.other_id);++ const from = await q.get.get(a.cluster_id);if (!split || split.status !== 'open' || !from || from.status !== 'open') throw new ApiError(409, 'cluster.changed', 'One of the clusters is no longer open');-- moved = items.filter((id) => (db.prepare('SELECT cluster_id FROM news_source_items WHERE id = ?').get(id) || {}).cluster_id === split.id);-- for (const id of moved) q.setItem.run(from.id, JSON.stringify({ rule: 'split_reversed', audit: a.id }), now, id);-- const left = q.members.all(split.id).length;-- if (!left) db.prepare("UPDATE news_story_clusters SET status = 'dissolved', updated_at = ? WHERE id = ?").run(now, split.id);-- const rev = recordAudit({ action: 'reverse_split', cluster_id: from.id, other_id: split.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer), reverses: a.id });-- db.prepare('UPDATE news_cluster_audit SET reversed_by = ? WHERE id = ?').run(rev.id, a.id);-- const c1 = recompute(from.id);-- const c2 = left ? recompute(split.id) : q.get.get(split.id);-- emit(c1, 'split_reversed', { audit_id: rev.id, dissolved: left ? null : split.id });-- result = { audit: shapeAudit(q.audit.get(rev.id)), clusters: [c1, c2] };++ const now2 = new Map((await db.prepare('SELECT id, cluster_id FROM news_source_items WHERE id = ANY(?)').all(items)).map((r) => [r.id, r.cluster_id]));++ moved = items.filter((id) => now2.get(id) === split.id);++ for (const id of moved) await q.setItem.run(from.id, JSON.stringify({ rule: 'split_reversed', audit: a.id }), now, id);++ const left = (await q.members.all(split.id)).length;++ if (!left) await db.prepare("UPDATE news_story_clusters SET status = 'dissolved', updated_at = ? WHERE id = ?").run(now, split.id);++ const rev = await recordAudit({ action: 'reverse_split', cluster_id: from.id, other_id: split.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer), reverses: a.id });++ await db.prepare('UPDATE news_cluster_audit SET reversed_by = ? WHERE id = ?').run(rev.id, a.id);++ const c1 = await recompute(from.id);++ const c2 = left ? await recompute(split.id) : await q.get.get(split.id);++ await emit(c1, 'split_reversed', { audit_id: rev.id, dissolved: left ? null : split.id });++ result = { audit: shapeAudit(await q.audit.get(rev.id)), clusters: [c1, c2] };}return result;});@@@ -13,11 +13,10 @@* a revision that no longer rests on it (comments could repeat the withdrawn material)** Writes that can change this call watch() inside their transaction, before they change anything.-- * better-sqlite3 transactions are synchronous, so the setImmediate flush runs after the commit (or-- * the rollback): the state is compared again and, when it changed, the thread is set hidden or-- * public (best effort, community.comment.moderate). A rolled-back write compares equal and calls-- * nothing; a story that never had a thread calls nothing either (threads are created only when an-- * open story is read).++ * After the commit (db.afterCommit; a rollback drops it) the state is compared again and, when it++ * changed, the thread is set hidden or public (best effort, community.comment.moderate). A rolled-back++ * write calls nothing; a story that never had a thread calls nothing either (threads are created only++ * when an open story is read).*/function createDiscussion({ store, publication, community, log = console }) {const { db } = store;@@@ -25,45 +24,35 @@ function createDiscussion({ store, publication, community, log = console }) {const citing = db.prepare('SELECT DISTINCT story_id FROM news_story_sources WHERE source_item_id = ?');/** { open, reason: null | not_published | retracted | source_removed } */-- function status(story) {++ async function status(story) {if (!story || !story.published_revision) return { open: false, reason: 'not_published' };if (story.state === 'retracted') return { open: false, reason: 'retracted' };if (story.state !== 'published') return { open: false, reason: 'not_published' };-- const rev = store.revisions.get(story.id, story.published_revision);++ const rev = await store.revisions.get(story.id, story.published_revision);if (!rev) return { open: false, reason: 'not_published' };-- if (publication.support(rev).unsupported > 0) return { open: false, reason: 'source_removed' };++ if ((await publication.support(rev)).unsupported > 0) return { open: false, reason: 'source_removed' };return { open: true, reason: null };}-- const pending = new Map(); // story id → open before the write-- let scheduled = false;-- let flushing = Promise.resolve();++ let flushing = Promise.resolve(); // visibility calls go one at a time, in commit order-- async function flush() {-- scheduled = false;-- const batch = [...pending];-- pending.clear();-- for (const [id, was] of batch) {-- const now = status(storyById.get(id)).open;-- if (now !== was) await community.setThreadVisibility(id, now ? 'public' : 'hidden');-- }++ async function follow(id, was) {++ const now = (await status(await storyById.get(id))).open;++ if (now !== was) await community.setThreadVisibility(id, now ? 'public' : 'hidden');}/** Remember whether a story is open now; after the commit, follow any change. Inside the write's transaction. */-- function watch(storyId) {++ async function watch(storyId) {if (!community.enabled) return;-- if (!pending.has(storyId)) pending.set(storyId, status(storyById.get(storyId)).open);-- if (!scheduled) {-- scheduled = true;-- setImmediate(() => { flushing = flushing.then(flush).catch((err) => log.warn(`[News] comment thread sync failed: ${err.message}`)); });-- }++ const was = (await status(await storyById.get(storyId))).open;++ db.afterCommit(() => { flushing = flushing.then(() => follow(storyId, was)).catch((err) => log.warn(`[News] comment thread sync failed: ${err.message}`)); });}return {status,watch,/** Every story that cites a source item (before the item changes upstream). */-- watchSource(itemId) { for (const r of citing.all(itemId)) watch(r.story_id); },++ async watchSource(itemId) { for (const r of await citing.all(itemId)) await watch(r.story_id); },/** Resolves once the scheduled visibility changes have been sent (tests, shutdown). */idle: () => new Promise((resolve) => setImmediate(() => flushing.then(resolve))),};@@@ -35,39 +35,39 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = nconst q = {bySourcesId: db.prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?'),byId: db.prepare('SELECT * FROM news_source_items WHERE id = ?'),-- byUrl: db.prepare("SELECT * FROM news_source_items WHERE url_key = ? AND status <> 'removed' ORDER BY first_seen_at, rowid LIMIT 1"),-- byHash: db.prepare("SELECT * FROM news_source_items WHERE content_hash = ? AND status <> 'removed' ORDER BY first_seen_at, rowid LIMIT 1"),-- recent: db.prepare("SELECT * FROM news_source_items WHERE status <> 'removed' AND first_seen_at >= ? ORDER BY first_seen_at, rowid"),++ byUrl: db.prepare("SELECT * FROM news_source_items WHERE url_key = ? AND status <> 'removed' ORDER BY first_seen_at, seq LIMIT 1"),++ byHash: db.prepare("SELECT * FROM news_source_items WHERE content_hash = ? AND status <> 'removed' ORDER BY first_seen_at, seq LIMIT 1"),++ recent: db.prepare("SELECT * FROM news_source_items WHERE status <> 'removed' AND first_seen_at >= ? ORDER BY first_seen_at, seq"),status: db.prepare('SELECT * FROM news_source_status WHERE source_key = ?'),putStatus: db.prepare(`INSERT INTO news_source_status (source_key, name, homepage_url, status, stale, last_success_at, refreshed_at)VALUES (@source_key, @name, @homepage_url, @status, @stale, @last_success_at, @now)-- ON CONFLICT (source_key) DO UPDATE SET name = COALESCE(excluded.name, name), homepage_url = COALESCE(excluded.homepage_url, homepage_url),-- status = COALESCE(excluded.status, status), stale = COALESCE(excluded.stale, stale),-- last_success_at = COALESCE(excluded.last_success_at, last_success_at), refreshed_at = excluded.refreshed_at`),++ ON CONFLICT (source_key) DO UPDATE SET name = COALESCE(excluded.name, news_source_status.name), homepage_url = COALESCE(excluded.homepage_url, news_source_status.homepage_url),++ status = COALESCE(excluded.status, news_source_status.status), stale = COALESCE(excluded.stale, news_source_status.stale),++ last_success_at = COALESCE(excluded.last_success_at, news_source_status.last_success_at), refreshed_at = excluded.refreshed_at`),run: db.prepare(`INSERT INTO news_ingest_runs (origin, state, source_key, sources_item_id, error_code, detail, counts, cursor_before, cursor_after, at)VALUES (@origin, @state, @source_key, @sources_item_id, @error_code, @detail, @counts, @cursor_before, @cursor_after, @at)`),lastPull: db.prepare("SELECT * FROM news_ingest_runs WHERE origin = 'pull' ORDER BY id DESC LIMIT 1"),};-- function recordRun(r) {-- q.run.run({ source_key: null, sources_item_id: null, error_code: null, detail: null, counts: null, cursor_before: null, cursor_after: null, ...r, counts: r.counts ? JSON.stringify(r.counts) : null, at: store.now() });++ async function recordRun(r) {++ await q.run.run({ source_key: null, sources_item_id: null, error_code: null, detail: null, counts: null, cursor_before: null, cursor_after: null, ...r, counts: r.counts ? JSON.stringify(r.counts) : null, at: store.now() });}-- function failedEvent({ origin, sourceKey = null, sourcesItemId = null, state, errorCode = null, detail = null, httpStatus = null, extra = {} }) {-- outbox.emit({++ async function failedEvent({ origin, sourceKey = null, sourcesItemId = null, state, errorCode = null, detail = null, httpStatus = null, extra = {} }) {++ await outbox.emit({event_type: 'news.source.failed', actor: { type: 'service', id: 'news' }, visibility: 'internal', priority: 'important',subject: sourceKey ? { type: 'source', id: sourceKey } : { type: 'ingest', id: origin },payload: { origin, source_key: sourceKey, sources_item_id: sourcesItemId, state, error_code: errorCode, http_status: httpStatus, detail: detail ? String(detail).slice(0, 500) : null, ...extra },});}-- function outletFor(item) {-- const st = q.status.get(item.source_key);++ async function outletFor(item) {++ const st = await q.status.get(item.source_key);return (st && st.name) || text.hostOf(item.canonical_url) || item.source_key;}/** The fields News keeps from a Sources item. Nothing else of the item is read. */-- function normalise(item) {++ async function normalise(item) {const prov = item.provenance || {};const headline = String(item.title || '').replace(/\s+/g, ' ').trim().slice(0, 500);const lic = text.licensedSummary(item.summary, { termsNote: prov.terms_note, licenseNote: prov.license_note, maxChars: config.licensing.summaryMaxChars });@@@ -77,7 +77,7 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = nheadline,canonical_url: item.canonical_url || null,url_key: text.urlKey(item.canonical_url),-- outlet: outletFor(item),++ outlet: await outletFor(item),authors: JSON.stringify(authors),published_at: published,summary: lic.summary,@@@ -91,19 +91,19 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n}/** Is this new item a duplicate of one we have? → null | { of, rule, detail } */-- function dedupe(n, sourceKey, at) {++ async function dedupe(n, sourceKey, at) {if (n.url_key) {-- const hit = q.byUrl.get(n.url_key);++ const hit = await q.byUrl.get(n.url_key);if (hit) return { of: hit.duplicate_of || hit.id, rule: 'canonical_url', detail: n.url_key };}if (n.content_hash) {-- const hit = q.byHash.get(n.content_hash);++ const hit = await q.byHash.get(n.content_hash);if (hit) return { of: hit.duplicate_of || hit.id, rule: 'content_hash', detail: n.content_hash };}const mine = text.shingles(n.headline);const host = text.hostOf(n.canonical_url);let best = null;-- for (const c of q.recent.all(Math.min(at, store.now()) - NEAR_TITLE.windowMs)) {++ for (const c of await q.recent.all(Math.min(at, store.now()) - NEAR_TITLE.windowMs)) {const t = clusters.itemTime(c);if (Math.abs(t - at) > NEAR_TITLE.windowMs) continue;const score = text.jaccard(mine, text.shingles(c.headline));@@@ -115,8 +115,8 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = nreturn null;}-- function ingestedEvent(row, action, extra = {}) {-- outbox.emit({++ async function ingestedEvent(row, action, extra = {}) {++ await outbox.emit({event_type: 'news.source.ingested', actor: { type: 'service', id: 'news' }, visibility: 'internal', priority: 'low',subject: { type: 'source_item', id: row.id },payload: {@@@ -132,38 +132,38 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n* Apply one Sources item (inside a transaction the caller may already hold). Returns* { outcome: created|updated|removed|unchanged|ignored|rejected, item?, reason? }.*/-- function apply(item, { origin = 'pull' } = {}) {-- return db.transaction(() => {++ async function apply(item, { origin = 'pull' } = {}) {++ return db.tx(async () => {if (!item || typeof item.id !== 'string') return { outcome: 'rejected', reason: 'item without id' };if (item.category && item.category !== config.sources.category) return { outcome: 'ignored', reason: `category ${item.category}` };const revision = Number.isInteger(item.revision) ? item.revision : 1;-- const cur = q.bySourcesId.get(item.id);++ const cur = await q.bySourcesId.get(item.id);if (item.removed) return applyRemoval({ sourcesItemId: item.id, reason: item.removed.reason || 'removed upstream', revision, origin });if (cur && cur.sources_revision >= revision) return { outcome: 'unchanged', item: cur };-- const n = normalise(item);++ const n = await normalise(item);if (!n.headline) {-- recordRun({ origin, state: 'rejected', source_key: item.source_key || null, sources_item_id: item.id, error_code: 'item.no_headline', detail: 'a Sources item without a title cannot be shown' });-- failedEvent({ origin, sourceKey: item.source_key || null, sourcesItemId: item.id, state: 'rejected', errorCode: 'item.no_headline' });++ await recordRun({ origin, state: 'rejected', source_key: item.source_key || null, sources_item_id: item.id, error_code: 'item.no_headline', detail: 'a Sources item without a title cannot be shown' });++ await failedEvent({ origin, sourceKey: item.source_key || null, sourcesItemId: item.id, state: 'rejected', errorCode: 'item.no_headline' });return { outcome: 'rejected', reason: 'no headline' };}const now = store.now();if (cur) {if (cur.status === 'removed') return { outcome: 'unchanged', item: cur }; // removal is sticky here too-- db.prepare(`UPDATE news_source_items SET sources_revision = @rev, canonical_url = @canonical_url, url_key = @url_key, headline = @headline,++ await db.prepare(`UPDATE news_source_items SET sources_revision = @rev, canonical_url = @canonical_url, url_key = @url_key, headline = @headline,outlet = @outlet, authors = @authors, published_at = @published_at, summary = @summary, summary_basis = @summary_basis,license_note = @license_note, terms_note = @terms_note, content_hash = @content_hash, title_key = @title_key,retrieved_at = @retrieved_at, upstream_updated_at = @now, updated_at = @now WHERE id = @id`).run({ ...n, rev: revision, now, id: cur.id });-- const row = q.byId.get(cur.id);-- if (row.cluster_id) clusters.recompute(clusters.resolve(row.cluster_id).id);-- ingestedEvent(row, 'updated', { previous_revision: cur.sources_revision });-- const affected = stories ? stories.onUpstreamChange(row, 'source_updated', { previous: cur }) : [];++ const row = await q.byId.get(cur.id);++ if (row.cluster_id) await clusters.recompute((await clusters.resolve(row.cluster_id)).id);++ await ingestedEvent(row, 'updated', { previous_revision: cur.sources_revision });++ const affected = stories ? await stories.onUpstreamChange(row, 'source_updated', { previous: cur }) : [];return { outcome: 'updated', item: row, stories: affected };}const at = (() => { const p = n.published_at ? Date.parse(n.published_at) : NaN; return Number.isFinite(p) ? p : now; })();-- const dup = dedupe(n, item.source_key, at);++ const dup = await dedupe(n, item.source_key, at);const id = newItemId(now);-- db.prepare(`INSERT INTO news_source_items (id, sources_item_id, sources_revision, source_key, canonical_url, url_key, headline, outlet, authors,++ await db.prepare(`INSERT INTO news_source_items (id, sources_item_id, sources_revision, source_key, canonical_url, url_key, headline, outlet, authors,published_at, summary, summary_basis, license_note, terms_note, content_hash, title_key, status, duplicate_of, dedupe,retrieved_at, first_seen_at, updated_at)VALUES (@id, @sources_item_id, @rev, @source_key, @canonical_url, @url_key, @headline, @outlet, @authors,@@@ -171,61 +171,61 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n@retrieved_at, @now, @now)`).run({ ...n, id, sources_item_id: item.id, rev: revision, source_key: String(item.source_key || 'unknown'),status: dup ? 'duplicate' : 'active', duplicate_of: dup ? dup.of : null, dedupe: dup ? JSON.stringify({ rule: dup.rule, detail: dup.detail }) : null, now });-- clusters.assign(q.byId.get(id));-- const row = q.byId.get(id);-- ingestedEvent(row, 'created');++ await clusters.assign(await q.byId.get(id));++ const row = await q.byId.get(id);++ await ingestedEvent(row, 'created');return { outcome: 'created', item: row };-- })();++ });}/** A Sources removal (takedown, licence, error): sticky; the stored summary is dropped. */-- function applyRemoval({ sourcesItemId, reason, revision = null, origin = 'webhook' }) {-- return db.transaction(() => {-- const cur = q.bySourcesId.get(sourcesItemId);++ async function applyRemoval({ sourcesItemId, reason, revision = null, origin = 'webhook' }) {++ return db.tx(async () => {++ const cur = await q.bySourcesId.get(sourcesItemId);if (!cur) return { outcome: 'ignored', reason: 'never ingested' };if (cur.status === 'removed') return { outcome: 'unchanged', item: cur };// A story resting on it may stop being open for comments (domain/discussion.js).-- if (discussion) discussion.watchSource(cur.id);++ if (discussion) await discussion.watchSource(cur.id);const now = store.now();-- db.prepare(`UPDATE news_source_items SET status = 'removed', summary = NULL, summary_basis = 'removed_upstream', removed_at = ?, removed_reason = ?,-- sources_revision = MAX(sources_revision, COALESCE(?, sources_revision)), updated_at = ? WHERE id = ?`)++ await db.prepare(`UPDATE news_source_items SET status = 'removed', summary = NULL, summary_basis = 'removed_upstream', removed_at = ?, removed_reason = ?,++ sources_revision = GREATEST(sources_revision, COALESCE(?, sources_revision)), updated_at = ? WHERE id = ?`).run(now, String(reason || 'removed upstream').slice(0, 500), revision, now, cur.id);-- const row = q.byId.get(cur.id);-- if (row.cluster_id) clusters.recompute(clusters.resolve(row.cluster_id).id);-- ingestedEvent(row, 'removed', { reason: row.removed_reason, origin });-- const affected = stories ? stories.onUpstreamChange(row, 'source_removed', { previous: cur }) : [];++ const row = await q.byId.get(cur.id);++ if (row.cluster_id) await clusters.recompute((await clusters.resolve(row.cluster_id)).id);++ await ingestedEvent(row, 'removed', { reason: row.removed_reason, origin });++ const affected = stories ? await stories.onUpstreamChange(row, 'source_removed', { previous: cur }) : [];return { outcome: 'removed', item: row, stories: affected };-- })();++ });}/** sources.fetch.failed for a news source: recorded and relayed; nothing else changes. */-- function upstreamFailure(payload) {-- return db.transaction(() => {++ async function upstreamFailure(payload) {++ return db.tx(async () => {if (payload.category && payload.category !== config.sources.category) return { outcome: 'ignored' };-- recordRun({ origin: 'sources', state: String(payload.state || 'failed').slice(0, 40), source_key: payload.source_key || null, error_code: payload.error_code || null,++ await recordRun({ origin: 'sources', state: String(payload.state || 'failed').slice(0, 40), source_key: payload.source_key || null, error_code: payload.error_code || null,detail: `run ${payload.run_id || '?'}${payload.http_status ? ` HTTP ${payload.http_status}` : ''}${payload.consecutive_failures ? `, ${payload.consecutive_failures} consecutive failures` : ''}` });-- q.putStatus.run({ source_key: payload.source_key || 'unknown', name: null, homepage_url: null, status: 'failing', stale: null, last_success_at: null, now: store.now() });-- failedEvent({ origin: 'sources', sourceKey: payload.source_key || null, state: payload.state || 'failed', errorCode: payload.error_code || null, httpStatus: payload.http_status || null,++ await q.putStatus.run({ source_key: payload.source_key || 'unknown', name: null, homepage_url: null, status: 'failing', stale: null, last_success_at: null, now: store.now() });++ await failedEvent({ origin: 'sources', sourceKey: payload.source_key || null, state: payload.state || 'failed', errorCode: payload.error_code || null, httpStatus: payload.http_status || null,extra: { run_id: payload.run_id || null, consecutive_failures: payload.consecutive_failures || null } });return { outcome: 'recorded' };-- })();++ });}/** Remember what a Sources page says about its sources' health (display only). */-- function rememberSources(map) {++ async function rememberSources(map) {for (const [key, h] of Object.entries(map || {})) {-- q.putStatus.run({ source_key: key, name: null, homepage_url: null, status: h.status || null, stale: h.stale == null ? null : (h.stale ? 1 : 0), last_success_at: h.last_success_at || null, now: store.now() });++ await q.putStatus.run({ source_key: key, name: null, homepage_url: null, status: h.status || null, stale: h.stale == null ? null : (h.stale ? 1 : 0), last_success_at: h.last_success_at || null, now: store.now() });}}/** Fetch registry names we do not know yet (needs sources.source.read; failure = keep the host name). */async function learnOutlets(keys) {for (const key of [...new Set(keys)]) {-- const st = q.status.get(key);++ const st = await q.status.get(key);if (st && st.name) continue;try {const s = await sources.getSource(key);-- q.putStatus.run({ source_key: key, name: s && s.name ? String(s.name).slice(0, 200) : null, homepage_url: s && s.homepage_url ? String(s.homepage_url) : null,++ await q.putStatus.run({ source_key: key, name: s && s.name ? String(s.name).slice(0, 200) : null, homepage_url: s && s.homepage_url ? String(s.homepage_url) : null,status: s && s.health ? s.health.status || null : null, stale: s && s.health && s.health.stale != null ? (s.health.stale ? 1 : 0) : null,last_success_at: s && s.health ? s.health.last_success_at || null : null, now: store.now() });} catch { /* optional grant; the outlet stays the URL's host */ }@@@ -240,33 +240,33 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = nasync function pull() {if (pulling) return pulling;pulling = (async () => {-- const before = store.getState('sources_cursor', 0);++ const before = await store.getState('sources_cursor', 0);let after = before;const counts = { created: 0, updated: 0, removed: 0, unchanged: 0, ignored: 0, rejected: 0, duplicates: 0 };try {for (let page = 0; page < config.sources.maxPagesPerRun; page++) {const body = await sources.listItems({ after, limit: config.sources.pageSize });-- rememberSources(body.sources);++ await rememberSources(body.sources);await learnOutlets(body.items.map((i) => i.source_key).filter(Boolean));-- store.tx(() => {++ await store.tx(async () => {for (const item of body.items) {-- const r = apply(item, { origin: 'pull' });++ const r = await apply(item, { origin: 'pull' });counts[r.outcome] = (counts[r.outcome] || 0) + 1;if (r.outcome === 'created' && r.item.status === 'duplicate') counts.duplicates++;}-- store.setState('sources_cursor', body.next_after);++ await store.setState('sources_cursor', body.next_after);});after = body.next_after;if (!body.more) break;}-- store.tx(() => recordRun({ origin: 'pull', state: 'ok', counts, cursor_before: before, cursor_after: after }));++ await store.tx(async () => await recordRun({ origin: 'pull', state: 'ok', counts, cursor_before: before, cursor_after: after }));return { ok: true, counts, cursor: after };} catch (err) {-- const prev = q.lastPull.get();-- store.tx(() => {-- recordRun({ origin: 'pull', state: 'failed', error_code: err.code || 'pull.error', detail: err.message, counts, cursor_before: before, cursor_after: after });++ const prev = await q.lastPull.get();++ await store.tx(async () => {++ await recordRun({ origin: 'pull', state: 'failed', error_code: err.code || 'pull.error', detail: err.message, counts, cursor_before: before, cursor_after: after });// One event per outage, not one per retry.-- if (!prev || prev.state !== 'failed') failedEvent({ origin: 'pull', state: 'failed', errorCode: err.code || 'pull.error', detail: err.message, httpStatus: err.status || null });++ if (!prev || prev.state !== 'failed') await failedEvent({ origin: 'pull', state: 'failed', errorCode: err.code || 'pull.error', detail: err.message, httpStatus: err.status || null });});log.warn(`[News] Sources pull failed (${err.code || 'error'}): ${err.message}`);return { ok: false, error: err.code || 'pull.error', detail: err.message, counts, cursor: after };@@@ -279,11 +279,11 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = nNEAR_TITLE,setStories(s) { stories = s; },normalise, dedupe, apply, applyRemoval, upstreamFailure, pull, recordRun, failedEvent, learnOutlets, rememberSources,-- get: (id) => q.byId.get(String(id || '')) || null,-- bySourcesId: (id) => q.bySourcesId.get(String(id || '')) || null,-- runs: ({ limit = 30 } = {}) => db.prepare('SELECT * FROM news_ingest_runs ORDER BY id DESC LIMIT ?').all(limit),-- sourceStatus: () => db.prepare('SELECT * FROM news_source_status ORDER BY source_key').all(),-- lastPull: () => q.lastPull.get() || null,++ get: async (id) => await q.byId.get(String(id || '')) || null,++ bySourcesId: async (id) => await q.bySourcesId.get(String(id || '')) || null,++ runs: async ({ limit = 30 } = {}) => await db.prepare('SELECT * FROM news_ingest_runs ORDER BY id DESC LIMIT ?').all(limit),++ sourceStatus: async () => await db.prepare('SELECT * FROM news_source_status ORDER BY source_key').all(),++ lastPull: async () => await q.lastPull.get() || null,};}@@@ -61,20 +61,20 @@ function createPublication({ store, config, outbox }) {const feedId = (story) => `tag:openvibe.news,2026:story/${story.id}`; // stable across slug changesconst authorshipOf = (rev) => (rev && rev.meta && rev.meta.authorship) || null;-- const reviewOf = (story, rev) => (rev ? store.reviews.latest(story.id, rev.number) : null);++ const reviewOf = async (story, rev) => (rev ? await store.reviews.latest(story.id, rev.number) : null);const paragraphsOf = (rev) => (rev && rev.fields && Array.isArray(rev.fields.paragraphs) ? rev.fields.paragraphs : []);const snapshotOf = (rev) => (rev && rev.fields && Array.isArray(rev.fields.sources) ? rev.fields.sources : []);/** Live upstream state of every source a revision names: n → news_source_items row (or null). */-- function liveSources(rev) {++ async function liveSources(rev) {const out = new Map();-- for (const s of snapshotOf(rev)) out.set(s.n, itemById.get(s.id) || null);++ for (const s of snapshotOf(rev)) out.set(s.n, await itemById.get(s.id) || null);return out;}/** What the revision rests on, against the live source records. */-- function support(rev) {-- const live = liveSources(rev);++ async function support(rev) {++ const live = await liveSources(rev);const alive = (n) => { const it = live.get(n); return Boolean(it && it.status !== 'removed'); };const cited = new Set();let unsupported = 0;@@@ -97,8 +97,8 @@ function createPublication({ store, config, outbox }) {}/** The gate's decision for a story at one revision (the published one by default). */-- function decide(story, rev, { state } = {}) {-- const sup = support(rev);++ async function decide(story, rev, { state } = {}) {++ const sup = await support(rev);const rec = authorshipOf(rev);const facts = {state: gateState(story, state),@@@ -110,25 +110,25 @@ function createPublication({ store, config, outbox }) {retracted: (state || story.state) === 'retracted',noindex: Boolean(story.noindex),};-- if (rec) Object.assign(facts, gateAuthorship(rec, reviewOf(story, rev)));++ if (rec) Object.assign(facts, gateAuthorship(rec, await reviewOf(story, rev)));return seo.evaluate(facts, { policy, now: store.now() });}-- function topicOf(story) { return story.topic_id ? topicById.get(story.topic_id) : null; }++ async function topicOf(story) { return story.topic_id ? await topicById.get(story.topic_id) : null; }/*** The index document as it would describe this story (for the product events), or a tombstone* when it is not published (or retracted).*/-- function documentFor(story, { forSearch }) {-- const rev = story.published_revision ? store.revisions.get(story.id, story.published_revision) : null;++ async function documentFor(story, { forSearch }) {++ const rev = story.published_revision ? await store.revisions.get(story.id, story.published_revision) : null;const identity = { owner: OWNER, type: 'story', id: story.id, revision: 0 };-- if (!rev || story.state !== 'published') return { doc: hooks.tombstone(identity), decision: rev ? decide(story, rev) : null };-- const decision = decide(story, rev);++ if (!rev || story.state !== 'published') return { doc: hooks.tombstone(identity), decision: rev ? await decide(story, rev) : null };++ const decision = await decide(story, rev);if (forSearch && !decision.listable) return { doc: hooks.tombstone(identity), decision };-- const topic = topicOf(story);++ const topic = await topicOf(story);const outlets = [...new Set(snapshotOf(rev).map((s) => s.outlet).filter(Boolean))].slice(0, 50);-- const citations = store.citations.forRevision(story.id, rev.number).filter((c) => !c.anchor || !c.anchor.startsWith('t'));++ const citations = (await store.citations.forRevision(story.id, rev.number)).filter((c) => !c.anchor || !c.anchor.startsWith('t'));const seen = new Set();const uniqueCites = citations.filter((c) => { const k = c.sourceItemId || c.url; if (seen.has(k)) return false; seen.add(k); return true; }).slice(0, 49);const doc = hooks.buildIndexDocument({@@@ -151,13 +151,13 @@ function createPublication({ store, config, outbox }) {}/** Stamp and enqueue the Search document when it changed. Inside the caller's transaction. */-- function syncIndex(story, { traceparent } = {}) {-- const { doc } = documentFor(story, { forSearch: true });-- const prev = store.sequencer.current(OWNER, 'story', story.id);++ async function syncIndex(story, { traceparent } = {}) {++ const { doc } = await documentFor(story, { forSearch: true });++ const prev = await store.sequencer.current(OWNER, 'story', story.id);if (doc.deleted && prev == null) return null;-- const stamped = store.sequencer.stamp(doc);++ const stamped = await store.sequencer.stamp(store.db, doc);if (prev != null && stamped.revision === prev) return null;-- return outbox.emit(hooks.indexEvent({ document: stamped, now: store.now() }), { traceparent });++ return await outbox.emit(hooks.indexEvent({ document: stamped, now: store.now() }), { traceparent });}function snapshot(story) {@@@ -168,13 +168,13 @@ function createPublication({ store, config, outbox }) {* Emit the product event for a transition (news.story.published|updated|unpublished|retracted)* and re-sync Search. Inside the caller's transaction, after the row changed.*/-- function afterChange(before, storyId, { actor, traceparent, extra = {} } = {}) {-- const story = storyById.get(storyId);++ async function afterChange(before, storyId, { actor, traceparent, extra = {} } = {}) {++ const story = await storyById.get(storyId);const after = snapshot(story);let event = null;if (after.state === 'retracted' && (!before || before.state !== 'retracted')) {-- const d = decide(story, store.revisions.get(story.id, story.published_revision));-- event = outbox.emit({++ const d = await decide(story, await store.revisions.get(story.id, story.published_revision));++ event = await outbox.emit({event_type: 'news.story.retracted', version: 1, source: OWNER, actor: actorRef(actor),timestamp: new Date(store.now()).toISOString(),visibility: d.listable ? 'public' : 'internal',@@@ -187,16 +187,16 @@ function createPublication({ store, config, outbox }) {let action = hooks.actionFor(norm(before), norm(after));if (!action && before && before.state === 'published' && after.state === 'published' && before.url !== after.url) action = 'updated';if (action) {-- const { doc, decision } = documentFor(story, { forSearch: false });-- const topic = topicOf(story);-- event = outbox.emit(hooks.publicationEvent({++ const { doc, decision } = await documentFor(story, { forSearch: false });++ const topic = await topicOf(story);++ event = await outbox.emit(hooks.publicationEvent({product: OWNER, type: 'story', action, id: story.id, revision: story.published_revision || 0,actor: actorRef(actor), document: doc, decision, now: store.now(),extra: { ...(topic ? { topic: topic.slug } : {}), ...extra },}), { traceparent });}}-- syncIndex(story, { traceparent });++ await syncIndex(story, { traceparent });return { story, event };}@@@ -18,11 +18,11 @@ const iso = (v) => (v == null ? null : new Date(v).toISOString());function createReading({ store, config, stories, publication, people, topics }) {/** The public model of a story at a revision (the published one unless given). */async function storyModel(story, { rev = null, editorView = false } = {}) {-- const r = rev || (story.published_revision ? store.revisions.get(story.id, story.published_revision) : null);++ const r = rev || (story.published_revision ? await store.revisions.get(story.id, story.published_revision) : null);if (!r) return null;-- const live = publication.liveSources(r);++ const live = await publication.liveSources(r);const snap = publication.snapshotOf(r);-- const upstream = stories.openUpstreamFlags(story);++ const upstream = await stories.openUpstreamFlags(story);const sources = snap.map((s) => {const it = live.get(s.n);const removed = !it || it.status === 'removed';@@@ -47,12 +47,12 @@ function createReading({ store, config, stories, publication, people, topics })g.sources.push(s.n);}const rec = publication.authorshipOf(r);-- const review = publication.reviewOf(story, r);++ const review = await publication.reviewOf(story, r);const authorSubjects = rec && Array.isArray(rec.authors) ? rec.authors : [];const who = authorSubjects.length ? await people.many(authorSubjects) : new Map();const authors = authorSubjects.map((s) => who.get(s)).filter((p) => p && p.known).map((p) => ({ name: p.name, username: p.username }));-- const topic = publication.topicOf(story);-- const flags = stories.publicFlags(story);++ const topic = await publication.topicOf(story);++ const flags = await stories.publicFlags(story);const retraction = flags.find((f) => f.kind === 'retraction') || null;return {story, rev: r, topic,@@@ -69,7 +69,7 @@ function createReading({ store, config, stories, publication, people, topics })return { kind: f.kind, n: s ? s.n : null, outlet: s ? s.outlet : null, at: iso(f.created_at) };}),authorship: rec, review, disclosure: rec ? disclosure(rec, review) : null, authors,-- decision: publication.decide(story, r),++ decision: await publication.decide(story, r),first_published_at: iso(story.first_published_at), updated_at: r.createdAt, published_at: iso(story.published_at),};}@@@ -109,7 +109,7 @@ function createReading({ store, config, stories, publication, people, topics })/** Feed items for published (and retracted, labelled) stories: our own text only, never a source's. */async function feedItems({ topicId = null, limit = 30 } = {}) {-- const { stories: rows } = stories.listPublished({ topicId, limit });++ const { stories: rows } = await stories.listPublished({ topicId, limit });const out = [];for (const story of rows) {const m = await storyModel(story);@@@ -130,10 +130,10 @@ function createReading({ store, config, stories, publication, people, topics })async function listItems(rows) {const out = [];for (const story of rows) {-- const r = store.revisions.get(story.id, story.published_revision);++ const r = await store.revisions.get(story.id, story.published_revision);if (!r) continue;const paragraphs = publication.paragraphsOf(r);-- const topic = publication.topicOf(story);++ const topic = await publication.topicOf(story);out.push({story, headline: r.fields.headline, url: publication.storyPath(story), lede: paragraphs[0] ? paragraphs[0].text : '',sourceCount: publication.snapshotOf(r).length, outlets: [...new Set(publication.snapshotOf(r).map((s) => s.outlet))].slice(0, 4),@@@ -67,39 +67,39 @@ function createStories({ store, config, publication, clusters, outbox, ai = nulllink: db.prepare('SELECT * FROM news_story_sources WHERE story_id = ? AND source_item_id = ?'),maxPos: db.prepare('SELECT COALESCE(MAX(position), 0) AS n FROM news_story_sources WHERE story_id = ?'),storiesCiting: db.prepare('SELECT DISTINCT story_id FROM news_story_sources WHERE source_item_id = ? AND detached_at IS NULL'),-- perspectives: db.prepare('SELECT * FROM news_perspectives WHERE story_id = ? AND removed_at IS NULL ORDER BY position, created_at, rowid'),++ perspectives: db.prepare('SELECT * FROM news_perspectives WHERE story_id = ? AND removed_at IS NULL ORDER BY position, created_at, seq'),perspective: db.prepare('SELECT * FROM news_perspectives WHERE id = ? AND story_id = ?'),-- timeline: db.prepare('SELECT * FROM news_timeline_entries WHERE story_id = ? AND removed_at IS NULL ORDER BY occurred_on, created_at, rowid'),-- flags: db.prepare('SELECT * FROM news_editorial_flags WHERE story_id = ? ORDER BY created_at, rowid'),++ timeline: db.prepare('SELECT * FROM news_timeline_entries WHERE story_id = ? AND removed_at IS NULL ORDER BY occurred_on, created_at, seq'),++ flags: db.prepare('SELECT * FROM news_editorial_flags WHERE story_id = ? ORDER BY created_at, seq'),openFlag: db.prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND source_item_id = ? AND kind = ? AND status = 'open' ORDER BY created_at DESC LIMIT 1"),insertFlag: db.prepare(`INSERT INTO news_editorial_flags (id, story_id, kind, status, note, source_item_id, pending_revision, revision, created_by, created_at, resolved_by, resolved_at)VALUES (@id, @story_id, @kind, @status, @note, @source_item_id, @pending_revision, @revision, @created_by, @now, @resolved_by, @resolved_at)`),};-- const touch = (id) => db.prepare('UPDATE news_stories SET updated_at = ? WHERE id = ?').run(store.now(), id);++ const touch = async (id) => await db.prepare('UPDATE news_stories SET updated_at = ? WHERE id = ?').run(store.now(), id);-- function mustItem(ref) {++ async function mustItem(ref) {const r = String(ref || '').trim();-- const it = r.startsWith('itm_') ? q.itemBySources.get(r) : q.item.get(r);++ const it = r.startsWith('itm_') ? await q.itemBySources.get(r) : await q.item.get(r);if (!it) throw new ApiError(404, 'source_item.not_found', `No source item ${r}`);return it;}-- function topicFrom(v) {++ async function topicFrom(v) {if (v == null || v === '') return null;const s = String(v);-- const t = s.startsWith('top_') ? q.topicById.get(s) : q.topicBySlug.get(s);++ const t = s.startsWith('top_') ? await q.topicById.get(s) : await q.topicBySlug.get(s);if (!t) throw new ApiError(422, 'topic.not_found', `No active topic ${s}`);return t;}-- function slugFor(wanted, storyId = null) {++ async function slugFor(wanted, storyId = null) {let base;try { base = slugify(wanted).slice(0, 90); } catch { throw new ApiError(422, 'story.invalid_slug', 'That headline has no letters or digits to make a URL from'); }if (!SLUG_RE.test(base)) throw new ApiError(422, 'story.invalid_slug', 'slug must be lowercase letters, digits and dashes');for (let i = 1; i < 1000; i++) {const s = i === 1 ? base : `${base.slice(0, 85)}-${i}`;-- const hit = q.bySlug.get(s);++ const hit = await q.bySlug.get(s);if (!hit || hit.id === storyId) return s;}throw new ApiError(409, 'story.slug_taken', 'No free slug for that headline');@@@ -110,8 +110,11 @@ function createStories({ store, config, publication, clusters, outbox, ai = null// ── The working source table, timeline and perspectives ─/** Attached sources with their items, in citation order: [{ n, item, perspective_id, detached_at }]. */-- function sourcesOf(story, { includeDetached = false } = {}) {-- return q.links.all(story.id).filter((l) => includeDetached || !l.detached_at).map((l) => ({ n: l.position, item: q.item.get(l.source_item_id), perspective_id: l.perspective_id, detached_at: l.detached_at, added_by: l.added_by }));++ async function sourcesOf(story, { includeDetached = false } = {}) {++ const links = (await q.links.all(story.id)).filter((l) => includeDetached || !l.detached_at);++ // Every item in one query (no N+1).++ const items = new Map((links.length ? await db.prepare('SELECT * FROM news_source_items WHERE id = ANY(?)').all(links.map((l) => l.source_item_id)) : []).map((r) => [r.id, r]));++ return links.map((l) => ({ n: l.position, item: items.get(l.source_item_id), perspective_id: l.perspective_id, detached_at: l.detached_at, added_by: l.added_by }));}function sourceEntry(n, item, perspectiveId) {@@@ -125,17 +128,17 @@ function createStories({ store, config, publication, clusters, outbox, ai = null}/** What the next revision snapshots. */-- function buildSnapshot(story) {-- const srcs = sourcesOf(story);++ async function buildSnapshot(story) {++ const srcs = await sourcesOf(story);const byItem = new Map(srcs.map((s) => [s.item.id, s.n]));return {sources: srcs.map((s) => sourceEntry(s.n, s.item, s.perspective_id)),-- timeline: q.timeline.all(story.id).filter((t) => byItem.has(t.source_item_id)).map((t) => ({ id: t.id, occurred_on: t.occurred_on, text: t.text, source: byItem.get(t.source_item_id) })),-- perspectives: q.perspectives.all(story.id).map((p) => ({ id: p.id, label: p.label, description: p.description, sources: srcs.filter((s) => s.perspective_id === p.id).map((s) => s.n) })),++ timeline: (await q.timeline.all(story.id)).filter((t) => byItem.has(t.source_item_id)).map((t) => ({ id: t.id, occurred_on: t.occurred_on, text: t.text, source: byItem.get(t.source_item_id) })),++ perspectives: (await q.perspectives.all(story.id)).map((p) => ({ id: p.id, label: p.label, description: p.description, sources: srcs.filter((s) => s.perspective_id === p.id).map((s) => s.n) })),};}-- function attachCitations(story, rev) {++ async function attachCitations(story, rev) {const byN = new Map((rev.fields.sources || []).map((s) => [s.n, s]));const list = [];(rev.fields.paragraphs || []).forEach((p, i) => {@@@ -148,7 +151,7 @@ function createStories({ store, config, publication, clusters, outbox, ai = nullconst s = byN.get(t.source);if (s) list.push({ anchor: `t:${t.id}`, sourceItemId: s.sources_item_id, url: s.url || null, title: s.headline, retrievedAt: s.retrieved_at || null, attachedBy: 'svc:news' });}-- if (list.length) store.citations.attachMany(story.id, rev.number, list.slice(0, 500));++ if (list.length) await store.citations.attachMany(story.id, rev.number, list.slice(0, 500));}function authorshipFor(viewer, input, previous) {@@@ -185,9 +188,9 @@ function createStories({ store, config, publication, clusters, outbox, ai = null}/** Paragraphs from the API (by item id or number) or from the form body (markers). */-- function paragraphsFrom(story, input) {++ async function paragraphsFrom(story, input) {if (Array.isArray(input.paragraphs)) {-- const numberOf = new Map(sourcesOf(story).map((s) => [s.item.id, s.n]).concat(sourcesOf(story).map((s) => [s.item.sources_item_id, s.n])));++ const numberOf = new Map((await sourcesOf(story)).map((s) => [s.item.id, s.n]).concat((await sourcesOf(story)).map((s) => [s.item.sources_item_id, s.n])));return input.paragraphs.map((p) => ({ text: p && p.text, sources: (Array.isArray(p && p.sources) ? p.sources : []).map((x) => (typeof x === 'number' ? x : numberOf.get(String(x)) || -1)) }));}if (input.body !== undefined) return parseBody(input.body);@@@ -195,15 +198,15 @@ function createStories({ store, config, publication, clusters, outbox, ai = null}/** Everything that stands between a revision and publication. → [{ code, detail }] */-- function problems(story, rev) {++ async function problems(story, rev) {const out = [];if (!rev) return [{ code: 'story.no_text', detail: 'Nothing has been written yet' }];const snap = rev.fields.sources || [];if (!snap.length) out.push({ code: 'story.unsourced', detail: 'A story with zero sources cannot be published: attach the source items it rests on' });const paragraphs = rev.fields.paragraphs || [];if (!paragraphs.length) out.push({ code: 'story.no_text', detail: 'The revision has no paragraphs' });-- const live = publication.liveSources(rev);-- const links = new Map(q.links.all(story.id).map((l) => [l.source_item_id, l]));++ const live = await publication.liveSources(rev);++ const links = new Map((await q.links.all(story.id)).map((l) => [l.source_item_id, l]));const cited = new Set();paragraphs.forEach((p, i) => {if (!p.sources || !p.sources.length) out.push({ code: 'story.claim_unsourced', detail: `Paragraph ${i + 1} cites no source` });@@@ -215,25 +218,25 @@ function createStories({ store, config, publication, clusters, outbox, ai = nullconst it = live.get(n);if (!entry || !it) { out.push({ code: 'story.unknown_source', detail: `[${n}] is not in the revision's source table` }); continue; }if (it.status === 'removed') out.push({ code: 'story.source_removed', detail: `[${n}] (${it.outlet}) was removed upstream: ${it.removed_reason || 'no reason given'} — revise the text so nothing rests on it` });-- const link = links.get(it.id);++ const link = await links.get(it.id);if (!link || link.detached_at) out.push({ code: 'story.source_detached', detail: `[${n}] was detached from the story` });}const rec = publication.authorshipOf(rev) || { mode: 'human' };-- const ok = canPublish(rec, publication.reviewOf(story, rev));++ const ok = canPublish(rec, await publication.reviewOf(story, rev));if (!ok.ok) out.push({ code: 'story.review_required', detail: `Revision ${rev.number} is ${rec.mode === 'hybrid' ? 'AI-assisted' : 'AI-generated'} (${ok.reason}): a person must review and approve it first` });if (!clean(rev.fields.headline, 300)) out.push({ code: 'story.no_headline', detail: 'The revision has no headline' });return out;}-- function flagRow(story, { kind, status = 'open', note = null, sourceItemId = null, pendingRevision = null, revision = null, createdBy, resolvedBy = null, resolvedAt = null }) {++ async function flagRow(story, { kind, status = 'open', note = null, sourceItemId = null, pendingRevision = null, revision = null, createdBy, resolvedBy = null, resolvedAt = null }) {const now = store.now();const id = newFlagId(now);-- q.insertFlag.run({ id, story_id: story.id, kind, status, note, source_item_id: sourceItemId, pending_revision: pendingRevision, revision, created_by: createdBy, now, resolved_by: resolvedBy, resolved_at: resolvedAt });-- return db.prepare('SELECT * FROM news_editorial_flags WHERE id = ?').get(id);++ await q.insertFlag.run({ id, story_id: story.id, kind, status, note, source_item_id: sourceItemId, pending_revision: pendingRevision, revision, created_by: createdBy, now, resolved_by: resolvedBy, resolved_at: resolvedAt });++ return await db.prepare('SELECT * FROM news_editorial_flags WHERE id = ?').get(id);}-- function flagEvent(story, flag, traceparent) {-- outbox.emit({++ async function flagEvent(story, flag, traceparent) {++ await outbox.emit({event_type: 'news.story.flagged', actor: flag.created_by === 'svc:news' ? { type: 'service', id: 'news' } : actorRef(flag.created_by),visibility: 'internal', priority: flag.kind === 'source_removed' ? 'important' : 'low', // the envelope allows important|low; 'normal' was refused by Eventssubject: { type: 'story', id: story.id, revision: flag.pending_revision || story.published_revision || 0 },@@@ -244,154 +247,155 @@ function createStories({ store, config, publication, clusters, outbox, ai = nullconst api = {parseBody, formatBody, AI_WORKFLOWS,-- get: (id) => q.byId.get(String(id || '')) || null,-- bySlug: (slug) => q.bySlug.get(String(slug || '')) || null,-- mustGet(id) {-- const s = api.get(id);++ get: async (id) => await q.byId.get(String(id || '')) || null,++ bySlug: async (slug) => await q.bySlug.get(String(slug || '')) || null,++ async mustGet(id) {++ const s = await api.get(id);if (!s) throw new ApiError(404, 'story.not_found', 'No such story');return s;},sources: sourcesOf,buildSnapshot,-- perspectives: (story) => q.perspectives.all(story.id),-- timeline: (story) => q.timeline.all(story.id),-- flags: (story) => q.flags.all(story.id),-- publicFlags: (story) => q.flags.all(story.id).filter((f) => f.status === 'published' && ['correction', 'update', 'retraction'].includes(f.kind)),-- openUpstreamFlags: (story) => q.flags.all(story.id).filter((f) => f.status === 'open' && (f.kind === 'source_updated' || f.kind === 'source_removed')),-- head: (story) => store.revisions.head(story.id),-- revision: (story, n) => store.revisions.get(story.id, n),-- revisions: (story, opts) => store.revisions.list(story.id, opts),-- diff: (story, from, to, mode) => store.revisions.diff(story.id, from, to, { mode: mode === 'word' ? 'word' : 'line' }),++ perspectives: async (story) => await q.perspectives.all(story.id),++ timeline: async (story) => await q.timeline.all(story.id),++ flags: async (story) => await q.flags.all(story.id),++ publicFlags: async (story) => (await q.flags.all(story.id)).filter((f) => f.status === 'published' && ['correction', 'update', 'retraction'].includes(f.kind)),++ openUpstreamFlags: async (story) => (await q.flags.all(story.id)).filter((f) => f.status === 'open' && (f.kind === 'source_updated' || f.kind === 'source_removed')),++ head: async (story) => await store.revisions.head(story.id),++ revision: async (story, n) => await store.revisions.get(story.id, n),++ revisions: async (story, opts) => await store.revisions.list(story.id, opts),++ diff: async (story, from, to, mode) => await store.revisions.diff(story.id, from, to, { mode: mode === 'word' ? 'word' : 'line' }),problems,/** Published and retracted stories (retracted ones stay listed with their notice). */-- listPublished({ topicId = null, limit = 20, offset = 0 } = {}) {++ async listPublished({ topicId = null, limit = 20, offset = 0 } = {}) {const where = ["state IN ('published','retracted')"];const args = [];if (topicId) { where.push('topic_id = ?'); args.push(topicId); }const w = where.join(' AND ');-- const total = db.prepare(`SELECT COUNT(*) AS n FROM news_stories WHERE ${w}`).get(...args).n;-- const rows = db.prepare(`SELECT * FROM news_stories WHERE ${w} ORDER BY published_at DESC, id DESC LIMIT ? OFFSET ?`).all(...args, limit, offset);++ const total = (await db.prepare(`SELECT COUNT(*) AS n FROM news_stories WHERE ${w}`).get(...args)).n;++ const rows = await db.prepare(`SELECT * FROM news_stories WHERE ${w} ORDER BY published_at DESC, id DESC LIMIT ? OFFSET ?`).all(...args, limit, offset);return { total, stories: rows };},/** Every story, for editors. */-- listAll({ limit = 200 } = {}) {-- return db.prepare('SELECT * FROM news_stories ORDER BY updated_at DESC, id DESC LIMIT ?').all(limit);++ async listAll({ limit = 200 } = {}) {++ return await db.prepare('SELECT * FROM news_stories ORDER BY updated_at DESC, id DESC LIMIT ?').all(limit);},-- byCluster: (clusterId) => db.prepare('SELECT * FROM news_stories WHERE cluster_id = ? ORDER BY created_at').all(clusterId),++ byCluster: async (clusterId) => await db.prepare('SELECT * FROM news_stories WHERE cluster_id = ? ORDER BY created_at').all(clusterId),/*** An editor opens a story from a cluster: the cluster's live, non-duplicate items are* attached as the first sources. Optional first text (headline + body) becomes revision 1.*/-- create(viewer, input = {}, { traceparent } = {}) {++ async create(viewer, input = {}, { traceparent } = {}) {const subject = actingEditor(viewer);const headline = clean(input.headline, 201);if (!headline) throw new ApiError(422, 'story.no_headline', 'A story needs a working headline');if (headline.length > 200) throw new ApiError(422, 'story.no_headline', 'A headline is at most 200 characters');let cluster = null;if (input.cluster || input.clusterId || input.cluster_id) {-- cluster = clusters.get(input.cluster || input.clusterId || input.cluster_id);++ cluster = await clusters.get(input.cluster || input.clusterId || input.cluster_id);if (!cluster) throw new ApiError(404, 'cluster.not_found', 'No such cluster');-- cluster = clusters.resolve(cluster.id);++ cluster = await clusters.resolve(cluster.id);}-- const topic = topicFrom(input.topic);-- return store.tx(() => {++ const topic = await topicFrom(input.topic);++ return await store.tx(async () => {const now = store.now();const id = newStoryId(now);-- const slug = slugFor(input.slug || headline);-- db.prepare(`INSERT INTO news_stories (id, slug, working_headline, cluster_id, topic_id, state, noindex, created_by, created_at, updated_at)++ const slug = await slugFor(input.slug || headline);++ await db.prepare(`INSERT INTO news_stories (id, slug, working_headline, cluster_id, topic_id, state, noindex, created_by, created_at, updated_at)VALUES (?, ?, ?, ?, ?, 'draft', 0, ?, ?, ?)`).run(id, slug, headline, cluster ? cluster.id : null, topic ? topic.id : null, subject, now, now);-- const story = q.byId.get(id);++ const story = await q.byId.get(id);if (cluster) {-- const items = clusters.members(cluster.id).filter((it) => it.status === 'active')++ const items = (await clusters.members(cluster.id)).filter((it) => it.status === 'active').sort((a, b) => String(a.published_at || '').localeCompare(String(b.published_at || '')) || a.first_seen_at - b.first_seen_at || (a.id < b.id ? -1 : 1));-- items.forEach((it, i) => db.prepare('INSERT INTO news_story_sources (story_id, source_item_id, position, added_by, added_at) VALUES (?, ?, ?, ?, ?)').run(id, it.id, i + 1, subject, now));++ const insertSource = db.prepare('INSERT INTO news_story_sources (story_id, source_item_id, position, added_by, added_at) VALUES (?, ?, ?, ?, ?)');++ for (const [i, it] of items.entries()) await insertSource.run(id, it.id, i + 1, subject, now);}-- outbox.emit({++ await outbox.emit({event_type: 'news.story.created', actor: actorRef(viewer), visibility: 'internal', priority: 'low',-- subject: { type: 'story', id }, payload: { cluster_id: cluster ? cluster.id : null, topic: topic ? topic.slug : null, sources: sourcesOf(story).length },++ subject: { type: 'story', id }, payload: { cluster_id: cluster ? cluster.id : null, topic: topic ? topic.slug : null, sources: (await sourcesOf(story)).length },}, { traceparent });let revision = null;if (input.body !== undefined || Array.isArray(input.paragraphs)) {-- revision = api.revise(viewer, story, { headline, body: input.body, paragraphs: input.paragraphs, expectedRevision: 0, message: 'First draft' }).revision;++ revision = (await api.revise(viewer, story, { headline, body: input.body, paragraphs: input.paragraphs, expectedRevision: 0, message: 'First draft' })).revision;}-- return { story: q.byId.get(id), revision };++ return { story: await q.byId.get(id), revision };});},-- attach(viewer, story, { item }) {++ async attach(viewer, story, { item }) {const subject = actingEditor(viewer);-- const it = mustItem(item);++ const it = await mustItem(item);if (it.status === 'removed') throw new ApiError(409, 'source_item.removed', `${it.id} was removed upstream (${it.removed_reason || 'no reason given'}) and cannot be cited`);-- return store.tx(() => {-- const link = q.link.get(story.id, it.id);++ return await store.tx(async () => {++ const link = await q.link.get(story.id, it.id);if (link && !link.detached_at) return { n: link.position, item: it, created: false };-- if (link) db.prepare('UPDATE news_story_sources SET detached_at = NULL WHERE story_id = ? AND source_item_id = ?').run(story.id, it.id);-- else db.prepare('INSERT INTO news_story_sources (story_id, source_item_id, position, added_by, added_at) VALUES (?, ?, ?, ?, ?)').run(story.id, it.id, q.maxPos.get(story.id).n + 1, subject, store.now());-- touch(story.id);-- return { n: q.link.get(story.id, it.id).position, item: it, created: true };++ if (link) await db.prepare('UPDATE news_story_sources SET detached_at = NULL WHERE story_id = ? AND source_item_id = ?').run(story.id, it.id);++ else await db.prepare('INSERT INTO news_story_sources (story_id, source_item_id, position, added_by, added_at) VALUES (?, ?, ?, ?, ?)').run(story.id, it.id, (await q.maxPos.get(story.id)).n + 1, subject, store.now());++ await touch(story.id);++ return { n: (await q.link.get(story.id, it.id)).position, item: it, created: true };});},/** Detach a source (its number is never reused). The next revision must not cite it. */-- detach(viewer, story, item) {++ async detach(viewer, story, item) {actingEditor(viewer);-- const it = mustItem(item);-- const link = q.link.get(story.id, it.id);++ const it = await mustItem(item);++ const link = await q.link.get(story.id, it.id);if (!link || link.detached_at) throw new ApiError(404, 'story.source_not_attached', `${it.id} is not attached to this story`);-- db.prepare('UPDATE news_story_sources SET detached_at = ? WHERE story_id = ? AND source_item_id = ?').run(store.now(), story.id, it.id);-- db.prepare('UPDATE news_timeline_entries SET removed_at = ? WHERE story_id = ? AND source_item_id = ? AND removed_at IS NULL').run(store.now(), story.id, it.id);-- touch(story.id);++ await db.prepare('UPDATE news_story_sources SET detached_at = ? WHERE story_id = ? AND source_item_id = ?').run(store.now(), story.id, it.id);++ await db.prepare('UPDATE news_timeline_entries SET removed_at = ? WHERE story_id = ? AND source_item_id = ? AND removed_at IS NULL').run(store.now(), story.id, it.id);++ await touch(story.id);return { detached: true, n: link.position };},-- addPerspective(viewer, story, { label, description = null }) {++ async addPerspective(viewer, story, { label, description = null }) {const subject = actingEditor(viewer);const l = clean(label, 81);if (!l) throw new ApiError(422, 'perspective.no_label', 'A perspective needs a label an editor writes (for example the outlet group or the position it states)');if (l.length > 80) throw new ApiError(422, 'perspective.label_too_long', 'A label is at most 80 characters');const now = store.now();const id = newPerspectiveId(now);-- const pos = q.perspectives.all(story.id).length;-- db.prepare('INSERT INTO news_perspectives (id, story_id, label, description, position, created_by, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')++ const pos = (await q.perspectives.all(story.id)).length;++ await db.prepare('INSERT INTO news_perspectives (id, story_id, label, description, position, created_by, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)').run(id, story.id, l, description ? clean(description, 500) : null, pos, subject, now, now);-- touch(story.id);-- return q.perspective.get(id, story.id);++ await touch(story.id);++ return await q.perspective.get(id, story.id);},-- removePerspective(viewer, story, perspectiveId) {++ async removePerspective(viewer, story, perspectiveId) {actingEditor(viewer);-- const p = q.perspective.get(String(perspectiveId || ''), story.id);++ const p = await q.perspective.get(String(perspectiveId || ''), story.id);if (!p || p.removed_at) throw new ApiError(404, 'perspective.not_found', 'No such perspective');-- store.tx(() => {-- db.prepare('UPDATE news_perspectives SET removed_at = ?, updated_at = ? WHERE id = ?').run(store.now(), store.now(), p.id);-- db.prepare('UPDATE news_story_sources SET perspective_id = NULL WHERE story_id = ? AND perspective_id = ?').run(story.id, p.id);++ await store.tx(async () => {++ await db.prepare('UPDATE news_perspectives SET removed_at = ?, updated_at = ? WHERE id = ?').run(store.now(), store.now(), p.id);++ await db.prepare('UPDATE news_story_sources SET perspective_id = NULL WHERE story_id = ? AND perspective_id = ?').run(story.id, p.id);});-- touch(story.id);++ await touch(story.id);return { removed: true };},/** Put an attached source in a perspective (or none). */-- assignPerspective(viewer, story, item, perspectiveId) {++ async assignPerspective(viewer, story, item, perspectiveId) {actingEditor(viewer);-- const it = mustItem(item);-- const link = q.link.get(story.id, it.id);++ const it = await mustItem(item);++ const link = await q.link.get(story.id, it.id);if (!link || link.detached_at) throw new ApiError(404, 'story.source_not_attached', `${it.id} is not attached to this story`);let pid = null;if (perspectiveId) {-- const p = q.perspective.get(String(perspectiveId), story.id);++ const p = await q.perspective.get(String(perspectiveId), story.id);if (!p || p.removed_at) throw new ApiError(404, 'perspective.not_found', 'No such perspective');pid = p.id;}-- db.prepare('UPDATE news_story_sources SET perspective_id = ? WHERE story_id = ? AND source_item_id = ?').run(pid, story.id, it.id);-- touch(story.id);++ await db.prepare('UPDATE news_story_sources SET perspective_id = ? WHERE story_id = ? AND source_item_id = ?').run(pid, story.id, it.id);++ await touch(story.id);return { n: link.position, perspective_id: pid };},/** A dated entry resting on one attached source. The date is what the source states. */-- addTimeline(viewer, story, { occurredOn, text: entryText, source }) {++ async addTimeline(viewer, story, { occurredOn, text: entryText, source }) {const subject = actingEditor(viewer);const on = String(occurredOn || '').trim();if (!DATE_RE.test(on) || !Number.isFinite(Date.parse(on))) throw new ApiError(422, 'timeline.invalid_date', 'occurred_on must be a date the source states (YYYY-MM-DD or an ISO 8601 time)');@@@ -400,27 +404,27 @@ function createStories({ store, config, publication, clusters, outbox, ai = nullif (t.length > 500) throw new ApiError(422, 'timeline.too_long', 'A timeline entry is at most 500 characters');let it;if (/^\d+$/.test(String(source))) {-- const s = sourcesOf(story).find((x) => x.n === parseInt(source, 10));++ const s = (await sourcesOf(story)).find((x) => x.n === parseInt(source, 10));if (!s) throw new ApiError(422, 'timeline.unknown_source', `[${source}] is not in this story's source table`);it = s.item;} else {-- it = mustItem(source);-- const link = q.link.get(story.id, it.id);++ it = await mustItem(source);++ const link = await q.link.get(story.id, it.id);if (!link || link.detached_at) throw new ApiError(422, 'timeline.unknown_source', 'The source must be attached to the story');}if (it.status === 'removed') throw new ApiError(409, 'source_item.removed', 'That source was removed upstream');const now = store.now();const id = newTimelineId(now);-- db.prepare('INSERT INTO news_timeline_entries (id, story_id, occurred_on, text, source_item_id, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)').run(id, story.id, on, t, it.id, subject, now);-- touch(story.id);-- return db.prepare('SELECT * FROM news_timeline_entries WHERE id = ?').get(id);++ await db.prepare('INSERT INTO news_timeline_entries (id, story_id, occurred_on, text, source_item_id, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)').run(id, story.id, on, t, it.id, subject, now);++ await touch(story.id);++ return await db.prepare('SELECT * FROM news_timeline_entries WHERE id = ?').get(id);},-- removeTimeline(viewer, story, entryId) {++ async removeTimeline(viewer, story, entryId) {actingEditor(viewer);-- const r = db.prepare('UPDATE news_timeline_entries SET removed_at = ? WHERE id = ? AND story_id = ? AND removed_at IS NULL').run(store.now(), String(entryId || ''), story.id);++ const r = await db.prepare('UPDATE news_timeline_entries SET removed_at = ? WHERE id = ? AND story_id = ? AND removed_at IS NULL').run(store.now(), String(entryId || ''), story.id);if (!r.changes) throw new ApiError(404, 'timeline.not_found', 'No such timeline entry');-- touch(story.id);++ await touch(story.id);return { removed: true };},@@@ -430,38 +434,38 @@ function createStories({ store, config, publication, clusters, outbox, ai = null* exists (412 revision.conflict on a stale base). Editors write human (or, after an AI draft,* hybrid) revisions; an AI delivery writes an AI revision that needs a person's review.*/-- revise(viewer, story, input = {}, { traceparent } = {}) {++ async revise(viewer, story, input = {}, { traceparent } = {}) {const ai = access.isAiDelivery(viewer);if (!ai) actingEditor(viewer);if (story.state === 'retracted') throw new ApiError(409, 'story.retracted', 'A retracted story is not revised; open a new story');-- return store.tx(() => {-- const head = store.revisions.head(story.id);++ return await store.tx(async () => {++ const head = await store.revisions.head(story.id);const expected = input.expectedRevision ?? input.expected_revision;if (head && (expected == null || expected === '')) throw new ApiError(428, 'revision.expected_required', 'Send expectedRevision (the revision you edited) so concurrent edits are not lost');const headline = input.headline !== undefined && input.headline !== null ? clean(input.headline, 201) : (head ? head.fields.headline : story.working_headline);if (!headline) throw new ApiError(422, 'story.no_headline', 'A revision needs a headline');if (headline.length > 200) throw new ApiError(422, 'story.no_headline', 'A headline is at most 200 characters');-- const attached = new Set(sourcesOf(story).map((s) => s.n));-- const given = paragraphsFrom(story, input);++ const attached = new Set((await sourcesOf(story)).map((s) => s.n));++ const given = await paragraphsFrom(story, input);const paragraphs = checkParagraphs(given || (head ? head.fields.paragraphs : []), attached);-- const snap = buildSnapshot(story);++ const snap = await buildSnapshot(story);const prevRec = head ? publication.authorshipOf(head) : null;const rec = authorshipFor(viewer, input, prevRec);const meta = { authorship: rec };if (ai && Array.isArray(input.gaps) && input.gaps.length) meta.gaps = input.gaps.map((g) => clean(g, 300)).slice(0, 20);-- const out = store.revisions.create({++ const out = await store.revisions.create({entityId: story.id, expectedRevision: head ? parseInt(expected, 10) : 0,content: formatBody(paragraphs), fields: { headline, paragraphs, ...snap }, meta,author: authorLabel(viewer), message: input.message ? clean(input.message, 200) : null,});-- if (out.created) attachCitations(story, out.revision);++ if (out.created) await attachCitations(story, out.revision);const sets = { working_headline: headline };-- if (!ai && input.topic !== undefined) { const t = topicFrom(input.topic); sets.topic_id = t ? t.id : null; }++ if (!ai && input.topic !== undefined) { const t = await topicFrom(input.topic); sets.topic_id = t ? t.id : null; }if (!ai && input.noindex !== undefined) sets.noindex = input.noindex === true || input.noindex === '1' || input.noindex === 1 ? 1 : 0;const keys = Object.keys(sets).filter((k) => k !== 'working_headline');-- db.prepare(`UPDATE news_stories SET ${Object.keys(sets).map((k) => `${k} = @${k}`).concat('updated_at = @now').join(', ')} WHERE id = @id`).run({ ...sets, now: store.now(), id: story.id });-- if (keys.length && story.state === 'published') publication.afterChange(publication.snapshot(story), story.id, { actor: viewer, traceparent });-- return { revision: out.revision, created: out.created, story: q.byId.get(story.id) };++ await db.prepare(`UPDATE news_stories SET ${Object.keys(sets).map((k) => `${k} = @${k}`).concat('updated_at = @now').join(', ')} WHERE id = @id`).run({ ...sets, now: store.now(), id: story.id });++ if (keys.length && story.state === 'published') await publication.afterChange(publication.snapshot(story), story.id, { actor: viewer, traceparent });++ return { revision: out.revision, created: out.created, story: await q.byId.get(story.id) };});},@@@ -470,15 +474,15 @@ function createStories({ store, config, publication, clusters, outbox, ai = null* Open correction/update flags are published with it; upstream flags the revision answers* are resolved. Republishing after a source was removed needs a correction or update note.*/-- publish(viewer, story, { revision, correction } = {}, { traceparent } = {}) {++ async publish(viewer, story, { revision, correction } = {}, { traceparent } = {}) {actingEditor(viewer);if (story.state === 'retracted') throw new ApiError(409, 'story.retracted', 'A retracted story stays retracted; open a new story');-- return store.tx(() => {-- const cur = q.byId.get(story.id);-- if (discussion) discussion.watch(story.id);-- const rev = revision == null || revision === '' ? store.revisions.head(story.id) : store.revisions.get(story.id, parseInt(revision, 10));++ return await store.tx(async () => {++ const cur = await q.byId.get(story.id);++ if (discussion) await discussion.watch(story.id);++ const rev = revision == null || revision === '' ? await store.revisions.head(story.id) : await store.revisions.get(story.id, parseInt(revision, 10));if (!rev && revision != null && revision !== '') throw new ApiError(404, 'revision.not_found', `No revision ${revision}`);-- const found = problems(cur, rev);++ const found = await problems(cur, rev);if (found.length) {const status = found.some((p) => p.code === 'story.review_required' || p.code === 'story.source_removed' || p.code === 'story.source_detached') ? 409 : 422;throw new ApiError(status, found[0].code, found.map((p) => p.detail).join(' · '), { problems: found });@@@ -486,87 +490,88 @@ function createStories({ store, config, publication, clusters, outbox, ai = nullif (cur.state === 'published' && cur.published_revision === rev.number && !correction) return { changed: false, story: cur };const citedIds = new Set();for (const p of rev.fields.paragraphs) for (const n of p.sources) { const s = rev.fields.sources.find((x) => x.n === n); if (s) citedIds.add(s.id); }-- const upstream = q.flags.all(story.id).filter((f) => f.status === 'open' && (f.kind === 'source_updated' || f.kind === 'source_removed'));-- const answers = (f) => {-- const it = q.item.get(f.source_item_id);++ const upstream = (await q.flags.all(story.id)).filter((f) => f.status === 'open' && (f.kind === 'source_updated' || f.kind === 'source_removed'));++ const answers = async (f) => {++ const it = await q.item.get(f.source_item_id);const entry = rev.fields.sources.find((s) => s.id === f.source_item_id);if (f.kind === 'source_removed') return !citedIds.has(f.source_item_id) && !(rev.fields.timeline || []).some((t) => entry && t.source === entry.n);return !entry || !it || entry.sources_revision >= it.sources_revision;};-- const resolving = upstream.filter(answers);++ const resolving = [];++ for (const f of upstream) if (await answers(f)) resolving.push(f);const note = correction && correction.note ? clean(correction.note, 2001) : '';const kind = correction && correction.kind ? String(correction.kind) : (note ? 'correction' : null);if (kind && !['correction', 'update'].includes(kind)) throw new ApiError(422, 'flag.invalid_kind', 'kind is correction or update');if (note.length > 2000) throw new ApiError(422, 'flag.note_too_long', 'A note is at most 2000 characters');-- if (cur.first_published_at && resolving.some((f) => f.kind === 'source_removed') && !note && !q.flags.all(story.id).some((f) => f.status === 'open' && (f.kind === 'correction' || f.kind === 'update') && f.note)) {++ if (cur.first_published_at && resolving.some((f) => f.kind === 'source_removed') && !note && !(await q.flags.all(story.id)).some((f) => f.status === 'open' && (f.kind === 'correction' || f.kind === 'update') && f.note)) {throw new ApiError(422, 'story.correction_note_required', 'A source this story cited was removed upstream: say what changed in a correction or update note readers will see');}const now = store.now();const before = publication.snapshot(cur);-- db.prepare(`UPDATE news_stories SET state = 'published', published_revision = ?, first_published_at = COALESCE(first_published_at, ?),++ await db.prepare(`UPDATE news_stories SET state = 'published', published_revision = ?, first_published_at = COALESCE(first_published_at, ?),published_at = ?, updated_at = ? WHERE id = ?`).run(rev.number, now, now, now, story.id);const subject = viewer.subject;-- for (const f of q.flags.all(story.id).filter((x) => x.status === 'open' && (x.kind === 'correction' || x.kind === 'update'))) {-- db.prepare("UPDATE news_editorial_flags SET status = 'published', revision = ?, resolved_by = ?, resolved_at = ? WHERE id = ?").run(rev.number, subject, now, f.id);++ for (const f of (await q.flags.all(story.id)).filter((x) => x.status === 'open' && (x.kind === 'correction' || x.kind === 'update'))) {++ await db.prepare("UPDATE news_editorial_flags SET status = 'published', revision = ?, resolved_by = ?, resolved_at = ? WHERE id = ?").run(rev.number, subject, now, f.id);}-- if (note) flagRow(cur, { kind, status: 'published', note, revision: rev.number, createdBy: subject, resolvedBy: subject, resolvedAt: now });-- for (const f of resolving) db.prepare("UPDATE news_editorial_flags SET status = 'resolved', revision = ?, resolved_by = ?, resolved_at = ? WHERE id = ?").run(rev.number, subject, now, f.id);-- const out = publication.afterChange(before, story.id, { actor: viewer, traceparent });++ if (note) await flagRow(cur, { kind, status: 'published', note, revision: rev.number, createdBy: subject, resolvedBy: subject, resolvedAt: now });++ for (const f of resolving) await db.prepare("UPDATE news_editorial_flags SET status = 'resolved', revision = ?, resolved_by = ?, resolved_at = ? WHERE id = ?").run(rev.number, subject, now, f.id);++ const out = await publication.afterChange(before, story.id, { actor: viewer, traceparent });return { changed: true, story: out.story, resolved: resolving.map((f) => f.id) };});},-- unpublish(viewer, story, { traceparent } = {}) {++ async unpublish(viewer, story, { traceparent } = {}) {actingEditor(viewer);-- return store.tx(() => {-- const cur = q.byId.get(story.id);++ return await store.tx(async () => {++ const cur = await q.byId.get(story.id);if (cur.state !== 'published' && cur.state !== 'retracted') return { changed: false, story: cur };-- if (discussion) discussion.watch(story.id);++ if (discussion) await discussion.watch(story.id);const before = publication.snapshot(cur);-- db.prepare("UPDATE news_stories SET state = 'unpublished', updated_at = ? WHERE id = ?").run(store.now(), story.id);-- return { changed: true, story: publication.afterChange(before, story.id, { actor: viewer, traceparent }).story };++ await db.prepare("UPDATE news_stories SET state = 'unpublished', updated_at = ? WHERE id = ?").run(store.now(), story.id);++ return { changed: true, story: (await publication.afterChange(before, story.id, { actor: viewer, traceparent })).story };});},/** Retract: the story stays at its URL with the retraction notice; noindex; out of Search. */-- retract(viewer, story, { note } = {}, { traceparent } = {}) {++ async retract(viewer, story, { note } = {}, { traceparent } = {}) {const subject = actingEditor(viewer);const n = clean(note, 2001);if (n.length < 10) throw new ApiError(422, 'story.retraction_note_required', 'A retraction says why, in at least a sentence readers will see');if (n.length > 2000) throw new ApiError(422, 'flag.note_too_long', 'A note is at most 2000 characters');-- return store.tx(() => {-- const cur = q.byId.get(story.id);++ return await store.tx(async () => {++ const cur = await q.byId.get(story.id);if (cur.state === 'retracted') return { changed: false, story: cur };if (cur.state !== 'published') throw new ApiError(409, 'story.not_published', 'Only a published story can be retracted');-- if (discussion) discussion.watch(story.id);++ if (discussion) await discussion.watch(story.id);const now = store.now();const before = publication.snapshot(cur);-- db.prepare("UPDATE news_stories SET state = 'retracted', retracted_at = ?, updated_at = ? WHERE id = ?").run(now, now, story.id);-- const flag = flagRow(cur, { kind: 'retraction', status: 'published', note: n, revision: cur.published_revision, createdBy: subject, resolvedBy: subject, resolvedAt: now });-- const out = publication.afterChange(before, story.id, { actor: viewer, traceparent, extra: { note: n, flag_id: flag.id } });++ await db.prepare("UPDATE news_stories SET state = 'retracted', retracted_at = ?, updated_at = ? WHERE id = ?").run(now, now, story.id);++ const flag = await flagRow(cur, { kind: 'retraction', status: 'published', note: n, revision: cur.published_revision, createdBy: subject, resolvedBy: subject, resolvedAt: now });++ const out = await publication.afterChange(before, story.id, { actor: viewer, traceparent, extra: { note: n, flag_id: flag.id } });return { changed: true, story: out.story, flag };});},/** A correction or update note, published with the next publication. */-- addFlag(viewer, story, { kind, note } = {}) {++ async addFlag(viewer, story, { kind, note } = {}) {const subject = actingEditor(viewer);if (!['correction', 'update'].includes(kind)) throw new ApiError(422, 'flag.invalid_kind', 'kind is correction or update (retractions use the retract action)');const n = clean(note, 2001);if (!n) throw new ApiError(422, 'flag.no_note', 'A correction or update note is public text: write it');if (n.length > 2000) throw new ApiError(422, 'flag.note_too_long', 'A note is at most 2000 characters');-- return flagRow(story, { kind, note: n, createdBy: subject });++ return await flagRow(story, { kind, note: n, createdBy: subject });},/** A person's review of one revision (what lets an AI draft be published and indexed). */-- review(viewer, story, { revision, decision, note } = {}, { traceparent } = {}) {++ async review(viewer, story, { revision, decision, note } = {}, { traceparent } = {}) {if (!viewer || viewer.kind !== 'user' || !viewer.subject) throw new ApiError(403, 'review.person_required', 'Only a signed-in person can review a revision');actingEditor(viewer);const n = parseInt(revision, 10);-- if (!store.revisions.get(story.id, n)) throw new ApiError(404, 'revision.not_found', `No revision ${revision}`);-- return store.tx(() => {-- const row = store.reviews.record({ entityId: story.id, revision: n, reviewer: viewer.subject, decision, note });-- if (story.state === 'published') publication.syncIndex(q.byId.get(story.id), { traceparent });++ if (!await store.revisions.get(story.id, n)) throw new ApiError(404, 'revision.not_found', `No revision ${revision}`);++ return await store.tx(async () => {++ const row = await store.reviews.record({ entityId: story.id, revision: n, reviewer: viewer.subject, decision, note });++ if (story.state === 'published') await publication.syncIndex(await q.byId.get(story.id), { traceparent });return row;});},@@@ -577,19 +582,19 @@ function createStories({ store, config, publication, clusters, outbox, ai = null* with the same paragraphs and the refreshed source entry. Nothing is published here.* Runs inside the ingest transaction.*/-- onUpstreamChange(item, kind) {++ async onUpstreamChange(item, kind) {const affected = [];-- for (const { story_id: sid } of q.storiesCiting.all(item.id)) {-- const story = q.byId.get(sid);++ for (const { story_id: sid } of await q.storiesCiting.all(item.id)) {++ const story = await q.byId.get(sid);if (!story) continue;-- const open = q.openFlag.get(story.id, item.id, kind);++ const open = await q.openFlag.get(story.id, item.id, kind);if (open && kind === 'source_removed') continue;-- const head = store.revisions.head(story.id);++ const head = await store.revisions.head(story.id);let pending = null;if (head) {const sources = (head.fields.sources || []).map((s) => (s.id === item.id ? sourceEntry(s.n, item, s.perspective) : s));const fields = { ...head.fields, sources };-- const out = store.revisions.create({++ const out = await store.revisions.create({entityId: story.id, expectedRevision: head.number, content: head.content, fields,meta: { authorship: head.meta.authorship, system: { reason: kind, source_item_id: item.id, sources_item_id: item.sources_item_id, sources_revision: item.sources_revision, based_on: head.number } },author: 'svc:news',@@@ -597,19 +602,19 @@ function createStories({ store, config, publication, clusters, outbox, ai = null});if (out.created) {pending = out.revision.number;-- store.citations.carryForward({ entityId: story.id, fromRevision: head.number, toRevision: pending, attachedBy: 'svc:news' });++ await store.citations.carryForward({ entityId: story.id, fromRevision: head.number, toRevision: pending, attachedBy: 'svc:news' });}}let flag;if (open) {-- if (pending) db.prepare('UPDATE news_editorial_flags SET pending_revision = ? WHERE id = ?').run(pending, open.id);-- flag = db.prepare('SELECT * FROM news_editorial_flags WHERE id = ?').get(open.id);++ if (pending) await db.prepare('UPDATE news_editorial_flags SET pending_revision = ? WHERE id = ?').run(pending, open.id);++ flag = await db.prepare('SELECT * FROM news_editorial_flags WHERE id = ?').get(open.id);} else {-- flag = flagRow(story, { kind, sourceItemId: item.id, pendingRevision: pending, revision: story.published_revision, createdBy: 'svc:news', note: kind === 'source_removed' ? (item.removed_reason || null) : null });++ flag = await flagRow(story, { kind, sourceItemId: item.id, pendingRevision: pending, revision: story.published_revision, createdBy: 'svc:news', note: kind === 'source_removed' ? (item.removed_reason || null) : null });}-- touch(story.id);-- flagEvent(story, flag);-- if (story.state === 'published') publication.syncIndex(q.byId.get(story.id));++ await touch(story.id);++ await flagEvent(story, flag);++ if (story.state === 'published') await publication.syncIndex(await q.byId.get(story.id));affected.push({ story: story.id, flag: flag.id, pending_revision: pending });log.warn && log.warn(`[News] ${kind} for ${item.id}: story ${story.id} flagged${pending ? `, pending revision ${pending}` : ''}`);}@@@ -623,11 +628,11 @@ function createStories({ store, config, publication, clusters, outbox, ai = nullasync aiDraft(viewer, story, workflow, { traceparent } = {}) {actingEditor(viewer);if (!ai || !ai.enabled) throw new ApiError(503, 'ai.not_configured', 'OpenVibe.AI is not configured on this server; write the story from the source items');-- const srcs = sourcesOf(story).filter((s) => s.item.status !== 'removed');++ const srcs = (await sourcesOf(story)).filter((s) => s.item.status !== 'removed');if (!srcs.length) throw new ApiError(422, 'story.unsourced', 'Attach source items first');if (workflow === 'news.compare_perspectives' && srcs.length < 2) throw new ApiError(422, 'story.too_few_sources', 'Comparing perspectives needs at least two sources');-- const topic = publication.topicOf(story);-- const head = store.revisions.head(story.id);++ const topic = await publication.topicOf(story);++ const head = await store.revisions.head(story.id);let result;try {result = await ai.run(workflow, {@@@ -642,9 +647,9 @@ function createStories({ store, config, publication, clusters, outbox, ai = nullconst draft = draftFromOutput(workflow, result.output, srcs.map((s) => s.n));if (!draft.paragraphs.length) throw new ApiError(422, 'ai.nothing_cited', `The ${workflow} run returned no cited text; no draft was made${draft.gaps.length ? ` (${draft.gaps.slice(0, 3).join('; ')})` : ''}`);const aiViewer = { kind: 'service', service: 'svc:ai', origin: 'ai', subject: null };-- const fresh = q.byId.get(story.id);-- const cur = store.revisions.head(story.id);-- return api.revise(aiViewer, fresh, {++ const fresh = await q.byId.get(story.id);++ const cur = await store.revisions.head(story.id);++ return await api.revise(aiViewer, fresh, {headline: draft.headline || (cur ? cur.fields.headline : fresh.working_headline),paragraphs: draft.paragraphs, expectedRevision: cur ? cur.number : 0, gaps: draft.gaps,authorship: { workflow: result.workflow, stubProvider: result.stub },@@@ -22,31 +22,31 @@ function createTopics({ store }) {insert: db.prepare('INSERT INTO news_topics (id, slug, name, description, status, created_at, updated_at) VALUES (?, ?, ?, ?, \'active\', ?, ?)'),};-- function create({ slug, name, description = null }) {++ async function create({ slug, name, description = null }) {const n = String(name || '').replace(/\s+/g, ' ').trim().slice(0, 80);if (!n) throw new ApiError(422, 'topic.no_name', 'A topic needs a name');let s;try { s = slugify(slug || n).slice(0, 60); } catch { throw new ApiError(422, 'topic.invalid_slug', 'That name has no letters or digits to make a URL from'); }-- if (q.bySlug.get(s)) return { topic: q.bySlug.get(s), created: false };++ if (await q.bySlug.get(s)) return { topic: await q.bySlug.get(s), created: false };const now = store.now();const id = `top_${ids.ulid(now)}`;-- q.insert.run(id, s, n, description ? String(description).replace(/\s+/g, ' ').trim().slice(0, 300) : null, now, now);-- return { topic: q.byId.get(id), created: true };++ await q.insert.run(id, s, n, description ? String(description).replace(/\s+/g, ' ').trim().slice(0, 300) : null, now, now);++ return { topic: await q.byId.get(id), created: true };}return {create,-- list: () => q.active.all(),-- bySlug: (slug) => q.bySlug.get(String(slug || '')) || null,-- byId: (id) => q.byId.get(String(id || '')) || null,++ list: async () => await q.active.all(),++ bySlug: async (slug) => await q.bySlug.get(String(slug || '')) || null,++ byId: async (id) => await q.byId.get(String(id || '')) || null,/** Insert the seeded topics that are missing. → { created, existing } */-- seed(file = SEED_FILE) {++ async seed(file = SEED_FILE) {const data = JSON.parse(fs.readFileSync(file, 'utf8'));let created = 0;let existing = 0;-- store.tx(() => {++ await store.tx(async () => {for (const t of data.topics || []) {-- const r = create(t);++ const r = await create(t);if (r.created) created++; else existing++;}});@@@ -23,7 +23,7 @@*/const { createClient } = require('openvibe-sdk/core');const { createServiceTokenClient } = require('openvibe-sdk/auth');--const { createEventsClient, createOutbox } = require('openvibe-sdk/events');++const { createEventsClient, createPgOutbox } = require('openvibe-sdk/events');function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {const enabled = Boolean(config.events.url && config.oauth.clientSecret);@@@ -39,7 +39,9 @@ function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {}const events = createEventsClient(createClient(clientOpts), { source: 'news' });let lastError = null;-- const outbox = createOutbox(db, {++ // The PostgreSQL outbox: rows are written in the change's own transaction (enqueue(db, …) joins the ambient++ // transaction); several processes relay one table safely (leases).++ const outbox = createPgOutbox(db, {events,intervalMs: config.events.intervalMs,now,@@@ -49,11 +51,10 @@ function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {lastError = msg;},});-- outbox.ensureSchema();/** Inside the caller's transaction. Returns the complete envelope (with its event_id). */-- function emit(envelope, { traceparent } = {}) {-- return outbox.enqueue(envelope, { traceparent });++ async function emit(envelope, { traceparent } = {}) {++ return await outbox.enqueue(db, envelope, { traceparent });}return {@@@ -62,8 +63,8 @@ function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {enabled,start() { if (enabled) outbox.start(); },stop: () => outbox.stop(),-- kick() { if (enabled) outbox.kick(); },-- status: () => ({ enabled, pending: outbox.pending(), rejected: outbox.rejected(), last_error: lastError }),++ async kick() { if (enabled) await outbox.kick(); },++ status: async () => ({ enabled, pending: await outbox.pending(), rejected: await outbox.rejected(), last_error: lastError }),};}@@@ -25,7 +25,7 @@* Never limited: /api/health, /api/ready, /release.json, /metrics, sign-in, the public pages and feeds* (nginx and the per-address limits bound them), and the signed Events deliveries at /internal/events.*/--const { createActorLimiter, defaultActor } = require('openvibe-sdk/limits');++const { createActorLimiter, createValkeyLimitStore, defaultActor } = require('openvibe-sdk/limits');const FIRST_PARTY = /^svc:/;@@@ -83,7 +83,7 @@ const BUDGETS = {* limits(name, own) middleware for one app, plus limits.reads(name) (the defaults on a counted* GET/HEAD) and limits.budget(name) (one of BUDGETS).*/--function createActorLimits({ config, now = () => Date.now(), registry = null, log = console }) {++function createActorLimits({ config, now = () => Date.now(), registry = null, log = console, valkey = null }) {const refused = registry? registry.counter({ name: 'news_rate_limited_total', help: 'Requests refused 429 by a per-actor limit, by limit name and window', labelNames: ['limit', 'window'] }): null;@@@ -91,6 +91,8 @@ function createActorLimits({ config, now = () => Date.now(), registry = null, lolimits: { minute: config.limits.minute, hour: config.limits.hour },actor,now,++ // Shared across processes on Valkey (ADR-035) when VALKEY_URL is set; in-process otherwise.++ ...(valkey ? { store: createValkeyLimitStore(valkey) } : {}),onLimited(e) {// The actor is a subject id, a principal or an address, never a token.log.warn(`[News] limit ${e.name}: ${e.actor} refused, over ${e.limit} per ${e.window}`);@@@ -87,181 +87,181 @@ function createApi(ctx) {/** Reads for editors: a service passed its capability guard; a browser must be an editor. */const editorRead = (req) => { if (req.viewer.kind !== 'service') access.requireEditor(config, req.viewer); };-- function clusterDto(c, { full = false } = {}) {++ async function clusterDto(c, { full = false } = {}) {const out = {id: c.id, label: c.label, status: c.status, merged_into: c.merged_into, split_from: c.split_from,window: { start: iso(c.window_start), end: iso(c.window_end) }, created_by: c.created_by,key_terms: Object.entries(parse(c.terms, {})).sort(([a, x], [b, y]) => (y - x) || (a < b ? -1 : 1)).slice(0, 15).map(([t, n]) => ({ term: t, items: n })),key_entities: Object.entries(parse(c.entities, {})).sort(([a, x], [b, y]) => (y - x) || (a < b ? -1 : 1)).slice(0, 10).map(([t, n]) => ({ entity: t, items: n })),-- item_count: clusters.members(c.id).length,++ item_count: (await clusters.members(c.id)).length,};-- if (full) Object.assign(out, { items: clusters.members(c.id).map(itemDto), audit: clusters.audit(c.id), stories: stories.byCluster(c.id).map((s) => ({ id: s.id, slug: s.slug, state: s.state })) });++ if (full) Object.assign(out, { items: (await clusters.members(c.id)).map(itemDto), audit: await clusters.audit(c.id), stories: (await stories.byCluster(c.id)).map((s) => ({ id: s.id, slug: s.slug, state: s.state })) });return out;}-- function storyDto(story) {-- const head = stories.head(story);-- const topic = publication.topicOf(story);++ async function storyDto(story) {++ const head = await stories.head(story);++ const topic = await publication.topicOf(story);return {id: story.id, slug: story.slug, url: publication.storyUrl(story), state: story.state, cluster_id: story.cluster_id,topic: topic ? topic.slug : null, working_headline: story.working_headline, noindex: Boolean(story.noindex),published_revision: story.published_revision, first_published_at: iso(story.first_published_at), published_at: iso(story.published_at), retracted_at: iso(story.retracted_at),head: head ? { revision: head.number, headline: head.fields.headline, paragraphs: head.fields.paragraphs, authorship: head.meta.authorship || null, system: head.meta.system || null, gaps: head.meta.gaps || null, author: head.author, created_at: head.createdAt } : null,-- publishable: head ? stories.problems(story, head) : [{ code: 'story.no_text', detail: 'Nothing has been written yet' }],-- indexability: story.published_revision ? (() => { const d = publication.decide(story, stories.revision(story, story.published_revision)); return { indexable: d.indexable, robots: d.robots, reasons: d.reasons }; })() : null,-- sources: stories.sources(story, { includeDetached: true }).map((s) => ({ n: s.n, perspective_id: s.perspective_id, detached_at: iso(s.detached_at), item: itemDto(s.item) })),-- timeline: stories.timeline(story).map((t) => ({ id: t.id, occurred_on: t.occurred_on, text: t.text, source_item_id: t.source_item_id })),-- perspectives: stories.perspectives(story).map((p) => ({ id: p.id, label: p.label, description: p.description })),-- flags: stories.flags(story).map((f) => ({ id: f.id, kind: f.kind, status: f.status, note: f.note, source_item_id: f.source_item_id, pending_revision: f.pending_revision, revision: f.revision, created_by: f.created_by, created_at: iso(f.created_at), resolved_at: iso(f.resolved_at) })),++ publishable: head ? await stories.problems(story, head) : [{ code: 'story.no_text', detail: 'Nothing has been written yet' }],++ indexability: story.published_revision ? (async () => { const d = await publication.decide(story, await stories.revision(story, story.published_revision)); return { indexable: d.indexable, robots: d.robots, reasons: d.reasons }; })() : null,++ sources: (await stories.sources(story, { includeDetached: true })).map((s) => ({ n: s.n, perspective_id: s.perspective_id, detached_at: iso(s.detached_at), item: itemDto(s.item) })),++ timeline: (await stories.timeline(story)).map((t) => ({ id: t.id, occurred_on: t.occurred_on, text: t.text, source_item_id: t.source_item_id })),++ perspectives: (await stories.perspectives(story)).map((p) => ({ id: p.id, label: p.label, description: p.description })),++ flags: (await stories.flags(story)).map((f) => ({ id: f.id, kind: f.kind, status: f.status, note: f.note, source_item_id: f.source_item_id, pending_revision: f.pending_revision, revision: f.revision, created_by: f.created_by, created_at: iso(f.created_at), resolved_at: iso(f.resolved_at) })),};}-- const mustStory = (req) => stories.mustGet(req.params.id);++ const mustStory = async (req) => await stories.mustGet(req.params.id);// ── Topics ──────────────────────────────────────────────-- router.get('/topics', run(() => ({ topics: topics.list().map((t) => ({ id: t.id, slug: t.slug, name: t.name, description: t.description, url: publication.abs(publication.topicPath(t)) })) })));++ router.get('/topics', run(async () => ({ topics: (await topics.list()).map((t) => ({ id: t.id, slug: t.slug, name: t.name, description: t.description, url: publication.abs(publication.topicPath(t)) })) })));-- router.post('/topics', guard('news.topic.manage'), B('news.topic.manage'), jsonBody, run((req) => {++ router.post('/topics', guard('news.topic.manage'), B('news.topic.manage'), jsonBody, run(async (req) => {access.requireEditor(config, req.viewer);const b = req.body || {};-- const r = topics.create({ name: b.name, slug: b.slug, description: b.description });++ const r = await topics.create({ name: b.name, slug: b.slug, description: b.description });return { topic: r.topic, created: r.created };}, (out) => (out.created ? 201 : 200)));// ── Stories ─────────────────────────────────────────────router.get('/stories', run(async (req) => {-- const topic = req.query.topic ? topics.bySlug(req.query.topic) : null;++ const topic = req.query.topic ? await topics.bySlug(req.query.topic) : null;if (req.query.topic && !topic) throw new ApiError(404, 'topic.not_found', 'No such topic');const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 20, 1), 100);const offset = Math.max(parseInt(req.query.offset, 10) || 0, 0);-- const { total, stories: rows } = stories.listPublished({ topicId: topic ? topic.id : null, limit, offset });++ const { total, stories: rows } = await stories.listPublished({ topicId: topic ? topic.id : null, limit, offset });const items = await reading.listItems(rows);return { total, stories: items.map((i) => ({ id: i.story.id, url: publication.abs(i.url), headline: i.headline, state: i.story.state, topic: i.topic ? i.topic.name : null, sources: i.sourceCount, first_published_at: iso(i.published_at), updated_at: i.updated_at })) };}));router.get('/stories/:id', run(async (req) => {-- const story = mustStory(req);++ const story = await mustStory(req);const privileged = req.viewer.kind === 'service'? contracts.capabilities.grants(req.viewer.claims && req.viewer.claims.cap, 'news.story.read'): access.isEditor(config, req.viewer);if (privileged) {const m = story.published_revision ? await reading.storyModel(story) : null;-- return { story: storyDto(story), published: m ? reading.storyJson(m) : null };++ return { story: await storyDto(story), published: m ? reading.storyJson(m) : null };}if (!story.published_revision || (story.state !== 'published' && story.state !== 'retracted')) throw new ApiError(404, 'story.not_found', 'No such story');return { published: reading.storyJson(await reading.storyModel(story)) };}));-- router.post('/stories', guard('news.story.create'), B('news.story.create'), jsonBody, run((req) => {++ router.post('/stories', guard('news.story.create'), B('news.story.create'), jsonBody, run(async (req) => {const b = req.body || {};-- const out = stories.create(req.viewer, { cluster: b.cluster || b.cluster_id, headline: b.headline, topic: b.topic, slug: b.slug, body: b.body, paragraphs: b.paragraphs }, tp(req));-- return { story: storyDto(out.story), revision: out.revision ? out.revision.number : null };++ const out = await stories.create(req.viewer, { cluster: b.cluster || b.cluster_id, headline: b.headline, topic: b.topic, slug: b.slug, body: b.body, paragraphs: b.paragraphs }, tp(req));++ return { story: await storyDto(out.story), revision: out.revision ? out.revision.number : null };}, 201));-- router.get('/stories/:id/revisions', guard('news.story.read'), run((req) => {++ router.get('/stories/:id/revisions', guard('news.story.read'), run(async (req) => {editorRead(req);-- const story = mustStory(req);-- return { revisions: stories.revisions(story, { limit: req.query.limit, before: req.query.before }).map((r) => ({ revision: r.number, kind: r.kind, author: r.author, message: r.message, created_at: r.createdAt, headline: r.fields.headline, authorship: r.meta.authorship || null, system: r.meta.system || null })) };++ const story = await mustStory(req);++ return { revisions: (await stories.revisions(story, { limit: req.query.limit, before: req.query.before })).map((r) => ({ revision: r.number, kind: r.kind, author: r.author, message: r.message, created_at: r.createdAt, headline: r.fields.headline, authorship: r.meta.authorship || null, system: r.meta.system || null })) };}));-- router.get('/stories/:id/revisions/:n', guard('news.story.read'), run((req) => {++ router.get('/stories/:id/revisions/:n', guard('news.story.read'), run(async (req) => {editorRead(req);-- const story = mustStory(req);-- const rev = stories.revision(story, parseInt(req.params.n, 10));++ const story = await mustStory(req);++ const rev = await stories.revision(story, parseInt(req.params.n, 10));if (!rev) throw new ApiError(404, 'revision.not_found', 'No such revision');-- return { revision: rev, citations: ctx.store.citations.forRevision(story.id, rev.number), problems: stories.problems(story, rev) };++ return { revision: rev, citations: await ctx.store.citations.forRevision(story.id, rev.number), problems: await stories.problems(story, rev) };}));-- router.get('/stories/:id/diff', guard('news.story.read'), limits.readBudget('news.story.diff'), run((req) => {++ router.get('/stories/:id/diff', guard('news.story.read'), limits.readBudget('news.story.diff'), run(async (req) => {editorRead(req);-- return { diff: stories.diff(mustStory(req), parseInt(req.query.from, 10), parseInt(req.query.to, 10), req.query.mode) };++ return { diff: await stories.diff(await mustStory(req), parseInt(req.query.from, 10), parseInt(req.query.to, 10), req.query.mode) };}));// guards: the capability guard, then the route's per-actor budget (both before the body is read).-- const write = (method, path, guards, fn, status = 200) => router[method](path, guards, jsonBody, run(async (req) => fn(req, mustStory(req), req.body || {}), status));++ const write = (method, path, guards, fn, status = 200) => router[method](path, guards, jsonBody, run(async (req) => fn(req, await mustStory(req), req.body || {}), status));-- write('post', '/stories/:id/revisions', [guard('news.story.revise'), B('news.story.revise')], (req, story, b) => {-- const out = stories.revise(req.viewer, story, {++ write('post', '/stories/:id/revisions', [guard('news.story.revise'), B('news.story.revise')], async (req, story, b) => {++ const out = await stories.revise(req.viewer, story, {headline: b.headline, body: b.body, paragraphs: b.paragraphs, expectedRevision: b.expected_revision ?? b.expectedRevision,topic: b.topic, noindex: b.noindex, message: b.message, authorship: b.authorship, gaps: b.gaps,}, tp(req));-- return { revision: out.revision.number, created: out.created, story: storyDto(out.story) };++ return { revision: out.revision.number, created: out.created, story: await storyDto(out.story) };}, 201);write('post', '/stories/:id/ai-drafts', [guard('news.story.revise'), B('news.story.ai_draft')], async (req, story, b) => {const out = await stories.aiDraft(req.viewer, story, String(b.workflow || 'news.summarize_story'), tp(req));-- return { revision: out.revision.number, created: out.created, story: storyDto(out.story) };++ return { revision: out.revision.number, created: out.created, story: await storyDto(out.story) };}, 201);-- write('post', '/stories/:id/publish', [guard('news.story.publish'), B('news.story.publish')], (req, story, b) => {-- const out = stories.publish(req.viewer, story, { revision: b.revision, correction: b.correction }, tp(req));-- return { changed: out.changed, resolved_flags: out.resolved || [], story: storyDto(out.story) };++ write('post', '/stories/:id/publish', [guard('news.story.publish'), B('news.story.publish')], async (req, story, b) => {++ const out = await stories.publish(req.viewer, story, { revision: b.revision, correction: b.correction }, tp(req));++ return { changed: out.changed, resolved_flags: out.resolved || [], story: await storyDto(out.story) };});-- write('post', '/stories/:id/unpublish', [guard('news.story.publish'), B('news.story.publish')], (req, story) => {-- const out = stories.unpublish(req.viewer, story, tp(req));-- return { changed: out.changed, story: storyDto(out.story) };++ write('post', '/stories/:id/unpublish', [guard('news.story.publish'), B('news.story.publish')], async (req, story) => {++ const out = await stories.unpublish(req.viewer, story, tp(req));++ return { changed: out.changed, story: await storyDto(out.story) };});-- write('post', '/stories/:id/retract', [guard('news.story.retract'), B('news.story.publish')], (req, story, b) => {-- const out = stories.retract(req.viewer, story, { note: b.note }, tp(req));-- return { changed: out.changed, story: storyDto(out.story) };++ write('post', '/stories/:id/retract', [guard('news.story.retract'), B('news.story.publish')], async (req, story, b) => {++ const out = await stories.retract(req.viewer, story, { note: b.note }, tp(req));++ return { changed: out.changed, story: await storyDto(out.story) };});-- router.post('/stories/:id/reviews', B('news.story.publish'), jsonBody, run((req) => {-- const story = mustStory(req);++ router.post('/stories/:id/reviews', B('news.story.publish'), jsonBody, run(async (req) => {++ const story = await mustStory(req);const b = req.body || {};-- return { review: stories.review(req.viewer, story, { revision: b.revision, decision: b.decision, note: b.note }, tp(req)) };++ return { review: await stories.review(req.viewer, story, { revision: b.revision, decision: b.decision, note: b.note }, tp(req)) };}, 201));-- write('post', '/stories/:id/flags', [guard('news.story.revise'), B('news.story.annotate')], (req, story, b) => ({ flag: stories.addFlag(req.viewer, story, { kind: b.kind, note: b.note }) }), 201);-- write('post', '/stories/:id/sources', [guard('news.source.attach'), B('news.source.attach')], (req, story, b) => {-- const out = stories.attach(req.viewer, story, { item: b.item || b.item_id });++ write('post', '/stories/:id/flags', [guard('news.story.revise'), B('news.story.annotate')], async (req, story, b) => ({ flag: await stories.addFlag(req.viewer, story, { kind: b.kind, note: b.note }) }), 201);++ write('post', '/stories/:id/sources', [guard('news.source.attach'), B('news.source.attach')], async (req, story, b) => {++ const out = await stories.attach(req.viewer, story, { item: b.item || b.item_id });return { n: out.n, created: out.created, item: itemDto(out.item) };}, 201);-- write('delete', '/stories/:id/sources/:item', [guard('news.source.attach'), B('news.source.attach')], (req, story) => stories.detach(req.viewer, story, req.params.item));-- write('post', '/stories/:id/perspectives', [guard('news.perspective.update'), B('news.story.annotate')], (req, story, b) => ({ perspective: stories.addPerspective(req.viewer, story, { label: b.label, description: b.description }) }), 201);-- write('delete', '/stories/:id/perspectives/:pid', [guard('news.perspective.update'), B('news.story.annotate')], (req, story) => stories.removePerspective(req.viewer, story, req.params.pid));-- write('put', '/stories/:id/sources/:item/perspective', [guard('news.perspective.update'), B('news.story.annotate')], (req, story, b) => stories.assignPerspective(req.viewer, story, req.params.item, b.perspective || null));-- write('post', '/stories/:id/timeline', [guard('news.timeline.update'), B('news.story.annotate')], (req, story, b) => ({ entry: stories.addTimeline(req.viewer, story, { occurredOn: b.occurred_on ?? b.occurredOn, text: b.text, source: b.source }) }), 201);-- write('delete', '/stories/:id/timeline/:eid', [guard('news.timeline.update'), B('news.story.annotate')], (req, story) => stories.removeTimeline(req.viewer, story, req.params.eid));++ write('delete', '/stories/:id/sources/:item', [guard('news.source.attach'), B('news.source.attach')], async (req, story) => await stories.detach(req.viewer, story, req.params.item));++ write('post', '/stories/:id/perspectives', [guard('news.perspective.update'), B('news.story.annotate')], async (req, story, b) => ({ perspective: await stories.addPerspective(req.viewer, story, { label: b.label, description: b.description }) }), 201);++ write('delete', '/stories/:id/perspectives/:pid', [guard('news.perspective.update'), B('news.story.annotate')], async (req, story) => await stories.removePerspective(req.viewer, story, req.params.pid));++ write('put', '/stories/:id/sources/:item/perspective', [guard('news.perspective.update'), B('news.story.annotate')], async (req, story, b) => await stories.assignPerspective(req.viewer, story, req.params.item, b.perspective || null));++ write('post', '/stories/:id/timeline', [guard('news.timeline.update'), B('news.story.annotate')], async (req, story, b) => ({ entry: await stories.addTimeline(req.viewer, story, { occurredOn: b.occurred_on ?? b.occurredOn, text: b.text, source: b.source }) }), 201);++ write('delete', '/stories/:id/timeline/:eid', [guard('news.timeline.update'), B('news.story.annotate')], async (req, story) => await stories.removeTimeline(req.viewer, story, req.params.eid));// ── Clusters, items, ingestion ──────────────────────────-- router.get('/clusters', guard('news.cluster.read'), run((req) => {++ router.get('/clusters', guard('news.cluster.read'), run(async (req) => {editorRead(req);const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 50, 1), 200);-- return { clusters: clusters.recent({ limit }).map((c) => clusterDto(c)) };++ return { clusters: await Promise.all((await clusters.recent({ limit })).map((c) => clusterDto(c))) };}));-- router.get('/clusters/:id', guard('news.cluster.read'), run((req) => {++ router.get('/clusters/:id', guard('news.cluster.read'), run(async (req) => {editorRead(req);-- const c = clusters.get(req.params.id);++ const c = await clusters.get(req.params.id);if (!c) throw new ApiError(404, 'cluster.not_found', 'No such cluster');-- return { cluster: clusterDto(c, { full: true }) };++ return { cluster: await clusterDto(c, { full: true }) };}));-- router.post('/clusters/audit/:auditId/reverse', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run((req) => {++ router.post('/clusters/audit/:auditId/reverse', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run(async (req) => {access.requireEditor(config, req.viewer);-- const out = clusters.reverse(req.viewer, req.params.auditId, { reason: (req.body || {}).reason });-- return { audit: out.audit, clusters: out.clusters.map((c) => clusterDto(c)) };++ const out = await clusters.reverse(req.viewer, req.params.auditId, { reason: (req.body || {}).reason });++ return { audit: out.audit, clusters: await Promise.all(out.clusters.map((c) => clusterDto(c))) };}));-- router.post('/clusters/:id/merge', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run((req) => {++ router.post('/clusters/:id/merge', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run(async (req) => {access.requireEditor(config, req.viewer);const b = req.body || {};-- const out = clusters.merge(req.viewer, req.params.id, b.other, { reason: b.reason });-- return { cluster: clusterDto(out.cluster, { full: true }), audit: out.audit };++ const out = await clusters.merge(req.viewer, req.params.id, b.other, { reason: b.reason });++ return { cluster: await clusterDto(out.cluster, { full: true }), audit: out.audit };}));-- router.post('/clusters/:id/split', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run((req) => {++ router.post('/clusters/:id/split', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run(async (req) => {access.requireEditor(config, req.viewer);const b = req.body || {};-- const out = clusters.split(req.viewer, req.params.id, b.items, { reason: b.reason });-- return { cluster: clusterDto(out.cluster), created: clusterDto(out.created, { full: true }), audit: out.audit };++ const out = await clusters.split(req.viewer, req.params.id, b.items, { reason: b.reason });++ return { cluster: await clusterDto(out.cluster), created: await clusterDto(out.created, { full: true }), audit: out.audit };}, 201));-- router.get('/source-items/:id', guard('news.cluster.read'), run((req) => {++ router.get('/source-items/:id', guard('news.cluster.read'), run(async (req) => {editorRead(req);-- const it = ingest.get(req.params.id) || ingest.bySourcesId(req.params.id);++ const it = await ingest.get(req.params.id) || await ingest.bySourcesId(req.params.id);if (!it) throw new ApiError(404, 'source_item.not_found', 'No such source item');return { item: itemDto(it) };}));-- router.get('/ingest', guard('news.cluster.read'), run((req) => {++ router.get('/ingest', guard('news.cluster.read'), run(async (req) => {editorRead(req);return {-- cursor: ctx.store.getState('sources_cursor', 0),-- runs: ingest.runs({ limit: 50 }).map((r) => ({ ...r, counts: parse(r.counts, null), at: iso(r.at) })),-- sources: ingest.sourceStatus().map((s) => ({ ...s, stale: s.stale == null ? null : Boolean(s.stale), refreshed_at: iso(s.refreshed_at) })),++ cursor: await ctx.store.getState('sources_cursor', 0),++ runs: (await ingest.runs({ limit: 50 })).map((r) => ({ ...r, counts: parse(r.counts, null), at: iso(r.at) })),++ sources: (await ingest.sourceStatus()).map((s) => ({ ...s, stale: s.stale == null ? null : Boolean(s.stale), refreshed_at: iso(s.refreshed_at) })),};}));@@@ -22,12 +22,13 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {const { db } = store;const abs = (p) => seo.canonicalUrl(config.baseUrl, p);-- function storyEntries() {-- return db.prepare("SELECT * FROM news_stories WHERE state IN ('published','retracted') AND published_revision IS NOT NULL ORDER BY published_at DESC LIMIT 50000").all()-- .map((story) => {-- const rev = store.revisions.get(story.id, story.published_revision);-- return { story, rev, decision: publication.decide(story, rev) };-- });++ async function storyEntries() {++ // Revisions in one query, whatever the number of stories (no N+1).++ const rows = await db.prepare("SELECT * FROM news_stories WHERE state IN ('published','retracted') AND published_revision IS NOT NULL ORDER BY published_at DESC, id LIMIT 50000").all();++ const revs = await store.revisions.getMany(rows.map((s) => ({ entityId: s.id, revision: s.published_revision })));++ const out = [];++ for (const [i, story] of rows.entries()) out.push({ story, rev: revs[i], decision: await publication.decide(story, revs[i]) });++ return out;}const xml = (res, body) => res.type('application/xml').set('Cache-Control', 'public, max-age=300').send(body);@@@ -64,8 +65,8 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {}));});-- router.get('/sitemap.xml', (_req, res) => {-- const entries = storyEntries().filter((e) => e.decision.indexable);++ router.get('/sitemap.xml', async (_req, res) => {++ const entries = (await storyEntries()).filter((e) => e.decision.indexable);const newest = entries.length ? entries.map((e) => e.rev.createdAt).sort().pop() : null;xml(res, seo.sitemapIndex([{ loc: abs('/sitemaps/stories.xml'), ...(newest ? { lastmod: newest } : {}) },@@@ -73,20 +74,20 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {]));});-- router.get('/sitemaps/stories.xml', (_req, res) => {-- xml(res, seo.sitemap(storyEntries().map((e) => ({ loc: publication.storyUrl(e.story), lastmod: e.rev.createdAt, decision: e.decision }))).files[0]);++ router.get('/sitemaps/stories.xml', async (_req, res) => {++ xml(res, seo.sitemap((await storyEntries()).map((e) => ({ loc: publication.storyUrl(e.story), lastmod: e.rev.createdAt, decision: e.decision }))).files[0]);});-- router.get('/sitemaps/topics.xml', (_req, res) => {++ router.get('/sitemaps/topics.xml', async (_req, res) => {const byTopic = new Map();-- for (const e of storyEntries()) {++ for (const e of await storyEntries()) {if (!e.decision.indexable || !e.story.topic_id) continue;const t = e.rev.createdAt;if (!byTopic.has(e.story.topic_id) || byTopic.get(e.story.topic_id) < t) byTopic.set(e.story.topic_id, t);}const entries = [];for (const [id, lastmod] of byTopic) {-- const topic = topics.byId(id);++ const topic = await topics.byId(id);if (!topic || topic.status !== 'active') continue;const path = publication.topicPath(topic);const decision = seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: abs(path), wordCount: 0 }, { policy: { minWords: 0 }, now: store.now() });@@@ -77,12 +77,12 @@ function createEditorRoutes(ctx) {// ── Dashboard ───────────────────────────────────────────router.get('/edit', wrap(async (req, res) => {-- const flags = store.db.prepare(`SELECT f.*, s.working_headline FROM news_editorial_flags f JOIN news_stories s ON s.id = f.story_id++ const flags = await store.db.prepare(`SELECT f.*, s.working_headline FROM news_editorial_flags f JOIN news_stories s ON s.id = f.story_idWHERE f.status = 'open' ORDER BY f.created_at DESC LIMIT 100`).all();page(req, res, 'Editor desk', editor.dashboard({-- clusters: clusters.recent({ limit: 40 }).map((c) => ({ ...c, count: clusters.members(c.id).length })),-- stories: stories.listAll({ limit: 100 }), flags, runs: ingest.runs({ limit: 20 }), sources: ingest.sourceStatus(),-- cursor: store.getState('sources_cursor', 0), csrf: csrf(req), message: flashOf(req), aiEnabled: Boolean(ai && ai.enabled),++ clusters: await Promise.all((await clusters.recent({ limit: 40 })).map(async (c) => ({ ...c, count: (await clusters.members(c.id)).length }))),++ stories: await stories.listAll({ limit: 100 }), flags, runs: await ingest.runs({ limit: 20 }), sources: await ingest.sourceStatus(),++ cursor: await store.getState('sources_cursor', 0), csrf: csrf(req), message: flashOf(req), aiEnabled: Boolean(ai && ai.enabled),pullOn: Boolean(config.worker.enabled && config.sources.pullIntervalMs),}));}));@@@ -92,114 +92,114 @@ function createEditorRoutes(ctx) {done(res, '/edit', r.ok ? `Pulled: ${JSON.stringify(r.counts)}` : `Pull failed (${r.error}): ${r.detail}`);}));-- router.get('/edit/topics', wrap(async (req, res) => page(req, res, 'Topics', editor.topicsPage({ topics: topics.list(), csrf: csrf(req), message: flashOf(req) }))));++ router.get('/edit/topics', wrap(async (req, res) => page(req, res, 'Topics', editor.topicsPage({ topics: await topics.list(), csrf: csrf(req), message: flashOf(req) }))));router.post('/edit/topics', B('news.topic.manage'), wrap(async (req, res) => {const b = input(req.body);-- const r = topics.create({ name: b.name, description: b.description });++ const r = await topics.create({ name: b.name, description: b.description });done(res, '/edit/topics', r.created ? `Added ${r.topic.name}` : `${r.topic.name} already exists`);}));// ── Clusters ────────────────────────────────────────────-- function mustCluster(req, res) {-- const c = clusters.get(req.params.id);++ async function mustCluster(req, res) {++ const c = await clusters.get(req.params.id);if (!c) { publicRoutes.notFound(req, res); return null; }return c;}router.get('/clusters/:id', wrap(async (req, res) => {-- const c = mustCluster(req, res);++ const c = await mustCluster(req, res);if (!c) return;page(req, res, `Cluster: ${c.label}`, editor.clusterPage({-- cluster: c, items: clusters.members(c.id), audit: clusters.audit(c.id),-- others: clusters.recent({ limit: 100 }).filter((o) => o.id !== c.id), storiesOf: stories.byCluster(c.id),-- topics: topics.list(), csrf: csrf(req), message: flashOf(req),++ cluster: c, items: await clusters.members(c.id), audit: await clusters.audit(c.id),++ others: (await clusters.recent({ limit: 100 })).filter((o) => o.id !== c.id), storiesOf: await stories.byCluster(c.id),++ topics: await topics.list(), csrf: csrf(req), message: flashOf(req),}));}));router.post('/clusters/:id/merge', B('news.cluster.manage'), wrap(async (req, res) => {const b = input(req.body);-- clusters.merge(req.viewer, req.params.id, b.other, { reason: b.reason });++ await clusters.merge(req.viewer, req.params.id, b.other, { reason: b.reason });done(res, `/clusters/${req.params.id}`, `Merged ${b.other} into this cluster`);}));router.post('/clusters/:id/split', B('news.cluster.manage'), wrap(async (req, res) => {const items = [].concat((req.body && req.body.items) || []);-- const out = clusters.split(req.viewer, req.params.id, items, { reason: one(req.body.reason) });++ const out = await clusters.split(req.viewer, req.params.id, items, { reason: one(req.body.reason) });done(res, `/clusters/${out.created.id}`, `Split ${items.length} items off ${req.params.id}`);}));router.post('/clusters/audit/:aid/reverse', B('news.cluster.manage'), wrap(async (req, res) => {-- const out = clusters.reverse(req.viewer, req.params.aid, { reason: one(req.body.reason) });++ const out = await clusters.reverse(req.viewer, req.params.aid, { reason: one(req.body.reason) });done(res, `/clusters/${out.clusters[0].id}`, `Reversed (${out.audit.action.replace('_', ' ')})`);}));router.post('/clusters/:id/stories', B('news.story.create'), wrap(async (req, res) => {const b = input(req.body);-- const out = stories.create(req.viewer, { cluster: req.params.id, headline: b.headline, topic: b.topic || null }, tp(req));++ const out = await stories.create(req.viewer, { cluster: req.params.id, headline: b.headline, topic: b.topic || null }, tp(req));done(res, `/edit/stories/${out.story.id}`, 'Story opened. Write the text; every paragraph cites its sources.');}));// ── Stories ─────────────────────────────────────────────-- function mustStory(req, res) {-- const s = stories.get(req.params.id);++ async function mustStory(req, res) {++ const s = await stories.get(req.params.id);if (!s) { publicRoutes.notFound(req, res); return null; }return s;}router.get('/edit/stories/:id', wrap(async (req, res) => {-- const story = mustStory(req, res);++ const story = await mustStory(req, res);if (!story) return;-- const head = stories.head(story);++ const head = await stories.head(story);page(req, res, story.working_headline, editor.storyEditor({-- story, head, problems: head ? stories.problems(story, head) : [],-- sources: stories.sources(story, { includeDetached: true }), timeline: stories.timeline(story), perspectives: stories.perspectives(story),-- flags: stories.flags(story), revisions: stories.revisions(story, { limit: 100 }), topics: topics.list(), topic: publication.topicOf(story),++ story, head, problems: head ? await stories.problems(story, head) : [],++ sources: await stories.sources(story, { includeDetached: true }), timeline: await stories.timeline(story), perspectives: await stories.perspectives(story),++ flags: await stories.flags(story), revisions: await stories.revisions(story, { limit: 100 }), topics: await topics.list(), topic: await publication.topicOf(story),csrf: csrf(req), message: flashOf(req), aiEnabled: Boolean(ai && ai.enabled), publicUrl: publication.storyPath(story),-- decision: story.published_revision ? publication.decide(story, stories.revision(story, story.published_revision)) : null,++ decision: story.published_revision ? await publication.decide(story, await stories.revision(story, story.published_revision)) : null,}));}));router.get('/edit/stories/:id/preview', wrap(async (req, res) => {-- const story = mustStory(req, res);++ const story = await mustStory(req, res);if (!story) return;const n = parseInt(req.query.revision, 10);-- const rev = Number.isInteger(n) ? stories.revision(story, n) : stories.head(story);++ const rev = Number.isInteger(n) ? await stories.revision(story, n) : await stories.head(story);if (!rev) return publicRoutes.notFound(req, res);await publicRoutes.renderStory(req, res, { story, rev, preview: true });return undefined;}));router.get('/edit/stories/:id/diff', B('news.story.diff'), wrap(async (req, res) => {-- const story = mustStory(req, res);++ const story = await mustStory(req, res);if (!story) return;-- page(req, res, 'Diff', editor.diffPage({ story, diff: stories.diff(story, parseInt(req.query.from, 10), parseInt(req.query.to, 10), 'word') }));++ page(req, res, 'Diff', editor.diffPage({ story, diff: await stories.diff(story, parseInt(req.query.from, 10), parseInt(req.query.to, 10), 'word') }));}));const act = (path, budget, fn) => router.post(`/edit/stories/:id/${path}`, B(budget), wrap(async (req, res) => {-- const story = mustStory(req, res);++ const story = await mustStory(req, res);if (!story) return;const text = await fn(req, story, input(req.body));done(res, `/edit/stories/${story.id}`, text);}));-- act('revise', 'news.story.revise', (req, story, b) => {-- const out = stories.revise(req.viewer, story, { headline: b.headline, body: b.body, expectedRevision: b.expectedRevision, topic: b.topic, noindex: b.noindex, message: b.message }, tp(req));++ act('revise', 'news.story.revise', async (req, story, b) => {++ const out = await stories.revise(req.viewer, story, { headline: b.headline, body: b.body, expectedRevision: b.expectedRevision, topic: b.topic, noindex: b.noindex, message: b.message }, tp(req));return out.created ? `Saved revision ${out.revision.number}` : 'Nothing changed';});-- act('publish', 'news.story.publish', (req, story, b) => {++ act('publish', 'news.story.publish', async (req, story, b) => {const correction = b.correctionKind || b.correctionNote ? { kind: b.correctionKind || 'correction', note: b.correctionNote } : undefined;-- const out = stories.publish(req.viewer, story, { revision: b.revision, correction }, tp(req));++ const out = await stories.publish(req.viewer, story, { revision: b.revision, correction }, tp(req));return out.changed ? `Published revision ${out.story.published_revision}` : 'Already published';});-- act('unpublish', 'news.story.publish', (req, story) => (stories.unpublish(req.viewer, story, tp(req)).changed ? 'Unpublished' : 'Not published'));-- act('retract', 'news.story.publish', (req, story, b) => (stories.retract(req.viewer, story, { note: b.note }, tp(req)).changed ? 'Retracted' : 'Already retracted'));-- act('review', 'news.story.publish', (req, story, b) => { stories.review(req.viewer, story, { revision: b.revision, decision: b.decision, note: b.note }, tp(req)); return `Review recorded for revision ${b.revision}`; });-- act('flag', 'news.story.annotate', (req, story, b) => { stories.addFlag(req.viewer, story, { kind: b.kind, note: b.note }); return 'Note added; it is published with the next publication'; });-- act('attach', 'news.source.attach', (req, story, b) => `Attached as [${stories.attach(req.viewer, story, { item: b.item }).n}]`);-- act('detach', 'news.source.attach', (req, story, b) => `Detached [${stories.detach(req.viewer, story, b.item).n}]`);-- act('assign', 'news.story.annotate', (req, story, b) => { stories.assignPerspective(req.viewer, story, b.item, b.perspective || null); return 'Perspective set'; });-- act('perspective', 'news.story.annotate', (req, story, b) => { stories.addPerspective(req.viewer, story, { label: b.label, description: b.description }); return 'Perspective added'; });-- act('perspective/remove', 'news.story.annotate', (req, story, b) => { stories.removePerspective(req.viewer, story, b.perspective); return 'Perspective removed'; });-- act('timeline', 'news.story.annotate', (req, story, b) => { stories.addTimeline(req.viewer, story, { occurredOn: b.occurredOn, text: b.text, source: b.source }); return 'Timeline entry added'; });-- act('timeline/remove', 'news.story.annotate', (req, story, b) => { stories.removeTimeline(req.viewer, story, b.entry); return 'Timeline entry removed'; });++ act('unpublish', 'news.story.publish', async (req, story) => ((await stories.unpublish(req.viewer, story, tp(req))).changed ? 'Unpublished' : 'Not published'));++ act('retract', 'news.story.publish', async (req, story, b) => ((await stories.retract(req.viewer, story, { note: b.note }, tp(req))).changed ? 'Retracted' : 'Already retracted'));++ act('review', 'news.story.publish', async (req, story, b) => { await stories.review(req.viewer, story, { revision: b.revision, decision: b.decision, note: b.note }, tp(req)); return `Review recorded for revision ${b.revision}`; });++ act('flag', 'news.story.annotate', async (req, story, b) => { await stories.addFlag(req.viewer, story, { kind: b.kind, note: b.note }); return 'Note added; it is published with the next publication'; });++ act('attach', 'news.source.attach', async (req, story, b) => `Attached as [${(await stories.attach(req.viewer, story, { item: b.item })).n}]`);++ act('detach', 'news.source.attach', async (req, story, b) => `Detached [${(await stories.detach(req.viewer, story, b.item)).n}]`);++ act('assign', 'news.story.annotate', async (req, story, b) => { await stories.assignPerspective(req.viewer, story, b.item, b.perspective || null); return 'Perspective set'; });++ act('perspective', 'news.story.annotate', async (req, story, b) => { await stories.addPerspective(req.viewer, story, { label: b.label, description: b.description }); return 'Perspective added'; });++ act('perspective/remove', 'news.story.annotate', async (req, story, b) => { await stories.removePerspective(req.viewer, story, b.perspective); return 'Perspective removed'; });++ act('timeline', 'news.story.annotate', async (req, story, b) => { await stories.addTimeline(req.viewer, story, { occurredOn: b.occurredOn, text: b.text, source: b.source }); return 'Timeline entry added'; });++ act('timeline/remove', 'news.story.annotate', async (req, story, b) => { await stories.removeTimeline(req.viewer, story, b.entry); return 'Timeline entry removed'; });act('ai', 'news.story.ai_draft', async (req, story, b) => {const out = await stories.aiDraft(req.viewer, story, b.workflow, tp(req));return `AI draft saved as revision ${out.revision.number}: review it before publishing`;@@@ -69,7 +69,7 @@ function createPublicRoutes(ctx) {async function listing(req, res, { topic = null }) {const page = pageNumber(req);const path = topic ? publication.topicPath(topic) : '/';-- const { total, stories: rows } = stories.listPublished({ topicId: topic ? topic.id : null, limit: PER_PAGE, offset: (page - 1) * PER_PAGE });++ const { total, stories: rows } = await stories.listPublished({ topicId: topic ? topic.id : null, limit: PER_PAGE, offset: (page - 1) * PER_PAGE });const pager = ssr.paginate({ page, perPage: PER_PAGE, total, href: (p) => (p === 1 ? path : `${path}?page=${p}`) });if (pager.outOfRange && total) return notFound(req, res);const items = await reading.listItems(rows);@@@ -85,7 +85,7 @@ function createPublicRoutes(ctx) {jsonLd: topic? [seo.structuredData.breadcrumbs(crumbs.map((c) => ({ name: c.name, url: c.url ? publication.abs(c.url) : canonical })))]: [{ '@context': 'https://schema.org', '@type': 'WebSite', name: 'OpenVibe.News', url: publication.abs('/') }],-- body: topic ? pages.topicPage({ topic, items, pager, feeds, breadcrumbs: crumbs }) : pages.home({ items, pager, topics: topics.list(), feeds }),++ body: topic ? pages.topicPage({ topic, items, pager, feeds, breadcrumbs: crumbs }) : pages.home({ items, pager, topics: await topics.list(), feeds }),}, { cacheable: true });}@@@ -95,16 +95,16 @@ function createPublicRoutes(ctx) {decision: pageDecision('/updates'), canonical: `${config.baseUrl}/updates`,body: frame.updatesBody({ service: 'news', siteName: 'OpenVibe.News' }) + `<script src="${ovServe.url('shipped.js')}" defer></script>`,}, { cacheable: true }));-- router.get('/', wrap((req, res) => listing(req, res, {})));++ router.get('/', wrap(async (req, res) => await listing(req, res, {})));router.get('/topics', wrap(async (req, res) => {-- send(req, res, 200, { title: 'Topics', description: 'Topics on OpenVibe.News.', decision: pageDecision('/topics'), canonical: publication.abs('/topics'), body: pages.topicsIndex({ topics: topics.list() }) }, { cacheable: true });++ send(req, res, 200, { title: 'Topics', description: 'Topics on OpenVibe.News.', decision: pageDecision('/topics'), canonical: publication.abs('/topics'), body: pages.topicsIndex({ topics: await topics.list() }) }, { cacheable: true });}));router.get('/topics/:slug', wrap(async (req, res) => {-- const topic = topics.bySlug(req.params.slug);++ const topic = await topics.bySlug(req.params.slug);if (!topic || topic.status !== 'active') return notFound(req, res);-- return listing(req, res, { topic });++ return await listing(req, res, { topic });}));// ── Feeds (published stories only, never viewer-dependent) ─@@@ -123,15 +123,15 @@ function createPublicRoutes(ctx) {if (kind === 'json') return res.type('application/feed+json').send(JSON.stringify(seo.jsonFeed({ title, link, feedUrl, description, language: 'en' }, items)));const listed = items.filter((i) => i.decision.listable);// An empty Atom feed still needs <updated>: the time the topic (or News' topic list) was set up, a real time.-- const updated = listed.length ? null : new Date(topic ? topic.updated_at : (store.db.prepare('SELECT MIN(created_at) AS t FROM news_topics').get().t || store.now())).toISOString();++ const updated = listed.length ? null : new Date(topic ? topic.updated_at : ((await store.db.prepare('SELECT MIN(created_at) AS t FROM news_topics').get()).t || store.now())).toISOString();return res.type('application/atom+xml').send(seo.atomFeed({ title, link, feedUrl, id: `tag:openvibe.news,2026:${topic ? `topic/${topic.id}` : 'all'}`, subtitle: description, ...(updated ? { updated } : {}) }, items));}for (const f of FEEDS) {-- router.get(`/${f.file}`, wrap((req, res) => feed(req, res, f.type, null)));-- router.get(`/topics/:slug/${f.file}`, wrap((req, res) => {-- const topic = topics.bySlug(req.params.slug);++ router.get(`/${f.file}`, wrap(async (req, res) => await feed(req, res, f.type, null)));++ router.get(`/topics/:slug/${f.file}`, wrap(async (req, res) => {++ const topic = await topics.bySlug(req.params.slug);if (!topic || topic.status !== 'active') return notFound(req, res);-- return feed(req, res, f.type, topic);++ return await feed(req, res, f.type, topic);}));}@@@ -139,7 +139,7 @@ function createPublicRoutes(ctx) {/** The story's Community thread as the page shows it: { state: ok|closed|off|unavailable, … }. */async function commentsFor(req, story, headline) {-- const st = discussion.status(story);++ const st = await discussion.status(story);if (!st.open) return { state: 'closed', reason: st.reason };if (!community.enabled) return { state: 'off' };try {@@@ -159,7 +159,7 @@ function createPublicRoutes(ctx) {const isEditor = access.isEditor(config, req.viewer);const m = await reading.storyModel(story, { rev, editorView: preview && isEditor });const group = ['outlet', 'perspective'].includes(req.query.group) ? req.query.group : 'number';-- const decision = preview ? publication.decide(story, m.rev, { state: 'draft' }) : m.decision;++ const decision = preview ? await publication.decide(story, m.rev, { state: 'draft' }) : m.decision;const crumbs = [{ name: 'OpenVibe.News', url: '/' }, ...(m.topic ? [{ name: m.topic.name, url: publication.topicPath(m.topic) }] : []), { name: m.headline }];const cacheable = !preview && (story.state === 'published' || story.state === 'retracted');const comments = preview ? null : await commentsFor(req, story, m.headline);@@@ -185,7 +185,7 @@ function createPublicRoutes(ctx) {router.get('/stories/:slug', wrap(async (req, res) => {const asJson = req.params.slug.endsWith('.json');const slug = asJson ? req.params.slug.slice(0, -5) : req.params.slug;-- const story = stories.bySlug(slug);++ const story = await stories.bySlug(slug);if (!story || !story.published_revision || (story.state !== 'published' && story.state !== 'retracted')) {if (story && story.state === 'unpublished') return messagePage(req, res, 410, 'Gone', 'This story was unpublished.', { href: '/', label: 'OpenVibe.News' });return notFound(req, res);@@@ -195,23 +195,23 @@ function createPublicRoutes(ctx) {cacheHeaders(res, { cacheable: req.viewer.kind === 'anonymous', robots: m.decision.robots });return res.json(reading.storyJson(m));}-- return renderStory(req, res, { story });++ return await renderStory(req, res, { story });}));// Per address (20 a minute), then per signed-in person (http/actor-limits.js) before the form is read.router.post('/stories/:slug/comments', rateLimit({ windowMs: 60_000, max: 20, standardHeaders: true, legacyHeaders: false }), ctx.limits.signedIn('news.discussion.comment'), express.urlencoded({ extended: false, limit: '32kb' }), wrap(async (req, res) => {-- const story = stories.bySlug(req.params.slug);++ const story = await stories.bySlug(req.params.slug);if (!story || !story.published_revision || (story.state !== 'published' && story.state !== 'retracted')) return notFound(req, res);const path = publication.storyPath(story);if (req.viewer.kind !== 'user' || !req.viewer.subject) return res.redirect(303, `/auth/login?next=${encodeURIComponent(path)}`);if (!checkCsrf(config, req.viewer, req.body && req.body._csrf)) return messagePage(req, res, 403, 'Form expired', 'Reload the story and try again.', { href: path, label: 'Back to the story' });-- if (!discussion.status(story).open) return messagePage(req, res, 409, 'Comments are closed', 'This story does not take comments.', { href: path, label: 'Back to the story' });++ if (!(await discussion.status(story)).open) return messagePage(req, res, 409, 'Comments are closed', 'This story does not take comments.', { href: path, label: 'Back to the story' });if (!community.enabled) return messagePage(req, res, 503, 'Comments are unavailable', 'Comments are not connected on this server.', { href: path, label: 'Back to the story' });const message = String((req.body && req.body.message) || '').trim();if (!message) return res.redirect(303, `${path}#comments`);if (message.length > 5000) return messagePage(req, res, 422, 'Comment too long', 'A comment is at most 5000 characters.', { href: path, label: 'Back to the story' });try {-- const rev = store.revisions.get(story.id, story.published_revision);++ const rev = await store.revisions.get(story.id, story.published_revision);const threadId = await community.threadFor(story, rev && rev.fields.headline, req.ov);if (!threadId) throw new Error('no thread');await community.comment(threadId, req.viewer.subject, { message }, req.ov);@@@ -16,15 +16,14 @@*/const express = require('express');const { http } = require('openvibe-contracts');--const { verifyDeliveryV2, createInbox } = require('openvibe-sdk/events');++const { verifyDeliveryV2, createPgInbox } = require('openvibe-sdk/events');const CONSUMER = 'news-sources';const EVT_RE = /^evt_[0-9A-HJKMNP-TV-Z]{26}$/;function createWebhook({ config, store, ingest, sources, log = console }) {const router = express.Router();-- const inbox = createInbox(store.db, { now: store.now });-- inbox.ensureSchema();++ const inbox = createPgInbox(store.db, { now: store.now }); // idempotency_receipts: migrations/0001_initial.sqlrouter.post('/internal/events', express.raw({ type: () => true, limit: '256kb' }), async (req, res) => {const ctx = req.ov;@@@ -47,24 +46,24 @@ function createWebhook({ config, store, ingest, sources, log = console }) {try {if (!fromSources || !/^sources\.(item\.(created|updated|removed)|fetch\.failed)$/.test(type)) {-- inbox.once(CONSUMER, event.event_id, () => null);++ await inbox.once(CONSUMER, event.event_id, () => null);return res.status(204).end();}if (type === 'sources.item.removed') {-- inbox.once(CONSUMER, event.event_id, () => {-- const r = ingest.applyRemoval({ sourcesItemId: String(p.item_id || ''), reason: p.reason, revision: Number.isInteger(p.revision) ? p.revision : null, origin: 'webhook' });-- ingest.recordRun({ origin: 'webhook', state: r.outcome, source_key: p.source_key || null, sources_item_id: p.item_id || null });++ await inbox.once(CONSUMER, event.event_id, async () => {++ const r = await ingest.applyRemoval({ sourcesItemId: String(p.item_id || ''), reason: p.reason, revision: Number.isInteger(p.revision) ? p.revision : null, origin: 'webhook' });++ await ingest.recordRun({ origin: 'webhook', state: r.outcome, source_key: p.source_key || null, sources_item_id: p.item_id || null });return r;});return res.status(204).end();}if (type === 'sources.fetch.failed') {-- inbox.once(CONSUMER, event.event_id, () => ingest.upstreamFailure(p));++ await inbox.once(CONSUMER, event.event_id, () => ingest.upstreamFailure(p));return res.status(204).end();}// created | updatedif (p.category && p.category !== config.sources.category) {-- inbox.once(CONSUMER, event.event_id, () => null);++ await inbox.once(CONSUMER, event.event_id, () => null);return res.status(204).end();}let fetched;@@@ -72,21 +71,21 @@ function createWebhook({ config, store, ingest, sources, log = console }) {if (p.source_key) await ingest.learnOutlets([String(p.source_key)]);fetched = await sources.getItem(String(p.item_id || ''));} catch (err) {-- store.tx(() => {-- ingest.recordRun({ origin: 'webhook', state: 'failed', source_key: p.source_key || null, sources_item_id: p.item_id || null, error_code: err.code || 'sources.error', detail: `${type} ${event.event_id} attempt ${attempt}: ${err.message}` });-- if (attempt === 1) ingest.failedEvent({ origin: 'webhook', sourceKey: p.source_key || null, sourcesItemId: p.item_id || null, state: 'failed', errorCode: err.code || 'sources.error', detail: err.message, httpStatus: err.status || null });++ await store.tx(async () => {++ await ingest.recordRun({ origin: 'webhook', state: 'failed', source_key: p.source_key || null, sources_item_id: p.item_id || null, error_code: err.code || 'sources.error', detail: `${type} ${event.event_id} attempt ${attempt}: ${err.message}` });++ if (attempt === 1) await ingest.failedEvent({ origin: 'webhook', sourceKey: p.source_key || null, sourcesItemId: p.item_id || null, state: 'failed', errorCode: err.code || 'sources.error', detail: err.message, httpStatus: err.status || null });});if (err.status === 404) {// Sources no longer has it: nothing to apply, and retrying cannot help.-- inbox.once(CONSUMER, event.event_id, () => null);++ await inbox.once(CONSUMER, event.event_id, () => null);return res.status(204).end();}return http.sendProblem(res, 503, 'news.sources_unavailable', { detail: 'OpenVibe.Sources could not be read; retry later', ctx });}-- if (fetched.source) ingest.rememberSources({ [fetched.item.source_key]: fetched.source });-- inbox.once(CONSUMER, event.event_id, () => {-- const r = ingest.apply(fetched.item, { origin: 'webhook' });-- ingest.recordRun({ origin: 'webhook', state: r.outcome, source_key: fetched.item.source_key || null, sources_item_id: fetched.item.id, detail: r.reason || null });++ if (fetched.source) await ingest.rememberSources({ [fetched.item.source_key]: fetched.source });++ await inbox.once(CONSUMER, event.event_id, async () => {++ const r = await ingest.apply(fetched.item, { origin: 'webhook' });++ await ingest.recordRun({ origin: 'webhook', state: r.outcome, source_key: fetched.item.source_key || null, sources_item_id: fetched.item.id, detail: r.reason || null });return r;});return res.status(204).end();@@@ -7,11 +7,12 @@*/const { createApp } = require('./app');--const { app, ctx } = createApp();++(async () => {++const { app, ctx } = await createApp();const { config } = ctx;const server = app.listen(config.port, config.host, () => {-- console.log(`[News] ${config.nodeEnv} on http://${config.host}:${config.port} → ${config.baseUrl} (db ${config.dbPath})`);++ console.log(`[News] ${config.nodeEnv} on http://${config.host}:${config.port} → ${config.baseUrl} (db ${ctx.store.db.store})`);console.log(`[News] events relay ${ctx.outbox.enabled ? `on → ${config.events.url}` : 'off (events wait in event_outbox)'}; webhook ${config.events.webhookSecrets.length ? 'on' : 'off (NEWS_EVENTS_SECRET unset)'}; Sources pull ${config.worker.enabled && config.sources.pullIntervalMs ? `every ${config.sources.pullIntervalMs} ms` : 'off'}; AI ${ctx.ai.enabled ? 'on' : 'off'}`);});server.keepAliveTimeout = 65_000;@@@ -23,10 +24,11 @@ function shutdown(signal) {ctx.worker.stop();server.close(async () => {try { await ctx.outbox.stop(); } catch { /* best effort */ }-- try { ctx.store.close(); } catch { /* already closed */ }++ try { await ctx.store.close(); } catch { /* already closed */ }process.exit(0);});setTimeout(() => process.exit(0), 5000).unref();}process.on('SIGTERM', () => shutdown('SIGTERM'));process.on('SIGINT', () => shutdown('SIGINT'));++})().catch((err) => { console.error('[News] failed to start:', err); process.exit(1); });@@@ -16,7 +16,7 @@const { createReadiness, skip } = require('openvibe-shared/ready');const { CHARTER_TABLES } = require('./db');--function createNewsReadiness({ store, auth, outbox, ingest, config, release = null }) {++function createNewsReadiness({ store, auth, outbox, ingest, config, valkey = null, release = null }) {const { db } = store;return createReadiness({service: 'news',@@@ -24,12 +24,16 @@ function createNewsReadiness({ store, auth, outbox, ingest, config, release = nuchecks: [{name: 'db', required: true,-- check: () => {-- const names = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table','view')").all().map((r) => r.name));++ check: async () => {++ // A real round trip that names the store (postgresql / pglite), and the charter tables present.++ const r = await db.ready();++ if (!r.ok) return r.error;++ const names = new Set((await db.prepare('SELECT table_name AS name FROM information_schema.tables WHERE table_schema = current_schema()').all()).map((x) => x.name));const missing = CHARTER_TABLES.filter((t) => !names.has(t));-- return missing.length ? `missing ${missing.join(', ')}` : true;++ return missing.length ? `missing ${missing.join(', ')} (migrations did not run)` : { ok: true, detail: r.detail };},},++ { name: 'valkey', required: false, check: async () => (valkey ? valkey.ready() : { skipped: 'VALKEY_URL not set: per-actor limits count in this process only' }) },{name: 'network_jwks', required: false,check: () => {@@@ -40,8 +44,8 @@ function createNewsReadiness({ store, auth, outbox, ingest, config, release = nu},{name: 'events_relay', required: false,-- check: () => {-- const s = outbox.status();++ check: async () => {++ const s = await outbox.status();if (!s.enabled) return `relay off (EVENTS_URL or OV_OAUTH_CLIENT_SECRET unset); ${s.pending} events waiting`;if (s.rejected) return `${s.rejected} events rejected by OpenVibe.Events`;return { ok: true, detail: { pending: s.pending } };@@@ -53,8 +57,8 @@ function createNewsReadiness({ store, auth, outbox, ingest, config, release = nu},{name: 'sources_pull', required: false,-- check: () => {-- const last = ingest.lastPull();++ check: async () => {++ const last = await ingest.lastPull();const off = !config.sources.pullIntervalMs || !config.worker.enabled;// Pull switched off and never run: nothing verified, so skipped, never ok (WS-Q task 7).if (!last) return off ? skip('pull off (NEWS_PULL_INTERVAL_MS=0 or NEWS_WORKER=off)', { pull: 'off' }) : 'no pull from OpenVibe.Sources has run yet';@@@ -14,7 +14,7 @@ function createWorker({ config, ingest, outbox, log = console }) {if (!ingest) return null;try {const r = await ingest.pull();-- outbox.kick();++ await outbox.kick();return r;} catch (err) {log.error('[News] pull tick failed:', err.message);@@@ -51,17 +51,17 @@ const { boot, check, done } = require('./helpers/boot');await check('writes: 10 topics a minute per editor, shared by the API and the desk form; nothing stored past it', async () => {clock = Date.UTC(2026, 8, 27, 12, 5, 0);-- const count = () => t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get().n;++ const count = async () => (await t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get()).n;for (let i = 0; i < 10; i++) {const r = await t.api('/topics', { json: { name: `Limits topic ${i}` } });assert.strictEqual(r.status, 201, `topic ${i + 1}: ${r.text}`);}-- const before = count();++ const before = await count();const form = await t.get('/edit/topics', { as: t.editor, form: { _csrf: t.csrf(t.editor), name: 'One too many' } });assert.deepStrictEqual([form.status, form.json().code, form.headers.get('retry-after')], [429, 'rate_limited', '60']);const api = await t.api('/topics', { json: { name: 'One too many' } });assert.strictEqual(api.status, 429);-- assert.strictEqual(count(), before, 'nothing stored');++ assert.strictEqual(await count(), before, 'nothing stored');const other = await t.get('/api/v1/topics', { as: admin, json: { name: 'Another editor' } });assert.strictEqual(other.status, 201, `another editor still writes: ${other.text}`);});@@@ -14,7 +14,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');const t = await boot({ ai: true });const reports = launchReports(t.sources, { a: { fields: { body: 'SECRET-BODY-FOR-AI-5c1d' } } });await t.pull();-- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;++ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;const story = (await t.api('/stories', { json: { cluster: clusterId, headline: 'Europa Clipper launch', topic: 'space' } })).json().story;const aiToken = t.network.serviceToken('ai', ['news.story.revise']);const aiHeaders = { 'x-ov-origin': 'ai' };@@@ -61,7 +61,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');const page = await t.get(`/stories/${story.slug}`);assert.match(page.text, /<strong>AI-generated\.<\/strong> AI-generated from the story’s source items by workflow news.summarize_story v1, reviewed by a person\./);assert.match(page.text, /<meta name="robots" content="index, follow">/);-- const doc = t.events('news.index_document.upserted').pop().payload;++ const doc = (await t.events('news.index_document.upserted')).pop().payload;assert.strictEqual(doc.authorship, 'ai_generated');assert.ok(doc.provenance.some((p) => p.service === 'ai' && p.type === 'run' && p.id === wf.runId));const json = (await t.get(`/stories/${story.slug}.json`)).json();@@@ -99,7 +99,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');const sent = JSON.stringify(t.ai.requests.pop());assert.ok(!sent.includes('SECRET-BODY-FOR-AI'), 'no body is ever sent to the model');assert.match(sent, /"source_type":"news.article"/);-- assert.strictEqual(t.ctx.stories.get(s2.id).state, 'draft');++ assert.strictEqual((await t.ctx.stories.get(s2.id)).state, 'draft');const edit = await t.get(`/edit/stories/${s2.id}`, { as: t.editor });assert.strictEqual(edit.status, 200, edit.text);assert.match(edit.text, /Ask OpenVibe.AI for a draft summary/);@@@ -108,7 +108,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');const csrf = t.csrf(t.editor);const rv = await t.get(`/edit/stories/${s2.id}/review`, { as: t.editor, form: { _csrf: csrf, revision: String(head.revision), decision: 'approved', note: '' } });assert.strictEqual(rv.status, 303, rv.text);-- assert.strictEqual(t.ctx.store.reviews.latest(s2.id, head.revision).reviewer, t.editor.subject);++ assert.strictEqual((await t.ctx.store.reviews.latest(s2.id, head.revision)).reviewer, t.editor.subject);});await check('a stub-provider run is held (stub_provider) and a failed run makes no text', async () => {@@@ -116,20 +116,20 @@ const { boot, check, done, launchReports } = require('./helpers/boot');t.ai.setNext({ status: 201, run: { id: 'run_01J8Z3V9Q6N1X2Y3Z4A5B6C7DA', status: 'failed', output: null } });let r = await t.api(`/stories/${s3.id}/ai-drafts`, { json: { workflow: 'news.compare_perspectives' } });assert.strictEqual(r.status, 502);-- assert.strictEqual(t.ctx.store.revisions.head(s3.id), null);++ assert.strictEqual(await t.ctx.store.revisions.head(s3.id), null);t.ai.setNext({ status: 201, run: { id: 'run_01J8Z3V9Q6N1X2Y3Z4A5B6C7DB', status: 'succeeded', synthetic: true, workflow: { key: 'news.compare_perspectives', version: 1 },output: { question: 'How did outlets frame the launch?', perspectives: [{ label: 'Launch', summary: 'Outlets described the launch.', citations: [0, 1] }], agreements: [], disagreements: [], citations: [], gaps: [] } } });r = await t.api(`/stories/${s3.id}/ai-drafts`, { json: { workflow: 'news.compare_perspectives' } });assert.strictEqual(r.status, 201, r.text);-- const head = t.ctx.store.revisions.head(s3.id);++ const head = await t.ctx.store.revisions.head(s3.id);assert.strictEqual(head.meta.authorship.stubProvider, true);-- const d = t.ctx.publication.decide(t.ctx.stories.get(s3.id), head, { state: 'published' });++ const d = await t.ctx.publication.decide(await t.ctx.stories.get(s3.id), head, { state: 'published' });assert.ok(d.codes.includes('stub_provider') && d.codes.includes('ai_generated_unreviewed'));-- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_perspectives WHERE story_id = ?').get(s3.id).n, 0, 'AI never creates perspective labels');++ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_perspectives WHERE story_id = ?').get(s3.id)).n, 0, 'AI never creates perspective labels');});await check('the published product events validate as events.event-envelope@1', async () => {-- for (const e of t.events(/^news\./)) {++ for (const e of await t.events(/^news\./)) {const v = contracts.validate('events.event-envelope@1', { ...e, event_id: e.event_id || 'evt_01J8Z3V9Q6N1X2Y3Z4A5B6C7D8' });assert.ok(v.valid, `${e.event_type}: ${JSON.stringify(v.errors)}`);}@@@ -141,7 +141,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');await check('with no AI configured, the AI seam is off and says so', async () => {const r2 = launchReports(t2.sources);await t2.pull();-- const cid = t2.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(r2.a.id).cluster_id;++ const cid = (await t2.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(r2.a.id)).cluster_id;const s = (await t2.api('/stories', { json: { cluster: cid, headline: 'Written by hand' } })).json().story;const r = await t2.api(`/stories/${s.id}/ai-drafts`, { json: { workflow: 'news.summarize_story' } });assert.strictEqual(r.status, 503);@@@ -13,28 +13,30 @@ const { boot, check, done, launchReports } = require('./helpers/boot');const other = t.sources.addItem({ source_key: 'paper-b', title: 'Clipper mission team answers questions about Jupiter cruise', url: 'https://paper-b.example/qa', published_at: '2026-09-22T10:30:00Z' });const unrelated = t.sources.addItem({ source_key: 'site-c', title: 'Local bakery wins regional sourdough prize', url: 'https://site-c.example/bakery', published_at: '2026-09-22T10:40:00Z' });await t.pull();-- const item = (itm) => t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(itm);-- const main = item(reports.a.id).cluster_id;-- const lone = item(unrelated.id).cluster_id;++ const item = async (itm) => await t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(itm);++ const main = (await item(reports.a.id)).cluster_id;++ const lone = (await item(unrelated.id)).cluster_id;await check('clustering is deterministic: rebuilding from the same items in the same order gives the same grouping', async () => {const { createClusters } = require('../server/domain/clusters');const cfg = t.ctx.config;-- const { openStore } = require('../server/db');-- const store = openStore(':memory:', { now: t.ctx.store.now });++ const { createStore } = require('../server/db');++ const fresh = await require('./helpers/db').testDb();++ const store = createStore(fresh.db, { now: t.ctx.store.now });const db = store.db;const events = [];const c2 = createClusters({ store, config: cfg, outbox: { emit: (e) => events.push(e) } });-- const rows = t.db().prepare("SELECT * FROM news_source_items WHERE status <> 'duplicate' ORDER BY first_seen_at, rowid").all();++ const rows = await t.db().prepare("SELECT * FROM news_source_items WHERE status <> 'duplicate' ORDER BY first_seen_at, seq").all();const groups = new Map();for (const r of rows) {-- db.prepare('INSERT INTO news_source_items (id, sources_item_id, sources_revision, source_key, headline, outlet, title_key, status, published_at, summary, first_seen_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)')++ await db.prepare('INSERT INTO news_source_items (id, sources_item_id, sources_revision, source_key, headline, outlet, title_key, status, published_at, summary, first_seen_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)').run(r.id, r.sources_item_id, r.sources_revision, r.source_key, r.headline, r.outlet, r.title_key, r.status, r.published_at, r.summary, r.first_seen_at, r.updated_at);-- const out = c2.assign(db.prepare('SELECT * FROM news_source_items WHERE id = ?').get(r.id));++ const out = await c2.assign(await db.prepare('SELECT * FROM news_source_items WHERE id = ?').get(r.id));groups.set(r.id, out.cluster.id);}const original = new Map(rows.map((r) => [r.id, r.cluster_id]));const same = (m) => { const byC = new Map(); for (const [i, c] of m) { if (!byC.has(c)) byC.set(c, []); byC.get(c).push(i); } return [...byC.values()].map((v) => v.sort().join(',')).sort(); };++ await fresh.close();assert.deepStrictEqual(same(groups), same(original));});@@@ -58,16 +60,16 @@ const { boot, check, done, launchReports } = require('./helpers/boot');let mergeAudit;await check('merge is audited: the absorbed cluster’s items move, it is marked merged', async () => {-- const before = t.ctx.clusters.members(lone).map((i) => i.id);++ const before = (await t.ctx.clusters.members(lone)).map((i) => i.id);const r = await t.api(`/clusters/${main}/merge`, { json: { other: lone, reason: 'testing a merge' } });assert.strictEqual(r.status, 200, r.text);mergeAudit = r.json().audit;assert.strictEqual(mergeAudit.action, 'merge');assert.deepStrictEqual(mergeAudit.item_ids.sort(), before.sort());-- assert.strictEqual(item(unrelated.id).cluster_id, main);-- assert.strictEqual(t.ctx.clusters.get(lone).status, 'merged');-- assert.strictEqual(t.ctx.clusters.get(lone).merged_into, main);-- const ev = t.events('news.cluster.updated').filter((e) => e.payload.action === 'merged');++ assert.strictEqual((await item(unrelated.id)).cluster_id, main);++ assert.strictEqual((await t.ctx.clusters.get(lone)).status, 'merged');++ assert.strictEqual((await t.ctx.clusters.get(lone)).merged_into, main);++ const ev = (await t.events('news.cluster.updated')).filter((e) => e.payload.action === 'merged');assert.strictEqual(ev.length, 1);});@@@ -75,9 +77,9 @@ const { boot, check, done, launchReports } = require('./helpers/boot');const r = await t.api(`/clusters/audit/${mergeAudit.id}/reverse`, { json: { reason: 'wrong merge' } });assert.strictEqual(r.status, 200, r.text);assert.strictEqual(r.json().audit.action, 'reverse_merge');-- assert.strictEqual(item(unrelated.id).cluster_id, lone);-- assert.strictEqual(item(reports.a.id).cluster_id, main);-- assert.strictEqual(t.ctx.clusters.get(lone).status, 'open');++ assert.strictEqual((await item(unrelated.id)).cluster_id, lone);++ assert.strictEqual((await item(reports.a.id)).cluster_id, main);++ assert.strictEqual((await t.ctx.clusters.get(lone)).status, 'open');const again = await t.api(`/clusters/audit/${mergeAudit.id}/reverse`, { json: {} });assert.strictEqual(again.status, 409);assert.strictEqual(again.json().code, 'cluster.already_reversed');@@@ -86,26 +88,26 @@ const { boot, check, done, launchReports } = require('./helpers/boot');let splitAudit;let splitId;await check('split is audited: chosen items move to a new cluster; at least one must stay', async () => {-- const all = t.ctx.clusters.members(main).map((i) => i.id);++ const all = (await t.ctx.clusters.members(main)).map((i) => i.id);let r = await t.api(`/clusters/${main}/split`, { json: { items: all } });assert.strictEqual(r.status, 422);assert.strictEqual(r.json().code, 'cluster.split_all');-- const moving = [item(other.id).id];++ const moving = [(await item(other.id)).id];r = await t.api(`/clusters/${main}/split`, { json: { items: moving, reason: 'a Q&A is a different story' } });assert.strictEqual(r.status, 201, r.text);splitAudit = r.json().audit;splitId = r.json().created.id;-- assert.strictEqual(item(other.id).cluster_id, splitId);-- assert.strictEqual(t.ctx.clusters.get(splitId).split_from, main);++ assert.strictEqual((await item(other.id)).cluster_id, splitId);++ assert.strictEqual((await t.ctx.clusters.get(splitId)).split_from, main);});await check('reversing the split puts the items back and dissolves the empty cluster (through the no-JS form)', async () => {const csrf = t.csrf(t.editor);const r = await t.get(`/clusters/audit/${splitAudit.id}/reverse`, { as: t.editor, form: { _csrf: csrf, reason: 'undo' } });assert.strictEqual(r.status, 303, r.text);-- assert.strictEqual(item(other.id).cluster_id, main);-- assert.strictEqual(t.ctx.clusters.get(splitId).status, 'dissolved');-- const audit = t.ctx.clusters.audit(main).map((a) => a.action);++ assert.strictEqual((await item(other.id)).cluster_id, main);++ assert.strictEqual((await t.ctx.clusters.get(splitId)).status, 'dissolved');++ const audit = (await t.ctx.clusters.audit(main)).map((a) => a.action);assert.deepStrictEqual(audit, ['merge', 'reverse_merge', 'split', 'reverse_split']);});@@@ -16,7 +16,7 @@ const MOD = 'mod:mod_01J8ZQ4Y7N3M2K1H0G9F8E7D6C';const reader = t.network.addUser('reader');const caps = ['news.topic.manage', 'news.story.create'];const token = (sub, actorType, extra) => t.network.signService({ sub, actorType, aud: ['openvibe.news'], cap: caps, extra });-- const topicsNamed = (name) => t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM news_topics WHERE name = ?').get(name).n;++ const topicsNamed = async (name) => (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM news_topics WHERE name = ?').get(name)).n;const newTopic = (as, name, headers) => t.get('/api/v1/topics', { as, headers, json: { name } });await check('an app or module cannot act as an editor by naming them in X-OV-Subject', async () => {@@@ -25,7 +25,7 @@ const MOD = 'mod:mod_01J8ZQ4Y7N3M2K1H0G9F8E7D6C';const r = await newTopic(token(sub, type, extra), `Planted by ${type}`, { 'x-ov-subject': t.editor.subject });assert.strictEqual(r.status, 403, `${type} ${JSON.stringify(extra)}: ${r.text}`);assert.strictEqual(r.json().code, 'subject.not_delegated');-- assert.strictEqual(topicsNamed(`Planted by ${type}`), 0);++ assert.strictEqual(await topicsNamed(`Planted by ${type}`), 0);}}});@@@ -21,7 +21,7 @@ const BODY = [const t = await boot();const reports = launchReports(t.sources);await t.pull();-- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;++ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;const open = async (headline, body = BODY) => {const r = await t.api('/stories', { json: { cluster: clusterId, headline, topic: 'space', body } });assert.strictEqual(r.status, 201, r.text);@@@ -31,9 +31,9 @@ const BODY = [const calls = (re, method) => t.community.calls.filter((c) => re.test(c.url.split('?')[0]) && (!method || c.method === method));const resolves = () => calls(/\/threads\/resolve$/, 'POST');const hides = () => calls(/\/visibility$/, 'PUT');-- const settle = () => t.ctx.discussion.idle();-- const refOf = (id) => t.db().prepare('SELECT * FROM news_story_discussion_refs WHERE entity_id = ?').get(id);-- t.community.setProbe(() => t.db().inTransaction);++ const settle = async () => await t.ctx.discussion.idle();++ const refOf = async (id) => await t.db().prepare('SELECT * FROM news_story_discussion_refs WHERE entity_id = ?').get(id);++ t.community.setProbe(() => t.db().stats().open > 0); // a News transaction still open when Community is calledconst story = await open('Europa Clipper is on its way to Jupiter');const url = `/stories/${story.slug}`;@@@ -45,7 +45,7 @@ const BODY = [assert.strictEqual(prev.status, 200, prev.text);assert.doesNotMatch(prev.text, /id="comments"/, 'the editor preview shows no thread');assert.strictEqual(resolves().length, 0);-- assert.strictEqual(refOf(story.id), undefined);++ assert.strictEqual(await refOf(story.id), undefined);});await check('a published story resolves its thread once, for the story’s EntityRef, and stores only the id', async () => {@@@ -60,7 +60,7 @@ const BODY = [const call = resolves()[0];assert.ok(call.cap.includes('community.comment.write'), 'resolved with the write capability');assert.deepStrictEqual(JSON.parse(call.body).ref, { service: 'news', type: 'story', id: story.id, label: 'Europa Clipper is on its way to Jupiter' });-- const ref = refOf(story.id);++ const ref = await refOf(story.id);assert.strictEqual(ref.thread_id, '1');assert.deepStrictEqual(Object.keys(ref).sort(), ['entity_id', 'ref', 'resolved_at', 'thread_id'], 'the reference table has no room for comments');page = await t.get(url);@@@ -90,8 +90,8 @@ const BODY = [const after = await t.get(url);assert.match(after.text, /<strong>Rita Reader<\/strong>/);assert.match(after.text, /Great summary; the 2030 arrival <b>date<\/b> matters\.<br>Thanks\./, 'escaped, read from Community');-- const dump = t.db().prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all()-- .map(({ name }) => JSON.stringify(t.db().prepare(`SELECT * FROM "${name}"`).all())).join('\n');++ const names = (await t.db().prepare("SELECT table_name AS name FROM information_schema.tables WHERE table_schema = current_schema() AND table_type = 'BASE TABLE'").all()).map((r) => r.name);++ const dump = (await Promise.all(names.map(async (name) => JSON.stringify(await t.db().prepare(`SELECT * FROM "${name}"`).all())))).join('\n');assert.doesNotMatch(dump, /Great summary|2030 arrival/, 'no comment text is copied into News');});@@@ -111,7 +111,7 @@ const BODY = [t.community.setDown(true);const p2 = await t.get(`/stories/${fresh.slug}`);assert.match(p2.text, /Comments are unavailable/);-- assert.strictEqual(refOf(fresh.id), undefined, 'a failed resolve stores no thread id');++ assert.strictEqual(await refOf(fresh.id), undefined, 'a failed resolve stores no thread id');t.community.setDown(false);});@@@ -136,9 +136,9 @@ const BODY = [const r = await t.get(`${url}/comments`, { as: member, form: { _csrf: t.csrf(member), message: 'Still here?' } });assert.strictEqual(r.status, 409);-- const itemA = t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).id;++ const itemA = (await t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).id;assert.strictEqual((await t.api(`/stories/${story.id}/sources/${itemA}`, { method: 'DELETE' })).status, 200);-- const head = t.ctx.store.revisions.head(story.id).number;++ const head = (await t.ctx.store.revisions.head(story.id)).number;const fixed = BODY.split('\n\n').slice(0, 2).map((p) => p.replace('[1, 2]', '[2, 3]')).join('\n\n');let rr = await t.api(`/stories/${story.id}/revisions`, { json: { body: fixed, expected_revision: head } });assert.strictEqual(rr.status, 201, rr.text);@@@ -155,7 +155,7 @@ const BODY = [const other = await open('Europa Clipper: the launch in numbers', BODY.split('\n\n').slice(0, 2).map((p) => p.replace(/\[[\d, ]+\]$/, '[1, 2]')).join('\n\n'));await publish(other);await t.get(`/stories/${other.slug}`);-- const id = refOf(other.id).thread_id;++ const id = (await refOf(other.id)).thread_id;let r = await t.api(`/stories/${other.id}/unpublish`, { json: {} });assert.strictEqual(r.status, 200, r.text);await settle();@@@ -51,11 +51,14 @@ async function boot(opts = {}) {const { createApp } = require('../../server/app');const quiet = { log() {}, warn() {}, error: (...a) => { if (process.env.VERBOSE) console.error(...a); } };++ const { createStore } = require('../../server/db');++ // One database per boot (PGlite, or NEWS_TEST_STORE=pg: the containers); a restart keeps it, like a file did.++ const testdb = await require('./db').testDb();let server = null;let built = null;async function start() {const config = configLib.load(env);-- built = createApp({ config, now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow });++ built = await createApp({ config, store: createStore(testdb.db, { now: clock.now }), now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow });await built.ctx.auth.ensureKey();server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });t.base = `http://127.0.0.1:${server.address().port}`;@@@ -64,7 +67,7 @@ async function boot(opts = {}) {}async function stop() {if (server) await new Promise((r) => server.close(r));-- if (built) { built.ctx.worker.stop(); await built.ctx.outbox.stop(); built.ctx.store.close(); }++ if (built) { built.ctx.worker.stop(); await built.ctx.outbox.stop(); }server = null; built = null;}@@@ -82,8 +85,8 @@ async function boot(opts = {}) {}/** Rows of event_outbox as parsed envelopes. */-- function events(type = null) {-- return t.ctx.store.db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all().map((r) => JSON.parse(r.envelope)).filter((e) => !type || e.event_type === type || (type instanceof RegExp && type.test(e.event_type)));++ async function events(type = null) {++ return (await t.ctx.store.db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all()).map((r) => (typeof r.envelope === 'string' ? JSON.parse(r.envelope) : r.envelope)).filter((e) => !type || e.event_type === type || (type instanceof RegExp && type.test(e.event_type)));}async function deliver(event, o = {}) {@@@ -93,15 +96,15 @@ async function boot(opts = {}) {}/** API call as the editor (browser JWT). */-- const api = (p, o = {}) => get(`/api/v1${p}`, { as: editor, ...o });++ const api = async (p, o = {}) => await get(`/api/v1${p}`, { as: editor, ...o });const t = {network, sources, ai, community, clock, dbPath, editor, get, api, events, deliver, SECRET,-- pull: () => t.ctx.ingest.pull(),++ pull: async () => await t.ctx.ingest.pull(),db: () => t.ctx.store.db,csrf: (user) => require('../../server/auth/forms').csrfToken({ formSecret: env.NEWS_FORM_SECRET }, user),async restart() { await stop(); await start(); },-- async close() { await stop(); await network.close(); await sources.close(); await ai.close(); await community.close(); fs.rmSync(dir, { recursive: true, force: true }); },++ async close() { await stop(); await testdb.close(); await network.close(); await sources.close(); await ai.close(); await community.close(); fs.rmSync(dir, { recursive: true, force: true }); },};await start();return t;nnew file mode 100644@@@ -0,0 +1,11 @@++'use strict';++/**++ * A migrated database for one test run (ADR-035), from openvibe-sdk/testing: PGlite by default; with++ * NEWS_TEST_STORE=pg (npm run test:pg) the PostgreSQL + PgBouncer containers, with roles and a schema of this run's own.++ */++const { createTestDb, pgAvailable } = require('openvibe-sdk/testing');++const { MIGRATIONS } = require('../../server/db');++++const testDb = ({ store = process.env.NEWS_TEST_STORE || 'pglite', max = 4 } = {}) => createTestDb({ migrations: MIGRATIONS, store, service: 'news', max });++++module.exports = { testDb, pgAvailable };@@@ -46,23 +46,23 @@ async function storyOn(t, headline, items) {const copy = S.addItem({ source_key: 'beta-daily', title: 'Water agency says two new treatment plants will open next spring', url: 'https://beta.example/b/syndicated-water' });const beta = S.addItem({ source_key: 'beta-daily', title: 'Budget vote clears way for valley water projects', url: 'https://beta.example/b/budget-vote' });await t.pull();-- const row = (it) => t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(it.id);++ const row = async (it) => await t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(it.id);const sitemap = async () => (await t.get('/sitemaps/stories.xml')).text;-- const indexEvent = (story) => t.events('news.index_document.upserted').filter((e) => e.payload.id === story.id).pop();++ const indexEvent = async (story) => (await t.events('news.index_document.upserted')).filter((e) => e.payload.id === story.id).pop();await check('independence is concrete: same Sources source, same publisher domain, same outlet or a copy of one report count once', async () => {assert.strictEqual(text.publisherDomain('https://tech.thepaper.example/x'), 'thepaper.example');assert.strictEqual(text.publisherDomain('https://news.bbc.co.uk/x'), 'bbc.co.uk');assert.strictEqual(text.publisherDomain('https://www.bbc.co.uk/x'), 'bbc.co.uk');-- assert.strictEqual(row(copy).status, 'duplicate', 'the syndicated copy is a near-duplicate of the Alpha report');-- assert.strictEqual(row(copy).duplicate_of, row(wire).id);-- const n = (items) => text.independentSources(items.map(row)).length;-- assert.strictEqual(n([feed1, feed2]), 1);-- assert.strictEqual(n([world, tech]), 1);-- assert.strictEqual(n([wire, copy]), 1);-- assert.strictEqual(n([wire, copy, beta]), 1, 'the copy ties Beta Daily to the Alpha report: still one');-- assert.strictEqual(n([solo, wire]), 2);-- assert.strictEqual(n([solo, feed1, world, wire]), 4);++ assert.strictEqual((await row(copy)).status, 'duplicate', 'the syndicated copy is a near-duplicate of the Alpha report');++ assert.strictEqual((await row(copy)).duplicate_of, (await row(wire)).id);++ const n = async (items) => text.independentSources(await Promise.all(items.map(row))).length;++ assert.strictEqual(await n([feed1, feed2]), 1);++ assert.strictEqual(await n([world, tech]), 1);++ assert.strictEqual(await n([wire, copy]), 1);++ assert.strictEqual(await n([wire, copy, beta]), 1, 'the copy ties Beta Daily to the Alpha report: still one');++ assert.strictEqual(await n([solo, wire]), 2);++ assert.strictEqual(await n([solo, feed1, world, wire]), 4);});await check('a single-source story is published and readable but noindex, and absent from the sitemap', async () => {@@@ -76,7 +76,7 @@ async function storyOn(t, headline, items) {assert.strictEqual(json.state, 'published');assert.deepStrictEqual(json.indexability.reasons.map((r) => [r.code, r.detail]), [['unsourced', '1 of 2 sources']]);assert.doesNotMatch(await sitemap(), new RegExp(story.slug));-- assert.strictEqual(indexEvent(story).payload.indexability.decision, 'noindex', 'Search is told it is noindex');++ assert.strictEqual((await indexEvent(story)).payload.indexability.decision, 'noindex', 'Search is told it is noindex');assert.match((await t.get('/')).text, new RegExp(`href="/stories/${story.slug}"`), 'still listed for readers');});@@@ -105,7 +105,7 @@ async function storyOn(t, headline, items) {assert.strictEqual(page.headers.get('x-robots-tag'), null);assert.deepStrictEqual((await t.get(`/stories/${story.slug}.json`)).json().indexability, { indexable: true, robots: 'index, follow', reasons: [] });assert.match(await sitemap(), new RegExp(`<loc>https://openvibe.news/stories/${story.slug}</loc>`));-- assert.strictEqual(indexEvent(story).payload.indexability.decision, 'index');++ assert.strictEqual((await indexEvent(story)).payload.indexability.decision, 'index');});await t.close();@@@ -9,12 +9,12 @@ const { boot, check, done, launchReports } = require('./helpers/boot');(async () => {const t = await boot();-- const count = (table) => t.db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n;-- const byItm = (id) => t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(id);++ const count = async (table) => (await t.db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).n;++ const byItm = async (id) => await t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(id);await check('only topics are seeded: no stories, source items or clusters exist on boot', async () => {-- assert.ok(count('news_topics') >= 10);-- for (const table of ['news_stories', 'news_source_items', 'news_story_clusters', 'news_story_revisions', 'news_editorial_flags']) assert.strictEqual(count(table), 0, table);++ assert.ok(await count('news_topics') >= 10);++ for (const table of ['news_stories', 'news_source_items', 'news_story_clusters', 'news_story_revisions', 'news_editorial_flags']) assert.strictEqual(await count(table), 0, table);const home = await t.get('/');assert.match(home.text, /No stories have been published yet/);});@@@ -26,10 +26,10 @@ const { boot, check, done, launchReports } = require('./helpers/boot');const r = await t.pull();assert.ok(r.ok, JSON.stringify(r));assert.strictEqual(r.counts.created, 3);-- assert.strictEqual(count('news_source_items'), 3);-- const a = byItm(reports.a.id);-- const b = byItm(reports.b.id);-- const c = byItm(reports.c.id);++ assert.strictEqual(await count('news_source_items'), 3);++ const a = await byItm(reports.a.id);++ const b = await byItm(reports.b.id);++ const c = await byItm(reports.c.id);assert.strictEqual(a.outlet, 'Wire A', 'outlet from the Sources registry name');assert.strictEqual(a.cluster_id, b.cluster_id);assert.strictEqual(a.cluster_id, c.cluster_id);@@@ -37,88 +37,88 @@ const { boot, check, done, launchReports } = require('./helpers/boot');assert.strictEqual(why.rule, 'shared_terms');assert.ok(why.shared_entities.includes('europa clipper'), JSON.stringify(why));assert.ok(why.score >= 3);-- assert.strictEqual(t.ctx.store.getState('sources_cursor'), reports.c.change_seq);-- assert.strictEqual(t.events('news.source.ingested').length, 3);-- assert.ok(t.events('news.cluster.updated').length >= 3);++ assert.strictEqual(await t.ctx.store.getState('sources_cursor'), reports.c.change_seq);++ assert.strictEqual((await t.events('news.source.ingested')).length, 3);++ assert.ok((await t.events('news.cluster.updated')).length >= 3);});await check('a second pull (and a replayed page) changes nothing and emits nothing', async () => {-- const before = t.events().length;++ const before = (await t.events()).length;const r = await t.pull();assert.strictEqual(r.counts.created || 0, 0);-- t.ctx.store.setState('sources_cursor', 0);++ await t.ctx.store.setState('sources_cursor', 0);const again = await t.pull();assert.strictEqual(again.counts.unchanged, 3);-- assert.strictEqual(t.events().length, before);-- assert.strictEqual(count('news_source_items'), 3);++ assert.strictEqual((await t.events()).length, before);++ assert.strictEqual(await count('news_source_items'), 3);});await check('dedupe: same canonical URL (tracking parameters and www. ignored) is a duplicate', async () => {const dup = t.sources.addItem({ source_key: 'paper-b', title: 'Europa Clipper launch: our live coverage', url: 'https://www.wire-a.example/space/europa-clipper-launch/?utm_source=x', published_at: '2026-09-22T11:00:00Z' });await t.pull();-- const row = byItm(dup.id);++ const row = await byItm(dup.id);assert.strictEqual(row.status, 'duplicate');-- assert.strictEqual(row.duplicate_of, byItm(reports.a.id).id);++ assert.strictEqual(row.duplicate_of, (await byItm(reports.a.id)).id);assert.strictEqual(JSON.parse(row.dedupe).rule, 'canonical_url');-- assert.strictEqual(row.cluster_id, byItm(reports.a.id).cluster_id, 'a duplicate follows its original into the cluster');++ assert.strictEqual(row.cluster_id, (await byItm(reports.a.id)).cluster_id, 'a duplicate follows its original into the cluster');});await check('dedupe: identical Sources content hash is a duplicate', async () => {const dup = t.sources.addItem({ source_key: 'site-c', title: 'Syndicated copy', url: 'https://mirror.example/copy', content_hash: reports.b.provenance.content_hash });await t.pull();-- const row = byItm(dup.id);++ const row = await byItm(dup.id);assert.strictEqual(row.status, 'duplicate');assert.strictEqual(JSON.parse(row.dedupe).rule, 'content_hash');-- assert.strictEqual(row.duplicate_of, byItm(reports.b.id).id);++ assert.strictEqual(row.duplicate_of, (await byItm(reports.b.id)).id);});await check('dedupe: a near-identical headline (2-shingle Jaccard) within 48 h is a duplicate; a different story is not', async () => {const near = t.sources.addItem({ source_key: 'wire-a', title: 'NASA launches Europa Clipper probe to Jupiter moon Europa, officials say', url: 'https://wire-a.example/space/europa-clipper-launch-updated', published_at: '2026-09-22T12:00:00Z' });const other = t.sources.addItem({ source_key: 'paper-b', title: 'City council approves new bike lanes downtown', url: 'https://paper-b.example/bike-lanes', published_at: '2026-09-22T12:00:00Z' });await t.pull();-- const n = byItm(near.id);++ const n = await byItm(near.id);assert.strictEqual(n.status, 'duplicate', JSON.stringify(n));assert.strictEqual(JSON.parse(n.dedupe).rule, 'near_title');assert.match(JSON.parse(n.dedupe).detail, /jaccard 0\.\d\d/);-- const o = byItm(other.id);++ const o = await byItm(other.id);assert.strictEqual(o.status, 'active');-- assert.notStrictEqual(o.cluster_id, byItm(reports.a.id).cluster_id, 'an unrelated story starts its own cluster');++ assert.notStrictEqual(o.cluster_id, (await byItm(reports.a.id)).cluster_id, 'an unrelated story starts its own cluster');assert.strictEqual(JSON.parse(o.cluster_reason).rule, 'new_cluster');});await check('a failed pull is recorded, relayed once per outage, and creates nothing', async () => {-- const items = count('news_source_items');-- const clusters = count('news_story_clusters');++ const items = await count('news_source_items');++ const clusters = await count('news_story_clusters');t.sources.addItem({ source_key: 'wire-a', title: 'Arrives while Sources is down' });t.sources.setDown(true);const r1 = await t.pull();const r2 = await t.pull();assert.strictEqual(r1.ok, false);assert.strictEqual(r2.ok, false);-- assert.strictEqual(count('news_source_items'), items);-- assert.strictEqual(count('news_story_clusters'), clusters);-- assert.strictEqual(count('news_stories'), 0);-- const runs = t.db().prepare("SELECT * FROM news_ingest_runs WHERE origin = 'pull' AND state = 'failed'").all();++ assert.strictEqual(await count('news_source_items'), items);++ assert.strictEqual(await count('news_story_clusters'), clusters);++ assert.strictEqual(await count('news_stories'), 0);++ const runs = await t.db().prepare("SELECT * FROM news_ingest_runs WHERE origin = 'pull' AND state = 'failed'").all();assert.strictEqual(runs.length, 2);-- assert.strictEqual(t.events('news.source.failed').filter((e) => e.payload.origin === 'pull').length, 1);++ assert.strictEqual((await t.events('news.source.failed')).filter((e) => e.payload.origin === 'pull').length, 1);const ready = await t.get('/api/ready');assert.match(ready.text, /last pull failed/);t.sources.setDown(false);const r3 = await t.pull();assert.ok(r3.ok);-- assert.strictEqual(count('news_source_items'), items + 1);++ assert.strictEqual(await count('news_source_items'), items + 1);});await check('webhook: a signed sources.item.created is applied once; a redelivery changes nothing', async () => {const it = t.sources.addItem({ source_key: 'site-c', title: 'Europa Clipper team describes first weeks of cruise to Jupiter', url: 'https://site-c.example/cruise', published_at: '2026-09-22T12:30:00Z' });const ev = { event_type: 'sources.item.created', subject: { type: 'item', id: it.id, revision: 1 }, payload: { item_id: it.id, source_key: 'site-c', category: 'news', revision: 1 } };const r = await t.deliver(ev);-- assert.strictEqual(r.status, 204, r.text + JSON.stringify(t.db().prepare('SELECT * FROM news_ingest_runs ORDER BY id DESC LIMIT 1').get()));-- assert.ok(byItm(it.id));-- const n = t.events('news.source.ingested').length;++ assert.strictEqual(r.status, 204, r.text + JSON.stringify(await t.db().prepare('SELECT * FROM news_ingest_runs ORDER BY id DESC LIMIT 1').get()));++ assert.ok(await byItm(it.id));++ const n = (await t.events('news.source.ingested')).length;const again = await fetch(`${t.base}/internal/events`, { method: 'POST', headers: r.headers, body: r.body });assert.strictEqual(again.status, 204);-- assert.strictEqual(t.events('news.source.ingested').length, n);++ assert.strictEqual((await t.events('news.source.ingested')).length, n);const pulled = await t.pull();assert.strictEqual(pulled.counts.created || 0, 0, 'the pull backstop sees it as already applied');});@@@ -134,38 +134,38 @@ const { boot, check, done, launchReports } = require('./helpers/boot');assert.strictEqual(other.status, 204);const forged = await t.deliver({ event_type: 'sources.item.removed', source: 'blog', payload: { item_id: reports.a.id, reason: 'forged' } });assert.strictEqual(forged.status, 204);-- assert.strictEqual(byItm(reports.a.id).status, 'active', 'only source "sources" may remove an item');++ assert.strictEqual((await byItm(reports.a.id)).status, 'active', 'only source "sources" may remove an item');});await check('webhook: when Sources cannot be read the delivery is 503 (retried), recorded, relayed once, and nothing is made', async () => {const it = t.sources.addItem({ source_key: 'wire-a', title: 'Delivered while Sources is unreachable' });t.sources.setDown(true);-- const before = count('news_source_items');-- const failedBefore = t.events('news.source.failed').length;++ const before = await count('news_source_items');++ const failedBefore = (await t.events('news.source.failed')).length;const ev = { event_type: 'sources.item.created', payload: { item_id: it.id, source_key: 'wire-a', category: 'news', revision: 1 } };const r1 = await t.deliver(ev, { attempt: 1 });const r2 = await t.deliver(ev, { attempt: 2 });assert.strictEqual(r1.status, 503);assert.strictEqual(r2.status, 503);-- assert.strictEqual(count('news_source_items'), before);-- assert.strictEqual(t.events('news.source.failed').length, failedBefore + 1);-- assert.ok(t.db().prepare("SELECT COUNT(*) AS n FROM news_ingest_runs WHERE origin = 'webhook' AND state = 'failed'").get().n >= 2);++ assert.strictEqual(await count('news_source_items'), before);++ assert.strictEqual((await t.events('news.source.failed')).length, failedBefore + 1);++ assert.ok((await t.db().prepare("SELECT COUNT(*) AS n FROM news_ingest_runs WHERE origin = 'webhook' AND state = 'failed'").get()).n >= 2);t.sources.setDown(false);});await check('sources.fetch.failed for a news source is recorded and relayed; no text is fabricated', async () => {-- const stories = count('news_stories');-- const items = count('news_source_items');++ const stories = await count('news_stories');++ const items = await count('news_source_items');const r = await t.deliver({ event_type: 'sources.fetch.failed', subject: { type: 'source', id: 'wire-a' }, payload: { source_key: 'wire-a', category: 'news', run_id: 'run_9', state: 'timeout', error_code: 'fetch.timeout', http_status: null, consecutive_failures: 3 } });assert.strictEqual(r.status, 204);-- const run = t.db().prepare("SELECT * FROM news_ingest_runs WHERE origin = 'sources' ORDER BY id DESC").get();++ const run = await t.db().prepare("SELECT * FROM news_ingest_runs WHERE origin = 'sources' ORDER BY id DESC").get();assert.strictEqual(run.state, 'timeout');assert.strictEqual(run.source_key, 'wire-a');-- const ev = t.events('news.source.failed').pop();++ const ev = (await t.events('news.source.failed')).pop();assert.strictEqual(ev.payload.origin, 'sources');assert.strictEqual(ev.payload.state, 'timeout');-- assert.strictEqual(count('news_stories'), stories);-- assert.strictEqual(count('news_source_items'), items);++ assert.strictEqual(await count('news_stories'), stories);++ assert.strictEqual(await count('news_source_items'), items);const desk = await t.get('/edit', { as: t.editor });assert.match(desk.text, /timeout/);});@@@ -173,8 +173,8 @@ const { boot, check, done, launchReports } = require('./helpers/boot');await check('an item without a title is rejected and recorded, never shown', async () => {const it = t.sources.addItem({ source_key: 'wire-a', title: '' });await t.pull();-- assert.strictEqual(byItm(it.id), undefined);-- assert.ok(t.db().prepare("SELECT COUNT(*) AS n FROM news_ingest_runs WHERE state = 'rejected' AND sources_item_id = ?").get(it.id).n === 1);++ assert.strictEqual(await byItm(it.id), undefined);++ assert.ok((await t.db().prepare("SELECT COUNT(*) AS n FROM news_ingest_runs WHERE state = 'rejected' AND sources_item_id = ?").get(it.id)).n === 1);});await t.close();@@@ -48,7 +48,7 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.republished_at: '2026-09-22T09:00:00Z',});await t.pull();-- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(allowed.id).cluster_id;++ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(allowed.id)).cluster_id;let r = await t.api('/stories', { json: { cluster: clusterId, headline: 'Storms delay Europa Clipper launch', topic: 'space', body: ['Storms over Florida delayed the launch of NASA’s Europa Clipper mission, according to reports from three outlets that covered the countdown at Kennedy Space Center. [1, 2, 3]','NASA said the launch would move to a later window once the weather clears, and the agency did not give a new date in the reports cited here. [2]',@@@ -59,9 +59,9 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.reassert.strictEqual(r.status, 200, r.text);await check('bodies are never stored: no column of News’ database holds the article text', async () => {-- const rows = t.db().prepare('SELECT * FROM news_source_items').all();++ const rows = await t.db().prepare('SELECT * FROM news_source_items').all();assert.strictEqual(rows.length, 3);-- const dump = JSON.stringify(rows) + JSON.stringify(t.db().prepare('SELECT fields, content FROM news_story_revisions').all());++ const dump = JSON.stringify(rows) + JSON.stringify(await t.db().prepare('SELECT fields, content FROM news_story_revisions').all());for (const s of [BODY_SENTINEL, TAIL_SENTINEL, FORBIDDEN_SENTINEL, UNLICENSED_SENTINEL]) assert.ok(!dump.includes(s), `${s} was stored`);const a = rows.find((x) => x.sources_item_id === allowed.id);assert.ok(a.summary && a.summary.length <= 280, 'the allowed summary is kept, short');@@@ -81,13 +81,13 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.reassert.strictEqual(res.status, 200, `${p} → ${res.status}`);seen += res.text;}-- seen += JSON.stringify(t.events());++ seen += JSON.stringify(await t.events());for (const s of [BODY_SENTINEL, TAIL_SENTINEL, FORBIDDEN_SENTINEL, UNLICENSED_SENTINEL, 'The full article text goes on']) assert.ok(!seen.includes(s), `${s} leaked`);const page = await t.get(`/stories/${story.slug}`);assert.match(page.text, /Storms delayed the Europa Clipper launch\./, 'the licensed short summary is shown in the source table');const feeds = (await t.get('/feed.xml')).text + (await t.get('/feed.json')).text;assert.ok(!feeds.includes('Storms delayed the Europa Clipper launch.'), 'feeds carry our text only, never a source summary');-- const doc = t.events('news.index_document.upserted').pop().payload;++ const doc = (await t.events('news.index_document.upserted')).pop().payload;assert.ok(!JSON.stringify(doc).includes('Storms delayed the Europa Clipper launch.'), 'the Search document carries our text only');});@@@ -16,7 +16,7 @@ const BODY = [const t = await boot();const reports = launchReports(t.sources);await t.pull();-- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;++ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;const ed = t.editor;const csrf = t.csrf(ed);let storyId;@@@ -42,7 +42,7 @@ const BODY = [assert.match(desk.headers.get('x-robots-tag'), /noindex/);const r = await t.get(`/clusters/${clusterId}/stories`, { as: ed, form: { headline: 'No token' } });assert.strictEqual(r.status, 403);-- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_stories').get().n, 0);++ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_stories').get()).n, 0);});await check('open a story from the cluster page, then write paragraphs with [n] markers', async () => {@@@ -68,8 +68,8 @@ const BODY = [assert.strictEqual(r.status, 303, r.text);r = await t.get(`/edit/stories/${storyId}/perspective`, { as: ed, form: { _csrf: csrf, label: 'Launch coverage', description: 'Reports about the launch itself.' } });assert.strictEqual(r.status, 303);-- const per = t.db().prepare('SELECT id FROM news_perspectives WHERE story_id = ?').get(storyId).id;-- const item = t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).id;++ const per = (await t.db().prepare('SELECT id FROM news_perspectives WHERE story_id = ?').get(storyId)).id;++ const item = (await t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).id;r = await t.get(`/edit/stories/${storyId}/assign`, { as: ed, form: { _csrf: csrf, item, perspective: per } });assert.strictEqual(r.status, 303);r = await t.get(`/edit/stories/${storyId}/revise`, { as: ed, form: { _csrf: csrf, expectedRevision: '1', headline: 'Europa Clipper begins its trip to Jupiter', body: BODY, topic: 'space' } });@@@ -72,14 +72,16 @@ const { boot, check, done } = require('./helpers/boot');});await check('the seed is topics only, idempotent, and never touches an existing topic', async () => {-- const n = t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get().n;-- t.db().prepare("UPDATE news_topics SET name = 'Space (renamed by an editor)' WHERE slug = 'space'").run();++ const n = (await t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get()).n;++ await t.db().prepare("UPDATE news_topics SET name = 'Space (renamed by an editor)' WHERE slug = 'space'").run();await t.restart();-- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get().n, n);-- assert.strictEqual(t.db().prepare("SELECT name FROM news_topics WHERE slug = 'space'").get().name, 'Space (renamed by an editor)');-- const out = execFileSync(process.execPath, [path.join(__dirname, '..', 'scripts', 'seed.js')], { env: { ...process.env, NEWS_DB_PATH: t.dbPath }, encoding: 'utf8' });-- assert.match(out, /topics: 0 added, \d+ already present/);-- for (const table of ['news_stories', 'news_source_items', 'news_story_clusters']) assert.strictEqual(t.db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n, 0, table);++ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get()).n, n);++ assert.strictEqual((await t.db().prepare("SELECT name FROM news_topics WHERE slug = 'space'").get()).name, 'Space (renamed by an editor)');++ // scripts/seed.js is this call on the configured database (DATABASE_URL); here, on the test's own.++ execFileSync(process.execPath, ['--check', path.join(__dirname, '..', 'scripts', 'seed.js')]);++ const r = await require('../server/domain/topics').createTopics({ store: t.ctx.store }).seed();++ assert.deepStrictEqual([r.created, r.existing], [0, n]);++ for (const table of ['news_stories', 'news_source_items', 'news_story_clusters']) assert.strictEqual((await t.db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).n, 0, table);});await t.close();@@@ -18,7 +18,7 @@ const BODY = [const t = await boot();const reports = launchReports(t.sources);await t.pull();-- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;++ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;let r = await t.api('/stories', { json: { cluster: clusterId, headline: 'Europa Clipper heads for Jupiter', topic: 'space', body: BODY } });const story = r.json().story;await t.api(`/stories/${story.id}/publish`, { json: {} });@@@ -55,10 +55,10 @@ const BODY = [await check('out of sitemaps and Search (tombstone), labelled in listings and feeds, event emitted', async () => {assert.doesNotMatch((await t.get('/sitemaps/stories.xml')).text, new RegExp(story.slug));-- const del = t.events('news.index_document.deleted');++ const del = await t.events('news.index_document.deleted');assert.strictEqual(del.length, 1);assert.strictEqual(del[0].payload.id, story.id);-- const ev = t.events('news.story.retracted');++ const ev = await t.events('news.story.retracted');assert.strictEqual(ev.length, 1);assert.strictEqual(ev[0].payload.publication_state, 'retracted');assert.ok(ev[0].payload.note.startsWith('The launch date'));@@@ -19,7 +19,7 @@ const BODY = [const t = await boot();const reports = launchReports(t.sources);await t.pull();-- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;++ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;const reader = t.network.addUser('reader');let story;@@@ -42,7 +42,7 @@ const BODY = [assert.strictEqual(story.state, 'draft');assert.deepStrictEqual(story.sources.map((s) => [s.n, s.item.sources_item.id]), [[1, reports.a.id], [2, reports.b.id], [3, reports.c.id]]);assert.strictEqual(story.head, null);-- assert.strictEqual(t.events('news.story.created').length, 1);++ assert.strictEqual((await t.events('news.story.created')).length, 1);});await check('a paragraph without a source, or citing a number not in the source table, is refused', async () => {@@@ -51,7 +51,7 @@ const BODY = [assert.strictEqual(r.json().code, 'story.claim_unsourced');r = await t.api(`/stories/${story.id}/revisions`, { json: { body: 'A claim citing a source that is not attached. [9]', expected_revision: 0 } });assert.strictEqual(r.json().code, 'story.unknown_source');-- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_story_revisions').get().n, 0);++ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_story_revisions').get()).n, 0);});await check('a story with zero sources cannot be published', async () => {@@@ -63,8 +63,8 @@ const BODY = [const pub = await t.api(`/stories/${lonely.id}/publish`, { json: {} });assert.strictEqual(pub.status, 422);assert.strictEqual(pub.json().code, 'story.no_text');-- assert.strictEqual(t.db().prepare("SELECT COUNT(*) AS n FROM news_stories WHERE state = 'published'").get().n, 0);-- const direct = t.ctx.stories.problems(t.ctx.stories.get(lonely.id), { number: 1, fields: { headline: 'x', paragraphs: [{ text: 'x', sources: [1] }], sources: [] }, meta: {} });++ assert.strictEqual((await t.db().prepare("SELECT COUNT(*) AS n FROM news_stories WHERE state = 'published'").get()).n, 0);++ const direct = await t.ctx.stories.problems(await t.ctx.stories.get(lonely.id), { number: 1, fields: { headline: 'x', paragraphs: [{ text: 'x', sources: [1] }], sources: [] }, meta: {} });assert.ok(direct.some((p) => p.code === 'story.unsourced'));});@@@ -121,17 +121,17 @@ const BODY = [assert.deepStrictEqual(json.headline_sources, [1, 2]);assert.strictEqual(json.indexability.indexable, true);-- const cites = t.db().prepare('SELECT anchor, source_item_id FROM news_story_citations WHERE entity_id = ? AND revision = 1 ORDER BY id').all(story.id);++ const cites = await t.db().prepare('SELECT anchor, source_item_id FROM news_story_citations WHERE entity_id = ? AND revision = 1 ORDER BY id').all(story.id);assert.deepStrictEqual(cites.filter((c) => c.anchor.startsWith('p')).map((c) => `${c.anchor}:${c.source_item_id}`), [`p1:${reports.a.id}`, `p1:${reports.b.id}`, `p2:${reports.b.id}`, `p3:${reports.c.id}`]);-- const idx = t.events('news.index_document.upserted').pop();++ const idx = (await t.events('news.index_document.upserted')).pop();assert.ok(idx, 'Search gets the document');const doc = idx.payload;assert.ok(contracts.validate('search.index-document@1', doc).valid, JSON.stringify(contracts.validate('search.index-document@1', doc).errors));const prov = doc.provenance.filter((p) => p.service === 'sources').map((p) => p.id).sort();assert.deepStrictEqual(prov, [reports.a.id, reports.b.id, reports.c.id].sort());assert.strictEqual(doc.indexability.decision, 'index');-- const pub = t.events('news.story.published');++ const pub = await t.events('news.story.published');assert.strictEqual(pub.length, 1);assert.strictEqual(pub[0].visibility, 'public');assert.strictEqual(pub[0].payload.canonical_url, 'https://openvibe.news/stories/europa-clipper-launches-for-jupiter');@@@ -167,7 +167,7 @@ const BODY = [assert.match(page.text, /about fifty times/);assert.match(page.text, /<strong>Correction<\/strong>/);assert.match(page.text, /revision 2\): An earlier version said/);-- assert.strictEqual(t.events('news.story.updated').length, 1);++ assert.strictEqual((await t.events('news.story.updated')).length, 1);const json = (await t.get('/stories/europa-clipper-launches-for-jupiter.json')).json();assert.strictEqual(json.corrections.length, 1);assert.strictEqual(json.revision, 2);@@@ -195,8 +195,8 @@ const BODY = [const r = await t.api(`/stories/${story.id}/unpublish`, { json: {} });assert.strictEqual(r.json().story.state, 'unpublished');assert.strictEqual((await t.get('/stories/europa-clipper-launches-for-jupiter')).status, 410);-- assert.strictEqual(t.events('news.index_document.deleted').length, 1);-- assert.strictEqual(t.events('news.story.unpublished').length, 1);++ assert.strictEqual((await t.events('news.index_document.deleted')).length, 1);++ assert.strictEqual((await t.events('news.story.unpublished')).length, 1);assert.doesNotMatch((await t.get('/sitemaps/stories.xml')).text, /europa-clipper-launches/);});@@@ -19,7 +19,7 @@ const BODY = [const t = await boot();const reports = launchReports(t.sources, { a: { summary: 'The Europa Clipper spacecraft lifted off on a Falcon Heavy rocket. ORIGINAL-SUMMARY-A.' } });await t.pull();-- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;++ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;let r = await t.api('/stories', { json: { cluster: clusterId, headline: 'Europa Clipper is on its way to Jupiter', topic: 'space', body: BODY } });const story = r.json().story;r = await t.api(`/stories/${story.id}/publish`, { json: {} });@@@ -30,16 +30,16 @@ const BODY = [await check('an upstream revision of a cited source: flag + pending revision; the published text is unchanged, with a notice', async () => {t.sources.updateItem(reports.b.id, { title: 'Europa Clipper lifts off; NASA now says arrival in 2031', summary: 'NASA revised its arrival estimate.' });await t.pull();-- const flags = t.db().prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND kind = 'source_updated'").all(story.id);++ const flags = await t.db().prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND kind = 'source_updated'").all(story.id);assert.strictEqual(flags.length, 1);assert.strictEqual(flags[0].status, 'open');assert.strictEqual(flags[0].pending_revision, 2);-- const rev2 = t.ctx.store.revisions.get(story.id, 2);++ const rev2 = await t.ctx.store.revisions.get(story.id, 2);assert.strictEqual(rev2.author, 'svc:news');assert.strictEqual(rev2.meta.system.reason, 'source_updated');-- assert.strictEqual(rev2.content, t.ctx.store.revisions.get(story.id, 1).content, 'the pending revision keeps the text for an editor to check');++ assert.strictEqual(rev2.content, (await t.ctx.store.revisions.get(story.id, 1)).content, 'the pending revision keeps the text for an editor to check');assert.strictEqual(rev2.fields.sources.find((s) => s.n === 2).headline, 'Europa Clipper lifts off; NASA now says arrival in 2031');-- const s = t.ctx.stories.get(story.id);++ const s = await t.ctx.stories.get(story.id);assert.strictEqual(s.published_revision, 1, 'nothing was published by the system');const page = await t.get(url);assert.match(page.text, /changed after publication\. Editors are checking/);@@@ -47,18 +47,18 @@ const BODY = [const body = (h) => (h.match(/<div class="story-body">[\s\S]*?<\/div>/) || [null])[0];assert.ok(body(published.text));assert.strictEqual(body(page.text), body(published.text), 'the published paragraphs are byte-for-byte the same');-- assert.strictEqual(t.events('news.story.flagged').length, 1);-- assert.strictEqual(t.events('news.story.updated').length, 0);++ assert.strictEqual((await t.events('news.story.flagged')).length, 1);++ assert.strictEqual((await t.events('news.story.updated')).length, 0);});await check('an upstream removal: flag + pending revision; the removed source’s headline, summary and link leave the page; noindex for unsupported claims', async () => {t.sources.removeItem(reports.a.id, 'licence withdrawn by the publisher');const d = await t.deliver({ event_type: 'sources.item.removed', subject: { type: 'item', id: reports.a.id, revision: 2 }, payload: { item_id: reports.a.id, source_key: 'wire-a', category: 'news', revision: 2, reason: 'licence withdrawn by the publisher' } });assert.strictEqual(d.status, 204, d.text);-- const item = t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id);++ const item = await t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id);assert.strictEqual(item.status, 'removed');assert.strictEqual(item.summary, null, 'the licensed summary is dropped');-- const flag = t.db().prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND kind = 'source_removed'").get(story.id);++ const flag = await t.db().prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND kind = 'source_removed'").get(story.id);assert.strictEqual(flag.status, 'open');assert.strictEqual(flag.pending_revision, 3);const page = await t.get(url);@@@ -75,12 +75,12 @@ const BODY = [assert.strictEqual(json.sources.find((s) => s.n === 1).headline, undefined);assert.strictEqual(json.claims[2].supported, false);assert.doesNotMatch(JSON.stringify(json), /ORIGINAL-SUMMARY-A|wire-a\.example/);-- const idx = t.events('news.index_document.upserted').pop();++ const idx = (await t.events('news.index_document.upserted')).pop();assert.strictEqual(idx.payload.indexability.decision, 'noindex', 'Search learns the story is noindex');assert.doesNotMatch((await t.get('/sitemaps/stories.xml')).text, /europa-clipper-is-on-its-way/);const replay = await t.deliver({ event_type: 'sources.item.removed', payload: { item_id: reports.a.id, reason: 'again' } });assert.strictEqual(replay.status, 204);-- assert.strictEqual(t.db().prepare("SELECT COUNT(*) AS n FROM news_editorial_flags WHERE kind = 'source_removed'").get().n, 1, 'a removal flags once');++ assert.strictEqual((await t.db().prepare("SELECT COUNT(*) AS n FROM news_editorial_flags WHERE kind = 'source_removed'").get()).n, 1, 'a removal flags once');});await check('the editor desk shows the open flags, the pending revision and why it cannot be published', async () => {@@@ -102,13 +102,13 @@ const BODY = [r = await t.api(`/stories/${story.id}/publish`, { json: { revision: 3 } });assert.strictEqual(r.status, 409);assert.strictEqual(r.json().code, 'story.source_removed');-- assert.strictEqual(t.ctx.stories.get(story.id).published_revision, 1);++ assert.strictEqual((await t.ctx.stories.get(story.id)).published_revision, 1);});await check('the editor revises without the removed source; publishing needs a correction note, then resolves both flags', async () => {// [1] leaves; the first paragraph now rests on [2] and [3], two independent outlets (the index gate needs two).const fixed = BODY.split('\n\n').slice(0, 2).map((p) => p.replace('[1, 2]', '[2, 3]')).join('\n\n');-- r = await t.api(`/stories/${story.id}/sources/${t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).id}`, { method: 'DELETE' });++ r = await t.api(`/stories/${story.id}/sources/${(await t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).id}`, { method: 'DELETE' });assert.strictEqual(r.status, 200, r.text);r = await t.api(`/stories/${story.id}/revisions`, { json: { body: fixed.replace('2030', '2031'), expected_revision: 3 } });assert.strictEqual(r.status, 201, r.text);@@@ -119,21 +119,21 @@ const BODY = [r = await t.api(`/stories/${story.id}/publish`, { json: { revision: n, correction: { kind: 'update', note: 'A source we cited withdrew its report, so the paragraph based only on it was removed; NASA now gives 2031 as the arrival year.' } } });assert.strictEqual(r.status, 200, r.text);assert.strictEqual(r.json().resolved_flags.length, 2);-- const open = t.db().prepare("SELECT COUNT(*) AS n FROM news_editorial_flags WHERE story_id = ? AND status = 'open'").get(story.id).n;++ const open = (await t.db().prepare("SELECT COUNT(*) AS n FROM news_editorial_flags WHERE story_id = ? AND status = 'open'").get(story.id)).n;assert.strictEqual(open, 0);const page = await t.get(url);assert.doesNotMatch(page.text, /Editors are (reviewing|checking)/);assert.match(page.text, /<strong>Update<\/strong>/);assert.match(page.text, /2031/);assert.match(page.text, /<meta name="robots" content="index, follow">/);-- assert.strictEqual(t.events('news.story.updated').length, 1);++ assert.strictEqual((await t.events('news.story.updated')).length, 1);});await check('an unchanged upstream refetch (same revision) flags nothing', async () => {-- const before = t.db().prepare('SELECT COUNT(*) AS n FROM news_editorial_flags').get().n;-- t.ctx.store.setState('sources_cursor', 0);++ const before = (await t.db().prepare('SELECT COUNT(*) AS n FROM news_editorial_flags').get()).n;++ await t.ctx.store.setState('sources_cursor', 0);await t.pull();-- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_editorial_flags').get().n, before);++ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_editorial_flags').get()).n, before);});await t.close();
@@ -19,6 +19,14 @@ TRUST_PROXY=2
@@ -19,6 +19,14 @@ TRUST_PROXY=2
#NEWS_LIMITS_HOUR=3000
#NEWS_LIMITS_HOUR=3000
# SQLite database (production unit: /var/lib/openvibe-news/news.db)
# SQLite database (production unit: /var/lib/openvibe-news/news.db)
+# PostgreSQL (ADR-035), written by OpenVibe.Host roles/data/add-service.sh news: the runtime role through PgBouncer,
+# the owner on a direct connection (migrations and the one-time import), and Valkey (shared limit counters).
+# Without DATABASE_URL, development uses an embedded PGlite database in data/pglite.
+DATABASE_URL=
+DATABASE_DIRECT_URL=
+VALKEY_URL=
+VALKEY_PREFIX=
+# The SQLite file of releases before PostgreSQL: read once by scripts/migrate-to-postgres.js.
NEWS_DB_PATH=./data/news.db
NEWS_DB_PATH=./data/news.db
# OpenVibe.Network — SSO (OAuth2), JWKS, client-credentials tokens
# OpenVibe.Network — SSO (OAuth2), JWKS, client-credentials tokens
@@ -9,6 +9,9 @@ jobs:
@@ -9,6 +9,9 @@ jobs:
uses: OpenVibers/OpenVibe.Shared/.github/workflows/test.yml@423fe519b5fe9594cc81b670e5ba5cb9f21f6176
uses: OpenVibers/OpenVibe.Shared/.github/workflows/test.yml@423fe519b5fe9594cc81b670e5ba5cb9f21f6176
with:
with:
contracts-service: news
contracts-service: news
+ # ADR-035: PostgreSQL 18 + PgBouncer (transaction mode) + Valkey 9 containers (openvibe-sdk
+ # scripts/test-services.sh). npm test runs every file on PGlite; test:pg runs them all through PgBouncer.
+ test: 'eval "$(node_modules/openvibe-sdk/scripts/test-services.sh up)" && npm test && npm run test:pg'
# Secret scan (gitleaks) and dependency audit, shared by every OpenVibers repo (Track Q).
# Secret scan (gitleaks) and dependency audit, shared by every OpenVibers repo (Track Q).
security:
security:
uses: OpenVibers/OpenVibe.Shared/.github/workflows/security.yml@423fe519b5fe9594cc81b670e5ba5cb9f21f6176
uses: OpenVibers/OpenVibe.Shared/.github/workflows/security.yml@423fe519b5fe9594cc81b670e5ba5cb9f21f6176
@@ -27,7 +27,7 @@ News publishes nothing until an editor does: the only seed is the topic list.
@@ -27,7 +27,7 @@ News publishes nothing until an editor does: the only seed is the topic list.
## Owns
## Owns
-The nine charter tables live in News' own SQLite (`NEWS_DB_PATH`):
+The nine charter tables live in News' own PostgreSQL database (`ov_news` on the host's data role, ADR-035; schema in [migrations/](migrations/)):
| Charter table | What it is |
| Charter table | What it is |
|---|---|
|---|---|
@@ -222,11 +222,13 @@ under [Grants the Network must hold](#grants-the-network-must-hold).
@@ -222,11 +222,13 @@ under [Grants the Network must hold](#grants-the-network-must-hold).
## Depends on
## Depends on
-- **Packages** (pinned by release tarball): `openvibe-publishing` v0.4.0 (revisions, citations,
- authorship, seo, index-hooks, ssr, taxonomy slugify), `openvibe-contracts` v0.49.0,
- `openvibe-shared` v1.25.0 (chrome, app icon, footer, legal, release, metrics, ready, seo),
- `openvibe-sdk` v0.12.0 (events outbox and inbox, webhook signatures v2, service tokens, per-actor
- limits).
+- **PostgreSQL 18 and Valkey 9** (OpenVibe.Host `roles/data/`, ADR-035): every read and write is async through
+ `openvibe-sdk/db`; Valkey holds the per-actor limit counters (optional: without `VALKEY_URL` they count per process).
+- **Packages** (pinned by release tarball): `openvibe-publishing` v1.0.0 (async PostgreSQL stores: revisions,
+ citations, authorship, seo, index-hooks, ssr, taxonomy slugify), `openvibe-contracts` v0.76.0,
+ `openvibe-shared` v1.25.0 (Frame, app icon, footer, legal, release, metrics, ready, seo),
+ `openvibe-sdk` v0.20.0 (db with after-commit hooks, PostgreSQL events outbox and inbox, webhook signatures v2,
+ service tokens, per-actor limits, testing).
- **OpenVibe.Sources** (4720): `sources.item.read`; optionally `sources.source.read` (outlet
- **OpenVibe.Sources** (4720): `sources.item.read`; optionally `sources.source.read` (outlet
names and source health; without it the outlet is the URL's host).
names and source health; without it the outlet is the URL's host).
- **OpenVibe.Events** (4300): `events.event.publish`; `events.subscription.manage` to create the
- **OpenVibe.Events** (4300): `events.event.publish`; `events.subscription.manage` to create the
@@ -374,10 +376,13 @@ fnm exec --using=22.22.1 npm run dev # http://localhost:4820 (set OV_OAUTH
@@ -374,10 +376,13 @@ fnm exec --using=22.22.1 npm run dev # http://localhost:4820 (set OV_OAUTH
Production deploys with `sudo ovhost deploy news` on the host (strategy `git-checkout`: fetch,
Production deploys with `sudo ovhost deploy news` on the host (strategy `git-checkout`: fetch,
fast-forward `/opt/openvibe.news`, install on a lockfile change, restart, wait for `/api/ready`).
fast-forward `/opt/openvibe.news`, install on a lockfile change, restart, wait for `/api/ready`).
-The unit is `openvibe-news.service` on `127.0.0.1:4820`, the env file `/etc/openvibe/news.env`.
+The unit is `openvibe-news.service` on `127.0.0.1:4820`, the env file `/etc/openvibe/news.env`. The database is
+`ov_news` on the host's data role (`sudo /opt/openvibe.host/roles/data/add-service.sh news` writes its settings); the
+release migrates it at boot. The one-time move from SQLite is `scripts/migrate-to-postgres.js` (openvibe-sdk
+`runSqliteMigration`, with a `--pglite` rehearsal mode), run while the service is stopped; the old
+`/var/lib/openvibe-news/news.db` stays read-only for 7 days as the rollback.
Rollback: ovhost puts the previous sha back by itself when `/api/ready` does not answer 2xx after the
Rollback: ovhost puts the previous sha back by itself when `/api/ready` does not answer 2xx after the
-restart; afterwards `sudo ovhost rollback news --to <sha>`. Nothing blocks a rollback: the schema
-code only adds tables and columns.
+restart; afterwards `sudo ovhost rollback news --to <sha>`. Migrations only add tables and columns.
First install (done once; kept for a rebuild):
First install (done once; kept for a rebuild):
@@ -4,11 +4,16 @@
@@ -4,11 +4,16 @@
"code": true,
"code": true,
"wave": "W17",
"wave": "W17",
"plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026)",
"plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026)",
- "runtime": "node 22, express 4, better-sqlite3 11",
+ "runtime": "node 22, express 4, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9",
"port": 4820,
"port": 4820,
"domain": "openvibe.news",
"domain": "openvibe.news",
- "contracts": "openvibe-contracts v0.49.0",
- "packages": ["openvibe-publishing v0.4.0", "openvibe-contracts v0.49.0", "openvibe-shared v1.25.0", "openvibe-sdk v0.12.0"],
+ "contracts": "openvibe-contracts v0.76.0",
+ "packages": [
+ "openvibe-publishing v1.0.0",
+ "openvibe-contracts v0.76.0",
+ "openvibe-shared v1.25.0",
+ "openvibe-sdk v0.20.0"
+ ],
"capabilities": [
"capabilities": [
"news.story.create",
"news.story.create",
"news.story.read",
"news.story.read",
@@ -3,8 +3,8 @@ Description=OpenVibe.News — source-backed stories from OpenVibe.Sources items
@@ -3,8 +3,8 @@ Description=OpenVibe.News — source-backed stories from OpenVibe.Sources items
# Network (sign-in, JWKS, service tokens), Sources (the items, over loopback) and Events (the outbox
# Network (sign-in, JWKS, service tokens), Sources (the items, over loopback) and Events (the outbox
# relay and the webhook) should be up first, but News degrades gracefully without them: pages and
# relay and the webhook) should be up first, but News degrades gracefully without them: pages and
# feeds serve; sign-in, ingestion and events wait and are reported by /api/ready.
# feeds serve; sign-in, ingestion and events wait and are reported by /api/ready.
-After=network.target
-Wants=openvibe-network.service
+After=network.target postgresql.service pgbouncer.service valkey-server.service
+Wants=openvibe-network.service postgresql.service pgbouncer.service valkey-server.service
[Service]
[Service]
Type=simple
Type=simple
new file mode 100644
new file mode 100644
@@ -0,0 +1,360 @@
@@ -0,0 +1,360 @@
+-- phase: expand
+-- OpenVibe.News on PostgreSQL (ADR-035, roadmap WS-X2): the tables as they were on SQLite (converted by openvibe-sdk
+-- tools/asyncify/sqlite-schema-to-pg: text COLLATE "C" compares like SQLite, integers are bigint, identities keep their ids),
+-- then the openvibe-publishing stores and the openvibe-sdk outbox. Generated once on 2026-09-28; never edited after it runs.
+
+CREATE TABLE news_topics (
+ id text COLLATE "C" PRIMARY KEY, -- top_<ULID>
+ slug text COLLATE "C" NOT NULL UNIQUE,
+ name text COLLATE "C" NOT NULL,
+ description text COLLATE "C",
+ status text COLLATE "C" NOT NULL DEFAULT 'active' CHECK (status IN ('active','archived')),
+ created_at bigint NOT NULL,
+ updated_at bigint NOT NULL
+);
+
+CREATE TABLE news_story_clusters (
+ id text COLLATE "C" PRIMARY KEY, -- clu_<ULID>
+ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)
+ label text COLLATE "C" NOT NULL, -- derived from the key terms, never invented
+ status text COLLATE "C" NOT NULL DEFAULT 'open' CHECK (status IN ('open','merged','dissolved')),
+ merged_into text COLLATE "C" REFERENCES news_story_clusters(id),
+ split_from text COLLATE "C" REFERENCES news_story_clusters(id),
+ terms text COLLATE "C" NOT NULL DEFAULT '{}', -- { term: count } over its members
+ entities text COLLATE "C" NOT NULL DEFAULT '{}', -- { entity: count } over its members
+ window_start bigint, -- earliest member time (published_at, else first seen)
+ window_end bigint, -- latest member time
+ created_by text COLLATE "C" NOT NULL, -- 'svc:news' (clustering) or usr_… (a split)
+ created_at bigint NOT NULL,
+ updated_at bigint NOT NULL
+);
+CREATE INDEX news_story_clusters_open ON news_story_clusters (status, window_end);
+
+CREATE TABLE news_source_items (
+ id text COLLATE "C" PRIMARY KEY, -- nsi_<ULID>
+ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)
+ sources_item_id text COLLATE "C" NOT NULL UNIQUE, -- itm_… in OpenVibe.Sources (a typed reference)
+ sources_revision bigint NOT NULL, -- the Sources revision these fields came from
+ source_key text COLLATE "C" NOT NULL,
+ canonical_url text COLLATE "C",
+ url_key text COLLATE "C", -- normalised URL for dedupe
+ headline text COLLATE "C" NOT NULL,
+ outlet text COLLATE "C" NOT NULL,
+ authors text COLLATE "C" NOT NULL DEFAULT '[]',
+ published_at text COLLATE "C", -- as the source states it; NULL when it does not
+ summary text COLLATE "C", -- a short summary ONLY when the terms allow it
+ summary_basis text COLLATE "C", -- why a summary is (not) stored
+ license_note text COLLATE "C",
+ terms_note text COLLATE "C",
+ content_hash text COLLATE "C", -- Sources' hash of the parsed fields
+ title_key text COLLATE "C" NOT NULL, -- normalised headline for near-duplicate checks
+ status text COLLATE "C" NOT NULL DEFAULT 'active' CHECK (status IN ('active','duplicate','removed')),
+ duplicate_of text COLLATE "C" REFERENCES news_source_items(id),
+ dedupe text COLLATE "C", -- { rule, detail } when a duplicate
+ cluster_id text COLLATE "C" REFERENCES news_story_clusters(id),
+ cluster_reason text COLLATE "C", -- { rule, shared_entities, shared_terms, score } (explanation)
+ retrieved_at text COLLATE "C",
+ first_seen_at bigint NOT NULL,
+ upstream_updated_at bigint, -- the last time Sources revised it after first ingest
+ removed_at bigint,
+ removed_reason text COLLATE "C",
+ updated_at bigint NOT NULL
+);
+CREATE INDEX news_source_items_url ON news_source_items (url_key);
+CREATE INDEX news_source_items_hash ON news_source_items (content_hash);
+CREATE INDEX news_source_items_cluster ON news_source_items (cluster_id);
+CREATE INDEX news_source_items_seen ON news_source_items (first_seen_at);
+
+CREATE TABLE news_stories (
+ id text COLLATE "C" PRIMARY KEY, -- sty_<ULID>
+ slug text COLLATE "C" NOT NULL UNIQUE,
+ working_headline text COLLATE "C" NOT NULL, -- the editors' latest headline (readers see the revision's)
+ cluster_id text COLLATE "C" REFERENCES news_story_clusters(id),
+ topic_id text COLLATE "C" REFERENCES news_topics(id),
+ state text COLLATE "C" NOT NULL DEFAULT 'draft' CHECK (state IN ('draft','published','unpublished','retracted')),
+ published_revision bigint, -- which revision readers see (NULL: never published)
+ first_published_at bigint,
+ published_at bigint, -- the latest publication of published_revision
+ retracted_at bigint,
+ noindex bigint NOT NULL DEFAULT 0, -- an editor asked search engines not to index it
+ created_by text COLLATE "C" NOT NULL, -- usr_… (the editor who opened the story)
+ created_at bigint NOT NULL,
+ updated_at bigint NOT NULL
+);
+CREATE INDEX news_stories_listing ON news_stories (state, published_at);
+CREATE INDEX news_stories_topic ON news_stories (topic_id, state, published_at);
+CREATE INDEX news_stories_cluster ON news_stories (cluster_id);
+
+CREATE TABLE news_perspectives (
+ id text COLLATE "C" PRIMARY KEY, -- per_<ULID>
+ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)
+ story_id text COLLATE "C" NOT NULL REFERENCES news_stories(id),
+ label text COLLATE "C" NOT NULL, -- written by an editor; never generated
+ description text COLLATE "C",
+ position bigint NOT NULL DEFAULT 0,
+ created_by text COLLATE "C" NOT NULL,
+ created_at bigint NOT NULL,
+ updated_at bigint NOT NULL,
+ removed_at bigint
+);
+CREATE INDEX news_perspectives_story ON news_perspectives (story_id, position);
+
+CREATE TABLE news_story_sources (
+ story_id text COLLATE "C" NOT NULL REFERENCES news_stories(id),
+ source_item_id text COLLATE "C" NOT NULL REFERENCES news_source_items(id),
+ position bigint NOT NULL, -- the [n] readers and editors cite
+ perspective_id text COLLATE "C" REFERENCES news_perspectives(id),
+ added_by text COLLATE "C" NOT NULL,
+ added_at bigint NOT NULL,
+ detached_at bigint,
+ PRIMARY KEY (story_id, source_item_id)
+);
+CREATE INDEX news_story_sources_item ON news_story_sources (source_item_id);
+
+CREATE TABLE news_timeline_entries (
+ id text COLLATE "C" PRIMARY KEY, -- tle_<ULID>
+ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)
+ story_id text COLLATE "C" NOT NULL REFERENCES news_stories(id),
+ occurred_on text COLLATE "C" NOT NULL, -- YYYY-MM-DD or an ISO instant, as the source states it
+ text text COLLATE "C" NOT NULL,
+ source_item_id text COLLATE "C" NOT NULL REFERENCES news_source_items(id),
+ created_by text COLLATE "C" NOT NULL,
+ created_at bigint NOT NULL,
+ removed_at bigint
+);
+CREATE INDEX news_timeline_story ON news_timeline_entries (story_id, occurred_on);
+
+CREATE TABLE news_editorial_flags (
+ id text COLLATE "C" PRIMARY KEY, -- flg_<ULID>
+ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)
+ story_id text COLLATE "C" NOT NULL REFERENCES news_stories(id),
+ kind text COLLATE "C" NOT NULL CHECK (kind IN ('correction','update','retraction','source_updated','source_removed')),
+ status text COLLATE "C" NOT NULL DEFAULT 'open' CHECK (status IN ('open','published','resolved')),
+ note text COLLATE "C", -- public text for correction/update/retraction
+ source_item_id text COLLATE "C" REFERENCES news_source_items(id),
+ pending_revision bigint, -- the revision the system prepared for an upstream change
+ revision bigint, -- the published revision it belongs to
+ created_by text COLLATE "C" NOT NULL,
+ created_at bigint NOT NULL,
+ resolved_by text COLLATE "C",
+ resolved_at bigint
+);
+CREATE INDEX news_flags_story ON news_editorial_flags (story_id, status, created_at);
+
+-- Every merge, split and reversal, with exactly which items moved (so each can be undone).
+CREATE TABLE news_cluster_audit (
+ id text COLLATE "C" PRIMARY KEY, -- cla_<ULID>
+ seq bigint GENERATED ALWAYS AS IDENTITY UNIQUE, -- insertion order (the SQLite rowid tiebreak)
+ action text COLLATE "C" NOT NULL CHECK (action IN ('merge','split','reverse_merge','reverse_split')),
+ cluster_id text COLLATE "C" NOT NULL, -- merge: the surviving cluster; split: the source cluster
+ other_id text COLLATE "C" NOT NULL, -- merge: the absorbed cluster; split: the new cluster
+ item_ids text COLLATE "C" NOT NULL, -- JSON array of news_source_items ids that moved
+ reason text COLLATE "C",
+ actor text COLLATE "C" NOT NULL,
+ reverses text COLLATE "C" REFERENCES news_cluster_audit(id),
+ reversed_by text COLLATE "C" REFERENCES news_cluster_audit(id),
+ created_at bigint NOT NULL
+);
+CREATE INDEX news_cluster_audit_cluster ON news_cluster_audit (cluster_id, created_at);
+
+-- What ingestion did: webhook deliveries, cursor pulls and upstream fetch failures.
+CREATE TABLE news_ingest_runs (
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ origin text COLLATE "C" NOT NULL CHECK (origin IN ('webhook','pull','sources')),
+ state text COLLATE "C" NOT NULL, -- ok | failed | ignored | upstream state (http_error, timeout, …)
+ source_key text COLLATE "C",
+ sources_item_id text COLLATE "C",
+ error_code text COLLATE "C",
+ detail text COLLATE "C",
+ counts text COLLATE "C", -- { created, updated, removed, duplicates, unchanged }
+ cursor_before bigint,
+ cursor_after bigint,
+ at bigint NOT NULL
+);
+CREATE INDEX news_ingest_runs_at ON news_ingest_runs (at);
+
+CREATE TABLE news_state (
+ key text COLLATE "C" PRIMARY KEY,
+ value text COLLATE "C" NOT NULL,
+ updated_at bigint NOT NULL
+);
+
+-- Display cache of OpenVibe.Sources' registry (outlet names) and health; never authority.
+CREATE TABLE news_source_status (
+ source_key text COLLATE "C" PRIMARY KEY,
+ name text COLLATE "C",
+ homepage_url text COLLATE "C",
+ status text COLLATE "C",
+ stale bigint,
+ last_success_at text COLLATE "C",
+ refreshed_at bigint NOT NULL
+);
+
+-- Display cache of Network names for subjects (from sign-in claims or identity.subject.resolve).
+CREATE TABLE subject_projections (
+ subject text COLLATE "C" PRIMARY KEY,
+ username text COLLATE "C",
+ display_name text COLLATE "C",
+ avatar_url text COLLATE "C",
+ refreshed_at bigint NOT NULL
+);
+CREATE INDEX subject_projections_username ON subject_projections (username);
+
+CREATE INDEX news_stories_published ON news_stories (published_at DESC, id) WHERE state = 'published';
+-- openvibe-publishing/revisions (prefix news_story)
+CREATE TABLE IF NOT EXISTS news_story_revisions (
+ id text PRIMARY KEY,
+ entity_id text COLLATE "C" NOT NULL,
+ number integer NOT NULL CHECK (number >= 1),
+ parent_id text,
+ parent_number integer,
+ kind text NOT NULL CHECK (kind IN ('edit','revert','import')),
+ reverted_to integer,
+ content text NOT NULL,
+ fields jsonb NOT NULL DEFAULT '{}',
+ meta jsonb NOT NULL DEFAULT '{}',
+ content_hash text NOT NULL,
+ author text,
+ message text,
+ created_at bigint NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS news_story_revisions_entity_num ON news_story_revisions (entity_id, number);
+CREATE TABLE IF NOT EXISTS news_story_drafts (
+ entity_id text COLLATE "C" NOT NULL,
+ owner text COLLATE "C" NOT NULL,
+ base_revision integer NOT NULL,
+ content text NOT NULL,
+ fields jsonb NOT NULL DEFAULT '{}',
+ meta jsonb NOT NULL DEFAULT '{}',
+ created_at bigint NOT NULL,
+ updated_at bigint NOT NULL,
+ PRIMARY KEY (entity_id, owner)
+);
+CREATE INDEX IF NOT EXISTS news_story_drafts_updated ON news_story_drafts (entity_id, updated_at DESC, owner);
+CREATE TABLE IF NOT EXISTS news_story_revision_purges (
+ entity_id text COLLATE "C" PRIMARY KEY,
+ reason text NOT NULL,
+ purged_by text,
+ purged_at bigint NOT NULL
+);
+CREATE OR REPLACE FUNCTION news_story_revisions_guard() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+ IF TG_OP = 'UPDATE' THEN
+ RAISE EXCEPTION 'news_story_revisions rows are immutable' USING ERRCODE = 'restrict_violation';
+ END IF;
+ IF NOT EXISTS (SELECT 1 FROM news_story_revision_purges WHERE entity_id = OLD.entity_id) THEN
+ RAISE EXCEPTION 'news_story_revisions rows are never deleted outside a recorded purge' USING ERRCODE = 'restrict_violation';
+ END IF;
+ RETURN OLD;
+END
+$$;
+CREATE OR REPLACE TRIGGER news_story_revisions_no_update BEFORE UPDATE ON news_story_revisions FOR EACH ROW EXECUTE FUNCTION news_story_revisions_guard();
+CREATE OR REPLACE TRIGGER news_story_revisions_no_delete BEFORE DELETE ON news_story_revisions FOR EACH ROW EXECUTE FUNCTION news_story_revisions_guard();
+
+-- openvibe-publishing/citations (prefix news_story)
+CREATE TABLE IF NOT EXISTS news_story_citations (
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ entity_id text COLLATE "C" NOT NULL,
+ revision integer NOT NULL CHECK (revision >= 1),
+ anchor text,
+ source_item_id text COLLATE "C",
+ url text,
+ title text,
+ retrieved_at timestamptz,
+ quote_text text,
+ quote_start integer,
+ quote_end integer,
+ license_note text,
+ carried_from bigint REFERENCES news_story_citations(id),
+ attached_by text,
+ attached_at bigint NOT NULL,
+ CHECK (source_item_id IS NOT NULL OR url IS NOT NULL)
+);
+CREATE INDEX IF NOT EXISTS news_story_citations_rev ON news_story_citations (entity_id, revision, id);
+CREATE INDEX IF NOT EXISTS news_story_citations_source ON news_story_citations (source_item_id, entity_id, revision, id) WHERE source_item_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS news_story_citations_carried ON news_story_citations (carried_from) WHERE carried_from IS NOT NULL;
+CREATE TABLE IF NOT EXISTS news_story_citation_purges (
+ entity_id text COLLATE "C" PRIMARY KEY,
+ reason text NOT NULL,
+ purged_by text,
+ purged_at bigint NOT NULL
+);
+CREATE OR REPLACE FUNCTION news_story_citations_guard() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+ IF TG_OP = 'UPDATE' THEN
+ RAISE EXCEPTION 'news_story_citations rows are immutable' USING ERRCODE = 'restrict_violation';
+ END IF;
+ IF NOT EXISTS (SELECT 1 FROM news_story_citation_purges WHERE entity_id = OLD.entity_id) THEN
+ RAISE EXCEPTION 'news_story_citations rows are never deleted outside a recorded purge' USING ERRCODE = 'restrict_violation';
+ END IF;
+ RETURN OLD;
+END
+$$;
+CREATE OR REPLACE TRIGGER news_story_citations_no_update BEFORE UPDATE ON news_story_citations FOR EACH ROW EXECUTE FUNCTION news_story_citations_guard();
+CREATE OR REPLACE TRIGGER news_story_citations_no_delete BEFORE DELETE ON news_story_citations FOR EACH ROW EXECUTE FUNCTION news_story_citations_guard();
+
+-- openvibe-publishing/authorship (prefix news_story)
+CREATE TABLE IF NOT EXISTS news_story_reviews (
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ entity_id text COLLATE "C" NOT NULL,
+ revision integer NOT NULL,
+ reviewer text NOT NULL,
+ decision text NOT NULL CHECK (decision IN ('approved','rejected')),
+ note text,
+ reviewed_at bigint NOT NULL
+);
+CREATE INDEX IF NOT EXISTS news_story_reviews_rev ON news_story_reviews (entity_id, revision, id);
+CREATE INDEX IF NOT EXISTS news_story_reviews_entity ON news_story_reviews (entity_id, id);
+CREATE OR REPLACE FUNCTION news_story_reviews_guard() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+ IF TG_OP = 'UPDATE' THEN
+ RAISE EXCEPTION 'news_story_reviews rows are immutable' USING ERRCODE = 'restrict_violation';
+ END IF;
+ RETURN OLD;
+END
+$$;
+CREATE OR REPLACE TRIGGER news_story_reviews_no_update BEFORE UPDATE ON news_story_reviews FOR EACH ROW EXECUTE FUNCTION news_story_reviews_guard();
+
+-- openvibe-publishing/index-hooks (prefix news)
+CREATE TABLE IF NOT EXISTS news_index_revisions (
+ owner text COLLATE "C" NOT NULL,
+ type text COLLATE "C" NOT NULL,
+ id text COLLATE "C" NOT NULL,
+ revision integer NOT NULL,
+ hash text NOT NULL,
+ updated_at bigint NOT NULL,
+ PRIMARY KEY (owner, type, id)
+);
+
+-- openvibe-publishing/discussion (prefix news_story)
+CREATE TABLE IF NOT EXISTS news_story_discussion_refs (
+ entity_id text COLLATE "C" PRIMARY KEY,
+ thread_id text NOT NULL,
+ ref jsonb NOT NULL,
+ resolved_at bigint NOT NULL
+);
+
+-- openvibe-sdk/events inbox: one receipt per (consumer, event) handled
+CREATE TABLE IF NOT EXISTS idempotency_receipts (
+ consumer text NOT NULL,
+ event_id text NOT NULL,
+ processed_at bigint NOT NULL,
+ PRIMARY KEY (consumer, event_id)
+);
+
+-- openvibe-sdk/events PostgreSQL outbox
+CREATE TABLE IF NOT EXISTS event_outbox (
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ event_id text NOT NULL UNIQUE,
+ envelope jsonb NOT NULL,
+ traceparent text,
+ created_at bigint NOT NULL,
+ attempts integer NOT NULL DEFAULT 0,
+ next_attempt_at bigint NOT NULL DEFAULT 0,
+ sent_at bigint,
+ seq bigint,
+ rejected_at bigint,
+ last_error text
+);
+CREATE INDEX IF NOT EXISTS event_outbox_due ON event_outbox (next_attempt_at, id) WHERE sent_at IS NULL AND rejected_at IS NULL;
+CREATE INDEX IF NOT EXISTS event_outbox_sent ON event_outbox (sent_at) WHERE sent_at IS NOT NULL;
@@ -15,18 +15,34 @@
@@ -15,18 +15,34 @@
"express": "^4.21.2",
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.49.0",
- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v0.4.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.12.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.25.0"
+ "iovalkey": "^0.4.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.76.0",
+ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.20.0",
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.25.0",
+ "pg": "^8.23.0"
},
},
"devDependencies": {
"devDependencies": {
+ "@electric-sql/pglite": "^0.5.8",
"jsonwebtoken": "^9.0.2"
"jsonwebtoken": "^9.0.2"
},
},
"engines": {
"engines": {
"node": ">=22.0.0"
"node": ">=22.0.0"
}
}
},
},
+ "node_modules/@electric-sql/pglite": {
+ "version": "0.5.8",
+ "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.5.8.tgz",
+ "integrity": "sha512-n9tsbUOhwx2epK1V0ZG9Ar4SHWUju04dhmzZXiSBXwBoleOvIfals33NAaWgagQVAL4Rbvx/Ptsu3P+pA09f6Q==",
+ "devOptional": true,
+ "license": "Apache-2.0"
+ },
+ "node_modules/@iovalkey/commands": {
+ "version": "0.1.0",
+ "resolved": "https://registry.npmjs.org/@iovalkey/commands/-/commands-0.1.0.tgz",
+ "integrity": "sha512-/B9W4qKSSITDii5nkBCHyPkIkAi+ealUtr1oqBJsLxjSRLka4pxun2VvMNSmcwgAMxgXtQfl0qRv7TE+udPJzg==",
+ "license": "MIT"
+ },
"node_modules/accepts": {
"node_modules/accepts": {
"version": "1.3.8",
"version": "1.3.8",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@@ -228,6 +244,15 @@
@@ -228,6 +244,15 @@
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
"license": "ISC"
"license": "ISC"
},
},
+ "node_modules/cluster-key-slot": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+ "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/content-disposition": {
"node_modules/content-disposition": {
"version": "0.5.4",
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -310,6 +335,15 @@
@@ -310,6 +335,15 @@
"node": ">=4.0.0"
"node": ">=4.0.0"
}
}
},
},
+ "node_modules/denque": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+ "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.10"
+ }
+ },
"node_modules/depd": {
"node_modules/depd": {
"version": "2.0.0",
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -746,6 +780,49 @@
@@ -746,6 +780,49 @@
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
"license": "ISC"
"license": "ISC"
},
},
+ "node_modules/iovalkey": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/iovalkey/-/iovalkey-0.4.0.tgz",
+ "integrity": "sha512-OSUKxJ+s44CLdUTaicX4+pVrBN/zHKIYwr2oKhRDuczr19FwcSCXEZHzcMUYWZNh1mNSCV9bNJHW/T6cHVm9Aw==",
+ "license": "MIT",
+ "dependencies": {
+ "@iovalkey/commands": "^0.1.0",
+ "cluster-key-slot": "^1.1.0",
+ "debug": "^4.3.4",
+ "denque": "^2.1.0",
+ "lodash.defaults": "^4.2.0",
+ "lodash.isarguments": "^3.1.0",
+ "redis-errors": "^1.2.0",
+ "redis-parser": "^3.0.0",
+ "standard-as-callback": "^2.1.0"
+ },
+ "engines": {
+ "node": ">=18.12.0"
+ }
+ },
+ "node_modules/iovalkey/node_modules/debug": {
+ "version": "4.4.3",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
+ "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
+ "license": "MIT",
+ "dependencies": {
+ "ms": "^2.1.3"
+ },
+ "engines": {
+ "node": ">=6.0"
+ },
+ "peerDependenciesMeta": {
+ "supports-color": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/iovalkey/node_modules/ms": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+ "license": "MIT"
+ },
"node_modules/ipaddr.js": {
"node_modules/ipaddr.js": {
"version": "1.9.1",
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -810,12 +887,24 @@
@@ -810,12 +887,24 @@
"safe-buffer": "^5.0.1"
"safe-buffer": "^5.0.1"
}
}
},
},
+ "node_modules/lodash.defaults": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
+ "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
+ "license": "MIT"
+ },
"node_modules/lodash.includes": {
"node_modules/lodash.includes": {
"version": "4.3.0",
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
"license": "MIT"
"license": "MIT"
},
},
+ "node_modules/lodash.isarguments": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
+ "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
+ "license": "MIT"
+ },
"node_modules/lodash.isboolean": {
"node_modules/lodash.isboolean": {
"version": "3.0.3",
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
@@ -1015,9 +1104,9 @@
@@ -1015,9 +1104,9 @@
}
}
},
},
"node_modules/openvibe-contracts": {
"node_modules/openvibe-contracts": {
- "version": "0.49.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.49.0",
- "integrity": "sha512-lapsLoBm3ZpndgiOMELdCBh2PJtuuyqj6SUhQsfwJkxnLdQfyhXfRFgiuLh+iQrg4kqtG9611191Mp8zAfk7fQ==",
+ "version": "0.76.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.76.0",
+ "integrity": "sha512-zTdx776KWrcJKm1JTnf+0+BnsvswdvXy1ldPyse1hSqhnW/SSr98sX696eRetFUhTl8qVM1xSRCykqIUbqItPw==",
"license": "AGPL-3.0-only",
"license": "AGPL-3.0-only",
"dependencies": {
"dependencies": {
"ajv": "^8.17.1",
"ajv": "^8.17.1",
@@ -1031,15 +1120,15 @@
@@ -1031,15 +1120,15 @@
}
}
},
},
"node_modules/openvibe-publishing": {
"node_modules/openvibe-publishing": {
- "version": "0.4.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v0.4.0",
- "integrity": "sha512-zipRyxNSb39fTk6ZtXfCygWgxSRVj/fOVGc31hynBANM5JCwErX/dYKsE/XmdYGlId29E5O3MyZLohgfocBP4w==",
+ "version": "1.0.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",
+ "integrity": "sha512-g4mU0PPtI4y38NcsdppEbrXTrCYp0rPmIYWyxLxh4GpEYhn48jMWq2+cMQA2uwZCeeZlB3UiXVrQNQyQLiliwg==",
"license": "MIT",
"license": "MIT",
"engines": {
"engines": {
"node": ">=20"
"node": ">=20"
},
},
"peerDependencies": {
"peerDependencies": {
- "better-sqlite3": ">=11",
+ "openvibe-sdk": ">=0.15.0",
"openvibe-shared": ">=1.5.0"
"openvibe-shared": ">=1.5.0"
},
},
"peerDependenciesMeta": {
"peerDependenciesMeta": {
@@ -1049,12 +1138,32 @@
@@ -1049,12 +1138,32 @@
}
}
},
},
"node_modules/openvibe-sdk": {
"node_modules/openvibe-sdk": {
- "version": "0.12.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.12.0",
- "integrity": "sha512-iG1h905O1f7y/PnP8xLA6GCVzXEBDfSaBj2tc1ucNreWg2sT/AJWosb8NW0WRMK5WS4sIQ6tMsprmDLj/fjjeQ==",
+ "version": "0.20.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.20.0",
+ "integrity": "sha512-CqZFfVuSv3ZPoyG1LuWCAdvo8TG2ZuUCeGaqTsxDKS3gsymSFV2QXPq/5QUotTd82WShTGUKKhTgcyp9ggUMfQ==",
"license": "MIT",
"license": "MIT",
"engines": {
"engines": {
"node": ">=20"
"node": ">=20"
+ },
+ "peerDependencies": {
+ "@electric-sql/pglite": "^0.5.8",
+ "better-sqlite3": ">=11",
+ "iovalkey": "^0.4.0",
+ "pg": "^8.23.0"
+ },
+ "peerDependenciesMeta": {
+ "@electric-sql/pglite": {
+ "optional": true
+ },
+ "better-sqlite3": {
+ "optional": true
+ },
+ "iovalkey": {
+ "optional": true
+ },
+ "pg": {
+ "optional": true
+ }
}
}
},
},
"node_modules/openvibe-shared": {
"node_modules/openvibe-shared": {
@@ -1081,6 +1190,134 @@
@@ -1081,6 +1190,134 @@
"integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
"integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
"license": "MIT"
"license": "MIT"
},
},
+ "node_modules/pg": {
+ "version": "8.23.0",
+ "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz",
+ "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==",
+ "license": "MIT",
+ "dependencies": {
+ "pg-connection-string": "^2.14.0",
+ "pg-pool": "^3.14.0",
+ "pg-protocol": "^1.16.0",
+ "pg-types": "2.2.0",
+ "pgpass": "1.0.5"
+ },
+ "engines": {
+ "node": ">= 16.0.0"
+ },
+ "optionalDependencies": {
+ "pg-cloudflare": "^1.4.0"
+ },
+ "peerDependencies": {
+ "pg-native": ">=3.0.1"
+ },
+ "peerDependenciesMeta": {
+ "pg-native": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/pg-cloudflare": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz",
+ "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==",
+ "license": "MIT",
+ "optional": true
+ },
+ "node_modules/pg-connection-string": {
+ "version": "2.14.0",
+ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz",
+ "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==",
+ "license": "MIT"
+ },
+ "node_modules/pg-int8": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+ "license": "ISC",
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
+ "node_modules/pg-pool": {
+ "version": "3.14.0",
+ "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
+ "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
+ "license": "MIT",
+ "peerDependencies": {
+ "pg": ">=8.0"
+ }
+ },
+ "node_modules/pg-protocol": {
+ "version": "1.16.0",
+ "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.0.tgz",
+ "integrity": "sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg==",
+ "license": "MIT"
+ },
+ "node_modules/pg-types": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+ "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+ "license": "MIT",
+ "dependencies": {
+ "pg-int8": "1.0.1",
+ "postgres-array": "~2.0.0",
+ "postgres-bytea": "~1.0.0",
+ "postgres-date": "~1.0.4",
+ "postgres-interval": "^1.1.0"
+ },
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/pgpass": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+ "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+ "license": "MIT",
+ "dependencies": {
+ "split2": "^4.1.0"
+ }
+ },
+ "node_modules/postgres-array": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+ "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/postgres-bytea": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
+ "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-date": {
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+ "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-interval": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+ "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+ "license": "MIT",
+ "dependencies": {
+ "xtend": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/prebuild-install": {
"node_modules/prebuild-install": {
"version": "7.1.3",
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
@@ -1204,6 +1441,27 @@
@@ -1204,6 +1441,27 @@
"node": ">= 6"
"node": ">= 6"
}
}
},
},
+ "node_modules/redis-errors": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+ "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/redis-parser": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
+ "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
+ "license": "MIT",
+ "dependencies": {
+ "redis-errors": "^1.0.0"
+ },
+ "engines": {
+ "node": ">=4"
+ }
+ },
"node_modules/require-from-string": {
"node_modules/require-from-string": {
"version": "2.0.2",
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
@@ -1419,6 +1677,21 @@
@@ -1419,6 +1677,21 @@
"simple-concat": "^1.0.0"
"simple-concat": "^1.0.0"
}
}
},
},
+ "node_modules/split2": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+ "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+ "license": "ISC",
+ "engines": {
+ "node": ">= 10.x"
+ }
+ },
+ "node_modules/standard-as-callback": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
+ "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
+ "license": "MIT"
+ },
"node_modules/statuses": {
"node_modules/statuses": {
"version": "2.0.2",
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
@@ -1546,6 +1819,15 @@
@@ -1546,6 +1819,15 @@
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
"license": "ISC"
"license": "ISC"
+ },
+ "node_modules/xtend": {
+ "version": "4.0.2",
+ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.4"
+ }
}
}
}
}
}
}
@@ -2,16 +2,23 @@
@@ -2,16 +2,23 @@
"name": "openvibe-news",
"name": "openvibe-news",
"version": "0.1.0",
"version": "0.1.0",
"private": true,
"private": true,
- "description": "OpenVibe.News — a source-backed publication product: source items from OpenVibe.Sources, dedupe, explainable clusters, editor-written stories whose every claim cites its sources, timelines, perspectives and correction/retraction revisions, server-rendered and useful without JavaScript.",
+ "description": "OpenVibe.News \u2014 a source-backed publication product: source items from OpenVibe.Sources, dedupe, explainable clusters, editor-written stories whose every claim cites its sources, timelines, perspectives and correction/retraction revisions, server-rendered and useful without JavaScript.",
"main": "server/index.js",
"main": "server/index.js",
"scripts": {
"scripts": {
"start": "node server/index.js",
"start": "node server/index.js",
"dev": "NODE_ENV=development node server/index.js",
"dev": "NODE_ENV=development node server/index.js",
"test": "node test/run.js",
"test": "node test/run.js",
"seed": "node scripts/seed.js",
"seed": "node scripts/seed.js",
- "subscribe": "node scripts/subscribe.js"
+ "subscribe": "node scripts/subscribe.js",
+ "test:pg": "NEWS_TEST_STORE=pg node test/run.js"
},
},
- "keywords": ["openvibe", "news", "publishing", "sources", "feeds"],
+ "keywords": [
+ "openvibe",
+ "news",
+ "publishing",
+ "sources",
+ "feeds"
+ ],
"author": "OpenVibers",
"author": "OpenVibers",
"license": "AGPL-3.0-only",
"license": "AGPL-3.0-only",
"repository": "github:OpenVibers/OpenVibe.News",
"repository": "github:OpenVibers/OpenVibe.News",
@@ -22,13 +29,16 @@
@@ -22,13 +29,16 @@
"express": "^4.21.2",
"express": "^4.21.2",
"express-rate-limit": "^7.5.0",
"express-rate-limit": "^7.5.0",
"helmet": "^8.1.0",
"helmet": "^8.1.0",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.49.0",
- "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v0.4.0",
- "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.12.0",
- "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.25.0"
+ "iovalkey": "^0.4.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.76.0",
+ "openvibe-publishing": "https://codeload.github.com/OpenVibers/OpenVibe.Publishing/tar.gz/refs/tags/v1.0.0",
+ "openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.20.0",
+ "openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v1.25.0",
+ "pg": "^8.23.0"
},
},
"devDependencies": {
"devDependencies": {
- "jsonwebtoken": "^9.0.2"
+ "jsonwebtoken": "^9.0.2",
+ "@electric-sql/pglite": "^0.5.8"
},
},
"engines": {
"engines": {
"node": ">=22.0.0"
"node": ">=22.0.0"
new file mode 100644
new file mode 100644
@@ -0,0 +1,26 @@
@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * The one-time move of News's SQLite database (NEWS_DB_PATH) into its PostgreSQL schema (ADR-035; the procedure is
+ * openvibe-sdk docs/migrating-to-postgresql.md, section 6, carried out by openvibe-sdk/db runSqliteMigration).
+ *
+ * node scripts/migrate-to-postgres.js [--sqlite <file>] [--pglite] [--json]
+ *
+ * Applies migrations/ as the owner (DATABASE_DIRECT_URL), copies every table into emptied tables, verifies row counts
+ * and checksums, and exits 1 unless everything verified. The SQLite file is opened read-only. Every table keeps its
+ * name and columns.
+ */
+require('dotenv').config();
+const { runSqliteMigration } = require('openvibe-sdk/db');
+const configLib = require('../server/config');
+const { MIGRATIONS } = require('../server/db');
+
+const TABLES = {};
+
+if (require.main === module) {
+ const config = configLib.load();
+ runSqliteMigration({ service: 'news', sqlite: config.dbPath, directUrl: config.db.directUrl, migrations: MIGRATIONS, tables: TABLES })
+ .then((code) => process.exit(code), (err) => { console.error(`migrate-to-postgres failed: ${err.message}`); process.exit(1); });
+}
+
+module.exports = { TABLES };
@@ -5,7 +5,7 @@
@@ -5,7 +5,7 @@
* existing ones are never renamed, re-described or re-activated. There are no seed stories,
* existing ones are never renamed, re-described or re-activated. There are no seed stories,
* source items or clusters: News publishes nothing until an editor does.
* source items or clusters: News publishes nothing until an editor does.
*
*
- * npm run seed (uses NEWS_DB_PATH from .env or /etc/openvibe/news.env)
+ * npm run seed (DATABASE_URL from .env or /etc/openvibe/news.env; without it, the development PGlite)
*/
*/
require('dotenv').config();
require('dotenv').config();
const { load } = require('../server/config');
const { load } = require('../server/config');
@@ -13,7 +13,9 @@ const { openStore } = require('../server/db');
@@ -13,7 +13,9 @@ const { openStore } = require('../server/db');
const { createTopics } = require('../server/domain/topics');
const { createTopics } = require('../server/domain/topics');
const config = load();
const config = load();
-const store = openStore(config.dbPath);
-const r = createTopics({ store }).seed();
-console.log(`topics: ${r.created} added, ${r.existing} already present (${config.dbPath})`);
-store.close();
+(async () => {
+ const store = await openStore(config);
+ const r = await createTopics({ store }).seed();
+ console.log(`topics: ${r.created} added, ${r.existing} already present (${store.db.store})`);
+ await store.close();
+})().catch((err) => { console.error(`[seed] ${err.message}`); process.exit(1); });
@@ -46,11 +46,12 @@ const VERSION = require('../package.json').version;
@@ -46,11 +46,12 @@ const VERSION = require('../package.json').version;
/** opts: config, store | dbPath, now (clock), fetchImpl, auth (a createAuthClient-like object), log,
/** opts: config, store | dbPath, now (clock), fetchImpl, auth (a createAuthClient-like object), log,
* limitsNow (the per-actor limiter's clock, tests; default the wall clock) */
* limitsNow (the per-actor limiter's clock, tests; default the wall clock) */
-function createApp(opts = {}) {
+async function createApp(opts = {}) {
const config = opts.config || configLib.load();
const config = opts.config || configLib.load();
const log = opts.log || console;
const log = opts.log || console;
const fetchImpl = opts.fetchImpl || globalThis.fetch;
const fetchImpl = opts.fetchImpl || globalThis.fetch;
- const store = opts.store || openStore(opts.dbPath || config.dbPath, { now: opts.now });
+ // PostgreSQL (ADR-035): opened and migrated here unless the caller (a test, a script) hands in a store.
+ const store = opts.store || await openStore(config, { now: opts.now, log });
const outbox = createNewsOutbox({ db: store.db, config, fetchImpl, now: store.now, log });
const outbox = createNewsOutbox({ db: store.db, config, fetchImpl, now: store.now, log });
const publication = createPublication({ store, config, outbox });
const publication = createPublication({ store, config, outbox });
@@ -64,7 +65,7 @@ function createApp(opts = {}) {
@@ -64,7 +65,7 @@ function createApp(opts = {}) {
ingest.setStories(stories);
ingest.setStories(stories);
const topics = createTopics({ store });
const topics = createTopics({ store });
// Topics are the only seed (idempotent: missing slugs are added, existing ones never changed).
// Topics are the only seed (idempotent: missing slugs are added, existing ones never changed).
- if (opts.seedTopics !== false) topics.seed();
+ if (opts.seedTopics !== false) await topics.seed();
const people = createPeople({ store, config, fetchImpl });
const people = createPeople({ store, config, fetchImpl });
const reading = createReading({ store, config, stories, publication, people, topics });
const reading = createReading({ store, config, stories, publication, people, topics });
const auth = opts.auth || createAuthClient(config);
const auth = opts.auth || createAuthClient(config);
@@ -82,7 +83,10 @@ function createApp(opts = {}) {
@@ -82,7 +83,10 @@ function createApp(opts = {}) {
app.locals.ctx = ctx;
app.locals.ctx = ctx;
// Per-actor limits (http/actor-limits.js) on /api/v1, the desk and the comment form, counted once
// Per-actor limits (http/actor-limits.js) on /api/v1, the desk and the comment form, counted once
// each router resolved req.viewer; the per-address limits below stay.
// each router resolved req.viewer; the per-address limits below stay.
- ctx.limits = createActorLimits({ config, now: opts.limitsNow || (() => Date.now()), registry: metrics.registry, log });
+ // Valkey (ADR-035): shared, never-authoritative state (per-actor limit counters). Optional.
+ const valkey = opts.valkey !== undefined ? opts.valkey : (config.valkey.url ? require('openvibe-sdk/valkey').createValkey({ url: config.valkey.url, prefix: config.valkey.prefix, log }) : null);
+ ctx.valkey = valkey;
+ ctx.limits = createActorLimits({ config, now: opts.limitsNow || (() => Date.now()), registry: metrics.registry, log, valkey });
app.use(contracts.http.middleware());
app.use(contracts.http.middleware());
app.use(helmet({
app.use(helmet({
@@ -112,7 +116,7 @@ function createApp(opts = {}) {
@@ -112,7 +116,7 @@ function createApp(opts = {}) {
app.get('/api/health', (_req, res) => res.json({ status: 'ok', service: 'openvibe-news', version: VERSION }));
app.get('/api/health', (_req, res) => res.json({ status: 'ok', service: 'openvibe-news', version: VERSION }));
// GET /release.json (ADR-016) and POST /release-metrics: open tabs' update reports into /metrics.
// GET /release.json (ADR-016) and POST /release-metrics: open tabs' update reports into /metrics.
release.mount(app, { registry: metrics.registry });
release.mount(app, { registry: metrics.registry });
- const readiness = createNewsReadiness({ store, auth, outbox, ingest, config, release: release.release });
+ const readiness = createNewsReadiness({ store, auth, outbox, ingest, config, release: release.release, valkey: ctx.valkey });
app.get('/api/ready', readiness.handler);
app.get('/api/ready', readiness.handler);
// ── Events webhook (raw body; before any other body parser) ─
// ── Events webhook (raw body; before any other body parser) ─
@@ -66,7 +66,7 @@ function createViewerResolver({ auth, config, people }) {
@@ -66,7 +66,7 @@ function createViewerResolver({ auth, config, people }) {
const claims = await auth.verify(token);
const claims = await auth.verify(token);
if (!claims || (typeof claims.sub === 'string' && PRINCIPAL_SUB.test(claims.sub))) return null;
if (!claims || (typeof claims.sub === 'string' && PRINCIPAL_SUB.test(claims.sub))) return null;
const subject = ids.isSubjectId('user', claims.subject_id) ? claims.subject_id : null;
const subject = ids.isSubjectId('user', claims.subject_id) ? claims.subject_id : null;
- if (subject && people) { try { people.rememberClaims(subject, claims); } catch { /* display cache only */ } }
+ if (subject && people) { try { await people.rememberClaims(subject, claims); } catch { /* display cache only */ } }
return { kind: 'user', subject, staff: staffMap.can(claims, 'staff.editorial.manage'), origin: 'user', user: claimsToUser(claims), token };
return { kind: 'user', subject, staff: staffMap.can(claims, 'staff.editorial.manage'), origin: 'user', user: claimsToUser(claims), token };
}
}
@@ -78,7 +78,7 @@ function createViewerResolver({ auth, config, people }) {
@@ -78,7 +78,7 @@ function createViewerResolver({ auth, config, people }) {
const payload = decodeJwtPayload(token);
const payload = decodeJwtPayload(token);
if (payload && typeof payload.sub === 'string' && PRINCIPAL_SUB.test(payload.sub)) {
if (payload && typeof payload.sub === 'string' && PRINCIPAL_SUB.test(payload.sub)) {
if (opts.services === false) return ANONYMOUS;
if (opts.services === false) return ANONYMOUS;
- return fromServiceToken(req, token);
+ return await fromServiceToken(req, token);
}
}
}
}
const token = extractToken(req);
const token = extractToken(req);
@@ -54,7 +54,7 @@ function createCommunity({ store, config, fetchImpl = globalThis.fetch, log = co
@@ -54,7 +54,7 @@ function createCommunity({ store, config, fetchImpl = globalThis.fetch, log = co
/** The stored thread id, or resolve it once through Community (the caller checks the story may have one). */
/** The stored thread id, or resolve it once through Community (the caller checks the story may have one). */
async threadFor(story, label, ctx) {
async threadFor(story, label, ctx) {
- const known = store.discussion.get(story.id);
+ const known = await store.discussion.get(story.id);
if (known) return known.threadId;
if (known) return known.threadId;
if (!discussion) return null;
if (!discussion) return null;
const out = await store.discussion.threadFor(story.id, refFor(story, label), { client: discussion, traceparent: ctx && ctx.traceparent, requestId: ctx && ctx.requestId });
const out = await store.discussion.threadFor(story.id, refFor(story, label), { client: discussion, traceparent: ctx && ctx.traceparent, requestId: ctx && ctx.requestId });
@@ -64,13 +64,13 @@ function createCommunity({ store, config, fetchImpl = globalThis.fetch, log = co
@@ -64,13 +64,13 @@ function createCommunity({ store, config, fetchImpl = globalThis.fetch, log = co
/** { thread, comments, next_cursor } read as an anonymous visitor (public data only). */
/** { thread, comments, next_cursor } read as an anonymous visitor (public data only). */
async readThread(threadId, { after, ctx } = {}) {
async readThread(threadId, { after, ctx } = {}) {
const qs = after ? `?after=${encodeURIComponent(after)}` : '';
const qs = after ? `?after=${encodeURIComponent(after)}` : '';
- return call('GET', `/api/v1/comments/threads/${encodeURIComponent(threadId)}${qs}`, { ctx });
+ return await call('GET', `/api/v1/comments/threads/${encodeURIComponent(threadId)}${qs}`, { ctx });
},
},
/** Comment as the signed-in member. */
/** Comment as the signed-in member. */
async comment(threadId, subject, { message, parentId } = {}, ctx) {
async comment(threadId, subject, { message, parentId } = {}, ctx) {
if (!writeTokens) { const e = new Error('comments are not configured'); e.status = 503; throw e; }
if (!writeTokens) { const e = new Error('comments are not configured'); e.status = 503; throw e; }
- return call('POST', `/api/v1/comments/threads/${encodeURIComponent(threadId)}/comments`, {
+ return await call('POST', `/api/v1/comments/threads/${encodeURIComponent(threadId)}/comments`, {
tokens: writeTokens, subject, body: { message, ...(parentId ? { parent_id: parentId } : {}) }, ctx,
tokens: writeTokens, subject, body: { message, ...(parentId ? { parent_id: parentId } : {}) }, ctx,
});
});
},
},
@@ -80,7 +80,7 @@ function createCommunity({ store, config, fetchImpl = globalThis.fetch, log = co
@@ -80,7 +80,7 @@ function createCommunity({ store, config, fetchImpl = globalThis.fetch, log = co
* had a thread. Never throws; a failure is logged.
* had a thread. Never throws; a failure is logged.
*/
*/
async setThreadVisibility(storyId, visibility, ctx) {
async setThreadVisibility(storyId, visibility, ctx) {
- const known = store.discussion.get(storyId);
+ const known = await store.discussion.get(storyId);
if (!known || !modTokens) return false;
if (!known || !modTokens) return false;
try {
try {
await call('PUT', `/api/v1/comments/threads/${encodeURIComponent(known.threadId)}/visibility`, { tokens: modTokens, body: { visibility }, ctx });
await call('PUT', `/api/v1/comments/threads/${encodeURIComponent(known.threadId)}/visibility`, { tokens: modTokens, body: { visibility }, ctx });
@@ -23,6 +23,7 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
@@ -23,6 +23,7 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
}) : null;
}) : null;
const q = {
const q = {
get: db.prepare('SELECT * FROM subject_projections WHERE subject = ?'),
get: db.prepare('SELECT * FROM subject_projections WHERE subject = ?'),
+ getMany: db.prepare('SELECT * FROM subject_projections WHERE subject = ANY(?)'),
byUsername: db.prepare('SELECT * FROM subject_projections WHERE username = ? ORDER BY refreshed_at DESC LIMIT 1'),
byUsername: db.prepare('SELECT * FROM subject_projections WHERE username = ? ORDER BY refreshed_at DESC LIMIT 1'),
put: db.prepare(`INSERT INTO subject_projections (subject, username, display_name, avatar_url, refreshed_at) VALUES (?, ?, ?, ?, ?)
put: db.prepare(`INSERT INTO subject_projections (subject, username, display_name, avatar_url, refreshed_at) VALUES (?, ?, ?, ?, ?)
ON CONFLICT (subject) DO UPDATE SET username = excluded.username, display_name = excluded.display_name,
ON CONFLICT (subject) DO UPDATE SET username = excluded.username, display_name = excluded.display_name,
@@ -30,9 +31,9 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
@@ -30,9 +31,9 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
};
};
const absAvatar = (u) => (!u ? null : /^https?:\/\//i.test(u) ? u : `${config.networkUrl}${u.startsWith('/') ? '' : '/'}${u}`);
const absAvatar = (u) => (!u ? null : /^https?:\/\//i.test(u) ? u : `${config.networkUrl}${u.startsWith('/') ? '' : '/'}${u}`);
- function remember(subject, p) {
+ async function remember(subject, p) {
if (!ids.isSubjectId('user', subject) || !p) return;
if (!ids.isSubjectId('user', subject) || !p) return;
- q.put.run(subject, p.username ? String(p.username).toLowerCase() : null, p.display_name || p.username || null, absAvatar(p.avatar_url), store.now());
+ await q.put.run(subject, p.username ? String(p.username).toLowerCase() : null, p.display_name || p.username || null, absAvatar(p.avatar_url), store.now());
}
}
async function resolve(subjects) {
async function resolve(subjects) {
@@ -46,7 +47,7 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
@@ -46,7 +47,7 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
if (res.status === 401) tokens.invalidate && tokens.invalidate();
if (res.status === 401) tokens.invalidate && tokens.invalidate();
const data = await res.json().catch(() => null);
const data = await res.json().catch(() => null);
if (!res.ok || !data || typeof data.results !== 'object') throw new Error(`resolve-batch answered ${res.status}`);
if (!res.ok || !data || typeof data.results !== 'object') throw new Error(`resolve-batch answered ${res.status}`);
- for (const s of subjects) if (data.results[s]) remember(s, data.results[s]);
+ for (const s of subjects) if (data.results[s]) await remember(s, data.results[s]);
}
}
const shape = (row, subject) => (row
const shape = (row, subject) => (row
@@ -56,12 +57,14 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
@@ -56,12 +57,14 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
return {
return {
enabled,
enabled,
/** From a verified sign-in: the member's own claims are a fresh projection. */
/** From a verified sign-in: the member's own claims are a fresh projection. */
- rememberClaims(subject, claims) { remember(subject, claims); },
+ async rememberClaims(subject, claims) { await remember(subject, claims); },
/** Map subject → { name, username, avatarUrl, known }. Missing entries are fetched (bounded wait). */
/** Map subject → { name, username, avatarUrl, known }. Missing entries are fetched (bounded wait). */
async many(subjects) {
async many(subjects) {
const list = [...new Set((subjects || []).filter((s) => ids.isSubjectId('user', s)))];
const list = [...new Set((subjects || []).filter((s) => ids.isSubjectId('user', s)))];
- const missing = list.filter((s) => { const r = q.get.get(s); return !r || store.now() - r.refreshed_at > TTL_MS; });
+ const read = async () => new Map((list.length ? await q.getMany.all(list) : []).map((r) => [r.subject, r]));
+ let rows = await read();
+ const missing = list.filter((s) => { const r = rows.get(s); return !r || store.now() - r.refreshed_at > TTL_MS; });
if (missing.length && tokens) {
if (missing.length && tokens) {
let timer;
let timer;
await Promise.race([
await Promise.race([
@@ -69,19 +72,20 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
@@ -69,19 +72,20 @@ function createPeople({ store, config, fetchImpl = globalThis.fetch }) {
new Promise((r) => { timer = setTimeout(r, WAIT_MS); }),
new Promise((r) => { timer = setTimeout(r, WAIT_MS); }),
]);
]);
clearTimeout(timer);
clearTimeout(timer);
+ rows = await read();
}
}
- return new Map(list.map((s) => [s, shape(q.get.get(s), s)]));
+ return new Map(list.map((s) => [s, shape(rows.get(s) || null, s)]));
},
},
async one(subject) { return (await this.many([subject])).get(subject) || shape(null, subject); },
async one(subject) { return (await this.many([subject])).get(subject) || shape(null, subject); },
/** The subject a username belongs to, from people News has seen. */
/** The subject a username belongs to, from people News has seen. */
- byUsername(username) {
- const row = q.byUsername.get(String(username || '').toLowerCase());
+ async byUsername(username) {
+ const row = await q.byUsername.get(String(username || '').toLowerCase());
return row ? shape(row, row.subject) : null;
return row ? shape(row, row.subject) : null;
},
},
- cached: (subject) => shape(q.get.get(subject), subject),
+ cached: async (subject) => shape(await q.get.get(subject), subject),
};
};
}
}
@@ -35,6 +35,10 @@ function load(env = process.env) {
@@ -35,6 +35,10 @@ function load(env = process.env) {
hour: Math.max(1, int(env.NEWS_LIMITS_HOUR, 3000)),
hour: Math.max(1, int(env.NEWS_LIMITS_HOUR, 3000)),
},
},
+ // PostgreSQL (ADR-035): DATABASE_URL serves (PgBouncer), DATABASE_DIRECT_URL migrates (owner role).
+ db: { url: env.DATABASE_URL || '', directUrl: env.DATABASE_DIRECT_URL || '' },
+ valkey: { url: env.VALKEY_URL || '', prefix: env.VALKEY_PREFIX || 'ov:news:' },
+ // The SQLite file of releases before the switch: read once by scripts/migrate-to-postgres.js.
dbPath: env.NEWS_DB_PATH || './data/news.db',
dbPath: env.NEWS_DB_PATH || './data/news.db',
// OpenVibe.Network: SSO (OAuth2 authorization server), JWKS, client-credentials tokens.
// OpenVibe.Network: SSO (OAuth2 authorization server), JWKS, client-credentials tokens.
@@ -1,7 +1,7 @@
@@ -1,7 +1,7 @@
'use strict';
'use strict';
/**
/**
- * News' own SQLite database: created on boot, idempotently. Nothing here is shared with another
+ * News' own PostgreSQL database (ADR-035, roadmap WS-X2): the schema is migrations/NNNN_*.sql, applied at boot. Nothing here is shared with another
* service; the publishing packages create their tables inside this database with News' prefixes.
* service; the publishing packages create their tables inside this database with News' prefixes.
*
*
* The nine charter tables (roadmap §15.13):
* The nine charter tables (roadmap §15.13):
@@ -28,220 +28,45 @@
@@ -28,220 +28,45 @@
*/
*/
const fs = require('fs');
const fs = require('fs');
const path = require('path');
const path = require('path');
-const Database = require('better-sqlite3');
+const { createDb } = require('openvibe-sdk/db');
const { createRevisionStore } = require('openvibe-publishing/revisions');
const { createRevisionStore } = require('openvibe-publishing/revisions');
const { createCitationStore } = require('openvibe-publishing/citations');
const { createCitationStore } = require('openvibe-publishing/citations');
const { createReviewLog } = require('openvibe-publishing/authorship');
const { createReviewLog } = require('openvibe-publishing/authorship');
const { createIndexSequencer } = require('openvibe-publishing/index-hooks');
const { createIndexSequencer } = require('openvibe-publishing/index-hooks');
const { createDiscussionRefs } = require('openvibe-publishing/discussion');
const { createDiscussionRefs } = require('openvibe-publishing/discussion');
-const SCHEMA = `
-CREATE TABLE IF NOT EXISTS news_topics (
- id TEXT PRIMARY KEY, -- top_<ULID>
- slug TEXT NOT NULL UNIQUE,
- name TEXT NOT NULL,
- description TEXT,
- status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','archived')),
- created_at INTEGER NOT NULL,
- updated_at INTEGER NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS news_story_clusters (
- id TEXT PRIMARY KEY, -- clu_<ULID>
- label TEXT NOT NULL, -- derived from the key terms, never invented
- status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('open','merged','dissolved')),
- merged_into TEXT REFERENCES news_story_clusters(id),
- split_from TEXT REFERENCES news_story_clusters(id),
- terms TEXT NOT NULL DEFAULT '{}', -- { term: count } over its members
- entities TEXT NOT NULL DEFAULT '{}', -- { entity: count } over its members
- window_start INTEGER, -- earliest member time (published_at, else first seen)
- window_end INTEGER, -- latest member time
- created_by TEXT NOT NULL, -- 'svc:news' (clustering) or usr_… (a split)
- created_at INTEGER NOT NULL,
- updated_at INTEGER NOT NULL
-);
-CREATE INDEX IF NOT EXISTS news_story_clusters_open ON news_story_clusters (status, window_end);
-
-CREATE TABLE IF NOT EXISTS news_source_items (
- id TEXT PRIMARY KEY, -- nsi_<ULID>
- sources_item_id TEXT NOT NULL UNIQUE, -- itm_… in OpenVibe.Sources (a typed reference)
- sources_revision INTEGER NOT NULL, -- the Sources revision these fields came from
- source_key TEXT NOT NULL,
- canonical_url TEXT,
- url_key TEXT, -- normalised URL for dedupe
- headline TEXT NOT NULL,
- outlet TEXT NOT NULL,
- authors TEXT NOT NULL DEFAULT '[]',
- published_at TEXT, -- as the source states it; NULL when it does not
- summary TEXT, -- a short summary ONLY when the terms allow it
- summary_basis TEXT, -- why a summary is (not) stored
- license_note TEXT,
- terms_note TEXT,
- content_hash TEXT, -- Sources' hash of the parsed fields
- title_key TEXT NOT NULL, -- normalised headline for near-duplicate checks
- status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','duplicate','removed')),
- duplicate_of TEXT REFERENCES news_source_items(id),
- dedupe TEXT, -- { rule, detail } when a duplicate
- cluster_id TEXT REFERENCES news_story_clusters(id),
- cluster_reason TEXT, -- { rule, shared_entities, shared_terms, score } (explanation)
- retrieved_at TEXT,
- first_seen_at INTEGER NOT NULL,
- upstream_updated_at INTEGER, -- the last time Sources revised it after first ingest
- removed_at INTEGER,
- removed_reason TEXT,
- updated_at INTEGER NOT NULL
-);
-CREATE INDEX IF NOT EXISTS news_source_items_url ON news_source_items (url_key);
-CREATE INDEX IF NOT EXISTS news_source_items_hash ON news_source_items (content_hash);
-CREATE INDEX IF NOT EXISTS news_source_items_cluster ON news_source_items (cluster_id);
-CREATE INDEX IF NOT EXISTS news_source_items_seen ON news_source_items (first_seen_at);
-
-CREATE TABLE IF NOT EXISTS news_stories (
- id TEXT PRIMARY KEY, -- sty_<ULID>
- slug TEXT NOT NULL UNIQUE,
- working_headline TEXT NOT NULL, -- the editors' latest headline (readers see the revision's)
- cluster_id TEXT REFERENCES news_story_clusters(id),
- topic_id TEXT REFERENCES news_topics(id),
- state TEXT NOT NULL DEFAULT 'draft' CHECK (state IN ('draft','published','unpublished','retracted')),
- published_revision INTEGER, -- which revision readers see (NULL: never published)
- first_published_at INTEGER,
- published_at INTEGER, -- the latest publication of published_revision
- retracted_at INTEGER,
- noindex INTEGER NOT NULL DEFAULT 0, -- an editor asked search engines not to index it
- created_by TEXT NOT NULL, -- usr_… (the editor who opened the story)
- created_at INTEGER NOT NULL,
- updated_at INTEGER NOT NULL
-);
-CREATE INDEX IF NOT EXISTS news_stories_listing ON news_stories (state, published_at);
-CREATE INDEX IF NOT EXISTS news_stories_topic ON news_stories (topic_id, state, published_at);
-CREATE INDEX IF NOT EXISTS news_stories_cluster ON news_stories (cluster_id);
-
-CREATE TABLE IF NOT EXISTS news_perspectives (
- id TEXT PRIMARY KEY, -- per_<ULID>
- story_id TEXT NOT NULL REFERENCES news_stories(id),
- label TEXT NOT NULL, -- written by an editor; never generated
- description TEXT,
- position INTEGER NOT NULL DEFAULT 0,
- created_by TEXT NOT NULL,
- created_at INTEGER NOT NULL,
- updated_at INTEGER NOT NULL,
- removed_at INTEGER
-);
-CREATE INDEX IF NOT EXISTS news_perspectives_story ON news_perspectives (story_id, position);
-
-CREATE TABLE IF NOT EXISTS news_story_sources (
- story_id TEXT NOT NULL REFERENCES news_stories(id),
- source_item_id TEXT NOT NULL REFERENCES news_source_items(id),
- position INTEGER NOT NULL, -- the [n] readers and editors cite
- perspective_id TEXT REFERENCES news_perspectives(id),
- added_by TEXT NOT NULL,
- added_at INTEGER NOT NULL,
- detached_at INTEGER,
- PRIMARY KEY (story_id, source_item_id)
-);
-CREATE INDEX IF NOT EXISTS news_story_sources_item ON news_story_sources (source_item_id);
-
-CREATE TABLE IF NOT EXISTS news_timeline_entries (
- id TEXT PRIMARY KEY, -- tle_<ULID>
- story_id TEXT NOT NULL REFERENCES news_stories(id),
- occurred_on TEXT NOT NULL, -- YYYY-MM-DD or an ISO instant, as the source states it
- text TEXT NOT NULL,
- source_item_id TEXT NOT NULL REFERENCES news_source_items(id),
- created_by TEXT NOT NULL,
- created_at INTEGER NOT NULL,
- removed_at INTEGER
-);
-CREATE INDEX IF NOT EXISTS news_timeline_story ON news_timeline_entries (story_id, occurred_on);
-
-CREATE TABLE IF NOT EXISTS news_editorial_flags (
- id TEXT PRIMARY KEY, -- flg_<ULID>
- story_id TEXT NOT NULL REFERENCES news_stories(id),
- kind TEXT NOT NULL CHECK (kind IN ('correction','update','retraction','source_updated','source_removed')),
- status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('open','published','resolved')),
- note TEXT, -- public text for correction/update/retraction
- source_item_id TEXT REFERENCES news_source_items(id),
- pending_revision INTEGER, -- the revision the system prepared for an upstream change
- revision INTEGER, -- the published revision it belongs to
- created_by TEXT NOT NULL,
- created_at INTEGER NOT NULL,
- resolved_by TEXT,
- resolved_at INTEGER
-);
-CREATE INDEX IF NOT EXISTS news_flags_story ON news_editorial_flags (story_id, status, created_at);
-
-CREATE TABLE IF NOT EXISTS news_cluster_audit (
- id TEXT PRIMARY KEY, -- cla_<ULID>
- action TEXT NOT NULL CHECK (action IN ('merge','split','reverse_merge','reverse_split')),
- cluster_id TEXT NOT NULL, -- merge: the surviving cluster; split: the source cluster
- other_id TEXT NOT NULL, -- merge: the absorbed cluster; split: the new cluster
- item_ids TEXT NOT NULL, -- JSON array of news_source_items ids that moved
- reason TEXT,
- actor TEXT NOT NULL,
- reverses TEXT REFERENCES news_cluster_audit(id),
- reversed_by TEXT REFERENCES news_cluster_audit(id),
- created_at INTEGER NOT NULL
-);
-CREATE INDEX IF NOT EXISTS news_cluster_audit_cluster ON news_cluster_audit (cluster_id, created_at);
-
-CREATE TABLE IF NOT EXISTS news_ingest_runs (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- origin TEXT NOT NULL CHECK (origin IN ('webhook','pull','sources')),
- state TEXT NOT NULL, -- ok | failed | ignored | upstream state (http_error, timeout, …)
- source_key TEXT,
- sources_item_id TEXT,
- error_code TEXT,
- detail TEXT,
- counts TEXT, -- { created, updated, removed, duplicates, unchanged }
- cursor_before INTEGER,
- cursor_after INTEGER,
- at INTEGER NOT NULL
-);
-CREATE INDEX IF NOT EXISTS news_ingest_runs_at ON news_ingest_runs (at);
-
-CREATE TABLE IF NOT EXISTS news_state (
- key TEXT PRIMARY KEY,
- value TEXT NOT NULL,
- updated_at INTEGER NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS news_source_status (
- source_key TEXT PRIMARY KEY,
- name TEXT,
- homepage_url TEXT,
- status TEXT,
- stale INTEGER,
- last_success_at TEXT,
- refreshed_at INTEGER NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS subject_projections (
- subject TEXT PRIMARY KEY,
- username TEXT,
- display_name TEXT,
- avatar_url TEXT,
- refreshed_at INTEGER NOT NULL
-);
-CREATE INDEX IF NOT EXISTS subject_projections_username ON subject_projections (username);
-`;
+const MIGRATIONS = path.join(__dirname, '..', 'migrations');
+const DEV_PGLITE = path.join(__dirname, '..', 'data', 'pglite');
/**
/**
- * Open (or create) the database and every store on it.
- * opts.now — injectable clock (epoch ms) shared by the stores, so tests and replays are deterministic.
+ * The serving handle (ADR-035): DATABASE_URL through PgBouncer; in development without it, an embedded PGlite database
+ * in data/pglite. Migrations run first, as the owner (DATABASE_DIRECT_URL), or on the embedded handle.
*/
*/
-function openStore(dbPath, { now = () => Date.now() } = {}) {
- if (dbPath !== ':memory:') fs.mkdirSync(path.dirname(path.resolve(dbPath)), { recursive: true });
- const db = new Database(dbPath);
- db.pragma('journal_mode = WAL');
- db.pragma('foreign_keys = ON');
- db.pragma('busy_timeout = 5000');
- db.exec(SCHEMA);
+async function openDb(config, { log = console, registry } = {}) {
+ if (!config.db.url) {
+ if (config.isProduction) throw new Error('DATABASE_URL is not set: production serves from PostgreSQL (OpenVibe.Host roles/data add-service.sh news)');
+ log.warn(`[News] DATABASE_URL unset: embedded PGlite database in ${DEV_PGLITE} (development only, one process)`);
+ fs.mkdirSync(DEV_PGLITE, { recursive: true });
+ const db = createDb({ pglite: DEV_PGLITE, service: 'news', registry, log });
+ await db.migrate({ dir: MIGRATIONS, log });
+ return db;
+ }
+ if (!config.db.directUrl) throw new Error('DATABASE_DIRECT_URL is not set: migrations run with the owner role on a direct connection');
+ const owner = createDb({ url: config.db.directUrl, service: 'news-migrate', max: 1, log });
+ try { await owner.migrate({ dir: MIGRATIONS, log }); } finally { await owner.close(); }
+ return createDb({ url: config.db.url, service: 'news', registry, log });
+}
+/**
+ * Every store on a migrated database handle. opts.now — injectable clock (epoch ms) shared by the stores, so tests
+ * and replays are deterministic. store.tx(fn) is a transaction; inside it, plain db calls join it (ambient).
+ */
+function createStore(db, { now = () => Date.now() } = {}) {
const revisions = createRevisionStore(db, { prefix: 'news_story', now });
const revisions = createRevisionStore(db, { prefix: 'news_story', now });
- const store = {
+ const getStateStmt = db.prepare('SELECT value FROM news_state WHERE key = ?');
+ const setStateStmt = db.prepare(`INSERT INTO news_state (key, value, updated_at) VALUES (?, ?, ?)
+ ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`);
+ return {
db,
db,
now,
now,
revisions,
revisions,
@@ -249,21 +74,24 @@ function openStore(dbPath, { now = () => Date.now() } = {}) {
@@ -249,21 +74,24 @@ function openStore(dbPath, { now = () => Date.now() } = {}) {
reviews: createReviewLog(db, { prefix: 'news_story', now }),
reviews: createReviewLog(db, { prefix: 'news_story', now }),
sequencer: createIndexSequencer(db, { prefix: 'news', now }),
sequencer: createIndexSequencer(db, { prefix: 'news', now }),
discussion: createDiscussionRefs(db, { prefix: 'news_story', now }),
discussion: createDiscussionRefs(db, { prefix: 'news_story', now }),
- tx: (fn) => db.transaction(fn)(),
- getState(key, def = null) {
- const r = db.prepare('SELECT value FROM news_state WHERE key = ?').get(key);
+ tx: async (fn) => await db.tx(() => fn()),
+ async getState(key, def = null) {
+ const r = await getStateStmt.get(key);
return r ? JSON.parse(r.value) : def;
return r ? JSON.parse(r.value) : def;
},
},
- setState(key, value) {
- db.prepare(`INSERT INTO news_state (key, value, updated_at) VALUES (?, ?, ?)
- ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`).run(key, JSON.stringify(value), now());
+ async setState(key, value) {
+ await setStateStmt.run(key, JSON.stringify(value), now());
},
},
close: () => db.close(),
close: () => db.close(),
};
};
- return store;
+}
+
+/** openDb + createStore. */
+async function openStore(config, { now, log } = {}) {
+ return createStore(await openDb(config, { log }), { now });
}
}
const CHARTER_TABLES = ['news_topics', 'news_source_items', 'news_story_clusters', 'news_stories', 'news_story_revisions',
const CHARTER_TABLES = ['news_topics', 'news_source_items', 'news_story_clusters', 'news_stories', 'news_story_revisions',
'news_story_sources', 'news_perspectives', 'news_timeline_entries', 'news_editorial_flags'];
'news_story_sources', 'news_perspectives', 'news_timeline_entries', 'news_editorial_flags'];
-module.exports = { openStore, CHARTER_TABLES };
+module.exports = { openDb, openStore, createStore, CHARTER_TABLES, MIGRATIONS };
@@ -59,32 +59,32 @@ function createClusters({ store, config, outbox }) {
@@ -59,32 +59,32 @@ function createClusters({ store, config, outbox }) {
const { db } = store;
const { db } = store;
const q = {
const q = {
get: db.prepare('SELECT * FROM news_story_clusters WHERE id = ?'),
get: db.prepare('SELECT * FROM news_story_clusters WHERE id = ?'),
- open: db.prepare(`SELECT * FROM news_story_clusters WHERE status = 'open' AND window_end >= ? AND window_start <= ? ORDER BY created_at, rowid`),
+ open: db.prepare(`SELECT * FROM news_story_clusters WHERE status = 'open' AND window_end >= ? AND window_start <= ? ORDER BY created_at, seq`),
insert: db.prepare(`INSERT INTO news_story_clusters (id, label, status, split_from, terms, entities, window_start, window_end, created_by, created_at, updated_at)
insert: db.prepare(`INSERT INTO news_story_clusters (id, label, status, split_from, terms, entities, window_start, window_end, created_by, created_at, updated_at)
VALUES (@id, @label, 'open', @split_from, @terms, @entities, @window_start, @window_end, @created_by, @now, @now)`),
VALUES (@id, @label, 'open', @split_from, @terms, @entities, @window_start, @window_end, @created_by, @now, @now)`),
- members: db.prepare("SELECT * FROM news_source_items WHERE cluster_id = ? ORDER BY first_seen_at, rowid"),
+ members: db.prepare("SELECT * FROM news_source_items WHERE cluster_id = ? ORDER BY first_seen_at, seq"),
setItem: db.prepare('UPDATE news_source_items SET cluster_id = ?, cluster_reason = ?, updated_at = ? WHERE id = ?'),
setItem: db.prepare('UPDATE news_source_items SET cluster_id = ?, cluster_reason = ?, updated_at = ? WHERE id = ?'),
audit: db.prepare('SELECT * FROM news_cluster_audit WHERE id = ?'),
audit: db.prepare('SELECT * FROM news_cluster_audit WHERE id = ?'),
- auditFor: db.prepare('SELECT * FROM news_cluster_audit WHERE cluster_id = ? OR other_id = ? ORDER BY created_at, rowid'),
+ auditFor: db.prepare('SELECT * FROM news_cluster_audit WHERE cluster_id = ? OR other_id = ? ORDER BY created_at, seq'),
insertAudit: db.prepare(`INSERT INTO news_cluster_audit (id, action, cluster_id, other_id, item_ids, reason, actor, reverses, created_at)
insertAudit: db.prepare(`INSERT INTO news_cluster_audit (id, action, cluster_id, other_id, item_ids, reason, actor, reverses, created_at)
VALUES (@id, @action, @cluster_id, @other_id, @item_ids, @reason, @actor, @reverses, @now)`),
VALUES (@id, @action, @cluster_id, @other_id, @item_ids, @reason, @actor, @reverses, @now)`),
};
};
- function emit(cluster, action, extra = {}) {
- outbox.emit({
+ async function emit(cluster, action, extra = {}) {
+ await outbox.emit({
event_type: 'news.cluster.updated', actor: { type: 'service', id: 'news' }, visibility: 'internal', priority: 'low',
event_type: 'news.cluster.updated', actor: { type: 'service', id: 'news' }, visibility: 'internal', priority: 'low',
subject: { type: 'cluster', id: cluster.id },
subject: { type: 'cluster', id: cluster.id },
- payload: { cluster_id: cluster.id, action, status: cluster.status, label: cluster.label, item_count: q.members.all(cluster.id).length, ...extra },
+ payload: { cluster_id: cluster.id, action, status: cluster.status, label: cluster.label, item_count: (await q.members.all(cluster.id)).length, ...extra },
});
});
}
}
/** Rebuild a cluster's terms, entities, window and label from its members (removed items do not count). */
/** Rebuild a cluster's terms, entities, window and label from its members (removed items do not count). */
- function recompute(clusterId) {
+ async function recompute(clusterId) {
const terms = {};
const terms = {};
const entities = {};
const entities = {};
let start = null;
let start = null;
let end = null;
let end = null;
- for (const it of q.members.all(clusterId)) {
+ for (const it of await q.members.all(clusterId)) {
if (it.status === 'removed') continue;
if (it.status === 'removed') continue;
const sig = signature(it);
const sig = signature(it);
for (const t of sig.terms) terms[t] = (terms[t] || 0) + 1;
for (const t of sig.terms) terms[t] = (terms[t] || 0) + 1;
@@ -93,23 +93,23 @@ function createClusters({ store, config, outbox }) {
@@ -93,23 +93,23 @@ function createClusters({ store, config, outbox }) {
start = start == null ? at : Math.min(start, at);
start = start == null ? at : Math.min(start, at);
end = end == null ? at : Math.max(end, at);
end = end == null ? at : Math.max(end, at);
}
}
- db.prepare('UPDATE news_story_clusters SET terms = ?, entities = ?, window_start = COALESCE(?, window_start), window_end = COALESCE(?, window_end), label = ?, updated_at = ? WHERE id = ?')
+ await db.prepare('UPDATE news_story_clusters SET terms = ?, entities = ?, window_start = COALESCE(?, window_start), window_end = COALESCE(?, window_end), label = ?, updated_at = ? WHERE id = ?')
.run(JSON.stringify(terms), JSON.stringify(entities), start, end, labelOf(entities, terms), store.now(), clusterId);
.run(JSON.stringify(terms), JSON.stringify(entities), start, end, labelOf(entities, terms), store.now(), clusterId);
- return q.get.get(clusterId);
+ return await q.get.get(clusterId);
}
}
/**
/**
* Place one item (inside the ingest transaction). Duplicates follow their original. Returns
* Place one item (inside the ingest transaction). Duplicates follow their original. Returns
* { cluster, created, reason }.
* { cluster, created, reason }.
*/
*/
- function assign(item) {
+ async function assign(item) {
if (item.status === 'duplicate' && item.duplicate_of) {
if (item.status === 'duplicate' && item.duplicate_of) {
- const original = db.prepare('SELECT * FROM news_source_items WHERE id = ?').get(item.duplicate_of);
+ const original = await db.prepare('SELECT * FROM news_source_items WHERE id = ?').get(item.duplicate_of);
if (original && original.cluster_id) {
if (original && original.cluster_id) {
- const cluster = resolve(original.cluster_id);
+ const cluster = await resolve(original.cluster_id);
const reason = { rule: 'duplicate_of', item: original.id };
const reason = { rule: 'duplicate_of', item: original.id };
- q.setItem.run(cluster.id, JSON.stringify(reason), store.now(), item.id);
- emit(cluster, 'item_added', { item_id: item.id, rule: 'duplicate_of' });
+ await q.setItem.run(cluster.id, JSON.stringify(reason), store.now(), item.id);
+ await emit(cluster, 'item_added', { item_id: item.id, rule: 'duplicate_of' });
return { cluster, created: false, reason };
return { cluster, created: false, reason };
}
}
}
}
@@ -117,37 +117,37 @@ function createClusters({ store, config, outbox }) {
@@ -117,37 +117,37 @@ function createClusters({ store, config, outbox }) {
const at = itemTime(item);
const at = itemTime(item);
const w = config.clustering.windowMs;
const w = config.clustering.windowMs;
let best = null;
let best = null;
- for (const c of q.open.all(at - w, at + w)) {
+ for (const c of await q.open.all(at - w, at + w)) {
const m = compare(sig, c);
const m = compare(sig, c);
if (!m.joins) continue;
if (!m.joins) continue;
- if (!best || m.score > best.m.score) best = { c, m }; // ORDER BY created_at, rowid: the oldest wins a tie
+ if (!best || m.score > best.m.score) best = { c, m }; // ORDER BY created_at, seq: the oldest wins a tie
}
}
if (best) {
if (best) {
const reason = { rule: 'shared_terms', cluster: best.c.id, shared_entities: best.m.sharedEntities, shared_terms: best.m.sharedTerms, score: best.m.score, window_hours: w / 3600000 };
const reason = { rule: 'shared_terms', cluster: best.c.id, shared_entities: best.m.sharedEntities, shared_terms: best.m.sharedTerms, score: best.m.score, window_hours: w / 3600000 };
- q.setItem.run(best.c.id, JSON.stringify(reason), store.now(), item.id);
- const cluster = recompute(best.c.id);
- emit(cluster, 'item_added', { item_id: item.id, rule: 'shared_terms', score: best.m.score });
+ await q.setItem.run(best.c.id, JSON.stringify(reason), store.now(), item.id);
+ const cluster = await recompute(best.c.id);
+ await emit(cluster, 'item_added', { item_id: item.id, rule: 'shared_terms', score: best.m.score });
return { cluster, created: false, reason };
return { cluster, created: false, reason };
}
}
const now = store.now();
const now = store.now();
const id = newClusterId(now);
const id = newClusterId(now);
- q.insert.run({ id, label: 'new', split_from: null, terms: '{}', entities: '{}', window_start: at, window_end: at, created_by: 'svc:news', now });
+ await q.insert.run({ id, label: 'new', split_from: null, terms: '{}', entities: '{}', window_start: at, window_end: at, created_by: 'svc:news', now });
const reason = { rule: 'new_cluster', cluster: id, terms: sig.terms.slice(0, 20), entities: sig.entities.slice(0, 10) };
const reason = { rule: 'new_cluster', cluster: id, terms: sig.terms.slice(0, 20), entities: sig.entities.slice(0, 10) };
- q.setItem.run(id, JSON.stringify(reason), now, item.id);
- const cluster = recompute(id);
- emit(cluster, 'created', { item_id: item.id });
+ await q.setItem.run(id, JSON.stringify(reason), now, item.id);
+ const cluster = await recompute(id);
+ await emit(cluster, 'created', { item_id: item.id });
return { cluster, created: true, reason };
return { cluster, created: true, reason };
}
}
/** Follow merges to the cluster that holds the items now. */
/** Follow merges to the cluster that holds the items now. */
- function resolve(id) {
- let c = q.get.get(id);
- for (let i = 0; c && c.status === 'merged' && c.merged_into && i < 50; i++) c = q.get.get(c.merged_into);
+ async function resolve(id) {
+ let c = await q.get.get(id);
+ for (let i = 0; c && c.status === 'merged' && c.merged_into && i < 50; i++) c = await q.get.get(c.merged_into);
return c;
return c;
}
}
- function mustOpen(id) {
- const c = q.get.get(String(id || ''));
+ async function mustOpen(id) {
+ const c = await q.get.get(String(id || ''));
if (!c) throw new ApiError(404, 'cluster.not_found', 'No such cluster');
if (!c) throw new ApiError(404, 'cluster.not_found', 'No such cluster');
if (c.status !== 'open') throw new ApiError(409, 'cluster.not_open', `Cluster ${c.id} is ${c.status}`);
if (c.status !== 'open') throw new ApiError(409, 'cluster.not_open', `Cluster ${c.id} is ${c.status}`);
return c;
return c;
@@ -155,11 +155,11 @@ function createClusters({ store, config, outbox }) {
@@ -155,11 +155,11 @@ function createClusters({ store, config, outbox }) {
function actorId(viewer) { return viewer && viewer.subject ? viewer.subject : (viewer && viewer.service) || 'svc:news'; }
function actorId(viewer) { return viewer && viewer.subject ? viewer.subject : (viewer && viewer.service) || 'svc:news'; }
- function recordAudit(row) {
+ async function recordAudit(row) {
const now = store.now();
const now = store.now();
const id = newAuditId(now);
const id = newAuditId(now);
- q.insertAudit.run({ id, reason: null, reverses: null, ...row, item_ids: JSON.stringify(row.item_ids), now });
- return q.audit.get(id);
+ await q.insertAudit.run({ id, reason: null, reverses: null, ...row, item_ids: JSON.stringify(row.item_ids), now });
+ return await q.audit.get(id);
}
}
function shapeAudit(a) {
function shapeAudit(a) {
@@ -169,51 +169,51 @@ function createClusters({ store, config, outbox }) {
@@ -169,51 +169,51 @@ function createClusters({ store, config, outbox }) {
const api = {
const api = {
signature, compare, itemTime, labelOf, assign, resolve, recompute,
signature, compare, itemTime, labelOf, assign, resolve, recompute,
- get: (id) => q.get.get(String(id || '')) || null,
- members: (id) => q.members.all(id),
- audit: (id) => q.auditFor.all(id, id).map(shapeAudit),
- auditEntry: (id) => shapeAudit(q.audit.get(String(id || ''))),
+ get: async (id) => await q.get.get(String(id || '')) || null,
+ members: async (id) => await q.members.all(id),
+ audit: async (id) => (await q.auditFor.all(id, id)).map(shapeAudit),
+ auditEntry: async (id) => shapeAudit(await q.audit.get(String(id || ''))),
/** Open clusters, newest activity first (the editor dashboard). */
/** Open clusters, newest activity first (the editor dashboard). */
- recent({ limit = 50 } = {}) {
- return db.prepare("SELECT * FROM news_story_clusters WHERE status = 'open' ORDER BY window_end DESC, id DESC LIMIT ?").all(limit);
+ async recent({ limit = 50 } = {}) {
+ return await db.prepare("SELECT * FROM news_story_clusters WHERE status = 'open' ORDER BY window_end DESC, id DESC LIMIT ?").all(limit);
},
},
/** Merge cluster `otherId` into `intoId`. */
/** Merge cluster `otherId` into `intoId`. */
- merge(viewer, intoId, otherId, { reason = null } = {}) {
- return store.tx(() => {
- const into = mustOpen(intoId);
- const other = mustOpen(otherId);
+ async merge(viewer, intoId, otherId, { reason = null } = {}) {
+ return await store.tx(async () => {
+ const into = await mustOpen(intoId);
+ const other = await mustOpen(otherId);
if (into.id === other.id) throw new ApiError(422, 'cluster.same', 'A cluster cannot be merged into itself');
if (into.id === other.id) throw new ApiError(422, 'cluster.same', 'A cluster cannot be merged into itself');
- const moved = q.members.all(other.id).map((i) => i.id);
+ const moved = (await q.members.all(other.id)).map((i) => i.id);
const now = store.now();
const now = store.now();
- for (const id of moved) q.setItem.run(into.id, JSON.stringify({ rule: 'merged', from: other.id }), now, id);
- db.prepare("UPDATE news_story_clusters SET status = 'merged', merged_into = ?, updated_at = ? WHERE id = ?").run(into.id, now, other.id);
- const audit = recordAudit({ action: 'merge', cluster_id: into.id, other_id: other.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer) });
- const c = recompute(into.id);
- emit(c, 'merged', { absorbed: other.id, audit_id: audit.id, moved: moved.length });
+ for (const id of moved) await q.setItem.run(into.id, JSON.stringify({ rule: 'merged', from: other.id }), now, id);
+ await db.prepare("UPDATE news_story_clusters SET status = 'merged', merged_into = ?, updated_at = ? WHERE id = ?").run(into.id, now, other.id);
+ const audit = await recordAudit({ action: 'merge', cluster_id: into.id, other_id: other.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer) });
+ const c = await recompute(into.id);
+ await emit(c, 'merged', { absorbed: other.id, audit_id: audit.id, moved: moved.length });
return { cluster: c, audit: shapeAudit(audit) };
return { cluster: c, audit: shapeAudit(audit) };
});
});
},
},
/** Move some items of a cluster into a new cluster. At least one item must stay. */
/** Move some items of a cluster into a new cluster. At least one item must stay. */
- split(viewer, clusterId, itemIds, { reason = null } = {}) {
- return store.tx(() => {
- const from = mustOpen(clusterId);
- const members = new Set(q.members.all(from.id).map((i) => i.id));
+ async split(viewer, clusterId, itemIds, { reason = null } = {}) {
+ return await store.tx(async () => {
+ const from = await mustOpen(clusterId);
+ const members = new Set((await q.members.all(from.id)).map((i) => i.id));
const wanted = [...new Set((itemIds || []).map(String))];
const wanted = [...new Set((itemIds || []).map(String))];
if (!wanted.length) throw new ApiError(422, 'cluster.split_empty', 'Choose the items to split off');
if (!wanted.length) throw new ApiError(422, 'cluster.split_empty', 'Choose the items to split off');
for (const id of wanted) if (!members.has(id)) throw new ApiError(422, 'cluster.split_foreign', `${id} is not in cluster ${from.id}`);
for (const id of wanted) if (!members.has(id)) throw new ApiError(422, 'cluster.split_foreign', `${id} is not in cluster ${from.id}`);
if (wanted.length >= members.size) throw new ApiError(422, 'cluster.split_all', 'At least one item must stay in the cluster');
if (wanted.length >= members.size) throw new ApiError(422, 'cluster.split_all', 'At least one item must stay in the cluster');
const now = store.now();
const now = store.now();
const id = newClusterId(now);
const id = newClusterId(now);
- q.insert.run({ id, label: 'new', split_from: from.id, terms: '{}', entities: '{}', window_start: null, window_end: null, created_by: actorId(viewer), now });
- for (const it of wanted) q.setItem.run(id, JSON.stringify({ rule: 'split', from: from.id }), now, it);
- const audit = recordAudit({ action: 'split', cluster_id: from.id, other_id: id, item_ids: wanted, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer) });
- const a = recompute(from.id);
- const b = recompute(id);
- emit(a, 'split', { new_cluster: id, audit_id: audit.id, moved: wanted.length });
- emit(b, 'created', { split_from: from.id, audit_id: audit.id });
+ await q.insert.run({ id, label: 'new', split_from: from.id, terms: '{}', entities: '{}', window_start: null, window_end: null, created_by: actorId(viewer), now });
+ for (const it of wanted) await q.setItem.run(id, JSON.stringify({ rule: 'split', from: from.id }), now, it);
+ const audit = await recordAudit({ action: 'split', cluster_id: from.id, other_id: id, item_ids: wanted, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer) });
+ const a = await recompute(from.id);
+ const b = await recompute(id);
+ await emit(a, 'split', { new_cluster: id, audit_id: audit.id, moved: wanted.length });
+ await emit(b, 'created', { split_from: from.id, audit_id: audit.id });
return { cluster: a, created: b, audit: shapeAudit(audit) };
return { cluster: a, created: b, audit: shapeAudit(audit) };
});
});
},
},
@@ -222,9 +222,9 @@ function createClusters({ store, config, outbox }) {
@@ -222,9 +222,9 @@ function createClusters({ store, config, outbox }) {
* Undo a merge or a split: the items that moved (and are still where they were moved to) go
* Undo a merge or a split: the items that moved (and are still where they were moved to) go
* back; the absorbed cluster reopens / the split-off cluster is dissolved. Audited as well.
* back; the absorbed cluster reopens / the split-off cluster is dissolved. Audited as well.
*/
*/
- reverse(viewer, auditId, { reason = null } = {}) {
- return store.tx(() => {
- const a = q.audit.get(String(auditId || ''));
+ async reverse(viewer, auditId, { reason = null } = {}) {
+ return await store.tx(async () => {
+ const a = await q.audit.get(String(auditId || ''));
if (!a) throw new ApiError(404, 'cluster.audit_not_found', 'No such merge or split');
if (!a) throw new ApiError(404, 'cluster.audit_not_found', 'No such merge or split');
if (a.reversed_by) throw new ApiError(409, 'cluster.already_reversed', `Already reversed by ${a.reversed_by}`);
if (a.reversed_by) throw new ApiError(409, 'cluster.already_reversed', `Already reversed by ${a.reversed_by}`);
if (a.action !== 'merge' && a.action !== 'split') throw new ApiError(422, 'cluster.not_reversible', 'Only merges and splits can be reversed');
if (a.action !== 'merge' && a.action !== 'split') throw new ApiError(422, 'cluster.not_reversible', 'Only merges and splits can be reversed');
@@ -233,32 +233,34 @@ function createClusters({ store, config, outbox }) {
@@ -233,32 +233,34 @@ function createClusters({ store, config, outbox }) {
let moved = [];
let moved = [];
let result;
let result;
if (a.action === 'merge') {
if (a.action === 'merge') {
- const other = q.get.get(a.other_id);
+ const other = await q.get.get(a.other_id);
if (!other || other.status !== 'merged' || other.merged_into !== a.cluster_id) throw new ApiError(409, 'cluster.changed', 'The absorbed cluster has changed since the merge');
if (!other || other.status !== 'merged' || other.merged_into !== a.cluster_id) throw new ApiError(409, 'cluster.changed', 'The absorbed cluster has changed since the merge');
- moved = items.filter((id) => (db.prepare('SELECT cluster_id FROM news_source_items WHERE id = ?').get(id) || {}).cluster_id === a.cluster_id);
- for (const id of moved) q.setItem.run(other.id, JSON.stringify({ rule: 'merge_reversed', audit: a.id }), now, id);
- db.prepare("UPDATE news_story_clusters SET status = 'open', merged_into = NULL, updated_at = ? WHERE id = ?").run(now, other.id);
- const rev = recordAudit({ action: 'reverse_merge', cluster_id: a.cluster_id, other_id: other.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer), reverses: a.id });
- db.prepare('UPDATE news_cluster_audit SET reversed_by = ? WHERE id = ?').run(rev.id, a.id);
- const c1 = recompute(a.cluster_id);
- const c2 = recompute(other.id);
- emit(c1, 'merge_reversed', { audit_id: rev.id, restored: other.id });
- emit(c2, 'reopened', { audit_id: rev.id });
- result = { audit: shapeAudit(q.audit.get(rev.id)), clusters: [c1, c2] };
+ const now1 = new Map((await db.prepare('SELECT id, cluster_id FROM news_source_items WHERE id = ANY(?)').all(items)).map((r) => [r.id, r.cluster_id]));
+ moved = items.filter((id) => now1.get(id) === a.cluster_id);
+ for (const id of moved) await q.setItem.run(other.id, JSON.stringify({ rule: 'merge_reversed', audit: a.id }), now, id);
+ await db.prepare("UPDATE news_story_clusters SET status = 'open', merged_into = NULL, updated_at = ? WHERE id = ?").run(now, other.id);
+ const rev = await recordAudit({ action: 'reverse_merge', cluster_id: a.cluster_id, other_id: other.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer), reverses: a.id });
+ await db.prepare('UPDATE news_cluster_audit SET reversed_by = ? WHERE id = ?').run(rev.id, a.id);
+ const c1 = await recompute(a.cluster_id);
+ const c2 = await recompute(other.id);
+ await emit(c1, 'merge_reversed', { audit_id: rev.id, restored: other.id });
+ await emit(c2, 'reopened', { audit_id: rev.id });
+ result = { audit: shapeAudit(await q.audit.get(rev.id)), clusters: [c1, c2] };
} else {
} else {
- const split = q.get.get(a.other_id);
- const from = q.get.get(a.cluster_id);
+ const split = await q.get.get(a.other_id);
+ const from = await q.get.get(a.cluster_id);
if (!split || split.status !== 'open' || !from || from.status !== 'open') throw new ApiError(409, 'cluster.changed', 'One of the clusters is no longer open');
if (!split || split.status !== 'open' || !from || from.status !== 'open') throw new ApiError(409, 'cluster.changed', 'One of the clusters is no longer open');
- moved = items.filter((id) => (db.prepare('SELECT cluster_id FROM news_source_items WHERE id = ?').get(id) || {}).cluster_id === split.id);
- for (const id of moved) q.setItem.run(from.id, JSON.stringify({ rule: 'split_reversed', audit: a.id }), now, id);
- const left = q.members.all(split.id).length;
- if (!left) db.prepare("UPDATE news_story_clusters SET status = 'dissolved', updated_at = ? WHERE id = ?").run(now, split.id);
- const rev = recordAudit({ action: 'reverse_split', cluster_id: from.id, other_id: split.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer), reverses: a.id });
- db.prepare('UPDATE news_cluster_audit SET reversed_by = ? WHERE id = ?').run(rev.id, a.id);
- const c1 = recompute(from.id);
- const c2 = left ? recompute(split.id) : q.get.get(split.id);
- emit(c1, 'split_reversed', { audit_id: rev.id, dissolved: left ? null : split.id });
- result = { audit: shapeAudit(q.audit.get(rev.id)), clusters: [c1, c2] };
+ const now2 = new Map((await db.prepare('SELECT id, cluster_id FROM news_source_items WHERE id = ANY(?)').all(items)).map((r) => [r.id, r.cluster_id]));
+ moved = items.filter((id) => now2.get(id) === split.id);
+ for (const id of moved) await q.setItem.run(from.id, JSON.stringify({ rule: 'split_reversed', audit: a.id }), now, id);
+ const left = (await q.members.all(split.id)).length;
+ if (!left) await db.prepare("UPDATE news_story_clusters SET status = 'dissolved', updated_at = ? WHERE id = ?").run(now, split.id);
+ const rev = await recordAudit({ action: 'reverse_split', cluster_id: from.id, other_id: split.id, item_ids: moved, reason: reason ? String(reason).slice(0, 500) : null, actor: actorId(viewer), reverses: a.id });
+ await db.prepare('UPDATE news_cluster_audit SET reversed_by = ? WHERE id = ?').run(rev.id, a.id);
+ const c1 = await recompute(from.id);
+ const c2 = left ? await recompute(split.id) : await q.get.get(split.id);
+ await emit(c1, 'split_reversed', { audit_id: rev.id, dissolved: left ? null : split.id });
+ result = { audit: shapeAudit(await q.audit.get(rev.id)), clusters: [c1, c2] };
}
}
return result;
return result;
});
});
@@ -13,11 +13,10 @@
@@ -13,11 +13,10 @@
* a revision that no longer rests on it (comments could repeat the withdrawn material)
* a revision that no longer rests on it (comments could repeat the withdrawn material)
*
*
* Writes that can change this call watch() inside their transaction, before they change anything.
* Writes that can change this call watch() inside their transaction, before they change anything.
- * better-sqlite3 transactions are synchronous, so the setImmediate flush runs after the commit (or
- * the rollback): the state is compared again and, when it changed, the thread is set hidden or
- * public (best effort, community.comment.moderate). A rolled-back write compares equal and calls
- * nothing; a story that never had a thread calls nothing either (threads are created only when an
- * open story is read).
+ * After the commit (db.afterCommit; a rollback drops it) the state is compared again and, when it
+ * changed, the thread is set hidden or public (best effort, community.comment.moderate). A rolled-back
+ * write calls nothing; a story that never had a thread calls nothing either (threads are created only
+ * when an open story is read).
*/
*/
function createDiscussion({ store, publication, community, log = console }) {
function createDiscussion({ store, publication, community, log = console }) {
const { db } = store;
const { db } = store;
@@ -25,45 +24,35 @@ function createDiscussion({ store, publication, community, log = console }) {
@@ -25,45 +24,35 @@ function createDiscussion({ store, publication, community, log = console }) {
const citing = db.prepare('SELECT DISTINCT story_id FROM news_story_sources WHERE source_item_id = ?');
const citing = db.prepare('SELECT DISTINCT story_id FROM news_story_sources WHERE source_item_id = ?');
/** { open, reason: null | not_published | retracted | source_removed } */
/** { open, reason: null | not_published | retracted | source_removed } */
- function status(story) {
+ async function status(story) {
if (!story || !story.published_revision) return { open: false, reason: 'not_published' };
if (!story || !story.published_revision) return { open: false, reason: 'not_published' };
if (story.state === 'retracted') return { open: false, reason: 'retracted' };
if (story.state === 'retracted') return { open: false, reason: 'retracted' };
if (story.state !== 'published') return { open: false, reason: 'not_published' };
if (story.state !== 'published') return { open: false, reason: 'not_published' };
- const rev = store.revisions.get(story.id, story.published_revision);
+ const rev = await store.revisions.get(story.id, story.published_revision);
if (!rev) return { open: false, reason: 'not_published' };
if (!rev) return { open: false, reason: 'not_published' };
- if (publication.support(rev).unsupported > 0) return { open: false, reason: 'source_removed' };
+ if ((await publication.support(rev)).unsupported > 0) return { open: false, reason: 'source_removed' };
return { open: true, reason: null };
return { open: true, reason: null };
}
}
- const pending = new Map(); // story id → open before the write
- let scheduled = false;
- let flushing = Promise.resolve();
+ let flushing = Promise.resolve(); // visibility calls go one at a time, in commit order
- async function flush() {
- scheduled = false;
- const batch = [...pending];
- pending.clear();
- for (const [id, was] of batch) {
- const now = status(storyById.get(id)).open;
- if (now !== was) await community.setThreadVisibility(id, now ? 'public' : 'hidden');
- }
+ async function follow(id, was) {
+ const now = (await status(await storyById.get(id))).open;
+ if (now !== was) await community.setThreadVisibility(id, now ? 'public' : 'hidden');
}
}
/** Remember whether a story is open now; after the commit, follow any change. Inside the write's transaction. */
/** Remember whether a story is open now; after the commit, follow any change. Inside the write's transaction. */
- function watch(storyId) {
+ async function watch(storyId) {
if (!community.enabled) return;
if (!community.enabled) return;
- if (!pending.has(storyId)) pending.set(storyId, status(storyById.get(storyId)).open);
- if (!scheduled) {
- scheduled = true;
- setImmediate(() => { flushing = flushing.then(flush).catch((err) => log.warn(`[News] comment thread sync failed: ${err.message}`)); });
- }
+ const was = (await status(await storyById.get(storyId))).open;
+ db.afterCommit(() => { flushing = flushing.then(() => follow(storyId, was)).catch((err) => log.warn(`[News] comment thread sync failed: ${err.message}`)); });
}
}
return {
return {
status,
status,
watch,
watch,
/** Every story that cites a source item (before the item changes upstream). */
/** Every story that cites a source item (before the item changes upstream). */
- watchSource(itemId) { for (const r of citing.all(itemId)) watch(r.story_id); },
+ async watchSource(itemId) { for (const r of await citing.all(itemId)) await watch(r.story_id); },
/** Resolves once the scheduled visibility changes have been sent (tests, shutdown). */
/** Resolves once the scheduled visibility changes have been sent (tests, shutdown). */
idle: () => new Promise((resolve) => setImmediate(() => flushing.then(resolve))),
idle: () => new Promise((resolve) => setImmediate(() => flushing.then(resolve))),
};
};
@@ -35,39 +35,39 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@@ -35,39 +35,39 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
const q = {
const q = {
bySourcesId: db.prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?'),
bySourcesId: db.prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?'),
byId: db.prepare('SELECT * FROM news_source_items WHERE id = ?'),
byId: db.prepare('SELECT * FROM news_source_items WHERE id = ?'),
- byUrl: db.prepare("SELECT * FROM news_source_items WHERE url_key = ? AND status <> 'removed' ORDER BY first_seen_at, rowid LIMIT 1"),
- byHash: db.prepare("SELECT * FROM news_source_items WHERE content_hash = ? AND status <> 'removed' ORDER BY first_seen_at, rowid LIMIT 1"),
- recent: db.prepare("SELECT * FROM news_source_items WHERE status <> 'removed' AND first_seen_at >= ? ORDER BY first_seen_at, rowid"),
+ byUrl: db.prepare("SELECT * FROM news_source_items WHERE url_key = ? AND status <> 'removed' ORDER BY first_seen_at, seq LIMIT 1"),
+ byHash: db.prepare("SELECT * FROM news_source_items WHERE content_hash = ? AND status <> 'removed' ORDER BY first_seen_at, seq LIMIT 1"),
+ recent: db.prepare("SELECT * FROM news_source_items WHERE status <> 'removed' AND first_seen_at >= ? ORDER BY first_seen_at, seq"),
status: db.prepare('SELECT * FROM news_source_status WHERE source_key = ?'),
status: db.prepare('SELECT * FROM news_source_status WHERE source_key = ?'),
putStatus: db.prepare(`INSERT INTO news_source_status (source_key, name, homepage_url, status, stale, last_success_at, refreshed_at)
putStatus: db.prepare(`INSERT INTO news_source_status (source_key, name, homepage_url, status, stale, last_success_at, refreshed_at)
VALUES (@source_key, @name, @homepage_url, @status, @stale, @last_success_at, @now)
VALUES (@source_key, @name, @homepage_url, @status, @stale, @last_success_at, @now)
- ON CONFLICT (source_key) DO UPDATE SET name = COALESCE(excluded.name, name), homepage_url = COALESCE(excluded.homepage_url, homepage_url),
- status = COALESCE(excluded.status, status), stale = COALESCE(excluded.stale, stale),
- last_success_at = COALESCE(excluded.last_success_at, last_success_at), refreshed_at = excluded.refreshed_at`),
+ ON CONFLICT (source_key) DO UPDATE SET name = COALESCE(excluded.name, news_source_status.name), homepage_url = COALESCE(excluded.homepage_url, news_source_status.homepage_url),
+ status = COALESCE(excluded.status, news_source_status.status), stale = COALESCE(excluded.stale, news_source_status.stale),
+ last_success_at = COALESCE(excluded.last_success_at, news_source_status.last_success_at), refreshed_at = excluded.refreshed_at`),
run: db.prepare(`INSERT INTO news_ingest_runs (origin, state, source_key, sources_item_id, error_code, detail, counts, cursor_before, cursor_after, at)
run: db.prepare(`INSERT INTO news_ingest_runs (origin, state, source_key, sources_item_id, error_code, detail, counts, cursor_before, cursor_after, at)
VALUES (@origin, @state, @source_key, @sources_item_id, @error_code, @detail, @counts, @cursor_before, @cursor_after, @at)`),
VALUES (@origin, @state, @source_key, @sources_item_id, @error_code, @detail, @counts, @cursor_before, @cursor_after, @at)`),
lastPull: db.prepare("SELECT * FROM news_ingest_runs WHERE origin = 'pull' ORDER BY id DESC LIMIT 1"),
lastPull: db.prepare("SELECT * FROM news_ingest_runs WHERE origin = 'pull' ORDER BY id DESC LIMIT 1"),
};
};
- function recordRun(r) {
- q.run.run({ source_key: null, sources_item_id: null, error_code: null, detail: null, counts: null, cursor_before: null, cursor_after: null, ...r, counts: r.counts ? JSON.stringify(r.counts) : null, at: store.now() });
+ async function recordRun(r) {
+ await q.run.run({ source_key: null, sources_item_id: null, error_code: null, detail: null, counts: null, cursor_before: null, cursor_after: null, ...r, counts: r.counts ? JSON.stringify(r.counts) : null, at: store.now() });
}
}
- function failedEvent({ origin, sourceKey = null, sourcesItemId = null, state, errorCode = null, detail = null, httpStatus = null, extra = {} }) {
- outbox.emit({
+ async function failedEvent({ origin, sourceKey = null, sourcesItemId = null, state, errorCode = null, detail = null, httpStatus = null, extra = {} }) {
+ await outbox.emit({
event_type: 'news.source.failed', actor: { type: 'service', id: 'news' }, visibility: 'internal', priority: 'important',
event_type: 'news.source.failed', actor: { type: 'service', id: 'news' }, visibility: 'internal', priority: 'important',
subject: sourceKey ? { type: 'source', id: sourceKey } : { type: 'ingest', id: origin },
subject: sourceKey ? { type: 'source', id: sourceKey } : { type: 'ingest', id: origin },
payload: { origin, source_key: sourceKey, sources_item_id: sourcesItemId, state, error_code: errorCode, http_status: httpStatus, detail: detail ? String(detail).slice(0, 500) : null, ...extra },
payload: { origin, source_key: sourceKey, sources_item_id: sourcesItemId, state, error_code: errorCode, http_status: httpStatus, detail: detail ? String(detail).slice(0, 500) : null, ...extra },
});
});
}
}
- function outletFor(item) {
- const st = q.status.get(item.source_key);
+ async function outletFor(item) {
+ const st = await q.status.get(item.source_key);
return (st && st.name) || text.hostOf(item.canonical_url) || item.source_key;
return (st && st.name) || text.hostOf(item.canonical_url) || item.source_key;
}
}
/** The fields News keeps from a Sources item. Nothing else of the item is read. */
/** The fields News keeps from a Sources item. Nothing else of the item is read. */
- function normalise(item) {
+ async function normalise(item) {
const prov = item.provenance || {};
const prov = item.provenance || {};
const headline = String(item.title || '').replace(/\s+/g, ' ').trim().slice(0, 500);
const headline = String(item.title || '').replace(/\s+/g, ' ').trim().slice(0, 500);
const lic = text.licensedSummary(item.summary, { termsNote: prov.terms_note, licenseNote: prov.license_note, maxChars: config.licensing.summaryMaxChars });
const lic = text.licensedSummary(item.summary, { termsNote: prov.terms_note, licenseNote: prov.license_note, maxChars: config.licensing.summaryMaxChars });
@@ -77,7 +77,7 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@@ -77,7 +77,7 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
headline,
headline,
canonical_url: item.canonical_url || null,
canonical_url: item.canonical_url || null,
url_key: text.urlKey(item.canonical_url),
url_key: text.urlKey(item.canonical_url),
- outlet: outletFor(item),
+ outlet: await outletFor(item),
authors: JSON.stringify(authors),
authors: JSON.stringify(authors),
published_at: published,
published_at: published,
summary: lic.summary,
summary: lic.summary,
@@ -91,19 +91,19 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@@ -91,19 +91,19 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
}
}
/** Is this new item a duplicate of one we have? → null | { of, rule, detail } */
/** Is this new item a duplicate of one we have? → null | { of, rule, detail } */
- function dedupe(n, sourceKey, at) {
+ async function dedupe(n, sourceKey, at) {
if (n.url_key) {
if (n.url_key) {
- const hit = q.byUrl.get(n.url_key);
+ const hit = await q.byUrl.get(n.url_key);
if (hit) return { of: hit.duplicate_of || hit.id, rule: 'canonical_url', detail: n.url_key };
if (hit) return { of: hit.duplicate_of || hit.id, rule: 'canonical_url', detail: n.url_key };
}
}
if (n.content_hash) {
if (n.content_hash) {
- const hit = q.byHash.get(n.content_hash);
+ const hit = await q.byHash.get(n.content_hash);
if (hit) return { of: hit.duplicate_of || hit.id, rule: 'content_hash', detail: n.content_hash };
if (hit) return { of: hit.duplicate_of || hit.id, rule: 'content_hash', detail: n.content_hash };
}
}
const mine = text.shingles(n.headline);
const mine = text.shingles(n.headline);
const host = text.hostOf(n.canonical_url);
const host = text.hostOf(n.canonical_url);
let best = null;
let best = null;
- for (const c of q.recent.all(Math.min(at, store.now()) - NEAR_TITLE.windowMs)) {
+ for (const c of await q.recent.all(Math.min(at, store.now()) - NEAR_TITLE.windowMs)) {
const t = clusters.itemTime(c);
const t = clusters.itemTime(c);
if (Math.abs(t - at) > NEAR_TITLE.windowMs) continue;
if (Math.abs(t - at) > NEAR_TITLE.windowMs) continue;
const score = text.jaccard(mine, text.shingles(c.headline));
const score = text.jaccard(mine, text.shingles(c.headline));
@@ -115,8 +115,8 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@@ -115,8 +115,8 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
return null;
return null;
}
}
- function ingestedEvent(row, action, extra = {}) {
- outbox.emit({
+ async function ingestedEvent(row, action, extra = {}) {
+ await outbox.emit({
event_type: 'news.source.ingested', actor: { type: 'service', id: 'news' }, visibility: 'internal', priority: 'low',
event_type: 'news.source.ingested', actor: { type: 'service', id: 'news' }, visibility: 'internal', priority: 'low',
subject: { type: 'source_item', id: row.id },
subject: { type: 'source_item', id: row.id },
payload: {
payload: {
@@ -132,38 +132,38 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@@ -132,38 +132,38 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
* Apply one Sources item (inside a transaction the caller may already hold). Returns
* Apply one Sources item (inside a transaction the caller may already hold). Returns
* { outcome: created|updated|removed|unchanged|ignored|rejected, item?, reason? }.
* { outcome: created|updated|removed|unchanged|ignored|rejected, item?, reason? }.
*/
*/
- function apply(item, { origin = 'pull' } = {}) {
- return db.transaction(() => {
+ async function apply(item, { origin = 'pull' } = {}) {
+ return db.tx(async () => {
if (!item || typeof item.id !== 'string') return { outcome: 'rejected', reason: 'item without id' };
if (!item || typeof item.id !== 'string') return { outcome: 'rejected', reason: 'item without id' };
if (item.category && item.category !== config.sources.category) return { outcome: 'ignored', reason: `category ${item.category}` };
if (item.category && item.category !== config.sources.category) return { outcome: 'ignored', reason: `category ${item.category}` };
const revision = Number.isInteger(item.revision) ? item.revision : 1;
const revision = Number.isInteger(item.revision) ? item.revision : 1;
- const cur = q.bySourcesId.get(item.id);
+ const cur = await q.bySourcesId.get(item.id);
if (item.removed) return applyRemoval({ sourcesItemId: item.id, reason: item.removed.reason || 'removed upstream', revision, origin });
if (item.removed) return applyRemoval({ sourcesItemId: item.id, reason: item.removed.reason || 'removed upstream', revision, origin });
if (cur && cur.sources_revision >= revision) return { outcome: 'unchanged', item: cur };
if (cur && cur.sources_revision >= revision) return { outcome: 'unchanged', item: cur };
- const n = normalise(item);
+ const n = await normalise(item);
if (!n.headline) {
if (!n.headline) {
- recordRun({ origin, state: 'rejected', source_key: item.source_key || null, sources_item_id: item.id, error_code: 'item.no_headline', detail: 'a Sources item without a title cannot be shown' });
- failedEvent({ origin, sourceKey: item.source_key || null, sourcesItemId: item.id, state: 'rejected', errorCode: 'item.no_headline' });
+ await recordRun({ origin, state: 'rejected', source_key: item.source_key || null, sources_item_id: item.id, error_code: 'item.no_headline', detail: 'a Sources item without a title cannot be shown' });
+ await failedEvent({ origin, sourceKey: item.source_key || null, sourcesItemId: item.id, state: 'rejected', errorCode: 'item.no_headline' });
return { outcome: 'rejected', reason: 'no headline' };
return { outcome: 'rejected', reason: 'no headline' };
}
}
const now = store.now();
const now = store.now();
if (cur) {
if (cur) {
if (cur.status === 'removed') return { outcome: 'unchanged', item: cur }; // removal is sticky here too
if (cur.status === 'removed') return { outcome: 'unchanged', item: cur }; // removal is sticky here too
- db.prepare(`UPDATE news_source_items SET sources_revision = @rev, canonical_url = @canonical_url, url_key = @url_key, headline = @headline,
+ await db.prepare(`UPDATE news_source_items SET sources_revision = @rev, canonical_url = @canonical_url, url_key = @url_key, headline = @headline,
outlet = @outlet, authors = @authors, published_at = @published_at, summary = @summary, summary_basis = @summary_basis,
outlet = @outlet, authors = @authors, published_at = @published_at, summary = @summary, summary_basis = @summary_basis,
license_note = @license_note, terms_note = @terms_note, content_hash = @content_hash, title_key = @title_key,
license_note = @license_note, terms_note = @terms_note, content_hash = @content_hash, title_key = @title_key,
retrieved_at = @retrieved_at, upstream_updated_at = @now, updated_at = @now WHERE id = @id`)
retrieved_at = @retrieved_at, upstream_updated_at = @now, updated_at = @now WHERE id = @id`)
.run({ ...n, rev: revision, now, id: cur.id });
.run({ ...n, rev: revision, now, id: cur.id });
- const row = q.byId.get(cur.id);
- if (row.cluster_id) clusters.recompute(clusters.resolve(row.cluster_id).id);
- ingestedEvent(row, 'updated', { previous_revision: cur.sources_revision });
- const affected = stories ? stories.onUpstreamChange(row, 'source_updated', { previous: cur }) : [];
+ const row = await q.byId.get(cur.id);
+ if (row.cluster_id) await clusters.recompute((await clusters.resolve(row.cluster_id)).id);
+ await ingestedEvent(row, 'updated', { previous_revision: cur.sources_revision });
+ const affected = stories ? await stories.onUpstreamChange(row, 'source_updated', { previous: cur }) : [];
return { outcome: 'updated', item: row, stories: affected };
return { outcome: 'updated', item: row, stories: affected };
}
}
const at = (() => { const p = n.published_at ? Date.parse(n.published_at) : NaN; return Number.isFinite(p) ? p : now; })();
const at = (() => { const p = n.published_at ? Date.parse(n.published_at) : NaN; return Number.isFinite(p) ? p : now; })();
- const dup = dedupe(n, item.source_key, at);
+ const dup = await dedupe(n, item.source_key, at);
const id = newItemId(now);
const id = newItemId(now);
- db.prepare(`INSERT INTO news_source_items (id, sources_item_id, sources_revision, source_key, canonical_url, url_key, headline, outlet, authors,
+ await db.prepare(`INSERT INTO news_source_items (id, sources_item_id, sources_revision, source_key, canonical_url, url_key, headline, outlet, authors,
published_at, summary, summary_basis, license_note, terms_note, content_hash, title_key, status, duplicate_of, dedupe,
published_at, summary, summary_basis, license_note, terms_note, content_hash, title_key, status, duplicate_of, dedupe,
retrieved_at, first_seen_at, updated_at)
retrieved_at, first_seen_at, updated_at)
VALUES (@id, @sources_item_id, @rev, @source_key, @canonical_url, @url_key, @headline, @outlet, @authors,
VALUES (@id, @sources_item_id, @rev, @source_key, @canonical_url, @url_key, @headline, @outlet, @authors,
@@ -171,61 +171,61 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@@ -171,61 +171,61 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@retrieved_at, @now, @now)`)
@retrieved_at, @now, @now)`)
.run({ ...n, id, sources_item_id: item.id, rev: revision, source_key: String(item.source_key || 'unknown'),
.run({ ...n, id, sources_item_id: item.id, rev: revision, source_key: String(item.source_key || 'unknown'),
status: dup ? 'duplicate' : 'active', duplicate_of: dup ? dup.of : null, dedupe: dup ? JSON.stringify({ rule: dup.rule, detail: dup.detail }) : null, now });
status: dup ? 'duplicate' : 'active', duplicate_of: dup ? dup.of : null, dedupe: dup ? JSON.stringify({ rule: dup.rule, detail: dup.detail }) : null, now });
- clusters.assign(q.byId.get(id));
- const row = q.byId.get(id);
- ingestedEvent(row, 'created');
+ await clusters.assign(await q.byId.get(id));
+ const row = await q.byId.get(id);
+ await ingestedEvent(row, 'created');
return { outcome: 'created', item: row };
return { outcome: 'created', item: row };
- })();
+ });
}
}
/** A Sources removal (takedown, licence, error): sticky; the stored summary is dropped. */
/** A Sources removal (takedown, licence, error): sticky; the stored summary is dropped. */
- function applyRemoval({ sourcesItemId, reason, revision = null, origin = 'webhook' }) {
- return db.transaction(() => {
- const cur = q.bySourcesId.get(sourcesItemId);
+ async function applyRemoval({ sourcesItemId, reason, revision = null, origin = 'webhook' }) {
+ return db.tx(async () => {
+ const cur = await q.bySourcesId.get(sourcesItemId);
if (!cur) return { outcome: 'ignored', reason: 'never ingested' };
if (!cur) return { outcome: 'ignored', reason: 'never ingested' };
if (cur.status === 'removed') return { outcome: 'unchanged', item: cur };
if (cur.status === 'removed') return { outcome: 'unchanged', item: cur };
// A story resting on it may stop being open for comments (domain/discussion.js).
// A story resting on it may stop being open for comments (domain/discussion.js).
- if (discussion) discussion.watchSource(cur.id);
+ if (discussion) await discussion.watchSource(cur.id);
const now = store.now();
const now = store.now();
- db.prepare(`UPDATE news_source_items SET status = 'removed', summary = NULL, summary_basis = 'removed_upstream', removed_at = ?, removed_reason = ?,
- sources_revision = MAX(sources_revision, COALESCE(?, sources_revision)), updated_at = ? WHERE id = ?`)
+ await db.prepare(`UPDATE news_source_items SET status = 'removed', summary = NULL, summary_basis = 'removed_upstream', removed_at = ?, removed_reason = ?,
+ sources_revision = GREATEST(sources_revision, COALESCE(?, sources_revision)), updated_at = ? WHERE id = ?`)
.run(now, String(reason || 'removed upstream').slice(0, 500), revision, now, cur.id);
.run(now, String(reason || 'removed upstream').slice(0, 500), revision, now, cur.id);
- const row = q.byId.get(cur.id);
- if (row.cluster_id) clusters.recompute(clusters.resolve(row.cluster_id).id);
- ingestedEvent(row, 'removed', { reason: row.removed_reason, origin });
- const affected = stories ? stories.onUpstreamChange(row, 'source_removed', { previous: cur }) : [];
+ const row = await q.byId.get(cur.id);
+ if (row.cluster_id) await clusters.recompute((await clusters.resolve(row.cluster_id)).id);
+ await ingestedEvent(row, 'removed', { reason: row.removed_reason, origin });
+ const affected = stories ? await stories.onUpstreamChange(row, 'source_removed', { previous: cur }) : [];
return { outcome: 'removed', item: row, stories: affected };
return { outcome: 'removed', item: row, stories: affected };
- })();
+ });
}
}
/** sources.fetch.failed for a news source: recorded and relayed; nothing else changes. */
/** sources.fetch.failed for a news source: recorded and relayed; nothing else changes. */
- function upstreamFailure(payload) {
- return db.transaction(() => {
+ async function upstreamFailure(payload) {
+ return db.tx(async () => {
if (payload.category && payload.category !== config.sources.category) return { outcome: 'ignored' };
if (payload.category && payload.category !== config.sources.category) return { outcome: 'ignored' };
- recordRun({ origin: 'sources', state: String(payload.state || 'failed').slice(0, 40), source_key: payload.source_key || null, error_code: payload.error_code || null,
+ await recordRun({ origin: 'sources', state: String(payload.state || 'failed').slice(0, 40), source_key: payload.source_key || null, error_code: payload.error_code || null,
detail: `run ${payload.run_id || '?'}${payload.http_status ? ` HTTP ${payload.http_status}` : ''}${payload.consecutive_failures ? `, ${payload.consecutive_failures} consecutive failures` : ''}` });
detail: `run ${payload.run_id || '?'}${payload.http_status ? ` HTTP ${payload.http_status}` : ''}${payload.consecutive_failures ? `, ${payload.consecutive_failures} consecutive failures` : ''}` });
- q.putStatus.run({ source_key: payload.source_key || 'unknown', name: null, homepage_url: null, status: 'failing', stale: null, last_success_at: null, now: store.now() });
- failedEvent({ origin: 'sources', sourceKey: payload.source_key || null, state: payload.state || 'failed', errorCode: payload.error_code || null, httpStatus: payload.http_status || null,
+ await q.putStatus.run({ source_key: payload.source_key || 'unknown', name: null, homepage_url: null, status: 'failing', stale: null, last_success_at: null, now: store.now() });
+ await failedEvent({ origin: 'sources', sourceKey: payload.source_key || null, state: payload.state || 'failed', errorCode: payload.error_code || null, httpStatus: payload.http_status || null,
extra: { run_id: payload.run_id || null, consecutive_failures: payload.consecutive_failures || null } });
extra: { run_id: payload.run_id || null, consecutive_failures: payload.consecutive_failures || null } });
return { outcome: 'recorded' };
return { outcome: 'recorded' };
- })();
+ });
}
}
/** Remember what a Sources page says about its sources' health (display only). */
/** Remember what a Sources page says about its sources' health (display only). */
- function rememberSources(map) {
+ async function rememberSources(map) {
for (const [key, h] of Object.entries(map || {})) {
for (const [key, h] of Object.entries(map || {})) {
- q.putStatus.run({ source_key: key, name: null, homepage_url: null, status: h.status || null, stale: h.stale == null ? null : (h.stale ? 1 : 0), last_success_at: h.last_success_at || null, now: store.now() });
+ await q.putStatus.run({ source_key: key, name: null, homepage_url: null, status: h.status || null, stale: h.stale == null ? null : (h.stale ? 1 : 0), last_success_at: h.last_success_at || null, now: store.now() });
}
}
}
}
/** Fetch registry names we do not know yet (needs sources.source.read; failure = keep the host name). */
/** Fetch registry names we do not know yet (needs sources.source.read; failure = keep the host name). */
async function learnOutlets(keys) {
async function learnOutlets(keys) {
for (const key of [...new Set(keys)]) {
for (const key of [...new Set(keys)]) {
- const st = q.status.get(key);
+ const st = await q.status.get(key);
if (st && st.name) continue;
if (st && st.name) continue;
try {
try {
const s = await sources.getSource(key);
const s = await sources.getSource(key);
- q.putStatus.run({ source_key: key, name: s && s.name ? String(s.name).slice(0, 200) : null, homepage_url: s && s.homepage_url ? String(s.homepage_url) : null,
+ await q.putStatus.run({ source_key: key, name: s && s.name ? String(s.name).slice(0, 200) : null, homepage_url: s && s.homepage_url ? String(s.homepage_url) : null,
status: s && s.health ? s.health.status || null : null, stale: s && s.health && s.health.stale != null ? (s.health.stale ? 1 : 0) : null,
status: s && s.health ? s.health.status || null : null, stale: s && s.health && s.health.stale != null ? (s.health.stale ? 1 : 0) : null,
last_success_at: s && s.health ? s.health.last_success_at || null : null, now: store.now() });
last_success_at: s && s.health ? s.health.last_success_at || null : null, now: store.now() });
} catch { /* optional grant; the outlet stays the URL's host */ }
} catch { /* optional grant; the outlet stays the URL's host */ }
@@ -240,33 +240,33 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@@ -240,33 +240,33 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
async function pull() {
async function pull() {
if (pulling) return pulling;
if (pulling) return pulling;
pulling = (async () => {
pulling = (async () => {
- const before = store.getState('sources_cursor', 0);
+ const before = await store.getState('sources_cursor', 0);
let after = before;
let after = before;
const counts = { created: 0, updated: 0, removed: 0, unchanged: 0, ignored: 0, rejected: 0, duplicates: 0 };
const counts = { created: 0, updated: 0, removed: 0, unchanged: 0, ignored: 0, rejected: 0, duplicates: 0 };
try {
try {
for (let page = 0; page < config.sources.maxPagesPerRun; page++) {
for (let page = 0; page < config.sources.maxPagesPerRun; page++) {
const body = await sources.listItems({ after, limit: config.sources.pageSize });
const body = await sources.listItems({ after, limit: config.sources.pageSize });
- rememberSources(body.sources);
+ await rememberSources(body.sources);
await learnOutlets(body.items.map((i) => i.source_key).filter(Boolean));
await learnOutlets(body.items.map((i) => i.source_key).filter(Boolean));
- store.tx(() => {
+ await store.tx(async () => {
for (const item of body.items) {
for (const item of body.items) {
- const r = apply(item, { origin: 'pull' });
+ const r = await apply(item, { origin: 'pull' });
counts[r.outcome] = (counts[r.outcome] || 0) + 1;
counts[r.outcome] = (counts[r.outcome] || 0) + 1;
if (r.outcome === 'created' && r.item.status === 'duplicate') counts.duplicates++;
if (r.outcome === 'created' && r.item.status === 'duplicate') counts.duplicates++;
}
}
- store.setState('sources_cursor', body.next_after);
+ await store.setState('sources_cursor', body.next_after);
});
});
after = body.next_after;
after = body.next_after;
if (!body.more) break;
if (!body.more) break;
}
}
- store.tx(() => recordRun({ origin: 'pull', state: 'ok', counts, cursor_before: before, cursor_after: after }));
+ await store.tx(async () => await recordRun({ origin: 'pull', state: 'ok', counts, cursor_before: before, cursor_after: after }));
return { ok: true, counts, cursor: after };
return { ok: true, counts, cursor: after };
} catch (err) {
} catch (err) {
- const prev = q.lastPull.get();
- store.tx(() => {
- recordRun({ origin: 'pull', state: 'failed', error_code: err.code || 'pull.error', detail: err.message, counts, cursor_before: before, cursor_after: after });
+ const prev = await q.lastPull.get();
+ await store.tx(async () => {
+ await recordRun({ origin: 'pull', state: 'failed', error_code: err.code || 'pull.error', detail: err.message, counts, cursor_before: before, cursor_after: after });
// One event per outage, not one per retry.
// One event per outage, not one per retry.
- if (!prev || prev.state !== 'failed') failedEvent({ origin: 'pull', state: 'failed', errorCode: err.code || 'pull.error', detail: err.message, httpStatus: err.status || null });
+ if (!prev || prev.state !== 'failed') await failedEvent({ origin: 'pull', state: 'failed', errorCode: err.code || 'pull.error', detail: err.message, httpStatus: err.status || null });
});
});
log.warn(`[News] Sources pull failed (${err.code || 'error'}): ${err.message}`);
log.warn(`[News] Sources pull failed (${err.code || 'error'}): ${err.message}`);
return { ok: false, error: err.code || 'pull.error', detail: err.message, counts, cursor: after };
return { ok: false, error: err.code || 'pull.error', detail: err.message, counts, cursor: after };
@@ -279,11 +279,11 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
@@ -279,11 +279,11 @@ function createIngest({ store, config, clusters, outbox, sources, discussion = n
NEAR_TITLE,
NEAR_TITLE,
setStories(s) { stories = s; },
setStories(s) { stories = s; },
normalise, dedupe, apply, applyRemoval, upstreamFailure, pull, recordRun, failedEvent, learnOutlets, rememberSources,
normalise, dedupe, apply, applyRemoval, upstreamFailure, pull, recordRun, failedEvent, learnOutlets, rememberSources,
- get: (id) => q.byId.get(String(id || '')) || null,
- bySourcesId: (id) => q.bySourcesId.get(String(id || '')) || null,
- runs: ({ limit = 30 } = {}) => db.prepare('SELECT * FROM news_ingest_runs ORDER BY id DESC LIMIT ?').all(limit),
- sourceStatus: () => db.prepare('SELECT * FROM news_source_status ORDER BY source_key').all(),
- lastPull: () => q.lastPull.get() || null,
+ get: async (id) => await q.byId.get(String(id || '')) || null,
+ bySourcesId: async (id) => await q.bySourcesId.get(String(id || '')) || null,
+ runs: async ({ limit = 30 } = {}) => await db.prepare('SELECT * FROM news_ingest_runs ORDER BY id DESC LIMIT ?').all(limit),
+ sourceStatus: async () => await db.prepare('SELECT * FROM news_source_status ORDER BY source_key').all(),
+ lastPull: async () => await q.lastPull.get() || null,
};
};
}
}
@@ -61,20 +61,20 @@ function createPublication({ store, config, outbox }) {
@@ -61,20 +61,20 @@ function createPublication({ store, config, outbox }) {
const feedId = (story) => `tag:openvibe.news,2026:story/${story.id}`; // stable across slug changes
const feedId = (story) => `tag:openvibe.news,2026:story/${story.id}`; // stable across slug changes
const authorshipOf = (rev) => (rev && rev.meta && rev.meta.authorship) || null;
const authorshipOf = (rev) => (rev && rev.meta && rev.meta.authorship) || null;
- const reviewOf = (story, rev) => (rev ? store.reviews.latest(story.id, rev.number) : null);
+ const reviewOf = async (story, rev) => (rev ? await store.reviews.latest(story.id, rev.number) : null);
const paragraphsOf = (rev) => (rev && rev.fields && Array.isArray(rev.fields.paragraphs) ? rev.fields.paragraphs : []);
const paragraphsOf = (rev) => (rev && rev.fields && Array.isArray(rev.fields.paragraphs) ? rev.fields.paragraphs : []);
const snapshotOf = (rev) => (rev && rev.fields && Array.isArray(rev.fields.sources) ? rev.fields.sources : []);
const snapshotOf = (rev) => (rev && rev.fields && Array.isArray(rev.fields.sources) ? rev.fields.sources : []);
/** Live upstream state of every source a revision names: n → news_source_items row (or null). */
/** Live upstream state of every source a revision names: n → news_source_items row (or null). */
- function liveSources(rev) {
+ async function liveSources(rev) {
const out = new Map();
const out = new Map();
- for (const s of snapshotOf(rev)) out.set(s.n, itemById.get(s.id) || null);
+ for (const s of snapshotOf(rev)) out.set(s.n, await itemById.get(s.id) || null);
return out;
return out;
}
}
/** What the revision rests on, against the live source records. */
/** What the revision rests on, against the live source records. */
- function support(rev) {
- const live = liveSources(rev);
+ async function support(rev) {
+ const live = await liveSources(rev);
const alive = (n) => { const it = live.get(n); return Boolean(it && it.status !== 'removed'); };
const alive = (n) => { const it = live.get(n); return Boolean(it && it.status !== 'removed'); };
const cited = new Set();
const cited = new Set();
let unsupported = 0;
let unsupported = 0;
@@ -97,8 +97,8 @@ function createPublication({ store, config, outbox }) {
@@ -97,8 +97,8 @@ function createPublication({ store, config, outbox }) {
}
}
/** The gate's decision for a story at one revision (the published one by default). */
/** The gate's decision for a story at one revision (the published one by default). */
- function decide(story, rev, { state } = {}) {
- const sup = support(rev);
+ async function decide(story, rev, { state } = {}) {
+ const sup = await support(rev);
const rec = authorshipOf(rev);
const rec = authorshipOf(rev);
const facts = {
const facts = {
state: gateState(story, state),
state: gateState(story, state),
@@ -110,25 +110,25 @@ function createPublication({ store, config, outbox }) {
@@ -110,25 +110,25 @@ function createPublication({ store, config, outbox }) {
retracted: (state || story.state) === 'retracted',
retracted: (state || story.state) === 'retracted',
noindex: Boolean(story.noindex),
noindex: Boolean(story.noindex),
};
};
- if (rec) Object.assign(facts, gateAuthorship(rec, reviewOf(story, rev)));
+ if (rec) Object.assign(facts, gateAuthorship(rec, await reviewOf(story, rev)));
return seo.evaluate(facts, { policy, now: store.now() });
return seo.evaluate(facts, { policy, now: store.now() });
}
}
- function topicOf(story) { return story.topic_id ? topicById.get(story.topic_id) : null; }
+ async function topicOf(story) { return story.topic_id ? await topicById.get(story.topic_id) : null; }
/**
/**
* The index document as it would describe this story (for the product events), or a tombstone
* The index document as it would describe this story (for the product events), or a tombstone
* when it is not published (or retracted).
* when it is not published (or retracted).
*/
*/
- function documentFor(story, { forSearch }) {
- const rev = story.published_revision ? store.revisions.get(story.id, story.published_revision) : null;
+ async function documentFor(story, { forSearch }) {
+ const rev = story.published_revision ? await store.revisions.get(story.id, story.published_revision) : null;
const identity = { owner: OWNER, type: 'story', id: story.id, revision: 0 };
const identity = { owner: OWNER, type: 'story', id: story.id, revision: 0 };
- if (!rev || story.state !== 'published') return { doc: hooks.tombstone(identity), decision: rev ? decide(story, rev) : null };
- const decision = decide(story, rev);
+ if (!rev || story.state !== 'published') return { doc: hooks.tombstone(identity), decision: rev ? await decide(story, rev) : null };
+ const decision = await decide(story, rev);
if (forSearch && !decision.listable) return { doc: hooks.tombstone(identity), decision };
if (forSearch && !decision.listable) return { doc: hooks.tombstone(identity), decision };
- const topic = topicOf(story);
+ const topic = await topicOf(story);
const outlets = [...new Set(snapshotOf(rev).map((s) => s.outlet).filter(Boolean))].slice(0, 50);
const outlets = [...new Set(snapshotOf(rev).map((s) => s.outlet).filter(Boolean))].slice(0, 50);
- const citations = store.citations.forRevision(story.id, rev.number).filter((c) => !c.anchor || !c.anchor.startsWith('t'));
+ const citations = (await store.citations.forRevision(story.id, rev.number)).filter((c) => !c.anchor || !c.anchor.startsWith('t'));
const seen = new Set();
const seen = new Set();
const uniqueCites = citations.filter((c) => { const k = c.sourceItemId || c.url; if (seen.has(k)) return false; seen.add(k); return true; }).slice(0, 49);
const uniqueCites = citations.filter((c) => { const k = c.sourceItemId || c.url; if (seen.has(k)) return false; seen.add(k); return true; }).slice(0, 49);
const doc = hooks.buildIndexDocument({
const doc = hooks.buildIndexDocument({
@@ -151,13 +151,13 @@ function createPublication({ store, config, outbox }) {
@@ -151,13 +151,13 @@ function createPublication({ store, config, outbox }) {
}
}
/** Stamp and enqueue the Search document when it changed. Inside the caller's transaction. */
/** Stamp and enqueue the Search document when it changed. Inside the caller's transaction. */
- function syncIndex(story, { traceparent } = {}) {
- const { doc } = documentFor(story, { forSearch: true });
- const prev = store.sequencer.current(OWNER, 'story', story.id);
+ async function syncIndex(story, { traceparent } = {}) {
+ const { doc } = await documentFor(story, { forSearch: true });
+ const prev = await store.sequencer.current(OWNER, 'story', story.id);
if (doc.deleted && prev == null) return null;
if (doc.deleted && prev == null) return null;
- const stamped = store.sequencer.stamp(doc);
+ const stamped = await store.sequencer.stamp(store.db, doc);
if (prev != null && stamped.revision === prev) return null;
if (prev != null && stamped.revision === prev) return null;
- return outbox.emit(hooks.indexEvent({ document: stamped, now: store.now() }), { traceparent });
+ return await outbox.emit(hooks.indexEvent({ document: stamped, now: store.now() }), { traceparent });
}
}
function snapshot(story) {
function snapshot(story) {
@@ -168,13 +168,13 @@ function createPublication({ store, config, outbox }) {
@@ -168,13 +168,13 @@ function createPublication({ store, config, outbox }) {
* Emit the product event for a transition (news.story.published|updated|unpublished|retracted)
* Emit the product event for a transition (news.story.published|updated|unpublished|retracted)
* and re-sync Search. Inside the caller's transaction, after the row changed.
* and re-sync Search. Inside the caller's transaction, after the row changed.
*/
*/
- function afterChange(before, storyId, { actor, traceparent, extra = {} } = {}) {
- const story = storyById.get(storyId);
+ async function afterChange(before, storyId, { actor, traceparent, extra = {} } = {}) {
+ const story = await storyById.get(storyId);
const after = snapshot(story);
const after = snapshot(story);
let event = null;
let event = null;
if (after.state === 'retracted' && (!before || before.state !== 'retracted')) {
if (after.state === 'retracted' && (!before || before.state !== 'retracted')) {
- const d = decide(story, store.revisions.get(story.id, story.published_revision));
- event = outbox.emit({
+ const d = await decide(story, await store.revisions.get(story.id, story.published_revision));
+ event = await outbox.emit({
event_type: 'news.story.retracted', version: 1, source: OWNER, actor: actorRef(actor),
event_type: 'news.story.retracted', version: 1, source: OWNER, actor: actorRef(actor),
timestamp: new Date(store.now()).toISOString(),
timestamp: new Date(store.now()).toISOString(),
visibility: d.listable ? 'public' : 'internal',
visibility: d.listable ? 'public' : 'internal',
@@ -187,16 +187,16 @@ function createPublication({ store, config, outbox }) {
@@ -187,16 +187,16 @@ function createPublication({ store, config, outbox }) {
let action = hooks.actionFor(norm(before), norm(after));
let action = hooks.actionFor(norm(before), norm(after));
if (!action && before && before.state === 'published' && after.state === 'published' && before.url !== after.url) action = 'updated';
if (!action && before && before.state === 'published' && after.state === 'published' && before.url !== after.url) action = 'updated';
if (action) {
if (action) {
- const { doc, decision } = documentFor(story, { forSearch: false });
- const topic = topicOf(story);
- event = outbox.emit(hooks.publicationEvent({
+ const { doc, decision } = await documentFor(story, { forSearch: false });
+ const topic = await topicOf(story);
+ event = await outbox.emit(hooks.publicationEvent({
product: OWNER, type: 'story', action, id: story.id, revision: story.published_revision || 0,
product: OWNER, type: 'story', action, id: story.id, revision: story.published_revision || 0,
actor: actorRef(actor), document: doc, decision, now: store.now(),
actor: actorRef(actor), document: doc, decision, now: store.now(),
extra: { ...(topic ? { topic: topic.slug } : {}), ...extra },
extra: { ...(topic ? { topic: topic.slug } : {}), ...extra },
}), { traceparent });
}), { traceparent });
}
}
}
}
- syncIndex(story, { traceparent });
+ await syncIndex(story, { traceparent });
return { story, event };
return { story, event };
}
}
@@ -18,11 +18,11 @@ const iso = (v) => (v == null ? null : new Date(v).toISOString());
@@ -18,11 +18,11 @@ const iso = (v) => (v == null ? null : new Date(v).toISOString());
function createReading({ store, config, stories, publication, people, topics }) {
function createReading({ store, config, stories, publication, people, topics }) {
/** The public model of a story at a revision (the published one unless given). */
/** The public model of a story at a revision (the published one unless given). */
async function storyModel(story, { rev = null, editorView = false } = {}) {
async function storyModel(story, { rev = null, editorView = false } = {}) {
- const r = rev || (story.published_revision ? store.revisions.get(story.id, story.published_revision) : null);
+ const r = rev || (story.published_revision ? await store.revisions.get(story.id, story.published_revision) : null);
if (!r) return null;
if (!r) return null;
- const live = publication.liveSources(r);
+ const live = await publication.liveSources(r);
const snap = publication.snapshotOf(r);
const snap = publication.snapshotOf(r);
- const upstream = stories.openUpstreamFlags(story);
+ const upstream = await stories.openUpstreamFlags(story);
const sources = snap.map((s) => {
const sources = snap.map((s) => {
const it = live.get(s.n);
const it = live.get(s.n);
const removed = !it || it.status === 'removed';
const removed = !it || it.status === 'removed';
@@ -47,12 +47,12 @@ function createReading({ store, config, stories, publication, people, topics })
@@ -47,12 +47,12 @@ function createReading({ store, config, stories, publication, people, topics })
g.sources.push(s.n);
g.sources.push(s.n);
}
}
const rec = publication.authorshipOf(r);
const rec = publication.authorshipOf(r);
- const review = publication.reviewOf(story, r);
+ const review = await publication.reviewOf(story, r);
const authorSubjects = rec && Array.isArray(rec.authors) ? rec.authors : [];
const authorSubjects = rec && Array.isArray(rec.authors) ? rec.authors : [];
const who = authorSubjects.length ? await people.many(authorSubjects) : new Map();
const who = authorSubjects.length ? await people.many(authorSubjects) : new Map();
const authors = authorSubjects.map((s) => who.get(s)).filter((p) => p && p.known).map((p) => ({ name: p.name, username: p.username }));
const authors = authorSubjects.map((s) => who.get(s)).filter((p) => p && p.known).map((p) => ({ name: p.name, username: p.username }));
- const topic = publication.topicOf(story);
- const flags = stories.publicFlags(story);
+ const topic = await publication.topicOf(story);
+ const flags = await stories.publicFlags(story);
const retraction = flags.find((f) => f.kind === 'retraction') || null;
const retraction = flags.find((f) => f.kind === 'retraction') || null;
return {
return {
story, rev: r, topic,
story, rev: r, topic,
@@ -69,7 +69,7 @@ function createReading({ store, config, stories, publication, people, topics })
@@ -69,7 +69,7 @@ function createReading({ store, config, stories, publication, people, topics })
return { kind: f.kind, n: s ? s.n : null, outlet: s ? s.outlet : null, at: iso(f.created_at) };
return { kind: f.kind, n: s ? s.n : null, outlet: s ? s.outlet : null, at: iso(f.created_at) };
}),
}),
authorship: rec, review, disclosure: rec ? disclosure(rec, review) : null, authors,
authorship: rec, review, disclosure: rec ? disclosure(rec, review) : null, authors,
- decision: publication.decide(story, r),
+ decision: await publication.decide(story, r),
first_published_at: iso(story.first_published_at), updated_at: r.createdAt, published_at: iso(story.published_at),
first_published_at: iso(story.first_published_at), updated_at: r.createdAt, published_at: iso(story.published_at),
};
};
}
}
@@ -109,7 +109,7 @@ function createReading({ store, config, stories, publication, people, topics })
@@ -109,7 +109,7 @@ function createReading({ store, config, stories, publication, people, topics })
/** Feed items for published (and retracted, labelled) stories: our own text only, never a source's. */
/** Feed items for published (and retracted, labelled) stories: our own text only, never a source's. */
async function feedItems({ topicId = null, limit = 30 } = {}) {
async function feedItems({ topicId = null, limit = 30 } = {}) {
- const { stories: rows } = stories.listPublished({ topicId, limit });
+ const { stories: rows } = await stories.listPublished({ topicId, limit });
const out = [];
const out = [];
for (const story of rows) {
for (const story of rows) {
const m = await storyModel(story);
const m = await storyModel(story);
@@ -130,10 +130,10 @@ function createReading({ store, config, stories, publication, people, topics })
@@ -130,10 +130,10 @@ function createReading({ store, config, stories, publication, people, topics })
async function listItems(rows) {
async function listItems(rows) {
const out = [];
const out = [];
for (const story of rows) {
for (const story of rows) {
- const r = store.revisions.get(story.id, story.published_revision);
+ const r = await store.revisions.get(story.id, story.published_revision);
if (!r) continue;
if (!r) continue;
const paragraphs = publication.paragraphsOf(r);
const paragraphs = publication.paragraphsOf(r);
- const topic = publication.topicOf(story);
+ const topic = await publication.topicOf(story);
out.push({
out.push({
story, headline: r.fields.headline, url: publication.storyPath(story), lede: paragraphs[0] ? paragraphs[0].text : '',
story, headline: r.fields.headline, url: publication.storyPath(story), lede: paragraphs[0] ? paragraphs[0].text : '',
sourceCount: publication.snapshotOf(r).length, outlets: [...new Set(publication.snapshotOf(r).map((s) => s.outlet))].slice(0, 4),
sourceCount: publication.snapshotOf(r).length, outlets: [...new Set(publication.snapshotOf(r).map((s) => s.outlet))].slice(0, 4),
@@ -67,39 +67,39 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -67,39 +67,39 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
link: db.prepare('SELECT * FROM news_story_sources WHERE story_id = ? AND source_item_id = ?'),
link: db.prepare('SELECT * FROM news_story_sources WHERE story_id = ? AND source_item_id = ?'),
maxPos: db.prepare('SELECT COALESCE(MAX(position), 0) AS n FROM news_story_sources WHERE story_id = ?'),
maxPos: db.prepare('SELECT COALESCE(MAX(position), 0) AS n FROM news_story_sources WHERE story_id = ?'),
storiesCiting: db.prepare('SELECT DISTINCT story_id FROM news_story_sources WHERE source_item_id = ? AND detached_at IS NULL'),
storiesCiting: db.prepare('SELECT DISTINCT story_id FROM news_story_sources WHERE source_item_id = ? AND detached_at IS NULL'),
- perspectives: db.prepare('SELECT * FROM news_perspectives WHERE story_id = ? AND removed_at IS NULL ORDER BY position, created_at, rowid'),
+ perspectives: db.prepare('SELECT * FROM news_perspectives WHERE story_id = ? AND removed_at IS NULL ORDER BY position, created_at, seq'),
perspective: db.prepare('SELECT * FROM news_perspectives WHERE id = ? AND story_id = ?'),
perspective: db.prepare('SELECT * FROM news_perspectives WHERE id = ? AND story_id = ?'),
- timeline: db.prepare('SELECT * FROM news_timeline_entries WHERE story_id = ? AND removed_at IS NULL ORDER BY occurred_on, created_at, rowid'),
- flags: db.prepare('SELECT * FROM news_editorial_flags WHERE story_id = ? ORDER BY created_at, rowid'),
+ timeline: db.prepare('SELECT * FROM news_timeline_entries WHERE story_id = ? AND removed_at IS NULL ORDER BY occurred_on, created_at, seq'),
+ flags: db.prepare('SELECT * FROM news_editorial_flags WHERE story_id = ? ORDER BY created_at, seq'),
openFlag: db.prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND source_item_id = ? AND kind = ? AND status = 'open' ORDER BY created_at DESC LIMIT 1"),
openFlag: db.prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND source_item_id = ? AND kind = ? AND status = 'open' ORDER BY created_at DESC LIMIT 1"),
insertFlag: db.prepare(`INSERT INTO news_editorial_flags (id, story_id, kind, status, note, source_item_id, pending_revision, revision, created_by, created_at, resolved_by, resolved_at)
insertFlag: db.prepare(`INSERT INTO news_editorial_flags (id, story_id, kind, status, note, source_item_id, pending_revision, revision, created_by, created_at, resolved_by, resolved_at)
VALUES (@id, @story_id, @kind, @status, @note, @source_item_id, @pending_revision, @revision, @created_by, @now, @resolved_by, @resolved_at)`),
VALUES (@id, @story_id, @kind, @status, @note, @source_item_id, @pending_revision, @revision, @created_by, @now, @resolved_by, @resolved_at)`),
};
};
- const touch = (id) => db.prepare('UPDATE news_stories SET updated_at = ? WHERE id = ?').run(store.now(), id);
+ const touch = async (id) => await db.prepare('UPDATE news_stories SET updated_at = ? WHERE id = ?').run(store.now(), id);
- function mustItem(ref) {
+ async function mustItem(ref) {
const r = String(ref || '').trim();
const r = String(ref || '').trim();
- const it = r.startsWith('itm_') ? q.itemBySources.get(r) : q.item.get(r);
+ const it = r.startsWith('itm_') ? await q.itemBySources.get(r) : await q.item.get(r);
if (!it) throw new ApiError(404, 'source_item.not_found', `No source item ${r}`);
if (!it) throw new ApiError(404, 'source_item.not_found', `No source item ${r}`);
return it;
return it;
}
}
- function topicFrom(v) {
+ async function topicFrom(v) {
if (v == null || v === '') return null;
if (v == null || v === '') return null;
const s = String(v);
const s = String(v);
- const t = s.startsWith('top_') ? q.topicById.get(s) : q.topicBySlug.get(s);
+ const t = s.startsWith('top_') ? await q.topicById.get(s) : await q.topicBySlug.get(s);
if (!t) throw new ApiError(422, 'topic.not_found', `No active topic ${s}`);
if (!t) throw new ApiError(422, 'topic.not_found', `No active topic ${s}`);
return t;
return t;
}
}
- function slugFor(wanted, storyId = null) {
+ async function slugFor(wanted, storyId = null) {
let base;
let base;
try { base = slugify(wanted).slice(0, 90); } catch { throw new ApiError(422, 'story.invalid_slug', 'That headline has no letters or digits to make a URL from'); }
try { base = slugify(wanted).slice(0, 90); } catch { throw new ApiError(422, 'story.invalid_slug', 'That headline has no letters or digits to make a URL from'); }
if (!SLUG_RE.test(base)) throw new ApiError(422, 'story.invalid_slug', 'slug must be lowercase letters, digits and dashes');
if (!SLUG_RE.test(base)) throw new ApiError(422, 'story.invalid_slug', 'slug must be lowercase letters, digits and dashes');
for (let i = 1; i < 1000; i++) {
for (let i = 1; i < 1000; i++) {
const s = i === 1 ? base : `${base.slice(0, 85)}-${i}`;
const s = i === 1 ? base : `${base.slice(0, 85)}-${i}`;
- const hit = q.bySlug.get(s);
+ const hit = await q.bySlug.get(s);
if (!hit || hit.id === storyId) return s;
if (!hit || hit.id === storyId) return s;
}
}
throw new ApiError(409, 'story.slug_taken', 'No free slug for that headline');
throw new ApiError(409, 'story.slug_taken', 'No free slug for that headline');
@@ -110,8 +110,11 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -110,8 +110,11 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
// ── The working source table, timeline and perspectives ─
// ── The working source table, timeline and perspectives ─
/** Attached sources with their items, in citation order: [{ n, item, perspective_id, detached_at }]. */
/** Attached sources with their items, in citation order: [{ n, item, perspective_id, detached_at }]. */
- function sourcesOf(story, { includeDetached = false } = {}) {
- return q.links.all(story.id).filter((l) => includeDetached || !l.detached_at).map((l) => ({ n: l.position, item: q.item.get(l.source_item_id), perspective_id: l.perspective_id, detached_at: l.detached_at, added_by: l.added_by }));
+ async function sourcesOf(story, { includeDetached = false } = {}) {
+ const links = (await q.links.all(story.id)).filter((l) => includeDetached || !l.detached_at);
+ // Every item in one query (no N+1).
+ const items = new Map((links.length ? await db.prepare('SELECT * FROM news_source_items WHERE id = ANY(?)').all(links.map((l) => l.source_item_id)) : []).map((r) => [r.id, r]));
+ return links.map((l) => ({ n: l.position, item: items.get(l.source_item_id), perspective_id: l.perspective_id, detached_at: l.detached_at, added_by: l.added_by }));
}
}
function sourceEntry(n, item, perspectiveId) {
function sourceEntry(n, item, perspectiveId) {
@@ -125,17 +128,17 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -125,17 +128,17 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
}
}
/** What the next revision snapshots. */
/** What the next revision snapshots. */
- function buildSnapshot(story) {
- const srcs = sourcesOf(story);
+ async function buildSnapshot(story) {
+ const srcs = await sourcesOf(story);
const byItem = new Map(srcs.map((s) => [s.item.id, s.n]));
const byItem = new Map(srcs.map((s) => [s.item.id, s.n]));
return {
return {
sources: srcs.map((s) => sourceEntry(s.n, s.item, s.perspective_id)),
sources: srcs.map((s) => sourceEntry(s.n, s.item, s.perspective_id)),
- timeline: q.timeline.all(story.id).filter((t) => byItem.has(t.source_item_id)).map((t) => ({ id: t.id, occurred_on: t.occurred_on, text: t.text, source: byItem.get(t.source_item_id) })),
- perspectives: q.perspectives.all(story.id).map((p) => ({ id: p.id, label: p.label, description: p.description, sources: srcs.filter((s) => s.perspective_id === p.id).map((s) => s.n) })),
+ timeline: (await q.timeline.all(story.id)).filter((t) => byItem.has(t.source_item_id)).map((t) => ({ id: t.id, occurred_on: t.occurred_on, text: t.text, source: byItem.get(t.source_item_id) })),
+ perspectives: (await q.perspectives.all(story.id)).map((p) => ({ id: p.id, label: p.label, description: p.description, sources: srcs.filter((s) => s.perspective_id === p.id).map((s) => s.n) })),
};
};
}
}
- function attachCitations(story, rev) {
+ async function attachCitations(story, rev) {
const byN = new Map((rev.fields.sources || []).map((s) => [s.n, s]));
const byN = new Map((rev.fields.sources || []).map((s) => [s.n, s]));
const list = [];
const list = [];
(rev.fields.paragraphs || []).forEach((p, i) => {
(rev.fields.paragraphs || []).forEach((p, i) => {
@@ -148,7 +151,7 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -148,7 +151,7 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
const s = byN.get(t.source);
const s = byN.get(t.source);
if (s) list.push({ anchor: `t:${t.id}`, sourceItemId: s.sources_item_id, url: s.url || null, title: s.headline, retrievedAt: s.retrieved_at || null, attachedBy: 'svc:news' });
if (s) list.push({ anchor: `t:${t.id}`, sourceItemId: s.sources_item_id, url: s.url || null, title: s.headline, retrievedAt: s.retrieved_at || null, attachedBy: 'svc:news' });
}
}
- if (list.length) store.citations.attachMany(story.id, rev.number, list.slice(0, 500));
+ if (list.length) await store.citations.attachMany(story.id, rev.number, list.slice(0, 500));
}
}
function authorshipFor(viewer, input, previous) {
function authorshipFor(viewer, input, previous) {
@@ -185,9 +188,9 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -185,9 +188,9 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
}
}
/** Paragraphs from the API (by item id or number) or from the form body (markers). */
/** Paragraphs from the API (by item id or number) or from the form body (markers). */
- function paragraphsFrom(story, input) {
+ async function paragraphsFrom(story, input) {
if (Array.isArray(input.paragraphs)) {
if (Array.isArray(input.paragraphs)) {
- const numberOf = new Map(sourcesOf(story).map((s) => [s.item.id, s.n]).concat(sourcesOf(story).map((s) => [s.item.sources_item_id, s.n])));
+ const numberOf = new Map((await sourcesOf(story)).map((s) => [s.item.id, s.n]).concat((await sourcesOf(story)).map((s) => [s.item.sources_item_id, s.n])));
return input.paragraphs.map((p) => ({ text: p && p.text, sources: (Array.isArray(p && p.sources) ? p.sources : []).map((x) => (typeof x === 'number' ? x : numberOf.get(String(x)) || -1)) }));
return input.paragraphs.map((p) => ({ text: p && p.text, sources: (Array.isArray(p && p.sources) ? p.sources : []).map((x) => (typeof x === 'number' ? x : numberOf.get(String(x)) || -1)) }));
}
}
if (input.body !== undefined) return parseBody(input.body);
if (input.body !== undefined) return parseBody(input.body);
@@ -195,15 +198,15 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -195,15 +198,15 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
}
}
/** Everything that stands between a revision and publication. → [{ code, detail }] */
/** Everything that stands between a revision and publication. → [{ code, detail }] */
- function problems(story, rev) {
+ async function problems(story, rev) {
const out = [];
const out = [];
if (!rev) return [{ code: 'story.no_text', detail: 'Nothing has been written yet' }];
if (!rev) return [{ code: 'story.no_text', detail: 'Nothing has been written yet' }];
const snap = rev.fields.sources || [];
const snap = rev.fields.sources || [];
if (!snap.length) out.push({ code: 'story.unsourced', detail: 'A story with zero sources cannot be published: attach the source items it rests on' });
if (!snap.length) out.push({ code: 'story.unsourced', detail: 'A story with zero sources cannot be published: attach the source items it rests on' });
const paragraphs = rev.fields.paragraphs || [];
const paragraphs = rev.fields.paragraphs || [];
if (!paragraphs.length) out.push({ code: 'story.no_text', detail: 'The revision has no paragraphs' });
if (!paragraphs.length) out.push({ code: 'story.no_text', detail: 'The revision has no paragraphs' });
- const live = publication.liveSources(rev);
- const links = new Map(q.links.all(story.id).map((l) => [l.source_item_id, l]));
+ const live = await publication.liveSources(rev);
+ const links = new Map((await q.links.all(story.id)).map((l) => [l.source_item_id, l]));
const cited = new Set();
const cited = new Set();
paragraphs.forEach((p, i) => {
paragraphs.forEach((p, i) => {
if (!p.sources || !p.sources.length) out.push({ code: 'story.claim_unsourced', detail: `Paragraph ${i + 1} cites no source` });
if (!p.sources || !p.sources.length) out.push({ code: 'story.claim_unsourced', detail: `Paragraph ${i + 1} cites no source` });
@@ -215,25 +218,25 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -215,25 +218,25 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
const it = live.get(n);
const it = live.get(n);
if (!entry || !it) { out.push({ code: 'story.unknown_source', detail: `[${n}] is not in the revision's source table` }); continue; }
if (!entry || !it) { out.push({ code: 'story.unknown_source', detail: `[${n}] is not in the revision's source table` }); continue; }
if (it.status === 'removed') out.push({ code: 'story.source_removed', detail: `[${n}] (${it.outlet}) was removed upstream: ${it.removed_reason || 'no reason given'} — revise the text so nothing rests on it` });
if (it.status === 'removed') out.push({ code: 'story.source_removed', detail: `[${n}] (${it.outlet}) was removed upstream: ${it.removed_reason || 'no reason given'} — revise the text so nothing rests on it` });
- const link = links.get(it.id);
+ const link = await links.get(it.id);
if (!link || link.detached_at) out.push({ code: 'story.source_detached', detail: `[${n}] was detached from the story` });
if (!link || link.detached_at) out.push({ code: 'story.source_detached', detail: `[${n}] was detached from the story` });
}
}
const rec = publication.authorshipOf(rev) || { mode: 'human' };
const rec = publication.authorshipOf(rev) || { mode: 'human' };
- const ok = canPublish(rec, publication.reviewOf(story, rev));
+ const ok = canPublish(rec, await publication.reviewOf(story, rev));
if (!ok.ok) out.push({ code: 'story.review_required', detail: `Revision ${rev.number} is ${rec.mode === 'hybrid' ? 'AI-assisted' : 'AI-generated'} (${ok.reason}): a person must review and approve it first` });
if (!ok.ok) out.push({ code: 'story.review_required', detail: `Revision ${rev.number} is ${rec.mode === 'hybrid' ? 'AI-assisted' : 'AI-generated'} (${ok.reason}): a person must review and approve it first` });
if (!clean(rev.fields.headline, 300)) out.push({ code: 'story.no_headline', detail: 'The revision has no headline' });
if (!clean(rev.fields.headline, 300)) out.push({ code: 'story.no_headline', detail: 'The revision has no headline' });
return out;
return out;
}
}
- function flagRow(story, { kind, status = 'open', note = null, sourceItemId = null, pendingRevision = null, revision = null, createdBy, resolvedBy = null, resolvedAt = null }) {
+ async function flagRow(story, { kind, status = 'open', note = null, sourceItemId = null, pendingRevision = null, revision = null, createdBy, resolvedBy = null, resolvedAt = null }) {
const now = store.now();
const now = store.now();
const id = newFlagId(now);
const id = newFlagId(now);
- q.insertFlag.run({ id, story_id: story.id, kind, status, note, source_item_id: sourceItemId, pending_revision: pendingRevision, revision, created_by: createdBy, now, resolved_by: resolvedBy, resolved_at: resolvedAt });
- return db.prepare('SELECT * FROM news_editorial_flags WHERE id = ?').get(id);
+ await q.insertFlag.run({ id, story_id: story.id, kind, status, note, source_item_id: sourceItemId, pending_revision: pendingRevision, revision, created_by: createdBy, now, resolved_by: resolvedBy, resolved_at: resolvedAt });
+ return await db.prepare('SELECT * FROM news_editorial_flags WHERE id = ?').get(id);
}
}
- function flagEvent(story, flag, traceparent) {
- outbox.emit({
+ async function flagEvent(story, flag, traceparent) {
+ await outbox.emit({
event_type: 'news.story.flagged', actor: flag.created_by === 'svc:news' ? { type: 'service', id: 'news' } : actorRef(flag.created_by),
event_type: 'news.story.flagged', actor: flag.created_by === 'svc:news' ? { type: 'service', id: 'news' } : actorRef(flag.created_by),
visibility: 'internal', priority: flag.kind === 'source_removed' ? 'important' : 'low', // the envelope allows important|low; 'normal' was refused by Events
visibility: 'internal', priority: flag.kind === 'source_removed' ? 'important' : 'low', // the envelope allows important|low; 'normal' was refused by Events
subject: { type: 'story', id: story.id, revision: flag.pending_revision || story.published_revision || 0 },
subject: { type: 'story', id: story.id, revision: flag.pending_revision || story.published_revision || 0 },
@@ -244,154 +247,155 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -244,154 +247,155 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
const api = {
const api = {
parseBody, formatBody, AI_WORKFLOWS,
parseBody, formatBody, AI_WORKFLOWS,
- get: (id) => q.byId.get(String(id || '')) || null,
- bySlug: (slug) => q.bySlug.get(String(slug || '')) || null,
- mustGet(id) {
- const s = api.get(id);
+ get: async (id) => await q.byId.get(String(id || '')) || null,
+ bySlug: async (slug) => await q.bySlug.get(String(slug || '')) || null,
+ async mustGet(id) {
+ const s = await api.get(id);
if (!s) throw new ApiError(404, 'story.not_found', 'No such story');
if (!s) throw new ApiError(404, 'story.not_found', 'No such story');
return s;
return s;
},
},
sources: sourcesOf,
sources: sourcesOf,
buildSnapshot,
buildSnapshot,
- perspectives: (story) => q.perspectives.all(story.id),
- timeline: (story) => q.timeline.all(story.id),
- flags: (story) => q.flags.all(story.id),
- publicFlags: (story) => q.flags.all(story.id).filter((f) => f.status === 'published' && ['correction', 'update', 'retraction'].includes(f.kind)),
- openUpstreamFlags: (story) => q.flags.all(story.id).filter((f) => f.status === 'open' && (f.kind === 'source_updated' || f.kind === 'source_removed')),
- head: (story) => store.revisions.head(story.id),
- revision: (story, n) => store.revisions.get(story.id, n),
- revisions: (story, opts) => store.revisions.list(story.id, opts),
- diff: (story, from, to, mode) => store.revisions.diff(story.id, from, to, { mode: mode === 'word' ? 'word' : 'line' }),
+ perspectives: async (story) => await q.perspectives.all(story.id),
+ timeline: async (story) => await q.timeline.all(story.id),
+ flags: async (story) => await q.flags.all(story.id),
+ publicFlags: async (story) => (await q.flags.all(story.id)).filter((f) => f.status === 'published' && ['correction', 'update', 'retraction'].includes(f.kind)),
+ openUpstreamFlags: async (story) => (await q.flags.all(story.id)).filter((f) => f.status === 'open' && (f.kind === 'source_updated' || f.kind === 'source_removed')),
+ head: async (story) => await store.revisions.head(story.id),
+ revision: async (story, n) => await store.revisions.get(story.id, n),
+ revisions: async (story, opts) => await store.revisions.list(story.id, opts),
+ diff: async (story, from, to, mode) => await store.revisions.diff(story.id, from, to, { mode: mode === 'word' ? 'word' : 'line' }),
problems,
problems,
/** Published and retracted stories (retracted ones stay listed with their notice). */
/** Published and retracted stories (retracted ones stay listed with their notice). */
- listPublished({ topicId = null, limit = 20, offset = 0 } = {}) {
+ async listPublished({ topicId = null, limit = 20, offset = 0 } = {}) {
const where = ["state IN ('published','retracted')"];
const where = ["state IN ('published','retracted')"];
const args = [];
const args = [];
if (topicId) { where.push('topic_id = ?'); args.push(topicId); }
if (topicId) { where.push('topic_id = ?'); args.push(topicId); }
const w = where.join(' AND ');
const w = where.join(' AND ');
- const total = db.prepare(`SELECT COUNT(*) AS n FROM news_stories WHERE ${w}`).get(...args).n;
- const rows = db.prepare(`SELECT * FROM news_stories WHERE ${w} ORDER BY published_at DESC, id DESC LIMIT ? OFFSET ?`).all(...args, limit, offset);
+ const total = (await db.prepare(`SELECT COUNT(*) AS n FROM news_stories WHERE ${w}`).get(...args)).n;
+ const rows = await db.prepare(`SELECT * FROM news_stories WHERE ${w} ORDER BY published_at DESC, id DESC LIMIT ? OFFSET ?`).all(...args, limit, offset);
return { total, stories: rows };
return { total, stories: rows };
},
},
/** Every story, for editors. */
/** Every story, for editors. */
- listAll({ limit = 200 } = {}) {
- return db.prepare('SELECT * FROM news_stories ORDER BY updated_at DESC, id DESC LIMIT ?').all(limit);
+ async listAll({ limit = 200 } = {}) {
+ return await db.prepare('SELECT * FROM news_stories ORDER BY updated_at DESC, id DESC LIMIT ?').all(limit);
},
},
- byCluster: (clusterId) => db.prepare('SELECT * FROM news_stories WHERE cluster_id = ? ORDER BY created_at').all(clusterId),
+ byCluster: async (clusterId) => await db.prepare('SELECT * FROM news_stories WHERE cluster_id = ? ORDER BY created_at').all(clusterId),
/**
/**
* An editor opens a story from a cluster: the cluster's live, non-duplicate items are
* An editor opens a story from a cluster: the cluster's live, non-duplicate items are
* attached as the first sources. Optional first text (headline + body) becomes revision 1.
* attached as the first sources. Optional first text (headline + body) becomes revision 1.
*/
*/
- create(viewer, input = {}, { traceparent } = {}) {
+ async create(viewer, input = {}, { traceparent } = {}) {
const subject = actingEditor(viewer);
const subject = actingEditor(viewer);
const headline = clean(input.headline, 201);
const headline = clean(input.headline, 201);
if (!headline) throw new ApiError(422, 'story.no_headline', 'A story needs a working headline');
if (!headline) throw new ApiError(422, 'story.no_headline', 'A story needs a working headline');
if (headline.length > 200) throw new ApiError(422, 'story.no_headline', 'A headline is at most 200 characters');
if (headline.length > 200) throw new ApiError(422, 'story.no_headline', 'A headline is at most 200 characters');
let cluster = null;
let cluster = null;
if (input.cluster || input.clusterId || input.cluster_id) {
if (input.cluster || input.clusterId || input.cluster_id) {
- cluster = clusters.get(input.cluster || input.clusterId || input.cluster_id);
+ cluster = await clusters.get(input.cluster || input.clusterId || input.cluster_id);
if (!cluster) throw new ApiError(404, 'cluster.not_found', 'No such cluster');
if (!cluster) throw new ApiError(404, 'cluster.not_found', 'No such cluster');
- cluster = clusters.resolve(cluster.id);
+ cluster = await clusters.resolve(cluster.id);
}
}
- const topic = topicFrom(input.topic);
- return store.tx(() => {
+ const topic = await topicFrom(input.topic);
+ return await store.tx(async () => {
const now = store.now();
const now = store.now();
const id = newStoryId(now);
const id = newStoryId(now);
- const slug = slugFor(input.slug || headline);
- db.prepare(`INSERT INTO news_stories (id, slug, working_headline, cluster_id, topic_id, state, noindex, created_by, created_at, updated_at)
+ const slug = await slugFor(input.slug || headline);
+ await db.prepare(`INSERT INTO news_stories (id, slug, working_headline, cluster_id, topic_id, state, noindex, created_by, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, 'draft', 0, ?, ?, ?)`).run(id, slug, headline, cluster ? cluster.id : null, topic ? topic.id : null, subject, now, now);
VALUES (?, ?, ?, ?, ?, 'draft', 0, ?, ?, ?)`).run(id, slug, headline, cluster ? cluster.id : null, topic ? topic.id : null, subject, now, now);
- const story = q.byId.get(id);
+ const story = await q.byId.get(id);
if (cluster) {
if (cluster) {
- const items = clusters.members(cluster.id).filter((it) => it.status === 'active')
+ const items = (await clusters.members(cluster.id)).filter((it) => it.status === 'active')
.sort((a, b) => String(a.published_at || '').localeCompare(String(b.published_at || '')) || a.first_seen_at - b.first_seen_at || (a.id < b.id ? -1 : 1));
.sort((a, b) => String(a.published_at || '').localeCompare(String(b.published_at || '')) || a.first_seen_at - b.first_seen_at || (a.id < b.id ? -1 : 1));
- items.forEach((it, i) => db.prepare('INSERT INTO news_story_sources (story_id, source_item_id, position, added_by, added_at) VALUES (?, ?, ?, ?, ?)').run(id, it.id, i + 1, subject, now));
+ const insertSource = db.prepare('INSERT INTO news_story_sources (story_id, source_item_id, position, added_by, added_at) VALUES (?, ?, ?, ?, ?)');
+ for (const [i, it] of items.entries()) await insertSource.run(id, it.id, i + 1, subject, now);
}
}
- outbox.emit({
+ await outbox.emit({
event_type: 'news.story.created', actor: actorRef(viewer), visibility: 'internal', priority: 'low',
event_type: 'news.story.created', actor: actorRef(viewer), visibility: 'internal', priority: 'low',
- subject: { type: 'story', id }, payload: { cluster_id: cluster ? cluster.id : null, topic: topic ? topic.slug : null, sources: sourcesOf(story).length },
+ subject: { type: 'story', id }, payload: { cluster_id: cluster ? cluster.id : null, topic: topic ? topic.slug : null, sources: (await sourcesOf(story)).length },
}, { traceparent });
}, { traceparent });
let revision = null;
let revision = null;
if (input.body !== undefined || Array.isArray(input.paragraphs)) {
if (input.body !== undefined || Array.isArray(input.paragraphs)) {
- revision = api.revise(viewer, story, { headline, body: input.body, paragraphs: input.paragraphs, expectedRevision: 0, message: 'First draft' }).revision;
+ revision = (await api.revise(viewer, story, { headline, body: input.body, paragraphs: input.paragraphs, expectedRevision: 0, message: 'First draft' })).revision;
}
}
- return { story: q.byId.get(id), revision };
+ return { story: await q.byId.get(id), revision };
});
});
},
},
- attach(viewer, story, { item }) {
+ async attach(viewer, story, { item }) {
const subject = actingEditor(viewer);
const subject = actingEditor(viewer);
- const it = mustItem(item);
+ const it = await mustItem(item);
if (it.status === 'removed') throw new ApiError(409, 'source_item.removed', `${it.id} was removed upstream (${it.removed_reason || 'no reason given'}) and cannot be cited`);
if (it.status === 'removed') throw new ApiError(409, 'source_item.removed', `${it.id} was removed upstream (${it.removed_reason || 'no reason given'}) and cannot be cited`);
- return store.tx(() => {
- const link = q.link.get(story.id, it.id);
+ return await store.tx(async () => {
+ const link = await q.link.get(story.id, it.id);
if (link && !link.detached_at) return { n: link.position, item: it, created: false };
if (link && !link.detached_at) return { n: link.position, item: it, created: false };
- if (link) db.prepare('UPDATE news_story_sources SET detached_at = NULL WHERE story_id = ? AND source_item_id = ?').run(story.id, it.id);
- else db.prepare('INSERT INTO news_story_sources (story_id, source_item_id, position, added_by, added_at) VALUES (?, ?, ?, ?, ?)').run(story.id, it.id, q.maxPos.get(story.id).n + 1, subject, store.now());
- touch(story.id);
- return { n: q.link.get(story.id, it.id).position, item: it, created: true };
+ if (link) await db.prepare('UPDATE news_story_sources SET detached_at = NULL WHERE story_id = ? AND source_item_id = ?').run(story.id, it.id);
+ else await db.prepare('INSERT INTO news_story_sources (story_id, source_item_id, position, added_by, added_at) VALUES (?, ?, ?, ?, ?)').run(story.id, it.id, (await q.maxPos.get(story.id)).n + 1, subject, store.now());
+ await touch(story.id);
+ return { n: (await q.link.get(story.id, it.id)).position, item: it, created: true };
});
});
},
},
/** Detach a source (its number is never reused). The next revision must not cite it. */
/** Detach a source (its number is never reused). The next revision must not cite it. */
- detach(viewer, story, item) {
+ async detach(viewer, story, item) {
actingEditor(viewer);
actingEditor(viewer);
- const it = mustItem(item);
- const link = q.link.get(story.id, it.id);
+ const it = await mustItem(item);
+ const link = await q.link.get(story.id, it.id);
if (!link || link.detached_at) throw new ApiError(404, 'story.source_not_attached', `${it.id} is not attached to this story`);
if (!link || link.detached_at) throw new ApiError(404, 'story.source_not_attached', `${it.id} is not attached to this story`);
- db.prepare('UPDATE news_story_sources SET detached_at = ? WHERE story_id = ? AND source_item_id = ?').run(store.now(), story.id, it.id);
- db.prepare('UPDATE news_timeline_entries SET removed_at = ? WHERE story_id = ? AND source_item_id = ? AND removed_at IS NULL').run(store.now(), story.id, it.id);
- touch(story.id);
+ await db.prepare('UPDATE news_story_sources SET detached_at = ? WHERE story_id = ? AND source_item_id = ?').run(store.now(), story.id, it.id);
+ await db.prepare('UPDATE news_timeline_entries SET removed_at = ? WHERE story_id = ? AND source_item_id = ? AND removed_at IS NULL').run(store.now(), story.id, it.id);
+ await touch(story.id);
return { detached: true, n: link.position };
return { detached: true, n: link.position };
},
},
- addPerspective(viewer, story, { label, description = null }) {
+ async addPerspective(viewer, story, { label, description = null }) {
const subject = actingEditor(viewer);
const subject = actingEditor(viewer);
const l = clean(label, 81);
const l = clean(label, 81);
if (!l) throw new ApiError(422, 'perspective.no_label', 'A perspective needs a label an editor writes (for example the outlet group or the position it states)');
if (!l) throw new ApiError(422, 'perspective.no_label', 'A perspective needs a label an editor writes (for example the outlet group or the position it states)');
if (l.length > 80) throw new ApiError(422, 'perspective.label_too_long', 'A label is at most 80 characters');
if (l.length > 80) throw new ApiError(422, 'perspective.label_too_long', 'A label is at most 80 characters');
const now = store.now();
const now = store.now();
const id = newPerspectiveId(now);
const id = newPerspectiveId(now);
- const pos = q.perspectives.all(story.id).length;
- db.prepare('INSERT INTO news_perspectives (id, story_id, label, description, position, created_by, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')
+ const pos = (await q.perspectives.all(story.id)).length;
+ await db.prepare('INSERT INTO news_perspectives (id, story_id, label, description, position, created_by, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')
.run(id, story.id, l, description ? clean(description, 500) : null, pos, subject, now, now);
.run(id, story.id, l, description ? clean(description, 500) : null, pos, subject, now, now);
- touch(story.id);
- return q.perspective.get(id, story.id);
+ await touch(story.id);
+ return await q.perspective.get(id, story.id);
},
},
- removePerspective(viewer, story, perspectiveId) {
+ async removePerspective(viewer, story, perspectiveId) {
actingEditor(viewer);
actingEditor(viewer);
- const p = q.perspective.get(String(perspectiveId || ''), story.id);
+ const p = await q.perspective.get(String(perspectiveId || ''), story.id);
if (!p || p.removed_at) throw new ApiError(404, 'perspective.not_found', 'No such perspective');
if (!p || p.removed_at) throw new ApiError(404, 'perspective.not_found', 'No such perspective');
- store.tx(() => {
- db.prepare('UPDATE news_perspectives SET removed_at = ?, updated_at = ? WHERE id = ?').run(store.now(), store.now(), p.id);
- db.prepare('UPDATE news_story_sources SET perspective_id = NULL WHERE story_id = ? AND perspective_id = ?').run(story.id, p.id);
+ await store.tx(async () => {
+ await db.prepare('UPDATE news_perspectives SET removed_at = ?, updated_at = ? WHERE id = ?').run(store.now(), store.now(), p.id);
+ await db.prepare('UPDATE news_story_sources SET perspective_id = NULL WHERE story_id = ? AND perspective_id = ?').run(story.id, p.id);
});
});
- touch(story.id);
+ await touch(story.id);
return { removed: true };
return { removed: true };
},
},
/** Put an attached source in a perspective (or none). */
/** Put an attached source in a perspective (or none). */
- assignPerspective(viewer, story, item, perspectiveId) {
+ async assignPerspective(viewer, story, item, perspectiveId) {
actingEditor(viewer);
actingEditor(viewer);
- const it = mustItem(item);
- const link = q.link.get(story.id, it.id);
+ const it = await mustItem(item);
+ const link = await q.link.get(story.id, it.id);
if (!link || link.detached_at) throw new ApiError(404, 'story.source_not_attached', `${it.id} is not attached to this story`);
if (!link || link.detached_at) throw new ApiError(404, 'story.source_not_attached', `${it.id} is not attached to this story`);
let pid = null;
let pid = null;
if (perspectiveId) {
if (perspectiveId) {
- const p = q.perspective.get(String(perspectiveId), story.id);
+ const p = await q.perspective.get(String(perspectiveId), story.id);
if (!p || p.removed_at) throw new ApiError(404, 'perspective.not_found', 'No such perspective');
if (!p || p.removed_at) throw new ApiError(404, 'perspective.not_found', 'No such perspective');
pid = p.id;
pid = p.id;
}
}
- db.prepare('UPDATE news_story_sources SET perspective_id = ? WHERE story_id = ? AND source_item_id = ?').run(pid, story.id, it.id);
- touch(story.id);
+ await db.prepare('UPDATE news_story_sources SET perspective_id = ? WHERE story_id = ? AND source_item_id = ?').run(pid, story.id, it.id);
+ await touch(story.id);
return { n: link.position, perspective_id: pid };
return { n: link.position, perspective_id: pid };
},
},
/** A dated entry resting on one attached source. The date is what the source states. */
/** A dated entry resting on one attached source. The date is what the source states. */
- addTimeline(viewer, story, { occurredOn, text: entryText, source }) {
+ async addTimeline(viewer, story, { occurredOn, text: entryText, source }) {
const subject = actingEditor(viewer);
const subject = actingEditor(viewer);
const on = String(occurredOn || '').trim();
const on = String(occurredOn || '').trim();
if (!DATE_RE.test(on) || !Number.isFinite(Date.parse(on))) throw new ApiError(422, 'timeline.invalid_date', 'occurred_on must be a date the source states (YYYY-MM-DD or an ISO 8601 time)');
if (!DATE_RE.test(on) || !Number.isFinite(Date.parse(on))) throw new ApiError(422, 'timeline.invalid_date', 'occurred_on must be a date the source states (YYYY-MM-DD or an ISO 8601 time)');
@@ -400,27 +404,27 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -400,27 +404,27 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
if (t.length > 500) throw new ApiError(422, 'timeline.too_long', 'A timeline entry is at most 500 characters');
if (t.length > 500) throw new ApiError(422, 'timeline.too_long', 'A timeline entry is at most 500 characters');
let it;
let it;
if (/^\d+$/.test(String(source))) {
if (/^\d+$/.test(String(source))) {
- const s = sourcesOf(story).find((x) => x.n === parseInt(source, 10));
+ const s = (await sourcesOf(story)).find((x) => x.n === parseInt(source, 10));
if (!s) throw new ApiError(422, 'timeline.unknown_source', `[${source}] is not in this story's source table`);
if (!s) throw new ApiError(422, 'timeline.unknown_source', `[${source}] is not in this story's source table`);
it = s.item;
it = s.item;
} else {
} else {
- it = mustItem(source);
- const link = q.link.get(story.id, it.id);
+ it = await mustItem(source);
+ const link = await q.link.get(story.id, it.id);
if (!link || link.detached_at) throw new ApiError(422, 'timeline.unknown_source', 'The source must be attached to the story');
if (!link || link.detached_at) throw new ApiError(422, 'timeline.unknown_source', 'The source must be attached to the story');
}
}
if (it.status === 'removed') throw new ApiError(409, 'source_item.removed', 'That source was removed upstream');
if (it.status === 'removed') throw new ApiError(409, 'source_item.removed', 'That source was removed upstream');
const now = store.now();
const now = store.now();
const id = newTimelineId(now);
const id = newTimelineId(now);
- db.prepare('INSERT INTO news_timeline_entries (id, story_id, occurred_on, text, source_item_id, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)').run(id, story.id, on, t, it.id, subject, now);
- touch(story.id);
- return db.prepare('SELECT * FROM news_timeline_entries WHERE id = ?').get(id);
+ await db.prepare('INSERT INTO news_timeline_entries (id, story_id, occurred_on, text, source_item_id, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)').run(id, story.id, on, t, it.id, subject, now);
+ await touch(story.id);
+ return await db.prepare('SELECT * FROM news_timeline_entries WHERE id = ?').get(id);
},
},
- removeTimeline(viewer, story, entryId) {
+ async removeTimeline(viewer, story, entryId) {
actingEditor(viewer);
actingEditor(viewer);
- const r = db.prepare('UPDATE news_timeline_entries SET removed_at = ? WHERE id = ? AND story_id = ? AND removed_at IS NULL').run(store.now(), String(entryId || ''), story.id);
+ const r = await db.prepare('UPDATE news_timeline_entries SET removed_at = ? WHERE id = ? AND story_id = ? AND removed_at IS NULL').run(store.now(), String(entryId || ''), story.id);
if (!r.changes) throw new ApiError(404, 'timeline.not_found', 'No such timeline entry');
if (!r.changes) throw new ApiError(404, 'timeline.not_found', 'No such timeline entry');
- touch(story.id);
+ await touch(story.id);
return { removed: true };
return { removed: true };
},
},
@@ -430,38 +434,38 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -430,38 +434,38 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
* exists (412 revision.conflict on a stale base). Editors write human (or, after an AI draft,
* exists (412 revision.conflict on a stale base). Editors write human (or, after an AI draft,
* hybrid) revisions; an AI delivery writes an AI revision that needs a person's review.
* hybrid) revisions; an AI delivery writes an AI revision that needs a person's review.
*/
*/
- revise(viewer, story, input = {}, { traceparent } = {}) {
+ async revise(viewer, story, input = {}, { traceparent } = {}) {
const ai = access.isAiDelivery(viewer);
const ai = access.isAiDelivery(viewer);
if (!ai) actingEditor(viewer);
if (!ai) actingEditor(viewer);
if (story.state === 'retracted') throw new ApiError(409, 'story.retracted', 'A retracted story is not revised; open a new story');
if (story.state === 'retracted') throw new ApiError(409, 'story.retracted', 'A retracted story is not revised; open a new story');
- return store.tx(() => {
- const head = store.revisions.head(story.id);
+ return await store.tx(async () => {
+ const head = await store.revisions.head(story.id);
const expected = input.expectedRevision ?? input.expected_revision;
const expected = input.expectedRevision ?? input.expected_revision;
if (head && (expected == null || expected === '')) throw new ApiError(428, 'revision.expected_required', 'Send expectedRevision (the revision you edited) so concurrent edits are not lost');
if (head && (expected == null || expected === '')) throw new ApiError(428, 'revision.expected_required', 'Send expectedRevision (the revision you edited) so concurrent edits are not lost');
const headline = input.headline !== undefined && input.headline !== null ? clean(input.headline, 201) : (head ? head.fields.headline : story.working_headline);
const headline = input.headline !== undefined && input.headline !== null ? clean(input.headline, 201) : (head ? head.fields.headline : story.working_headline);
if (!headline) throw new ApiError(422, 'story.no_headline', 'A revision needs a headline');
if (!headline) throw new ApiError(422, 'story.no_headline', 'A revision needs a headline');
if (headline.length > 200) throw new ApiError(422, 'story.no_headline', 'A headline is at most 200 characters');
if (headline.length > 200) throw new ApiError(422, 'story.no_headline', 'A headline is at most 200 characters');
- const attached = new Set(sourcesOf(story).map((s) => s.n));
- const given = paragraphsFrom(story, input);
+ const attached = new Set((await sourcesOf(story)).map((s) => s.n));
+ const given = await paragraphsFrom(story, input);
const paragraphs = checkParagraphs(given || (head ? head.fields.paragraphs : []), attached);
const paragraphs = checkParagraphs(given || (head ? head.fields.paragraphs : []), attached);
- const snap = buildSnapshot(story);
+ const snap = await buildSnapshot(story);
const prevRec = head ? publication.authorshipOf(head) : null;
const prevRec = head ? publication.authorshipOf(head) : null;
const rec = authorshipFor(viewer, input, prevRec);
const rec = authorshipFor(viewer, input, prevRec);
const meta = { authorship: rec };
const meta = { authorship: rec };
if (ai && Array.isArray(input.gaps) && input.gaps.length) meta.gaps = input.gaps.map((g) => clean(g, 300)).slice(0, 20);
if (ai && Array.isArray(input.gaps) && input.gaps.length) meta.gaps = input.gaps.map((g) => clean(g, 300)).slice(0, 20);
- const out = store.revisions.create({
+ const out = await store.revisions.create({
entityId: story.id, expectedRevision: head ? parseInt(expected, 10) : 0,
entityId: story.id, expectedRevision: head ? parseInt(expected, 10) : 0,
content: formatBody(paragraphs), fields: { headline, paragraphs, ...snap }, meta,
content: formatBody(paragraphs), fields: { headline, paragraphs, ...snap }, meta,
author: authorLabel(viewer), message: input.message ? clean(input.message, 200) : null,
author: authorLabel(viewer), message: input.message ? clean(input.message, 200) : null,
});
});
- if (out.created) attachCitations(story, out.revision);
+ if (out.created) await attachCitations(story, out.revision);
const sets = { working_headline: headline };
const sets = { working_headline: headline };
- if (!ai && input.topic !== undefined) { const t = topicFrom(input.topic); sets.topic_id = t ? t.id : null; }
+ if (!ai && input.topic !== undefined) { const t = await topicFrom(input.topic); sets.topic_id = t ? t.id : null; }
if (!ai && input.noindex !== undefined) sets.noindex = input.noindex === true || input.noindex === '1' || input.noindex === 1 ? 1 : 0;
if (!ai && input.noindex !== undefined) sets.noindex = input.noindex === true || input.noindex === '1' || input.noindex === 1 ? 1 : 0;
const keys = Object.keys(sets).filter((k) => k !== 'working_headline');
const keys = Object.keys(sets).filter((k) => k !== 'working_headline');
- db.prepare(`UPDATE news_stories SET ${Object.keys(sets).map((k) => `${k} = @${k}`).concat('updated_at = @now').join(', ')} WHERE id = @id`).run({ ...sets, now: store.now(), id: story.id });
- if (keys.length && story.state === 'published') publication.afterChange(publication.snapshot(story), story.id, { actor: viewer, traceparent });
- return { revision: out.revision, created: out.created, story: q.byId.get(story.id) };
+ await db.prepare(`UPDATE news_stories SET ${Object.keys(sets).map((k) => `${k} = @${k}`).concat('updated_at = @now').join(', ')} WHERE id = @id`).run({ ...sets, now: store.now(), id: story.id });
+ if (keys.length && story.state === 'published') await publication.afterChange(publication.snapshot(story), story.id, { actor: viewer, traceparent });
+ return { revision: out.revision, created: out.created, story: await q.byId.get(story.id) };
});
});
},
},
@@ -470,15 +474,15 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -470,15 +474,15 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
* Open correction/update flags are published with it; upstream flags the revision answers
* Open correction/update flags are published with it; upstream flags the revision answers
* are resolved. Republishing after a source was removed needs a correction or update note.
* are resolved. Republishing after a source was removed needs a correction or update note.
*/
*/
- publish(viewer, story, { revision, correction } = {}, { traceparent } = {}) {
+ async publish(viewer, story, { revision, correction } = {}, { traceparent } = {}) {
actingEditor(viewer);
actingEditor(viewer);
if (story.state === 'retracted') throw new ApiError(409, 'story.retracted', 'A retracted story stays retracted; open a new story');
if (story.state === 'retracted') throw new ApiError(409, 'story.retracted', 'A retracted story stays retracted; open a new story');
- return store.tx(() => {
- const cur = q.byId.get(story.id);
- if (discussion) discussion.watch(story.id);
- const rev = revision == null || revision === '' ? store.revisions.head(story.id) : store.revisions.get(story.id, parseInt(revision, 10));
+ return await store.tx(async () => {
+ const cur = await q.byId.get(story.id);
+ if (discussion) await discussion.watch(story.id);
+ const rev = revision == null || revision === '' ? await store.revisions.head(story.id) : await store.revisions.get(story.id, parseInt(revision, 10));
if (!rev && revision != null && revision !== '') throw new ApiError(404, 'revision.not_found', `No revision ${revision}`);
if (!rev && revision != null && revision !== '') throw new ApiError(404, 'revision.not_found', `No revision ${revision}`);
- const found = problems(cur, rev);
+ const found = await problems(cur, rev);
if (found.length) {
if (found.length) {
const status = found.some((p) => p.code === 'story.review_required' || p.code === 'story.source_removed' || p.code === 'story.source_detached') ? 409 : 422;
const status = found.some((p) => p.code === 'story.review_required' || p.code === 'story.source_removed' || p.code === 'story.source_detached') ? 409 : 422;
throw new ApiError(status, found[0].code, found.map((p) => p.detail).join(' · '), { problems: found });
throw new ApiError(status, found[0].code, found.map((p) => p.detail).join(' · '), { problems: found });
@@ -486,87 +490,88 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -486,87 +490,88 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
if (cur.state === 'published' && cur.published_revision === rev.number && !correction) return { changed: false, story: cur };
if (cur.state === 'published' && cur.published_revision === rev.number && !correction) return { changed: false, story: cur };
const citedIds = new Set();
const citedIds = new Set();
for (const p of rev.fields.paragraphs) for (const n of p.sources) { const s = rev.fields.sources.find((x) => x.n === n); if (s) citedIds.add(s.id); }
for (const p of rev.fields.paragraphs) for (const n of p.sources) { const s = rev.fields.sources.find((x) => x.n === n); if (s) citedIds.add(s.id); }
- const upstream = q.flags.all(story.id).filter((f) => f.status === 'open' && (f.kind === 'source_updated' || f.kind === 'source_removed'));
- const answers = (f) => {
- const it = q.item.get(f.source_item_id);
+ const upstream = (await q.flags.all(story.id)).filter((f) => f.status === 'open' && (f.kind === 'source_updated' || f.kind === 'source_removed'));
+ const answers = async (f) => {
+ const it = await q.item.get(f.source_item_id);
const entry = rev.fields.sources.find((s) => s.id === f.source_item_id);
const entry = rev.fields.sources.find((s) => s.id === f.source_item_id);
if (f.kind === 'source_removed') return !citedIds.has(f.source_item_id) && !(rev.fields.timeline || []).some((t) => entry && t.source === entry.n);
if (f.kind === 'source_removed') return !citedIds.has(f.source_item_id) && !(rev.fields.timeline || []).some((t) => entry && t.source === entry.n);
return !entry || !it || entry.sources_revision >= it.sources_revision;
return !entry || !it || entry.sources_revision >= it.sources_revision;
};
};
- const resolving = upstream.filter(answers);
+ const resolving = [];
+ for (const f of upstream) if (await answers(f)) resolving.push(f);
const note = correction && correction.note ? clean(correction.note, 2001) : '';
const note = correction && correction.note ? clean(correction.note, 2001) : '';
const kind = correction && correction.kind ? String(correction.kind) : (note ? 'correction' : null);
const kind = correction && correction.kind ? String(correction.kind) : (note ? 'correction' : null);
if (kind && !['correction', 'update'].includes(kind)) throw new ApiError(422, 'flag.invalid_kind', 'kind is correction or update');
if (kind && !['correction', 'update'].includes(kind)) throw new ApiError(422, 'flag.invalid_kind', 'kind is correction or update');
if (note.length > 2000) throw new ApiError(422, 'flag.note_too_long', 'A note is at most 2000 characters');
if (note.length > 2000) throw new ApiError(422, 'flag.note_too_long', 'A note is at most 2000 characters');
- if (cur.first_published_at && resolving.some((f) => f.kind === 'source_removed') && !note && !q.flags.all(story.id).some((f) => f.status === 'open' && (f.kind === 'correction' || f.kind === 'update') && f.note)) {
+ if (cur.first_published_at && resolving.some((f) => f.kind === 'source_removed') && !note && !(await q.flags.all(story.id)).some((f) => f.status === 'open' && (f.kind === 'correction' || f.kind === 'update') && f.note)) {
throw new ApiError(422, 'story.correction_note_required', 'A source this story cited was removed upstream: say what changed in a correction or update note readers will see');
throw new ApiError(422, 'story.correction_note_required', 'A source this story cited was removed upstream: say what changed in a correction or update note readers will see');
}
}
const now = store.now();
const now = store.now();
const before = publication.snapshot(cur);
const before = publication.snapshot(cur);
- db.prepare(`UPDATE news_stories SET state = 'published', published_revision = ?, first_published_at = COALESCE(first_published_at, ?),
+ await db.prepare(`UPDATE news_stories SET state = 'published', published_revision = ?, first_published_at = COALESCE(first_published_at, ?),
published_at = ?, updated_at = ? WHERE id = ?`).run(rev.number, now, now, now, story.id);
published_at = ?, updated_at = ? WHERE id = ?`).run(rev.number, now, now, now, story.id);
const subject = viewer.subject;
const subject = viewer.subject;
- for (const f of q.flags.all(story.id).filter((x) => x.status === 'open' && (x.kind === 'correction' || x.kind === 'update'))) {
- db.prepare("UPDATE news_editorial_flags SET status = 'published', revision = ?, resolved_by = ?, resolved_at = ? WHERE id = ?").run(rev.number, subject, now, f.id);
+ for (const f of (await q.flags.all(story.id)).filter((x) => x.status === 'open' && (x.kind === 'correction' || x.kind === 'update'))) {
+ await db.prepare("UPDATE news_editorial_flags SET status = 'published', revision = ?, resolved_by = ?, resolved_at = ? WHERE id = ?").run(rev.number, subject, now, f.id);
}
}
- if (note) flagRow(cur, { kind, status: 'published', note, revision: rev.number, createdBy: subject, resolvedBy: subject, resolvedAt: now });
- for (const f of resolving) db.prepare("UPDATE news_editorial_flags SET status = 'resolved', revision = ?, resolved_by = ?, resolved_at = ? WHERE id = ?").run(rev.number, subject, now, f.id);
- const out = publication.afterChange(before, story.id, { actor: viewer, traceparent });
+ if (note) await flagRow(cur, { kind, status: 'published', note, revision: rev.number, createdBy: subject, resolvedBy: subject, resolvedAt: now });
+ for (const f of resolving) await db.prepare("UPDATE news_editorial_flags SET status = 'resolved', revision = ?, resolved_by = ?, resolved_at = ? WHERE id = ?").run(rev.number, subject, now, f.id);
+ const out = await publication.afterChange(before, story.id, { actor: viewer, traceparent });
return { changed: true, story: out.story, resolved: resolving.map((f) => f.id) };
return { changed: true, story: out.story, resolved: resolving.map((f) => f.id) };
});
});
},
},
- unpublish(viewer, story, { traceparent } = {}) {
+ async unpublish(viewer, story, { traceparent } = {}) {
actingEditor(viewer);
actingEditor(viewer);
- return store.tx(() => {
- const cur = q.byId.get(story.id);
+ return await store.tx(async () => {
+ const cur = await q.byId.get(story.id);
if (cur.state !== 'published' && cur.state !== 'retracted') return { changed: false, story: cur };
if (cur.state !== 'published' && cur.state !== 'retracted') return { changed: false, story: cur };
- if (discussion) discussion.watch(story.id);
+ if (discussion) await discussion.watch(story.id);
const before = publication.snapshot(cur);
const before = publication.snapshot(cur);
- db.prepare("UPDATE news_stories SET state = 'unpublished', updated_at = ? WHERE id = ?").run(store.now(), story.id);
- return { changed: true, story: publication.afterChange(before, story.id, { actor: viewer, traceparent }).story };
+ await db.prepare("UPDATE news_stories SET state = 'unpublished', updated_at = ? WHERE id = ?").run(store.now(), story.id);
+ return { changed: true, story: (await publication.afterChange(before, story.id, { actor: viewer, traceparent })).story };
});
});
},
},
/** Retract: the story stays at its URL with the retraction notice; noindex; out of Search. */
/** Retract: the story stays at its URL with the retraction notice; noindex; out of Search. */
- retract(viewer, story, { note } = {}, { traceparent } = {}) {
+ async retract(viewer, story, { note } = {}, { traceparent } = {}) {
const subject = actingEditor(viewer);
const subject = actingEditor(viewer);
const n = clean(note, 2001);
const n = clean(note, 2001);
if (n.length < 10) throw new ApiError(422, 'story.retraction_note_required', 'A retraction says why, in at least a sentence readers will see');
if (n.length < 10) throw new ApiError(422, 'story.retraction_note_required', 'A retraction says why, in at least a sentence readers will see');
if (n.length > 2000) throw new ApiError(422, 'flag.note_too_long', 'A note is at most 2000 characters');
if (n.length > 2000) throw new ApiError(422, 'flag.note_too_long', 'A note is at most 2000 characters');
- return store.tx(() => {
- const cur = q.byId.get(story.id);
+ return await store.tx(async () => {
+ const cur = await q.byId.get(story.id);
if (cur.state === 'retracted') return { changed: false, story: cur };
if (cur.state === 'retracted') return { changed: false, story: cur };
if (cur.state !== 'published') throw new ApiError(409, 'story.not_published', 'Only a published story can be retracted');
if (cur.state !== 'published') throw new ApiError(409, 'story.not_published', 'Only a published story can be retracted');
- if (discussion) discussion.watch(story.id);
+ if (discussion) await discussion.watch(story.id);
const now = store.now();
const now = store.now();
const before = publication.snapshot(cur);
const before = publication.snapshot(cur);
- db.prepare("UPDATE news_stories SET state = 'retracted', retracted_at = ?, updated_at = ? WHERE id = ?").run(now, now, story.id);
- const flag = flagRow(cur, { kind: 'retraction', status: 'published', note: n, revision: cur.published_revision, createdBy: subject, resolvedBy: subject, resolvedAt: now });
- const out = publication.afterChange(before, story.id, { actor: viewer, traceparent, extra: { note: n, flag_id: flag.id } });
+ await db.prepare("UPDATE news_stories SET state = 'retracted', retracted_at = ?, updated_at = ? WHERE id = ?").run(now, now, story.id);
+ const flag = await flagRow(cur, { kind: 'retraction', status: 'published', note: n, revision: cur.published_revision, createdBy: subject, resolvedBy: subject, resolvedAt: now });
+ const out = await publication.afterChange(before, story.id, { actor: viewer, traceparent, extra: { note: n, flag_id: flag.id } });
return { changed: true, story: out.story, flag };
return { changed: true, story: out.story, flag };
});
});
},
},
/** A correction or update note, published with the next publication. */
/** A correction or update note, published with the next publication. */
- addFlag(viewer, story, { kind, note } = {}) {
+ async addFlag(viewer, story, { kind, note } = {}) {
const subject = actingEditor(viewer);
const subject = actingEditor(viewer);
if (!['correction', 'update'].includes(kind)) throw new ApiError(422, 'flag.invalid_kind', 'kind is correction or update (retractions use the retract action)');
if (!['correction', 'update'].includes(kind)) throw new ApiError(422, 'flag.invalid_kind', 'kind is correction or update (retractions use the retract action)');
const n = clean(note, 2001);
const n = clean(note, 2001);
if (!n) throw new ApiError(422, 'flag.no_note', 'A correction or update note is public text: write it');
if (!n) throw new ApiError(422, 'flag.no_note', 'A correction or update note is public text: write it');
if (n.length > 2000) throw new ApiError(422, 'flag.note_too_long', 'A note is at most 2000 characters');
if (n.length > 2000) throw new ApiError(422, 'flag.note_too_long', 'A note is at most 2000 characters');
- return flagRow(story, { kind, note: n, createdBy: subject });
+ return await flagRow(story, { kind, note: n, createdBy: subject });
},
},
/** A person's review of one revision (what lets an AI draft be published and indexed). */
/** A person's review of one revision (what lets an AI draft be published and indexed). */
- review(viewer, story, { revision, decision, note } = {}, { traceparent } = {}) {
+ async review(viewer, story, { revision, decision, note } = {}, { traceparent } = {}) {
if (!viewer || viewer.kind !== 'user' || !viewer.subject) throw new ApiError(403, 'review.person_required', 'Only a signed-in person can review a revision');
if (!viewer || viewer.kind !== 'user' || !viewer.subject) throw new ApiError(403, 'review.person_required', 'Only a signed-in person can review a revision');
actingEditor(viewer);
actingEditor(viewer);
const n = parseInt(revision, 10);
const n = parseInt(revision, 10);
- if (!store.revisions.get(story.id, n)) throw new ApiError(404, 'revision.not_found', `No revision ${revision}`);
- return store.tx(() => {
- const row = store.reviews.record({ entityId: story.id, revision: n, reviewer: viewer.subject, decision, note });
- if (story.state === 'published') publication.syncIndex(q.byId.get(story.id), { traceparent });
+ if (!await store.revisions.get(story.id, n)) throw new ApiError(404, 'revision.not_found', `No revision ${revision}`);
+ return await store.tx(async () => {
+ const row = await store.reviews.record({ entityId: story.id, revision: n, reviewer: viewer.subject, decision, note });
+ if (story.state === 'published') await publication.syncIndex(await q.byId.get(story.id), { traceparent });
return row;
return row;
});
});
},
},
@@ -577,19 +582,19 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -577,19 +582,19 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
* with the same paragraphs and the refreshed source entry. Nothing is published here.
* with the same paragraphs and the refreshed source entry. Nothing is published here.
* Runs inside the ingest transaction.
* Runs inside the ingest transaction.
*/
*/
- onUpstreamChange(item, kind) {
+ async onUpstreamChange(item, kind) {
const affected = [];
const affected = [];
- for (const { story_id: sid } of q.storiesCiting.all(item.id)) {
- const story = q.byId.get(sid);
+ for (const { story_id: sid } of await q.storiesCiting.all(item.id)) {
+ const story = await q.byId.get(sid);
if (!story) continue;
if (!story) continue;
- const open = q.openFlag.get(story.id, item.id, kind);
+ const open = await q.openFlag.get(story.id, item.id, kind);
if (open && kind === 'source_removed') continue;
if (open && kind === 'source_removed') continue;
- const head = store.revisions.head(story.id);
+ const head = await store.revisions.head(story.id);
let pending = null;
let pending = null;
if (head) {
if (head) {
const sources = (head.fields.sources || []).map((s) => (s.id === item.id ? sourceEntry(s.n, item, s.perspective) : s));
const sources = (head.fields.sources || []).map((s) => (s.id === item.id ? sourceEntry(s.n, item, s.perspective) : s));
const fields = { ...head.fields, sources };
const fields = { ...head.fields, sources };
- const out = store.revisions.create({
+ const out = await store.revisions.create({
entityId: story.id, expectedRevision: head.number, content: head.content, fields,
entityId: story.id, expectedRevision: head.number, content: head.content, fields,
meta: { authorship: head.meta.authorship, system: { reason: kind, source_item_id: item.id, sources_item_id: item.sources_item_id, sources_revision: item.sources_revision, based_on: head.number } },
meta: { authorship: head.meta.authorship, system: { reason: kind, source_item_id: item.id, sources_item_id: item.sources_item_id, sources_revision: item.sources_revision, based_on: head.number } },
author: 'svc:news',
author: 'svc:news',
@@ -597,19 +602,19 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -597,19 +602,19 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
});
});
if (out.created) {
if (out.created) {
pending = out.revision.number;
pending = out.revision.number;
- store.citations.carryForward({ entityId: story.id, fromRevision: head.number, toRevision: pending, attachedBy: 'svc:news' });
+ await store.citations.carryForward({ entityId: story.id, fromRevision: head.number, toRevision: pending, attachedBy: 'svc:news' });
}
}
}
}
let flag;
let flag;
if (open) {
if (open) {
- if (pending) db.prepare('UPDATE news_editorial_flags SET pending_revision = ? WHERE id = ?').run(pending, open.id);
- flag = db.prepare('SELECT * FROM news_editorial_flags WHERE id = ?').get(open.id);
+ if (pending) await db.prepare('UPDATE news_editorial_flags SET pending_revision = ? WHERE id = ?').run(pending, open.id);
+ flag = await db.prepare('SELECT * FROM news_editorial_flags WHERE id = ?').get(open.id);
} else {
} else {
- flag = flagRow(story, { kind, sourceItemId: item.id, pendingRevision: pending, revision: story.published_revision, createdBy: 'svc:news', note: kind === 'source_removed' ? (item.removed_reason || null) : null });
+ flag = await flagRow(story, { kind, sourceItemId: item.id, pendingRevision: pending, revision: story.published_revision, createdBy: 'svc:news', note: kind === 'source_removed' ? (item.removed_reason || null) : null });
}
}
- touch(story.id);
- flagEvent(story, flag);
- if (story.state === 'published') publication.syncIndex(q.byId.get(story.id));
+ await touch(story.id);
+ await flagEvent(story, flag);
+ if (story.state === 'published') await publication.syncIndex(await q.byId.get(story.id));
affected.push({ story: story.id, flag: flag.id, pending_revision: pending });
affected.push({ story: story.id, flag: flag.id, pending_revision: pending });
log.warn && log.warn(`[News] ${kind} for ${item.id}: story ${story.id} flagged${pending ? `, pending revision ${pending}` : ''}`);
log.warn && log.warn(`[News] ${kind} for ${item.id}: story ${story.id} flagged${pending ? `, pending revision ${pending}` : ''}`);
}
}
@@ -623,11 +628,11 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -623,11 +628,11 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
async aiDraft(viewer, story, workflow, { traceparent } = {}) {
async aiDraft(viewer, story, workflow, { traceparent } = {}) {
actingEditor(viewer);
actingEditor(viewer);
if (!ai || !ai.enabled) throw new ApiError(503, 'ai.not_configured', 'OpenVibe.AI is not configured on this server; write the story from the source items');
if (!ai || !ai.enabled) throw new ApiError(503, 'ai.not_configured', 'OpenVibe.AI is not configured on this server; write the story from the source items');
- const srcs = sourcesOf(story).filter((s) => s.item.status !== 'removed');
+ const srcs = (await sourcesOf(story)).filter((s) => s.item.status !== 'removed');
if (!srcs.length) throw new ApiError(422, 'story.unsourced', 'Attach source items first');
if (!srcs.length) throw new ApiError(422, 'story.unsourced', 'Attach source items first');
if (workflow === 'news.compare_perspectives' && srcs.length < 2) throw new ApiError(422, 'story.too_few_sources', 'Comparing perspectives needs at least two sources');
if (workflow === 'news.compare_perspectives' && srcs.length < 2) throw new ApiError(422, 'story.too_few_sources', 'Comparing perspectives needs at least two sources');
- const topic = publication.topicOf(story);
- const head = store.revisions.head(story.id);
+ const topic = await publication.topicOf(story);
+ const head = await store.revisions.head(story.id);
let result;
let result;
try {
try {
result = await ai.run(workflow, {
result = await ai.run(workflow, {
@@ -642,9 +647,9 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
@@ -642,9 +647,9 @@ function createStories({ store, config, publication, clusters, outbox, ai = null
const draft = draftFromOutput(workflow, result.output, srcs.map((s) => s.n));
const draft = draftFromOutput(workflow, result.output, srcs.map((s) => s.n));
if (!draft.paragraphs.length) throw new ApiError(422, 'ai.nothing_cited', `The ${workflow} run returned no cited text; no draft was made${draft.gaps.length ? ` (${draft.gaps.slice(0, 3).join('; ')})` : ''}`);
if (!draft.paragraphs.length) throw new ApiError(422, 'ai.nothing_cited', `The ${workflow} run returned no cited text; no draft was made${draft.gaps.length ? ` (${draft.gaps.slice(0, 3).join('; ')})` : ''}`);
const aiViewer = { kind: 'service', service: 'svc:ai', origin: 'ai', subject: null };
const aiViewer = { kind: 'service', service: 'svc:ai', origin: 'ai', subject: null };
- const fresh = q.byId.get(story.id);
- const cur = store.revisions.head(story.id);
- return api.revise(aiViewer, fresh, {
+ const fresh = await q.byId.get(story.id);
+ const cur = await store.revisions.head(story.id);
+ return await api.revise(aiViewer, fresh, {
headline: draft.headline || (cur ? cur.fields.headline : fresh.working_headline),
headline: draft.headline || (cur ? cur.fields.headline : fresh.working_headline),
paragraphs: draft.paragraphs, expectedRevision: cur ? cur.number : 0, gaps: draft.gaps,
paragraphs: draft.paragraphs, expectedRevision: cur ? cur.number : 0, gaps: draft.gaps,
authorship: { workflow: result.workflow, stubProvider: result.stub },
authorship: { workflow: result.workflow, stubProvider: result.stub },
@@ -22,31 +22,31 @@ function createTopics({ store }) {
@@ -22,31 +22,31 @@ function createTopics({ store }) {
insert: db.prepare('INSERT INTO news_topics (id, slug, name, description, status, created_at, updated_at) VALUES (?, ?, ?, ?, \'active\', ?, ?)'),
insert: db.prepare('INSERT INTO news_topics (id, slug, name, description, status, created_at, updated_at) VALUES (?, ?, ?, ?, \'active\', ?, ?)'),
};
};
- function create({ slug, name, description = null }) {
+ async function create({ slug, name, description = null }) {
const n = String(name || '').replace(/\s+/g, ' ').trim().slice(0, 80);
const n = String(name || '').replace(/\s+/g, ' ').trim().slice(0, 80);
if (!n) throw new ApiError(422, 'topic.no_name', 'A topic needs a name');
if (!n) throw new ApiError(422, 'topic.no_name', 'A topic needs a name');
let s;
let s;
try { s = slugify(slug || n).slice(0, 60); } catch { throw new ApiError(422, 'topic.invalid_slug', 'That name has no letters or digits to make a URL from'); }
try { s = slugify(slug || n).slice(0, 60); } catch { throw new ApiError(422, 'topic.invalid_slug', 'That name has no letters or digits to make a URL from'); }
- if (q.bySlug.get(s)) return { topic: q.bySlug.get(s), created: false };
+ if (await q.bySlug.get(s)) return { topic: await q.bySlug.get(s), created: false };
const now = store.now();
const now = store.now();
const id = `top_${ids.ulid(now)}`;
const id = `top_${ids.ulid(now)}`;
- q.insert.run(id, s, n, description ? String(description).replace(/\s+/g, ' ').trim().slice(0, 300) : null, now, now);
- return { topic: q.byId.get(id), created: true };
+ await q.insert.run(id, s, n, description ? String(description).replace(/\s+/g, ' ').trim().slice(0, 300) : null, now, now);
+ return { topic: await q.byId.get(id), created: true };
}
}
return {
return {
create,
create,
- list: () => q.active.all(),
- bySlug: (slug) => q.bySlug.get(String(slug || '')) || null,
- byId: (id) => q.byId.get(String(id || '')) || null,
+ list: async () => await q.active.all(),
+ bySlug: async (slug) => await q.bySlug.get(String(slug || '')) || null,
+ byId: async (id) => await q.byId.get(String(id || '')) || null,
/** Insert the seeded topics that are missing. → { created, existing } */
/** Insert the seeded topics that are missing. → { created, existing } */
- seed(file = SEED_FILE) {
+ async seed(file = SEED_FILE) {
const data = JSON.parse(fs.readFileSync(file, 'utf8'));
const data = JSON.parse(fs.readFileSync(file, 'utf8'));
let created = 0;
let created = 0;
let existing = 0;
let existing = 0;
- store.tx(() => {
+ await store.tx(async () => {
for (const t of data.topics || []) {
for (const t of data.topics || []) {
- const r = create(t);
+ const r = await create(t);
if (r.created) created++; else existing++;
if (r.created) created++; else existing++;
}
}
});
});
@@ -23,7 +23,7 @@
@@ -23,7 +23,7 @@
*/
*/
const { createClient } = require('openvibe-sdk/core');
const { createClient } = require('openvibe-sdk/core');
const { createServiceTokenClient } = require('openvibe-sdk/auth');
const { createServiceTokenClient } = require('openvibe-sdk/auth');
-const { createEventsClient, createOutbox } = require('openvibe-sdk/events');
+const { createEventsClient, createPgOutbox } = require('openvibe-sdk/events');
function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {
function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {
const enabled = Boolean(config.events.url && config.oauth.clientSecret);
const enabled = Boolean(config.events.url && config.oauth.clientSecret);
@@ -39,7 +39,9 @@ function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {
@@ -39,7 +39,9 @@ function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {
}
}
const events = createEventsClient(createClient(clientOpts), { source: 'news' });
const events = createEventsClient(createClient(clientOpts), { source: 'news' });
let lastError = null;
let lastError = null;
- const outbox = createOutbox(db, {
+ // The PostgreSQL outbox: rows are written in the change's own transaction (enqueue(db, …) joins the ambient
+ // transaction); several processes relay one table safely (leases).
+ const outbox = createPgOutbox(db, {
events,
events,
intervalMs: config.events.intervalMs,
intervalMs: config.events.intervalMs,
now,
now,
@@ -49,11 +51,10 @@ function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {
@@ -49,11 +51,10 @@ function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {
lastError = msg;
lastError = msg;
},
},
});
});
- outbox.ensureSchema();
/** Inside the caller's transaction. Returns the complete envelope (with its event_id). */
/** Inside the caller's transaction. Returns the complete envelope (with its event_id). */
- function emit(envelope, { traceparent } = {}) {
- return outbox.enqueue(envelope, { traceparent });
+ async function emit(envelope, { traceparent } = {}) {
+ return await outbox.enqueue(db, envelope, { traceparent });
}
}
return {
return {
@@ -62,8 +63,8 @@ function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {
@@ -62,8 +63,8 @@ function createNewsOutbox({ db, config, fetchImpl, now, log = console }) {
enabled,
enabled,
start() { if (enabled) outbox.start(); },
start() { if (enabled) outbox.start(); },
stop: () => outbox.stop(),
stop: () => outbox.stop(),
- kick() { if (enabled) outbox.kick(); },
- status: () => ({ enabled, pending: outbox.pending(), rejected: outbox.rejected(), last_error: lastError }),
+ async kick() { if (enabled) await outbox.kick(); },
+ status: async () => ({ enabled, pending: await outbox.pending(), rejected: await outbox.rejected(), last_error: lastError }),
};
};
}
}
@@ -25,7 +25,7 @@
@@ -25,7 +25,7 @@
* Never limited: /api/health, /api/ready, /release.json, /metrics, sign-in, the public pages and feeds
* Never limited: /api/health, /api/ready, /release.json, /metrics, sign-in, the public pages and feeds
* (nginx and the per-address limits bound them), and the signed Events deliveries at /internal/events.
* (nginx and the per-address limits bound them), and the signed Events deliveries at /internal/events.
*/
*/
-const { createActorLimiter, defaultActor } = require('openvibe-sdk/limits');
+const { createActorLimiter, createValkeyLimitStore, defaultActor } = require('openvibe-sdk/limits');
const FIRST_PARTY = /^svc:/;
const FIRST_PARTY = /^svc:/;
@@ -83,7 +83,7 @@ const BUDGETS = {
@@ -83,7 +83,7 @@ const BUDGETS = {
* limits(name, own) middleware for one app, plus limits.reads(name) (the defaults on a counted
* limits(name, own) middleware for one app, plus limits.reads(name) (the defaults on a counted
* GET/HEAD) and limits.budget(name) (one of BUDGETS).
* GET/HEAD) and limits.budget(name) (one of BUDGETS).
*/
*/
-function createActorLimits({ config, now = () => Date.now(), registry = null, log = console }) {
+function createActorLimits({ config, now = () => Date.now(), registry = null, log = console, valkey = null }) {
const refused = registry
const refused = registry
? registry.counter({ name: 'news_rate_limited_total', help: 'Requests refused 429 by a per-actor limit, by limit name and window', labelNames: ['limit', 'window'] })
? registry.counter({ name: 'news_rate_limited_total', help: 'Requests refused 429 by a per-actor limit, by limit name and window', labelNames: ['limit', 'window'] })
: null;
: null;
@@ -91,6 +91,8 @@ function createActorLimits({ config, now = () => Date.now(), registry = null, lo
@@ -91,6 +91,8 @@ function createActorLimits({ config, now = () => Date.now(), registry = null, lo
limits: { minute: config.limits.minute, hour: config.limits.hour },
limits: { minute: config.limits.minute, hour: config.limits.hour },
actor,
actor,
now,
now,
+ // Shared across processes on Valkey (ADR-035) when VALKEY_URL is set; in-process otherwise.
+ ...(valkey ? { store: createValkeyLimitStore(valkey) } : {}),
onLimited(e) {
onLimited(e) {
// The actor is a subject id, a principal or an address, never a token.
// The actor is a subject id, a principal or an address, never a token.
log.warn(`[News] limit ${e.name}: ${e.actor} refused, over ${e.limit} per ${e.window}`);
log.warn(`[News] limit ${e.name}: ${e.actor} refused, over ${e.limit} per ${e.window}`);
@@ -87,181 +87,181 @@ function createApi(ctx) {
@@ -87,181 +87,181 @@ function createApi(ctx) {
/** Reads for editors: a service passed its capability guard; a browser must be an editor. */
/** Reads for editors: a service passed its capability guard; a browser must be an editor. */
const editorRead = (req) => { if (req.viewer.kind !== 'service') access.requireEditor(config, req.viewer); };
const editorRead = (req) => { if (req.viewer.kind !== 'service') access.requireEditor(config, req.viewer); };
- function clusterDto(c, { full = false } = {}) {
+ async function clusterDto(c, { full = false } = {}) {
const out = {
const out = {
id: c.id, label: c.label, status: c.status, merged_into: c.merged_into, split_from: c.split_from,
id: c.id, label: c.label, status: c.status, merged_into: c.merged_into, split_from: c.split_from,
window: { start: iso(c.window_start), end: iso(c.window_end) }, created_by: c.created_by,
window: { start: iso(c.window_start), end: iso(c.window_end) }, created_by: c.created_by,
key_terms: Object.entries(parse(c.terms, {})).sort(([a, x], [b, y]) => (y - x) || (a < b ? -1 : 1)).slice(0, 15).map(([t, n]) => ({ term: t, items: n })),
key_terms: Object.entries(parse(c.terms, {})).sort(([a, x], [b, y]) => (y - x) || (a < b ? -1 : 1)).slice(0, 15).map(([t, n]) => ({ term: t, items: n })),
key_entities: Object.entries(parse(c.entities, {})).sort(([a, x], [b, y]) => (y - x) || (a < b ? -1 : 1)).slice(0, 10).map(([t, n]) => ({ entity: t, items: n })),
key_entities: Object.entries(parse(c.entities, {})).sort(([a, x], [b, y]) => (y - x) || (a < b ? -1 : 1)).slice(0, 10).map(([t, n]) => ({ entity: t, items: n })),
- item_count: clusters.members(c.id).length,
+ item_count: (await clusters.members(c.id)).length,
};
};
- if (full) Object.assign(out, { items: clusters.members(c.id).map(itemDto), audit: clusters.audit(c.id), stories: stories.byCluster(c.id).map((s) => ({ id: s.id, slug: s.slug, state: s.state })) });
+ if (full) Object.assign(out, { items: (await clusters.members(c.id)).map(itemDto), audit: await clusters.audit(c.id), stories: (await stories.byCluster(c.id)).map((s) => ({ id: s.id, slug: s.slug, state: s.state })) });
return out;
return out;
}
}
- function storyDto(story) {
- const head = stories.head(story);
- const topic = publication.topicOf(story);
+ async function storyDto(story) {
+ const head = await stories.head(story);
+ const topic = await publication.topicOf(story);
return {
return {
id: story.id, slug: story.slug, url: publication.storyUrl(story), state: story.state, cluster_id: story.cluster_id,
id: story.id, slug: story.slug, url: publication.storyUrl(story), state: story.state, cluster_id: story.cluster_id,
topic: topic ? topic.slug : null, working_headline: story.working_headline, noindex: Boolean(story.noindex),
topic: topic ? topic.slug : null, working_headline: story.working_headline, noindex: Boolean(story.noindex),
published_revision: story.published_revision, first_published_at: iso(story.first_published_at), published_at: iso(story.published_at), retracted_at: iso(story.retracted_at),
published_revision: story.published_revision, first_published_at: iso(story.first_published_at), published_at: iso(story.published_at), retracted_at: iso(story.retracted_at),
head: head ? { revision: head.number, headline: head.fields.headline, paragraphs: head.fields.paragraphs, authorship: head.meta.authorship || null, system: head.meta.system || null, gaps: head.meta.gaps || null, author: head.author, created_at: head.createdAt } : null,
head: head ? { revision: head.number, headline: head.fields.headline, paragraphs: head.fields.paragraphs, authorship: head.meta.authorship || null, system: head.meta.system || null, gaps: head.meta.gaps || null, author: head.author, created_at: head.createdAt } : null,
- publishable: head ? stories.problems(story, head) : [{ code: 'story.no_text', detail: 'Nothing has been written yet' }],
- indexability: story.published_revision ? (() => { const d = publication.decide(story, stories.revision(story, story.published_revision)); return { indexable: d.indexable, robots: d.robots, reasons: d.reasons }; })() : null,
- sources: stories.sources(story, { includeDetached: true }).map((s) => ({ n: s.n, perspective_id: s.perspective_id, detached_at: iso(s.detached_at), item: itemDto(s.item) })),
- timeline: stories.timeline(story).map((t) => ({ id: t.id, occurred_on: t.occurred_on, text: t.text, source_item_id: t.source_item_id })),
- perspectives: stories.perspectives(story).map((p) => ({ id: p.id, label: p.label, description: p.description })),
- flags: stories.flags(story).map((f) => ({ id: f.id, kind: f.kind, status: f.status, note: f.note, source_item_id: f.source_item_id, pending_revision: f.pending_revision, revision: f.revision, created_by: f.created_by, created_at: iso(f.created_at), resolved_at: iso(f.resolved_at) })),
+ publishable: head ? await stories.problems(story, head) : [{ code: 'story.no_text', detail: 'Nothing has been written yet' }],
+ indexability: story.published_revision ? (async () => { const d = await publication.decide(story, await stories.revision(story, story.published_revision)); return { indexable: d.indexable, robots: d.robots, reasons: d.reasons }; })() : null,
+ sources: (await stories.sources(story, { includeDetached: true })).map((s) => ({ n: s.n, perspective_id: s.perspective_id, detached_at: iso(s.detached_at), item: itemDto(s.item) })),
+ timeline: (await stories.timeline(story)).map((t) => ({ id: t.id, occurred_on: t.occurred_on, text: t.text, source_item_id: t.source_item_id })),
+ perspectives: (await stories.perspectives(story)).map((p) => ({ id: p.id, label: p.label, description: p.description })),
+ flags: (await stories.flags(story)).map((f) => ({ id: f.id, kind: f.kind, status: f.status, note: f.note, source_item_id: f.source_item_id, pending_revision: f.pending_revision, revision: f.revision, created_by: f.created_by, created_at: iso(f.created_at), resolved_at: iso(f.resolved_at) })),
};
};
}
}
- const mustStory = (req) => stories.mustGet(req.params.id);
+ const mustStory = async (req) => await stories.mustGet(req.params.id);
// ── Topics ──────────────────────────────────────────────
// ── Topics ──────────────────────────────────────────────
- router.get('/topics', run(() => ({ topics: topics.list().map((t) => ({ id: t.id, slug: t.slug, name: t.name, description: t.description, url: publication.abs(publication.topicPath(t)) })) })));
+ router.get('/topics', run(async () => ({ topics: (await topics.list()).map((t) => ({ id: t.id, slug: t.slug, name: t.name, description: t.description, url: publication.abs(publication.topicPath(t)) })) })));
- router.post('/topics', guard('news.topic.manage'), B('news.topic.manage'), jsonBody, run((req) => {
+ router.post('/topics', guard('news.topic.manage'), B('news.topic.manage'), jsonBody, run(async (req) => {
access.requireEditor(config, req.viewer);
access.requireEditor(config, req.viewer);
const b = req.body || {};
const b = req.body || {};
- const r = topics.create({ name: b.name, slug: b.slug, description: b.description });
+ const r = await topics.create({ name: b.name, slug: b.slug, description: b.description });
return { topic: r.topic, created: r.created };
return { topic: r.topic, created: r.created };
}, (out) => (out.created ? 201 : 200)));
}, (out) => (out.created ? 201 : 200)));
// ── Stories ─────────────────────────────────────────────
// ── Stories ─────────────────────────────────────────────
router.get('/stories', run(async (req) => {
router.get('/stories', run(async (req) => {
- const topic = req.query.topic ? topics.bySlug(req.query.topic) : null;
+ const topic = req.query.topic ? await topics.bySlug(req.query.topic) : null;
if (req.query.topic && !topic) throw new ApiError(404, 'topic.not_found', 'No such topic');
if (req.query.topic && !topic) throw new ApiError(404, 'topic.not_found', 'No such topic');
const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 20, 1), 100);
const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 20, 1), 100);
const offset = Math.max(parseInt(req.query.offset, 10) || 0, 0);
const offset = Math.max(parseInt(req.query.offset, 10) || 0, 0);
- const { total, stories: rows } = stories.listPublished({ topicId: topic ? topic.id : null, limit, offset });
+ const { total, stories: rows } = await stories.listPublished({ topicId: topic ? topic.id : null, limit, offset });
const items = await reading.listItems(rows);
const items = await reading.listItems(rows);
return { total, stories: items.map((i) => ({ id: i.story.id, url: publication.abs(i.url), headline: i.headline, state: i.story.state, topic: i.topic ? i.topic.name : null, sources: i.sourceCount, first_published_at: iso(i.published_at), updated_at: i.updated_at })) };
return { total, stories: items.map((i) => ({ id: i.story.id, url: publication.abs(i.url), headline: i.headline, state: i.story.state, topic: i.topic ? i.topic.name : null, sources: i.sourceCount, first_published_at: iso(i.published_at), updated_at: i.updated_at })) };
}));
}));
router.get('/stories/:id', run(async (req) => {
router.get('/stories/:id', run(async (req) => {
- const story = mustStory(req);
+ const story = await mustStory(req);
const privileged = req.viewer.kind === 'service'
const privileged = req.viewer.kind === 'service'
? contracts.capabilities.grants(req.viewer.claims && req.viewer.claims.cap, 'news.story.read')
? contracts.capabilities.grants(req.viewer.claims && req.viewer.claims.cap, 'news.story.read')
: access.isEditor(config, req.viewer);
: access.isEditor(config, req.viewer);
if (privileged) {
if (privileged) {
const m = story.published_revision ? await reading.storyModel(story) : null;
const m = story.published_revision ? await reading.storyModel(story) : null;
- return { story: storyDto(story), published: m ? reading.storyJson(m) : null };
+ return { story: await storyDto(story), published: m ? reading.storyJson(m) : null };
}
}
if (!story.published_revision || (story.state !== 'published' && story.state !== 'retracted')) throw new ApiError(404, 'story.not_found', 'No such story');
if (!story.published_revision || (story.state !== 'published' && story.state !== 'retracted')) throw new ApiError(404, 'story.not_found', 'No such story');
return { published: reading.storyJson(await reading.storyModel(story)) };
return { published: reading.storyJson(await reading.storyModel(story)) };
}));
}));
- router.post('/stories', guard('news.story.create'), B('news.story.create'), jsonBody, run((req) => {
+ router.post('/stories', guard('news.story.create'), B('news.story.create'), jsonBody, run(async (req) => {
const b = req.body || {};
const b = req.body || {};
- const out = stories.create(req.viewer, { cluster: b.cluster || b.cluster_id, headline: b.headline, topic: b.topic, slug: b.slug, body: b.body, paragraphs: b.paragraphs }, tp(req));
- return { story: storyDto(out.story), revision: out.revision ? out.revision.number : null };
+ const out = await stories.create(req.viewer, { cluster: b.cluster || b.cluster_id, headline: b.headline, topic: b.topic, slug: b.slug, body: b.body, paragraphs: b.paragraphs }, tp(req));
+ return { story: await storyDto(out.story), revision: out.revision ? out.revision.number : null };
}, 201));
}, 201));
- router.get('/stories/:id/revisions', guard('news.story.read'), run((req) => {
+ router.get('/stories/:id/revisions', guard('news.story.read'), run(async (req) => {
editorRead(req);
editorRead(req);
- const story = mustStory(req);
- return { revisions: stories.revisions(story, { limit: req.query.limit, before: req.query.before }).map((r) => ({ revision: r.number, kind: r.kind, author: r.author, message: r.message, created_at: r.createdAt, headline: r.fields.headline, authorship: r.meta.authorship || null, system: r.meta.system || null })) };
+ const story = await mustStory(req);
+ return { revisions: (await stories.revisions(story, { limit: req.query.limit, before: req.query.before })).map((r) => ({ revision: r.number, kind: r.kind, author: r.author, message: r.message, created_at: r.createdAt, headline: r.fields.headline, authorship: r.meta.authorship || null, system: r.meta.system || null })) };
}));
}));
- router.get('/stories/:id/revisions/:n', guard('news.story.read'), run((req) => {
+ router.get('/stories/:id/revisions/:n', guard('news.story.read'), run(async (req) => {
editorRead(req);
editorRead(req);
- const story = mustStory(req);
- const rev = stories.revision(story, parseInt(req.params.n, 10));
+ const story = await mustStory(req);
+ const rev = await stories.revision(story, parseInt(req.params.n, 10));
if (!rev) throw new ApiError(404, 'revision.not_found', 'No such revision');
if (!rev) throw new ApiError(404, 'revision.not_found', 'No such revision');
- return { revision: rev, citations: ctx.store.citations.forRevision(story.id, rev.number), problems: stories.problems(story, rev) };
+ return { revision: rev, citations: await ctx.store.citations.forRevision(story.id, rev.number), problems: await stories.problems(story, rev) };
}));
}));
- router.get('/stories/:id/diff', guard('news.story.read'), limits.readBudget('news.story.diff'), run((req) => {
+ router.get('/stories/:id/diff', guard('news.story.read'), limits.readBudget('news.story.diff'), run(async (req) => {
editorRead(req);
editorRead(req);
- return { diff: stories.diff(mustStory(req), parseInt(req.query.from, 10), parseInt(req.query.to, 10), req.query.mode) };
+ return { diff: await stories.diff(await mustStory(req), parseInt(req.query.from, 10), parseInt(req.query.to, 10), req.query.mode) };
}));
}));
// guards: the capability guard, then the route's per-actor budget (both before the body is read).
// guards: the capability guard, then the route's per-actor budget (both before the body is read).
- const write = (method, path, guards, fn, status = 200) => router[method](path, guards, jsonBody, run(async (req) => fn(req, mustStory(req), req.body || {}), status));
+ const write = (method, path, guards, fn, status = 200) => router[method](path, guards, jsonBody, run(async (req) => fn(req, await mustStory(req), req.body || {}), status));
- write('post', '/stories/:id/revisions', [guard('news.story.revise'), B('news.story.revise')], (req, story, b) => {
- const out = stories.revise(req.viewer, story, {
+ write('post', '/stories/:id/revisions', [guard('news.story.revise'), B('news.story.revise')], async (req, story, b) => {
+ const out = await stories.revise(req.viewer, story, {
headline: b.headline, body: b.body, paragraphs: b.paragraphs, expectedRevision: b.expected_revision ?? b.expectedRevision,
headline: b.headline, body: b.body, paragraphs: b.paragraphs, expectedRevision: b.expected_revision ?? b.expectedRevision,
topic: b.topic, noindex: b.noindex, message: b.message, authorship: b.authorship, gaps: b.gaps,
topic: b.topic, noindex: b.noindex, message: b.message, authorship: b.authorship, gaps: b.gaps,
}, tp(req));
}, tp(req));
- return { revision: out.revision.number, created: out.created, story: storyDto(out.story) };
+ return { revision: out.revision.number, created: out.created, story: await storyDto(out.story) };
}, 201);
}, 201);
write('post', '/stories/:id/ai-drafts', [guard('news.story.revise'), B('news.story.ai_draft')], async (req, story, b) => {
write('post', '/stories/:id/ai-drafts', [guard('news.story.revise'), B('news.story.ai_draft')], async (req, story, b) => {
const out = await stories.aiDraft(req.viewer, story, String(b.workflow || 'news.summarize_story'), tp(req));
const out = await stories.aiDraft(req.viewer, story, String(b.workflow || 'news.summarize_story'), tp(req));
- return { revision: out.revision.number, created: out.created, story: storyDto(out.story) };
+ return { revision: out.revision.number, created: out.created, story: await storyDto(out.story) };
}, 201);
}, 201);
- write('post', '/stories/:id/publish', [guard('news.story.publish'), B('news.story.publish')], (req, story, b) => {
- const out = stories.publish(req.viewer, story, { revision: b.revision, correction: b.correction }, tp(req));
- return { changed: out.changed, resolved_flags: out.resolved || [], story: storyDto(out.story) };
+ write('post', '/stories/:id/publish', [guard('news.story.publish'), B('news.story.publish')], async (req, story, b) => {
+ const out = await stories.publish(req.viewer, story, { revision: b.revision, correction: b.correction }, tp(req));
+ return { changed: out.changed, resolved_flags: out.resolved || [], story: await storyDto(out.story) };
});
});
- write('post', '/stories/:id/unpublish', [guard('news.story.publish'), B('news.story.publish')], (req, story) => {
- const out = stories.unpublish(req.viewer, story, tp(req));
- return { changed: out.changed, story: storyDto(out.story) };
+ write('post', '/stories/:id/unpublish', [guard('news.story.publish'), B('news.story.publish')], async (req, story) => {
+ const out = await stories.unpublish(req.viewer, story, tp(req));
+ return { changed: out.changed, story: await storyDto(out.story) };
});
});
- write('post', '/stories/:id/retract', [guard('news.story.retract'), B('news.story.publish')], (req, story, b) => {
- const out = stories.retract(req.viewer, story, { note: b.note }, tp(req));
- return { changed: out.changed, story: storyDto(out.story) };
+ write('post', '/stories/:id/retract', [guard('news.story.retract'), B('news.story.publish')], async (req, story, b) => {
+ const out = await stories.retract(req.viewer, story, { note: b.note }, tp(req));
+ return { changed: out.changed, story: await storyDto(out.story) };
});
});
- router.post('/stories/:id/reviews', B('news.story.publish'), jsonBody, run((req) => {
- const story = mustStory(req);
+ router.post('/stories/:id/reviews', B('news.story.publish'), jsonBody, run(async (req) => {
+ const story = await mustStory(req);
const b = req.body || {};
const b = req.body || {};
- return { review: stories.review(req.viewer, story, { revision: b.revision, decision: b.decision, note: b.note }, tp(req)) };
+ return { review: await stories.review(req.viewer, story, { revision: b.revision, decision: b.decision, note: b.note }, tp(req)) };
}, 201));
}, 201));
- write('post', '/stories/:id/flags', [guard('news.story.revise'), B('news.story.annotate')], (req, story, b) => ({ flag: stories.addFlag(req.viewer, story, { kind: b.kind, note: b.note }) }), 201);
- write('post', '/stories/:id/sources', [guard('news.source.attach'), B('news.source.attach')], (req, story, b) => {
- const out = stories.attach(req.viewer, story, { item: b.item || b.item_id });
+ write('post', '/stories/:id/flags', [guard('news.story.revise'), B('news.story.annotate')], async (req, story, b) => ({ flag: await stories.addFlag(req.viewer, story, { kind: b.kind, note: b.note }) }), 201);
+ write('post', '/stories/:id/sources', [guard('news.source.attach'), B('news.source.attach')], async (req, story, b) => {
+ const out = await stories.attach(req.viewer, story, { item: b.item || b.item_id });
return { n: out.n, created: out.created, item: itemDto(out.item) };
return { n: out.n, created: out.created, item: itemDto(out.item) };
}, 201);
}, 201);
- write('delete', '/stories/:id/sources/:item', [guard('news.source.attach'), B('news.source.attach')], (req, story) => stories.detach(req.viewer, story, req.params.item));
- write('post', '/stories/:id/perspectives', [guard('news.perspective.update'), B('news.story.annotate')], (req, story, b) => ({ perspective: stories.addPerspective(req.viewer, story, { label: b.label, description: b.description }) }), 201);
- write('delete', '/stories/:id/perspectives/:pid', [guard('news.perspective.update'), B('news.story.annotate')], (req, story) => stories.removePerspective(req.viewer, story, req.params.pid));
- write('put', '/stories/:id/sources/:item/perspective', [guard('news.perspective.update'), B('news.story.annotate')], (req, story, b) => stories.assignPerspective(req.viewer, story, req.params.item, b.perspective || null));
- write('post', '/stories/:id/timeline', [guard('news.timeline.update'), B('news.story.annotate')], (req, story, b) => ({ entry: stories.addTimeline(req.viewer, story, { occurredOn: b.occurred_on ?? b.occurredOn, text: b.text, source: b.source }) }), 201);
- write('delete', '/stories/:id/timeline/:eid', [guard('news.timeline.update'), B('news.story.annotate')], (req, story) => stories.removeTimeline(req.viewer, story, req.params.eid));
+ write('delete', '/stories/:id/sources/:item', [guard('news.source.attach'), B('news.source.attach')], async (req, story) => await stories.detach(req.viewer, story, req.params.item));
+ write('post', '/stories/:id/perspectives', [guard('news.perspective.update'), B('news.story.annotate')], async (req, story, b) => ({ perspective: await stories.addPerspective(req.viewer, story, { label: b.label, description: b.description }) }), 201);
+ write('delete', '/stories/:id/perspectives/:pid', [guard('news.perspective.update'), B('news.story.annotate')], async (req, story) => await stories.removePerspective(req.viewer, story, req.params.pid));
+ write('put', '/stories/:id/sources/:item/perspective', [guard('news.perspective.update'), B('news.story.annotate')], async (req, story, b) => await stories.assignPerspective(req.viewer, story, req.params.item, b.perspective || null));
+ write('post', '/stories/:id/timeline', [guard('news.timeline.update'), B('news.story.annotate')], async (req, story, b) => ({ entry: await stories.addTimeline(req.viewer, story, { occurredOn: b.occurred_on ?? b.occurredOn, text: b.text, source: b.source }) }), 201);
+ write('delete', '/stories/:id/timeline/:eid', [guard('news.timeline.update'), B('news.story.annotate')], async (req, story) => await stories.removeTimeline(req.viewer, story, req.params.eid));
// ── Clusters, items, ingestion ──────────────────────────
// ── Clusters, items, ingestion ──────────────────────────
- router.get('/clusters', guard('news.cluster.read'), run((req) => {
+ router.get('/clusters', guard('news.cluster.read'), run(async (req) => {
editorRead(req);
editorRead(req);
const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 50, 1), 200);
const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 50, 1), 200);
- return { clusters: clusters.recent({ limit }).map((c) => clusterDto(c)) };
+ return { clusters: await Promise.all((await clusters.recent({ limit })).map((c) => clusterDto(c))) };
}));
}));
- router.get('/clusters/:id', guard('news.cluster.read'), run((req) => {
+ router.get('/clusters/:id', guard('news.cluster.read'), run(async (req) => {
editorRead(req);
editorRead(req);
- const c = clusters.get(req.params.id);
+ const c = await clusters.get(req.params.id);
if (!c) throw new ApiError(404, 'cluster.not_found', 'No such cluster');
if (!c) throw new ApiError(404, 'cluster.not_found', 'No such cluster');
- return { cluster: clusterDto(c, { full: true }) };
+ return { cluster: await clusterDto(c, { full: true }) };
}));
}));
- router.post('/clusters/audit/:auditId/reverse', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run((req) => {
+ router.post('/clusters/audit/:auditId/reverse', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run(async (req) => {
access.requireEditor(config, req.viewer);
access.requireEditor(config, req.viewer);
- const out = clusters.reverse(req.viewer, req.params.auditId, { reason: (req.body || {}).reason });
- return { audit: out.audit, clusters: out.clusters.map((c) => clusterDto(c)) };
+ const out = await clusters.reverse(req.viewer, req.params.auditId, { reason: (req.body || {}).reason });
+ return { audit: out.audit, clusters: await Promise.all(out.clusters.map((c) => clusterDto(c))) };
}));
}));
- router.post('/clusters/:id/merge', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run((req) => {
+ router.post('/clusters/:id/merge', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run(async (req) => {
access.requireEditor(config, req.viewer);
access.requireEditor(config, req.viewer);
const b = req.body || {};
const b = req.body || {};
- const out = clusters.merge(req.viewer, req.params.id, b.other, { reason: b.reason });
- return { cluster: clusterDto(out.cluster, { full: true }), audit: out.audit };
+ const out = await clusters.merge(req.viewer, req.params.id, b.other, { reason: b.reason });
+ return { cluster: await clusterDto(out.cluster, { full: true }), audit: out.audit };
}));
}));
- router.post('/clusters/:id/split', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run((req) => {
+ router.post('/clusters/:id/split', guard('news.cluster.manage'), B('news.cluster.manage'), jsonBody, run(async (req) => {
access.requireEditor(config, req.viewer);
access.requireEditor(config, req.viewer);
const b = req.body || {};
const b = req.body || {};
- const out = clusters.split(req.viewer, req.params.id, b.items, { reason: b.reason });
- return { cluster: clusterDto(out.cluster), created: clusterDto(out.created, { full: true }), audit: out.audit };
+ const out = await clusters.split(req.viewer, req.params.id, b.items, { reason: b.reason });
+ return { cluster: await clusterDto(out.cluster), created: await clusterDto(out.created, { full: true }), audit: out.audit };
}, 201));
}, 201));
- router.get('/source-items/:id', guard('news.cluster.read'), run((req) => {
+ router.get('/source-items/:id', guard('news.cluster.read'), run(async (req) => {
editorRead(req);
editorRead(req);
- const it = ingest.get(req.params.id) || ingest.bySourcesId(req.params.id);
+ const it = await ingest.get(req.params.id) || await ingest.bySourcesId(req.params.id);
if (!it) throw new ApiError(404, 'source_item.not_found', 'No such source item');
if (!it) throw new ApiError(404, 'source_item.not_found', 'No such source item');
return { item: itemDto(it) };
return { item: itemDto(it) };
}));
}));
- router.get('/ingest', guard('news.cluster.read'), run((req) => {
+ router.get('/ingest', guard('news.cluster.read'), run(async (req) => {
editorRead(req);
editorRead(req);
return {
return {
- cursor: ctx.store.getState('sources_cursor', 0),
- runs: ingest.runs({ limit: 50 }).map((r) => ({ ...r, counts: parse(r.counts, null), at: iso(r.at) })),
- sources: ingest.sourceStatus().map((s) => ({ ...s, stale: s.stale == null ? null : Boolean(s.stale), refreshed_at: iso(s.refreshed_at) })),
+ cursor: await ctx.store.getState('sources_cursor', 0),
+ runs: (await ingest.runs({ limit: 50 })).map((r) => ({ ...r, counts: parse(r.counts, null), at: iso(r.at) })),
+ sources: (await ingest.sourceStatus()).map((s) => ({ ...s, stale: s.stale == null ? null : Boolean(s.stale), refreshed_at: iso(s.refreshed_at) })),
};
};
}));
}));
@@ -22,12 +22,13 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
@@ -22,12 +22,13 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
const { db } = store;
const { db } = store;
const abs = (p) => seo.canonicalUrl(config.baseUrl, p);
const abs = (p) => seo.canonicalUrl(config.baseUrl, p);
- function storyEntries() {
- return db.prepare("SELECT * FROM news_stories WHERE state IN ('published','retracted') AND published_revision IS NOT NULL ORDER BY published_at DESC LIMIT 50000").all()
- .map((story) => {
- const rev = store.revisions.get(story.id, story.published_revision);
- return { story, rev, decision: publication.decide(story, rev) };
- });
+ async function storyEntries() {
+ // Revisions in one query, whatever the number of stories (no N+1).
+ const rows = await db.prepare("SELECT * FROM news_stories WHERE state IN ('published','retracted') AND published_revision IS NOT NULL ORDER BY published_at DESC, id LIMIT 50000").all();
+ const revs = await store.revisions.getMany(rows.map((s) => ({ entityId: s.id, revision: s.published_revision })));
+ const out = [];
+ for (const [i, story] of rows.entries()) out.push({ story, rev: revs[i], decision: await publication.decide(story, revs[i]) });
+ return out;
}
}
const xml = (res, body) => res.type('application/xml').set('Cache-Control', 'public, max-age=300').send(body);
const xml = (res, body) => res.type('application/xml').set('Cache-Control', 'public, max-age=300').send(body);
@@ -64,8 +65,8 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
@@ -64,8 +65,8 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
}));
}));
});
});
- router.get('/sitemap.xml', (_req, res) => {
- const entries = storyEntries().filter((e) => e.decision.indexable);
+ router.get('/sitemap.xml', async (_req, res) => {
+ const entries = (await storyEntries()).filter((e) => e.decision.indexable);
const newest = entries.length ? entries.map((e) => e.rev.createdAt).sort().pop() : null;
const newest = entries.length ? entries.map((e) => e.rev.createdAt).sort().pop() : null;
xml(res, seo.sitemapIndex([
xml(res, seo.sitemapIndex([
{ loc: abs('/sitemaps/stories.xml'), ...(newest ? { lastmod: newest } : {}) },
{ loc: abs('/sitemaps/stories.xml'), ...(newest ? { lastmod: newest } : {}) },
@@ -73,20 +74,20 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
@@ -73,20 +74,20 @@ function createDiscoveryRoutes({ config, store, publication, topics }) {
]));
]));
});
});
- router.get('/sitemaps/stories.xml', (_req, res) => {
- xml(res, seo.sitemap(storyEntries().map((e) => ({ loc: publication.storyUrl(e.story), lastmod: e.rev.createdAt, decision: e.decision }))).files[0]);
+ router.get('/sitemaps/stories.xml', async (_req, res) => {
+ xml(res, seo.sitemap((await storyEntries()).map((e) => ({ loc: publication.storyUrl(e.story), lastmod: e.rev.createdAt, decision: e.decision }))).files[0]);
});
});
- router.get('/sitemaps/topics.xml', (_req, res) => {
+ router.get('/sitemaps/topics.xml', async (_req, res) => {
const byTopic = new Map();
const byTopic = new Map();
- for (const e of storyEntries()) {
+ for (const e of await storyEntries()) {
if (!e.decision.indexable || !e.story.topic_id) continue;
if (!e.decision.indexable || !e.story.topic_id) continue;
const t = e.rev.createdAt;
const t = e.rev.createdAt;
if (!byTopic.has(e.story.topic_id) || byTopic.get(e.story.topic_id) < t) byTopic.set(e.story.topic_id, t);
if (!byTopic.has(e.story.topic_id) || byTopic.get(e.story.topic_id) < t) byTopic.set(e.story.topic_id, t);
}
}
const entries = [];
const entries = [];
for (const [id, lastmod] of byTopic) {
for (const [id, lastmod] of byTopic) {
- const topic = topics.byId(id);
+ const topic = await topics.byId(id);
if (!topic || topic.status !== 'active') continue;
if (!topic || topic.status !== 'active') continue;
const path = publication.topicPath(topic);
const path = publication.topicPath(topic);
const decision = seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: abs(path), wordCount: 0 }, { policy: { minWords: 0 }, now: store.now() });
const decision = seo.evaluate({ state: 'published', visibility: 'public', canonicalUrl: abs(path), wordCount: 0 }, { policy: { minWords: 0 }, now: store.now() });
@@ -77,12 +77,12 @@ function createEditorRoutes(ctx) {
@@ -77,12 +77,12 @@ function createEditorRoutes(ctx) {
// ── Dashboard ───────────────────────────────────────────
// ── Dashboard ───────────────────────────────────────────
router.get('/edit', wrap(async (req, res) => {
router.get('/edit', wrap(async (req, res) => {
- const flags = store.db.prepare(`SELECT f.*, s.working_headline FROM news_editorial_flags f JOIN news_stories s ON s.id = f.story_id
+ const flags = await store.db.prepare(`SELECT f.*, s.working_headline FROM news_editorial_flags f JOIN news_stories s ON s.id = f.story_id
WHERE f.status = 'open' ORDER BY f.created_at DESC LIMIT 100`).all();
WHERE f.status = 'open' ORDER BY f.created_at DESC LIMIT 100`).all();
page(req, res, 'Editor desk', editor.dashboard({
page(req, res, 'Editor desk', editor.dashboard({
- clusters: clusters.recent({ limit: 40 }).map((c) => ({ ...c, count: clusters.members(c.id).length })),
- stories: stories.listAll({ limit: 100 }), flags, runs: ingest.runs({ limit: 20 }), sources: ingest.sourceStatus(),
- cursor: store.getState('sources_cursor', 0), csrf: csrf(req), message: flashOf(req), aiEnabled: Boolean(ai && ai.enabled),
+ clusters: await Promise.all((await clusters.recent({ limit: 40 })).map(async (c) => ({ ...c, count: (await clusters.members(c.id)).length }))),
+ stories: await stories.listAll({ limit: 100 }), flags, runs: await ingest.runs({ limit: 20 }), sources: await ingest.sourceStatus(),
+ cursor: await store.getState('sources_cursor', 0), csrf: csrf(req), message: flashOf(req), aiEnabled: Boolean(ai && ai.enabled),
pullOn: Boolean(config.worker.enabled && config.sources.pullIntervalMs),
pullOn: Boolean(config.worker.enabled && config.sources.pullIntervalMs),
}));
}));
}));
}));
@@ -92,114 +92,114 @@ function createEditorRoutes(ctx) {
@@ -92,114 +92,114 @@ function createEditorRoutes(ctx) {
done(res, '/edit', r.ok ? `Pulled: ${JSON.stringify(r.counts)}` : `Pull failed (${r.error}): ${r.detail}`);
done(res, '/edit', r.ok ? `Pulled: ${JSON.stringify(r.counts)}` : `Pull failed (${r.error}): ${r.detail}`);
}));
}));
- router.get('/edit/topics', wrap(async (req, res) => page(req, res, 'Topics', editor.topicsPage({ topics: topics.list(), csrf: csrf(req), message: flashOf(req) }))));
+ router.get('/edit/topics', wrap(async (req, res) => page(req, res, 'Topics', editor.topicsPage({ topics: await topics.list(), csrf: csrf(req), message: flashOf(req) }))));
router.post('/edit/topics', B('news.topic.manage'), wrap(async (req, res) => {
router.post('/edit/topics', B('news.topic.manage'), wrap(async (req, res) => {
const b = input(req.body);
const b = input(req.body);
- const r = topics.create({ name: b.name, description: b.description });
+ const r = await topics.create({ name: b.name, description: b.description });
done(res, '/edit/topics', r.created ? `Added ${r.topic.name}` : `${r.topic.name} already exists`);
done(res, '/edit/topics', r.created ? `Added ${r.topic.name}` : `${r.topic.name} already exists`);
}));
}));
// ── Clusters ────────────────────────────────────────────
// ── Clusters ────────────────────────────────────────────
- function mustCluster(req, res) {
- const c = clusters.get(req.params.id);
+ async function mustCluster(req, res) {
+ const c = await clusters.get(req.params.id);
if (!c) { publicRoutes.notFound(req, res); return null; }
if (!c) { publicRoutes.notFound(req, res); return null; }
return c;
return c;
}
}
router.get('/clusters/:id', wrap(async (req, res) => {
router.get('/clusters/:id', wrap(async (req, res) => {
- const c = mustCluster(req, res);
+ const c = await mustCluster(req, res);
if (!c) return;
if (!c) return;
page(req, res, `Cluster: ${c.label}`, editor.clusterPage({
page(req, res, `Cluster: ${c.label}`, editor.clusterPage({
- cluster: c, items: clusters.members(c.id), audit: clusters.audit(c.id),
- others: clusters.recent({ limit: 100 }).filter((o) => o.id !== c.id), storiesOf: stories.byCluster(c.id),
- topics: topics.list(), csrf: csrf(req), message: flashOf(req),
+ cluster: c, items: await clusters.members(c.id), audit: await clusters.audit(c.id),
+ others: (await clusters.recent({ limit: 100 })).filter((o) => o.id !== c.id), storiesOf: await stories.byCluster(c.id),
+ topics: await topics.list(), csrf: csrf(req), message: flashOf(req),
}));
}));
}));
}));
router.post('/clusters/:id/merge', B('news.cluster.manage'), wrap(async (req, res) => {
router.post('/clusters/:id/merge', B('news.cluster.manage'), wrap(async (req, res) => {
const b = input(req.body);
const b = input(req.body);
- clusters.merge(req.viewer, req.params.id, b.other, { reason: b.reason });
+ await clusters.merge(req.viewer, req.params.id, b.other, { reason: b.reason });
done(res, `/clusters/${req.params.id}`, `Merged ${b.other} into this cluster`);
done(res, `/clusters/${req.params.id}`, `Merged ${b.other} into this cluster`);
}));
}));
router.post('/clusters/:id/split', B('news.cluster.manage'), wrap(async (req, res) => {
router.post('/clusters/:id/split', B('news.cluster.manage'), wrap(async (req, res) => {
const items = [].concat((req.body && req.body.items) || []);
const items = [].concat((req.body && req.body.items) || []);
- const out = clusters.split(req.viewer, req.params.id, items, { reason: one(req.body.reason) });
+ const out = await clusters.split(req.viewer, req.params.id, items, { reason: one(req.body.reason) });
done(res, `/clusters/${out.created.id}`, `Split ${items.length} items off ${req.params.id}`);
done(res, `/clusters/${out.created.id}`, `Split ${items.length} items off ${req.params.id}`);
}));
}));
router.post('/clusters/audit/:aid/reverse', B('news.cluster.manage'), wrap(async (req, res) => {
router.post('/clusters/audit/:aid/reverse', B('news.cluster.manage'), wrap(async (req, res) => {
- const out = clusters.reverse(req.viewer, req.params.aid, { reason: one(req.body.reason) });
+ const out = await clusters.reverse(req.viewer, req.params.aid, { reason: one(req.body.reason) });
done(res, `/clusters/${out.clusters[0].id}`, `Reversed (${out.audit.action.replace('_', ' ')})`);
done(res, `/clusters/${out.clusters[0].id}`, `Reversed (${out.audit.action.replace('_', ' ')})`);
}));
}));
router.post('/clusters/:id/stories', B('news.story.create'), wrap(async (req, res) => {
router.post('/clusters/:id/stories', B('news.story.create'), wrap(async (req, res) => {
const b = input(req.body);
const b = input(req.body);
- const out = stories.create(req.viewer, { cluster: req.params.id, headline: b.headline, topic: b.topic || null }, tp(req));
+ const out = await stories.create(req.viewer, { cluster: req.params.id, headline: b.headline, topic: b.topic || null }, tp(req));
done(res, `/edit/stories/${out.story.id}`, 'Story opened. Write the text; every paragraph cites its sources.');
done(res, `/edit/stories/${out.story.id}`, 'Story opened. Write the text; every paragraph cites its sources.');
}));
}));
// ── Stories ─────────────────────────────────────────────
// ── Stories ─────────────────────────────────────────────
- function mustStory(req, res) {
- const s = stories.get(req.params.id);
+ async function mustStory(req, res) {
+ const s = await stories.get(req.params.id);
if (!s) { publicRoutes.notFound(req, res); return null; }
if (!s) { publicRoutes.notFound(req, res); return null; }
return s;
return s;
}
}
router.get('/edit/stories/:id', wrap(async (req, res) => {
router.get('/edit/stories/:id', wrap(async (req, res) => {
- const story = mustStory(req, res);
+ const story = await mustStory(req, res);
if (!story) return;
if (!story) return;
- const head = stories.head(story);
+ const head = await stories.head(story);
page(req, res, story.working_headline, editor.storyEditor({
page(req, res, story.working_headline, editor.storyEditor({
- story, head, problems: head ? stories.problems(story, head) : [],
- sources: stories.sources(story, { includeDetached: true }), timeline: stories.timeline(story), perspectives: stories.perspectives(story),
- flags: stories.flags(story), revisions: stories.revisions(story, { limit: 100 }), topics: topics.list(), topic: publication.topicOf(story),
+ story, head, problems: head ? await stories.problems(story, head) : [],
+ sources: await stories.sources(story, { includeDetached: true }), timeline: await stories.timeline(story), perspectives: await stories.perspectives(story),
+ flags: await stories.flags(story), revisions: await stories.revisions(story, { limit: 100 }), topics: await topics.list(), topic: await publication.topicOf(story),
csrf: csrf(req), message: flashOf(req), aiEnabled: Boolean(ai && ai.enabled), publicUrl: publication.storyPath(story),
csrf: csrf(req), message: flashOf(req), aiEnabled: Boolean(ai && ai.enabled), publicUrl: publication.storyPath(story),
- decision: story.published_revision ? publication.decide(story, stories.revision(story, story.published_revision)) : null,
+ decision: story.published_revision ? await publication.decide(story, await stories.revision(story, story.published_revision)) : null,
}));
}));
}));
}));
router.get('/edit/stories/:id/preview', wrap(async (req, res) => {
router.get('/edit/stories/:id/preview', wrap(async (req, res) => {
- const story = mustStory(req, res);
+ const story = await mustStory(req, res);
if (!story) return;
if (!story) return;
const n = parseInt(req.query.revision, 10);
const n = parseInt(req.query.revision, 10);
- const rev = Number.isInteger(n) ? stories.revision(story, n) : stories.head(story);
+ const rev = Number.isInteger(n) ? await stories.revision(story, n) : await stories.head(story);
if (!rev) return publicRoutes.notFound(req, res);
if (!rev) return publicRoutes.notFound(req, res);
await publicRoutes.renderStory(req, res, { story, rev, preview: true });
await publicRoutes.renderStory(req, res, { story, rev, preview: true });
return undefined;
return undefined;
}));
}));
router.get('/edit/stories/:id/diff', B('news.story.diff'), wrap(async (req, res) => {
router.get('/edit/stories/:id/diff', B('news.story.diff'), wrap(async (req, res) => {
- const story = mustStory(req, res);
+ const story = await mustStory(req, res);
if (!story) return;
if (!story) return;
- page(req, res, 'Diff', editor.diffPage({ story, diff: stories.diff(story, parseInt(req.query.from, 10), parseInt(req.query.to, 10), 'word') }));
+ page(req, res, 'Diff', editor.diffPage({ story, diff: await stories.diff(story, parseInt(req.query.from, 10), parseInt(req.query.to, 10), 'word') }));
}));
}));
const act = (path, budget, fn) => router.post(`/edit/stories/:id/${path}`, B(budget), wrap(async (req, res) => {
const act = (path, budget, fn) => router.post(`/edit/stories/:id/${path}`, B(budget), wrap(async (req, res) => {
- const story = mustStory(req, res);
+ const story = await mustStory(req, res);
if (!story) return;
if (!story) return;
const text = await fn(req, story, input(req.body));
const text = await fn(req, story, input(req.body));
done(res, `/edit/stories/${story.id}`, text);
done(res, `/edit/stories/${story.id}`, text);
}));
}));
- act('revise', 'news.story.revise', (req, story, b) => {
- const out = stories.revise(req.viewer, story, { headline: b.headline, body: b.body, expectedRevision: b.expectedRevision, topic: b.topic, noindex: b.noindex, message: b.message }, tp(req));
+ act('revise', 'news.story.revise', async (req, story, b) => {
+ const out = await stories.revise(req.viewer, story, { headline: b.headline, body: b.body, expectedRevision: b.expectedRevision, topic: b.topic, noindex: b.noindex, message: b.message }, tp(req));
return out.created ? `Saved revision ${out.revision.number}` : 'Nothing changed';
return out.created ? `Saved revision ${out.revision.number}` : 'Nothing changed';
});
});
- act('publish', 'news.story.publish', (req, story, b) => {
+ act('publish', 'news.story.publish', async (req, story, b) => {
const correction = b.correctionKind || b.correctionNote ? { kind: b.correctionKind || 'correction', note: b.correctionNote } : undefined;
const correction = b.correctionKind || b.correctionNote ? { kind: b.correctionKind || 'correction', note: b.correctionNote } : undefined;
- const out = stories.publish(req.viewer, story, { revision: b.revision, correction }, tp(req));
+ const out = await stories.publish(req.viewer, story, { revision: b.revision, correction }, tp(req));
return out.changed ? `Published revision ${out.story.published_revision}` : 'Already published';
return out.changed ? `Published revision ${out.story.published_revision}` : 'Already published';
});
});
- act('unpublish', 'news.story.publish', (req, story) => (stories.unpublish(req.viewer, story, tp(req)).changed ? 'Unpublished' : 'Not published'));
- act('retract', 'news.story.publish', (req, story, b) => (stories.retract(req.viewer, story, { note: b.note }, tp(req)).changed ? 'Retracted' : 'Already retracted'));
- act('review', 'news.story.publish', (req, story, b) => { stories.review(req.viewer, story, { revision: b.revision, decision: b.decision, note: b.note }, tp(req)); return `Review recorded for revision ${b.revision}`; });
- act('flag', 'news.story.annotate', (req, story, b) => { stories.addFlag(req.viewer, story, { kind: b.kind, note: b.note }); return 'Note added; it is published with the next publication'; });
- act('attach', 'news.source.attach', (req, story, b) => `Attached as [${stories.attach(req.viewer, story, { item: b.item }).n}]`);
- act('detach', 'news.source.attach', (req, story, b) => `Detached [${stories.detach(req.viewer, story, b.item).n}]`);
- act('assign', 'news.story.annotate', (req, story, b) => { stories.assignPerspective(req.viewer, story, b.item, b.perspective || null); return 'Perspective set'; });
- act('perspective', 'news.story.annotate', (req, story, b) => { stories.addPerspective(req.viewer, story, { label: b.label, description: b.description }); return 'Perspective added'; });
- act('perspective/remove', 'news.story.annotate', (req, story, b) => { stories.removePerspective(req.viewer, story, b.perspective); return 'Perspective removed'; });
- act('timeline', 'news.story.annotate', (req, story, b) => { stories.addTimeline(req.viewer, story, { occurredOn: b.occurredOn, text: b.text, source: b.source }); return 'Timeline entry added'; });
- act('timeline/remove', 'news.story.annotate', (req, story, b) => { stories.removeTimeline(req.viewer, story, b.entry); return 'Timeline entry removed'; });
+ act('unpublish', 'news.story.publish', async (req, story) => ((await stories.unpublish(req.viewer, story, tp(req))).changed ? 'Unpublished' : 'Not published'));
+ act('retract', 'news.story.publish', async (req, story, b) => ((await stories.retract(req.viewer, story, { note: b.note }, tp(req))).changed ? 'Retracted' : 'Already retracted'));
+ act('review', 'news.story.publish', async (req, story, b) => { await stories.review(req.viewer, story, { revision: b.revision, decision: b.decision, note: b.note }, tp(req)); return `Review recorded for revision ${b.revision}`; });
+ act('flag', 'news.story.annotate', async (req, story, b) => { await stories.addFlag(req.viewer, story, { kind: b.kind, note: b.note }); return 'Note added; it is published with the next publication'; });
+ act('attach', 'news.source.attach', async (req, story, b) => `Attached as [${(await stories.attach(req.viewer, story, { item: b.item })).n}]`);
+ act('detach', 'news.source.attach', async (req, story, b) => `Detached [${(await stories.detach(req.viewer, story, b.item)).n}]`);
+ act('assign', 'news.story.annotate', async (req, story, b) => { await stories.assignPerspective(req.viewer, story, b.item, b.perspective || null); return 'Perspective set'; });
+ act('perspective', 'news.story.annotate', async (req, story, b) => { await stories.addPerspective(req.viewer, story, { label: b.label, description: b.description }); return 'Perspective added'; });
+ act('perspective/remove', 'news.story.annotate', async (req, story, b) => { await stories.removePerspective(req.viewer, story, b.perspective); return 'Perspective removed'; });
+ act('timeline', 'news.story.annotate', async (req, story, b) => { await stories.addTimeline(req.viewer, story, { occurredOn: b.occurredOn, text: b.text, source: b.source }); return 'Timeline entry added'; });
+ act('timeline/remove', 'news.story.annotate', async (req, story, b) => { await stories.removeTimeline(req.viewer, story, b.entry); return 'Timeline entry removed'; });
act('ai', 'news.story.ai_draft', async (req, story, b) => {
act('ai', 'news.story.ai_draft', async (req, story, b) => {
const out = await stories.aiDraft(req.viewer, story, b.workflow, tp(req));
const out = await stories.aiDraft(req.viewer, story, b.workflow, tp(req));
return `AI draft saved as revision ${out.revision.number}: review it before publishing`;
return `AI draft saved as revision ${out.revision.number}: review it before publishing`;
@@ -69,7 +69,7 @@ function createPublicRoutes(ctx) {
@@ -69,7 +69,7 @@ function createPublicRoutes(ctx) {
async function listing(req, res, { topic = null }) {
async function listing(req, res, { topic = null }) {
const page = pageNumber(req);
const page = pageNumber(req);
const path = topic ? publication.topicPath(topic) : '/';
const path = topic ? publication.topicPath(topic) : '/';
- const { total, stories: rows } = stories.listPublished({ topicId: topic ? topic.id : null, limit: PER_PAGE, offset: (page - 1) * PER_PAGE });
+ const { total, stories: rows } = await stories.listPublished({ topicId: topic ? topic.id : null, limit: PER_PAGE, offset: (page - 1) * PER_PAGE });
const pager = ssr.paginate({ page, perPage: PER_PAGE, total, href: (p) => (p === 1 ? path : `${path}?page=${p}`) });
const pager = ssr.paginate({ page, perPage: PER_PAGE, total, href: (p) => (p === 1 ? path : `${path}?page=${p}`) });
if (pager.outOfRange && total) return notFound(req, res);
if (pager.outOfRange && total) return notFound(req, res);
const items = await reading.listItems(rows);
const items = await reading.listItems(rows);
@@ -85,7 +85,7 @@ function createPublicRoutes(ctx) {
@@ -85,7 +85,7 @@ function createPublicRoutes(ctx) {
jsonLd: topic
jsonLd: topic
? [seo.structuredData.breadcrumbs(crumbs.map((c) => ({ name: c.name, url: c.url ? publication.abs(c.url) : canonical })))]
? [seo.structuredData.breadcrumbs(crumbs.map((c) => ({ name: c.name, url: c.url ? publication.abs(c.url) : canonical })))]
: [{ '@context': 'https://schema.org', '@type': 'WebSite', name: 'OpenVibe.News', url: publication.abs('/') }],
: [{ '@context': 'https://schema.org', '@type': 'WebSite', name: 'OpenVibe.News', url: publication.abs('/') }],
- body: topic ? pages.topicPage({ topic, items, pager, feeds, breadcrumbs: crumbs }) : pages.home({ items, pager, topics: topics.list(), feeds }),
+ body: topic ? pages.topicPage({ topic, items, pager, feeds, breadcrumbs: crumbs }) : pages.home({ items, pager, topics: await topics.list(), feeds }),
}, { cacheable: true });
}, { cacheable: true });
}
}
@@ -95,16 +95,16 @@ function createPublicRoutes(ctx) {
@@ -95,16 +95,16 @@ function createPublicRoutes(ctx) {
decision: pageDecision('/updates'), canonical: `${config.baseUrl}/updates`,
decision: pageDecision('/updates'), canonical: `${config.baseUrl}/updates`,
body: frame.updatesBody({ service: 'news', siteName: 'OpenVibe.News' }) + `<script src="${ovServe.url('shipped.js')}" defer></script>`,
body: frame.updatesBody({ service: 'news', siteName: 'OpenVibe.News' }) + `<script src="${ovServe.url('shipped.js')}" defer></script>`,
}, { cacheable: true }));
}, { cacheable: true }));
- router.get('/', wrap((req, res) => listing(req, res, {})));
+ router.get('/', wrap(async (req, res) => await listing(req, res, {})));
router.get('/topics', wrap(async (req, res) => {
router.get('/topics', wrap(async (req, res) => {
- send(req, res, 200, { title: 'Topics', description: 'Topics on OpenVibe.News.', decision: pageDecision('/topics'), canonical: publication.abs('/topics'), body: pages.topicsIndex({ topics: topics.list() }) }, { cacheable: true });
+ send(req, res, 200, { title: 'Topics', description: 'Topics on OpenVibe.News.', decision: pageDecision('/topics'), canonical: publication.abs('/topics'), body: pages.topicsIndex({ topics: await topics.list() }) }, { cacheable: true });
}));
}));
router.get('/topics/:slug', wrap(async (req, res) => {
router.get('/topics/:slug', wrap(async (req, res) => {
- const topic = topics.bySlug(req.params.slug);
+ const topic = await topics.bySlug(req.params.slug);
if (!topic || topic.status !== 'active') return notFound(req, res);
if (!topic || topic.status !== 'active') return notFound(req, res);
- return listing(req, res, { topic });
+ return await listing(req, res, { topic });
}));
}));
// ── Feeds (published stories only, never viewer-dependent) ─
// ── Feeds (published stories only, never viewer-dependent) ─
@@ -123,15 +123,15 @@ function createPublicRoutes(ctx) {
@@ -123,15 +123,15 @@ function createPublicRoutes(ctx) {
if (kind === 'json') return res.type('application/feed+json').send(JSON.stringify(seo.jsonFeed({ title, link, feedUrl, description, language: 'en' }, items)));
if (kind === 'json') return res.type('application/feed+json').send(JSON.stringify(seo.jsonFeed({ title, link, feedUrl, description, language: 'en' }, items)));
const listed = items.filter((i) => i.decision.listable);
const listed = items.filter((i) => i.decision.listable);
// An empty Atom feed still needs <updated>: the time the topic (or News' topic list) was set up, a real time.
// An empty Atom feed still needs <updated>: the time the topic (or News' topic list) was set up, a real time.
- const updated = listed.length ? null : new Date(topic ? topic.updated_at : (store.db.prepare('SELECT MIN(created_at) AS t FROM news_topics').get().t || store.now())).toISOString();
+ const updated = listed.length ? null : new Date(topic ? topic.updated_at : ((await store.db.prepare('SELECT MIN(created_at) AS t FROM news_topics').get()).t || store.now())).toISOString();
return res.type('application/atom+xml').send(seo.atomFeed({ title, link, feedUrl, id: `tag:openvibe.news,2026:${topic ? `topic/${topic.id}` : 'all'}`, subtitle: description, ...(updated ? { updated } : {}) }, items));
return res.type('application/atom+xml').send(seo.atomFeed({ title, link, feedUrl, id: `tag:openvibe.news,2026:${topic ? `topic/${topic.id}` : 'all'}`, subtitle: description, ...(updated ? { updated } : {}) }, items));
}
}
for (const f of FEEDS) {
for (const f of FEEDS) {
- router.get(`/${f.file}`, wrap((req, res) => feed(req, res, f.type, null)));
- router.get(`/topics/:slug/${f.file}`, wrap((req, res) => {
- const topic = topics.bySlug(req.params.slug);
+ router.get(`/${f.file}`, wrap(async (req, res) => await feed(req, res, f.type, null)));
+ router.get(`/topics/:slug/${f.file}`, wrap(async (req, res) => {
+ const topic = await topics.bySlug(req.params.slug);
if (!topic || topic.status !== 'active') return notFound(req, res);
if (!topic || topic.status !== 'active') return notFound(req, res);
- return feed(req, res, f.type, topic);
+ return await feed(req, res, f.type, topic);
}));
}));
}
}
@@ -139,7 +139,7 @@ function createPublicRoutes(ctx) {
@@ -139,7 +139,7 @@ function createPublicRoutes(ctx) {
/** The story's Community thread as the page shows it: { state: ok|closed|off|unavailable, … }. */
/** The story's Community thread as the page shows it: { state: ok|closed|off|unavailable, … }. */
async function commentsFor(req, story, headline) {
async function commentsFor(req, story, headline) {
- const st = discussion.status(story);
+ const st = await discussion.status(story);
if (!st.open) return { state: 'closed', reason: st.reason };
if (!st.open) return { state: 'closed', reason: st.reason };
if (!community.enabled) return { state: 'off' };
if (!community.enabled) return { state: 'off' };
try {
try {
@@ -159,7 +159,7 @@ function createPublicRoutes(ctx) {
@@ -159,7 +159,7 @@ function createPublicRoutes(ctx) {
const isEditor = access.isEditor(config, req.viewer);
const isEditor = access.isEditor(config, req.viewer);
const m = await reading.storyModel(story, { rev, editorView: preview && isEditor });
const m = await reading.storyModel(story, { rev, editorView: preview && isEditor });
const group = ['outlet', 'perspective'].includes(req.query.group) ? req.query.group : 'number';
const group = ['outlet', 'perspective'].includes(req.query.group) ? req.query.group : 'number';
- const decision = preview ? publication.decide(story, m.rev, { state: 'draft' }) : m.decision;
+ const decision = preview ? await publication.decide(story, m.rev, { state: 'draft' }) : m.decision;
const crumbs = [{ name: 'OpenVibe.News', url: '/' }, ...(m.topic ? [{ name: m.topic.name, url: publication.topicPath(m.topic) }] : []), { name: m.headline }];
const crumbs = [{ name: 'OpenVibe.News', url: '/' }, ...(m.topic ? [{ name: m.topic.name, url: publication.topicPath(m.topic) }] : []), { name: m.headline }];
const cacheable = !preview && (story.state === 'published' || story.state === 'retracted');
const cacheable = !preview && (story.state === 'published' || story.state === 'retracted');
const comments = preview ? null : await commentsFor(req, story, m.headline);
const comments = preview ? null : await commentsFor(req, story, m.headline);
@@ -185,7 +185,7 @@ function createPublicRoutes(ctx) {
@@ -185,7 +185,7 @@ function createPublicRoutes(ctx) {
router.get('/stories/:slug', wrap(async (req, res) => {
router.get('/stories/:slug', wrap(async (req, res) => {
const asJson = req.params.slug.endsWith('.json');
const asJson = req.params.slug.endsWith('.json');
const slug = asJson ? req.params.slug.slice(0, -5) : req.params.slug;
const slug = asJson ? req.params.slug.slice(0, -5) : req.params.slug;
- const story = stories.bySlug(slug);
+ const story = await stories.bySlug(slug);
if (!story || !story.published_revision || (story.state !== 'published' && story.state !== 'retracted')) {
if (!story || !story.published_revision || (story.state !== 'published' && story.state !== 'retracted')) {
if (story && story.state === 'unpublished') return messagePage(req, res, 410, 'Gone', 'This story was unpublished.', { href: '/', label: 'OpenVibe.News' });
if (story && story.state === 'unpublished') return messagePage(req, res, 410, 'Gone', 'This story was unpublished.', { href: '/', label: 'OpenVibe.News' });
return notFound(req, res);
return notFound(req, res);
@@ -195,23 +195,23 @@ function createPublicRoutes(ctx) {
@@ -195,23 +195,23 @@ function createPublicRoutes(ctx) {
cacheHeaders(res, { cacheable: req.viewer.kind === 'anonymous', robots: m.decision.robots });
cacheHeaders(res, { cacheable: req.viewer.kind === 'anonymous', robots: m.decision.robots });
return res.json(reading.storyJson(m));
return res.json(reading.storyJson(m));
}
}
- return renderStory(req, res, { story });
+ return await renderStory(req, res, { story });
}));
}));
// Per address (20 a minute), then per signed-in person (http/actor-limits.js) before the form is read.
// Per address (20 a minute), then per signed-in person (http/actor-limits.js) before the form is read.
router.post('/stories/:slug/comments', rateLimit({ windowMs: 60_000, max: 20, standardHeaders: true, legacyHeaders: false }), ctx.limits.signedIn('news.discussion.comment'), express.urlencoded({ extended: false, limit: '32kb' }), wrap(async (req, res) => {
router.post('/stories/:slug/comments', rateLimit({ windowMs: 60_000, max: 20, standardHeaders: true, legacyHeaders: false }), ctx.limits.signedIn('news.discussion.comment'), express.urlencoded({ extended: false, limit: '32kb' }), wrap(async (req, res) => {
- const story = stories.bySlug(req.params.slug);
+ const story = await stories.bySlug(req.params.slug);
if (!story || !story.published_revision || (story.state !== 'published' && story.state !== 'retracted')) return notFound(req, res);
if (!story || !story.published_revision || (story.state !== 'published' && story.state !== 'retracted')) return notFound(req, res);
const path = publication.storyPath(story);
const path = publication.storyPath(story);
if (req.viewer.kind !== 'user' || !req.viewer.subject) return res.redirect(303, `/auth/login?next=${encodeURIComponent(path)}`);
if (req.viewer.kind !== 'user' || !req.viewer.subject) return res.redirect(303, `/auth/login?next=${encodeURIComponent(path)}`);
if (!checkCsrf(config, req.viewer, req.body && req.body._csrf)) return messagePage(req, res, 403, 'Form expired', 'Reload the story and try again.', { href: path, label: 'Back to the story' });
if (!checkCsrf(config, req.viewer, req.body && req.body._csrf)) return messagePage(req, res, 403, 'Form expired', 'Reload the story and try again.', { href: path, label: 'Back to the story' });
- if (!discussion.status(story).open) return messagePage(req, res, 409, 'Comments are closed', 'This story does not take comments.', { href: path, label: 'Back to the story' });
+ if (!(await discussion.status(story)).open) return messagePage(req, res, 409, 'Comments are closed', 'This story does not take comments.', { href: path, label: 'Back to the story' });
if (!community.enabled) return messagePage(req, res, 503, 'Comments are unavailable', 'Comments are not connected on this server.', { href: path, label: 'Back to the story' });
if (!community.enabled) return messagePage(req, res, 503, 'Comments are unavailable', 'Comments are not connected on this server.', { href: path, label: 'Back to the story' });
const message = String((req.body && req.body.message) || '').trim();
const message = String((req.body && req.body.message) || '').trim();
if (!message) return res.redirect(303, `${path}#comments`);
if (!message) return res.redirect(303, `${path}#comments`);
if (message.length > 5000) return messagePage(req, res, 422, 'Comment too long', 'A comment is at most 5000 characters.', { href: path, label: 'Back to the story' });
if (message.length > 5000) return messagePage(req, res, 422, 'Comment too long', 'A comment is at most 5000 characters.', { href: path, label: 'Back to the story' });
try {
try {
- const rev = store.revisions.get(story.id, story.published_revision);
+ const rev = await store.revisions.get(story.id, story.published_revision);
const threadId = await community.threadFor(story, rev && rev.fields.headline, req.ov);
const threadId = await community.threadFor(story, rev && rev.fields.headline, req.ov);
if (!threadId) throw new Error('no thread');
if (!threadId) throw new Error('no thread');
await community.comment(threadId, req.viewer.subject, { message }, req.ov);
await community.comment(threadId, req.viewer.subject, { message }, req.ov);
@@ -16,15 +16,14 @@
@@ -16,15 +16,14 @@
*/
*/
const express = require('express');
const express = require('express');
const { http } = require('openvibe-contracts');
const { http } = require('openvibe-contracts');
-const { verifyDeliveryV2, createInbox } = require('openvibe-sdk/events');
+const { verifyDeliveryV2, createPgInbox } = require('openvibe-sdk/events');
const CONSUMER = 'news-sources';
const CONSUMER = 'news-sources';
const EVT_RE = /^evt_[0-9A-HJKMNP-TV-Z]{26}$/;
const EVT_RE = /^evt_[0-9A-HJKMNP-TV-Z]{26}$/;
function createWebhook({ config, store, ingest, sources, log = console }) {
function createWebhook({ config, store, ingest, sources, log = console }) {
const router = express.Router();
const router = express.Router();
- const inbox = createInbox(store.db, { now: store.now });
- inbox.ensureSchema();
+ const inbox = createPgInbox(store.db, { now: store.now }); // idempotency_receipts: migrations/0001_initial.sql
router.post('/internal/events', express.raw({ type: () => true, limit: '256kb' }), async (req, res) => {
router.post('/internal/events', express.raw({ type: () => true, limit: '256kb' }), async (req, res) => {
const ctx = req.ov;
const ctx = req.ov;
@@ -47,24 +46,24 @@ function createWebhook({ config, store, ingest, sources, log = console }) {
@@ -47,24 +46,24 @@ function createWebhook({ config, store, ingest, sources, log = console }) {
try {
try {
if (!fromSources || !/^sources\.(item\.(created|updated|removed)|fetch\.failed)$/.test(type)) {
if (!fromSources || !/^sources\.(item\.(created|updated|removed)|fetch\.failed)$/.test(type)) {
- inbox.once(CONSUMER, event.event_id, () => null);
+ await inbox.once(CONSUMER, event.event_id, () => null);
return res.status(204).end();
return res.status(204).end();
}
}
if (type === 'sources.item.removed') {
if (type === 'sources.item.removed') {
- inbox.once(CONSUMER, event.event_id, () => {
- const r = ingest.applyRemoval({ sourcesItemId: String(p.item_id || ''), reason: p.reason, revision: Number.isInteger(p.revision) ? p.revision : null, origin: 'webhook' });
- ingest.recordRun({ origin: 'webhook', state: r.outcome, source_key: p.source_key || null, sources_item_id: p.item_id || null });
+ await inbox.once(CONSUMER, event.event_id, async () => {
+ const r = await ingest.applyRemoval({ sourcesItemId: String(p.item_id || ''), reason: p.reason, revision: Number.isInteger(p.revision) ? p.revision : null, origin: 'webhook' });
+ await ingest.recordRun({ origin: 'webhook', state: r.outcome, source_key: p.source_key || null, sources_item_id: p.item_id || null });
return r;
return r;
});
});
return res.status(204).end();
return res.status(204).end();
}
}
if (type === 'sources.fetch.failed') {
if (type === 'sources.fetch.failed') {
- inbox.once(CONSUMER, event.event_id, () => ingest.upstreamFailure(p));
+ await inbox.once(CONSUMER, event.event_id, () => ingest.upstreamFailure(p));
return res.status(204).end();
return res.status(204).end();
}
}
// created | updated
// created | updated
if (p.category && p.category !== config.sources.category) {
if (p.category && p.category !== config.sources.category) {
- inbox.once(CONSUMER, event.event_id, () => null);
+ await inbox.once(CONSUMER, event.event_id, () => null);
return res.status(204).end();
return res.status(204).end();
}
}
let fetched;
let fetched;
@@ -72,21 +71,21 @@ function createWebhook({ config, store, ingest, sources, log = console }) {
@@ -72,21 +71,21 @@ function createWebhook({ config, store, ingest, sources, log = console }) {
if (p.source_key) await ingest.learnOutlets([String(p.source_key)]);
if (p.source_key) await ingest.learnOutlets([String(p.source_key)]);
fetched = await sources.getItem(String(p.item_id || ''));
fetched = await sources.getItem(String(p.item_id || ''));
} catch (err) {
} catch (err) {
- store.tx(() => {
- ingest.recordRun({ origin: 'webhook', state: 'failed', source_key: p.source_key || null, sources_item_id: p.item_id || null, error_code: err.code || 'sources.error', detail: `${type} ${event.event_id} attempt ${attempt}: ${err.message}` });
- if (attempt === 1) ingest.failedEvent({ origin: 'webhook', sourceKey: p.source_key || null, sourcesItemId: p.item_id || null, state: 'failed', errorCode: err.code || 'sources.error', detail: err.message, httpStatus: err.status || null });
+ await store.tx(async () => {
+ await ingest.recordRun({ origin: 'webhook', state: 'failed', source_key: p.source_key || null, sources_item_id: p.item_id || null, error_code: err.code || 'sources.error', detail: `${type} ${event.event_id} attempt ${attempt}: ${err.message}` });
+ if (attempt === 1) await ingest.failedEvent({ origin: 'webhook', sourceKey: p.source_key || null, sourcesItemId: p.item_id || null, state: 'failed', errorCode: err.code || 'sources.error', detail: err.message, httpStatus: err.status || null });
});
});
if (err.status === 404) {
if (err.status === 404) {
// Sources no longer has it: nothing to apply, and retrying cannot help.
// Sources no longer has it: nothing to apply, and retrying cannot help.
- inbox.once(CONSUMER, event.event_id, () => null);
+ await inbox.once(CONSUMER, event.event_id, () => null);
return res.status(204).end();
return res.status(204).end();
}
}
return http.sendProblem(res, 503, 'news.sources_unavailable', { detail: 'OpenVibe.Sources could not be read; retry later', ctx });
return http.sendProblem(res, 503, 'news.sources_unavailable', { detail: 'OpenVibe.Sources could not be read; retry later', ctx });
}
}
- if (fetched.source) ingest.rememberSources({ [fetched.item.source_key]: fetched.source });
- inbox.once(CONSUMER, event.event_id, () => {
- const r = ingest.apply(fetched.item, { origin: 'webhook' });
- ingest.recordRun({ origin: 'webhook', state: r.outcome, source_key: fetched.item.source_key || null, sources_item_id: fetched.item.id, detail: r.reason || null });
+ if (fetched.source) await ingest.rememberSources({ [fetched.item.source_key]: fetched.source });
+ await inbox.once(CONSUMER, event.event_id, async () => {
+ const r = await ingest.apply(fetched.item, { origin: 'webhook' });
+ await ingest.recordRun({ origin: 'webhook', state: r.outcome, source_key: fetched.item.source_key || null, sources_item_id: fetched.item.id, detail: r.reason || null });
return r;
return r;
});
});
return res.status(204).end();
return res.status(204).end();
@@ -7,11 +7,12 @@
@@ -7,11 +7,12 @@
*/
*/
const { createApp } = require('./app');
const { createApp } = require('./app');
-const { app, ctx } = createApp();
+(async () => {
+const { app, ctx } = await createApp();
const { config } = ctx;
const { config } = ctx;
const server = app.listen(config.port, config.host, () => {
const server = app.listen(config.port, config.host, () => {
- console.log(`[News] ${config.nodeEnv} on http://${config.host}:${config.port} → ${config.baseUrl} (db ${config.dbPath})`);
+ console.log(`[News] ${config.nodeEnv} on http://${config.host}:${config.port} → ${config.baseUrl} (db ${ctx.store.db.store})`);
console.log(`[News] events relay ${ctx.outbox.enabled ? `on → ${config.events.url}` : 'off (events wait in event_outbox)'}; webhook ${config.events.webhookSecrets.length ? 'on' : 'off (NEWS_EVENTS_SECRET unset)'}; Sources pull ${config.worker.enabled && config.sources.pullIntervalMs ? `every ${config.sources.pullIntervalMs} ms` : 'off'}; AI ${ctx.ai.enabled ? 'on' : 'off'}`);
console.log(`[News] events relay ${ctx.outbox.enabled ? `on → ${config.events.url}` : 'off (events wait in event_outbox)'}; webhook ${config.events.webhookSecrets.length ? 'on' : 'off (NEWS_EVENTS_SECRET unset)'}; Sources pull ${config.worker.enabled && config.sources.pullIntervalMs ? `every ${config.sources.pullIntervalMs} ms` : 'off'}; AI ${ctx.ai.enabled ? 'on' : 'off'}`);
});
});
server.keepAliveTimeout = 65_000;
server.keepAliveTimeout = 65_000;
@@ -23,10 +24,11 @@ function shutdown(signal) {
@@ -23,10 +24,11 @@ function shutdown(signal) {
ctx.worker.stop();
ctx.worker.stop();
server.close(async () => {
server.close(async () => {
try { await ctx.outbox.stop(); } catch { /* best effort */ }
try { await ctx.outbox.stop(); } catch { /* best effort */ }
- try { ctx.store.close(); } catch { /* already closed */ }
+ try { await ctx.store.close(); } catch { /* already closed */ }
process.exit(0);
process.exit(0);
});
});
setTimeout(() => process.exit(0), 5000).unref();
setTimeout(() => process.exit(0), 5000).unref();
}
}
process.on('SIGTERM', () => shutdown('SIGTERM'));
process.on('SIGTERM', () => shutdown('SIGTERM'));
process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGINT', () => shutdown('SIGINT'));
+})().catch((err) => { console.error('[News] failed to start:', err); process.exit(1); });
@@ -16,7 +16,7 @@
@@ -16,7 +16,7 @@
const { createReadiness, skip } = require('openvibe-shared/ready');
const { createReadiness, skip } = require('openvibe-shared/ready');
const { CHARTER_TABLES } = require('./db');
const { CHARTER_TABLES } = require('./db');
-function createNewsReadiness({ store, auth, outbox, ingest, config, release = null }) {
+function createNewsReadiness({ store, auth, outbox, ingest, config, valkey = null, release = null }) {
const { db } = store;
const { db } = store;
return createReadiness({
return createReadiness({
service: 'news',
service: 'news',
@@ -24,12 +24,16 @@ function createNewsReadiness({ store, auth, outbox, ingest, config, release = nu
@@ -24,12 +24,16 @@ function createNewsReadiness({ store, auth, outbox, ingest, config, release = nu
checks: [
checks: [
{
{
name: 'db', required: true,
name: 'db', required: true,
- check: () => {
- const names = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table','view')").all().map((r) => r.name));
+ check: async () => {
+ // A real round trip that names the store (postgresql / pglite), and the charter tables present.
+ const r = await db.ready();
+ if (!r.ok) return r.error;
+ const names = new Set((await db.prepare('SELECT table_name AS name FROM information_schema.tables WHERE table_schema = current_schema()').all()).map((x) => x.name));
const missing = CHARTER_TABLES.filter((t) => !names.has(t));
const missing = CHARTER_TABLES.filter((t) => !names.has(t));
- return missing.length ? `missing ${missing.join(', ')}` : true;
+ return missing.length ? `missing ${missing.join(', ')} (migrations did not run)` : { ok: true, detail: r.detail };
},
},
},
},
+ { name: 'valkey', required: false, check: async () => (valkey ? valkey.ready() : { skipped: 'VALKEY_URL not set: per-actor limits count in this process only' }) },
{
{
name: 'network_jwks', required: false,
name: 'network_jwks', required: false,
check: () => {
check: () => {
@@ -40,8 +44,8 @@ function createNewsReadiness({ store, auth, outbox, ingest, config, release = nu
@@ -40,8 +44,8 @@ function createNewsReadiness({ store, auth, outbox, ingest, config, release = nu
},
},
{
{
name: 'events_relay', required: false,
name: 'events_relay', required: false,
- check: () => {
- const s = outbox.status();
+ check: async () => {
+ const s = await outbox.status();
if (!s.enabled) return `relay off (EVENTS_URL or OV_OAUTH_CLIENT_SECRET unset); ${s.pending} events waiting`;
if (!s.enabled) return `relay off (EVENTS_URL or OV_OAUTH_CLIENT_SECRET unset); ${s.pending} events waiting`;
if (s.rejected) return `${s.rejected} events rejected by OpenVibe.Events`;
if (s.rejected) return `${s.rejected} events rejected by OpenVibe.Events`;
return { ok: true, detail: { pending: s.pending } };
return { ok: true, detail: { pending: s.pending } };
@@ -53,8 +57,8 @@ function createNewsReadiness({ store, auth, outbox, ingest, config, release = nu
@@ -53,8 +57,8 @@ function createNewsReadiness({ store, auth, outbox, ingest, config, release = nu
},
},
{
{
name: 'sources_pull', required: false,
name: 'sources_pull', required: false,
- check: () => {
- const last = ingest.lastPull();
+ check: async () => {
+ const last = await ingest.lastPull();
const off = !config.sources.pullIntervalMs || !config.worker.enabled;
const off = !config.sources.pullIntervalMs || !config.worker.enabled;
// Pull switched off and never run: nothing verified, so skipped, never ok (WS-Q task 7).
// Pull switched off and never run: nothing verified, so skipped, never ok (WS-Q task 7).
if (!last) return off ? skip('pull off (NEWS_PULL_INTERVAL_MS=0 or NEWS_WORKER=off)', { pull: 'off' }) : 'no pull from OpenVibe.Sources has run yet';
if (!last) return off ? skip('pull off (NEWS_PULL_INTERVAL_MS=0 or NEWS_WORKER=off)', { pull: 'off' }) : 'no pull from OpenVibe.Sources has run yet';
@@ -14,7 +14,7 @@ function createWorker({ config, ingest, outbox, log = console }) {
@@ -14,7 +14,7 @@ function createWorker({ config, ingest, outbox, log = console }) {
if (!ingest) return null;
if (!ingest) return null;
try {
try {
const r = await ingest.pull();
const r = await ingest.pull();
- outbox.kick();
+ await outbox.kick();
return r;
return r;
} catch (err) {
} catch (err) {
log.error('[News] pull tick failed:', err.message);
log.error('[News] pull tick failed:', err.message);
@@ -51,17 +51,17 @@ const { boot, check, done } = require('./helpers/boot');
@@ -51,17 +51,17 @@ const { boot, check, done } = require('./helpers/boot');
await check('writes: 10 topics a minute per editor, shared by the API and the desk form; nothing stored past it', async () => {
await check('writes: 10 topics a minute per editor, shared by the API and the desk form; nothing stored past it', async () => {
clock = Date.UTC(2026, 8, 27, 12, 5, 0);
clock = Date.UTC(2026, 8, 27, 12, 5, 0);
- const count = () => t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get().n;
+ const count = async () => (await t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get()).n;
for (let i = 0; i < 10; i++) {
for (let i = 0; i < 10; i++) {
const r = await t.api('/topics', { json: { name: `Limits topic ${i}` } });
const r = await t.api('/topics', { json: { name: `Limits topic ${i}` } });
assert.strictEqual(r.status, 201, `topic ${i + 1}: ${r.text}`);
assert.strictEqual(r.status, 201, `topic ${i + 1}: ${r.text}`);
}
}
- const before = count();
+ const before = await count();
const form = await t.get('/edit/topics', { as: t.editor, form: { _csrf: t.csrf(t.editor), name: 'One too many' } });
const form = await t.get('/edit/topics', { as: t.editor, form: { _csrf: t.csrf(t.editor), name: 'One too many' } });
assert.deepStrictEqual([form.status, form.json().code, form.headers.get('retry-after')], [429, 'rate_limited', '60']);
assert.deepStrictEqual([form.status, form.json().code, form.headers.get('retry-after')], [429, 'rate_limited', '60']);
const api = await t.api('/topics', { json: { name: 'One too many' } });
const api = await t.api('/topics', { json: { name: 'One too many' } });
assert.strictEqual(api.status, 429);
assert.strictEqual(api.status, 429);
- assert.strictEqual(count(), before, 'nothing stored');
+ assert.strictEqual(await count(), before, 'nothing stored');
const other = await t.get('/api/v1/topics', { as: admin, json: { name: 'Another editor' } });
const other = await t.get('/api/v1/topics', { as: admin, json: { name: 'Another editor' } });
assert.strictEqual(other.status, 201, `another editor still writes: ${other.text}`);
assert.strictEqual(other.status, 201, `another editor still writes: ${other.text}`);
});
});
@@ -14,7 +14,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -14,7 +14,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
const t = await boot({ ai: true });
const t = await boot({ ai: true });
const reports = launchReports(t.sources, { a: { fields: { body: 'SECRET-BODY-FOR-AI-5c1d' } } });
const reports = launchReports(t.sources, { a: { fields: { body: 'SECRET-BODY-FOR-AI-5c1d' } } });
await t.pull();
await t.pull();
- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;
+ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;
const story = (await t.api('/stories', { json: { cluster: clusterId, headline: 'Europa Clipper launch', topic: 'space' } })).json().story;
const story = (await t.api('/stories', { json: { cluster: clusterId, headline: 'Europa Clipper launch', topic: 'space' } })).json().story;
const aiToken = t.network.serviceToken('ai', ['news.story.revise']);
const aiToken = t.network.serviceToken('ai', ['news.story.revise']);
const aiHeaders = { 'x-ov-origin': 'ai' };
const aiHeaders = { 'x-ov-origin': 'ai' };
@@ -61,7 +61,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -61,7 +61,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
const page = await t.get(`/stories/${story.slug}`);
const page = await t.get(`/stories/${story.slug}`);
assert.match(page.text, /<strong>AI-generated\.<\/strong> AI-generated from the story’s source items by workflow news.summarize_story v1, reviewed by a person\./);
assert.match(page.text, /<strong>AI-generated\.<\/strong> AI-generated from the story’s source items by workflow news.summarize_story v1, reviewed by a person\./);
assert.match(page.text, /<meta name="robots" content="index, follow">/);
assert.match(page.text, /<meta name="robots" content="index, follow">/);
- const doc = t.events('news.index_document.upserted').pop().payload;
+ const doc = (await t.events('news.index_document.upserted')).pop().payload;
assert.strictEqual(doc.authorship, 'ai_generated');
assert.strictEqual(doc.authorship, 'ai_generated');
assert.ok(doc.provenance.some((p) => p.service === 'ai' && p.type === 'run' && p.id === wf.runId));
assert.ok(doc.provenance.some((p) => p.service === 'ai' && p.type === 'run' && p.id === wf.runId));
const json = (await t.get(`/stories/${story.slug}.json`)).json();
const json = (await t.get(`/stories/${story.slug}.json`)).json();
@@ -99,7 +99,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -99,7 +99,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
const sent = JSON.stringify(t.ai.requests.pop());
const sent = JSON.stringify(t.ai.requests.pop());
assert.ok(!sent.includes('SECRET-BODY-FOR-AI'), 'no body is ever sent to the model');
assert.ok(!sent.includes('SECRET-BODY-FOR-AI'), 'no body is ever sent to the model');
assert.match(sent, /"source_type":"news.article"/);
assert.match(sent, /"source_type":"news.article"/);
- assert.strictEqual(t.ctx.stories.get(s2.id).state, 'draft');
+ assert.strictEqual((await t.ctx.stories.get(s2.id)).state, 'draft');
const edit = await t.get(`/edit/stories/${s2.id}`, { as: t.editor });
const edit = await t.get(`/edit/stories/${s2.id}`, { as: t.editor });
assert.strictEqual(edit.status, 200, edit.text);
assert.strictEqual(edit.status, 200, edit.text);
assert.match(edit.text, /Ask OpenVibe.AI for a draft summary/);
assert.match(edit.text, /Ask OpenVibe.AI for a draft summary/);
@@ -108,7 +108,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -108,7 +108,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
const csrf = t.csrf(t.editor);
const csrf = t.csrf(t.editor);
const rv = await t.get(`/edit/stories/${s2.id}/review`, { as: t.editor, form: { _csrf: csrf, revision: String(head.revision), decision: 'approved', note: '' } });
const rv = await t.get(`/edit/stories/${s2.id}/review`, { as: t.editor, form: { _csrf: csrf, revision: String(head.revision), decision: 'approved', note: '' } });
assert.strictEqual(rv.status, 303, rv.text);
assert.strictEqual(rv.status, 303, rv.text);
- assert.strictEqual(t.ctx.store.reviews.latest(s2.id, head.revision).reviewer, t.editor.subject);
+ assert.strictEqual((await t.ctx.store.reviews.latest(s2.id, head.revision)).reviewer, t.editor.subject);
});
});
await check('a stub-provider run is held (stub_provider) and a failed run makes no text', async () => {
await check('a stub-provider run is held (stub_provider) and a failed run makes no text', async () => {
@@ -116,20 +116,20 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -116,20 +116,20 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
t.ai.setNext({ status: 201, run: { id: 'run_01J8Z3V9Q6N1X2Y3Z4A5B6C7DA', status: 'failed', output: null } });
t.ai.setNext({ status: 201, run: { id: 'run_01J8Z3V9Q6N1X2Y3Z4A5B6C7DA', status: 'failed', output: null } });
let r = await t.api(`/stories/${s3.id}/ai-drafts`, { json: { workflow: 'news.compare_perspectives' } });
let r = await t.api(`/stories/${s3.id}/ai-drafts`, { json: { workflow: 'news.compare_perspectives' } });
assert.strictEqual(r.status, 502);
assert.strictEqual(r.status, 502);
- assert.strictEqual(t.ctx.store.revisions.head(s3.id), null);
+ assert.strictEqual(await t.ctx.store.revisions.head(s3.id), null);
t.ai.setNext({ status: 201, run: { id: 'run_01J8Z3V9Q6N1X2Y3Z4A5B6C7DB', status: 'succeeded', synthetic: true, workflow: { key: 'news.compare_perspectives', version: 1 },
t.ai.setNext({ status: 201, run: { id: 'run_01J8Z3V9Q6N1X2Y3Z4A5B6C7DB', status: 'succeeded', synthetic: true, workflow: { key: 'news.compare_perspectives', version: 1 },
output: { question: 'How did outlets frame the launch?', perspectives: [{ label: 'Launch', summary: 'Outlets described the launch.', citations: [0, 1] }], agreements: [], disagreements: [], citations: [], gaps: [] } } });
output: { question: 'How did outlets frame the launch?', perspectives: [{ label: 'Launch', summary: 'Outlets described the launch.', citations: [0, 1] }], agreements: [], disagreements: [], citations: [], gaps: [] } } });
r = await t.api(`/stories/${s3.id}/ai-drafts`, { json: { workflow: 'news.compare_perspectives' } });
r = await t.api(`/stories/${s3.id}/ai-drafts`, { json: { workflow: 'news.compare_perspectives' } });
assert.strictEqual(r.status, 201, r.text);
assert.strictEqual(r.status, 201, r.text);
- const head = t.ctx.store.revisions.head(s3.id);
+ const head = await t.ctx.store.revisions.head(s3.id);
assert.strictEqual(head.meta.authorship.stubProvider, true);
assert.strictEqual(head.meta.authorship.stubProvider, true);
- const d = t.ctx.publication.decide(t.ctx.stories.get(s3.id), head, { state: 'published' });
+ const d = await t.ctx.publication.decide(await t.ctx.stories.get(s3.id), head, { state: 'published' });
assert.ok(d.codes.includes('stub_provider') && d.codes.includes('ai_generated_unreviewed'));
assert.ok(d.codes.includes('stub_provider') && d.codes.includes('ai_generated_unreviewed'));
- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_perspectives WHERE story_id = ?').get(s3.id).n, 0, 'AI never creates perspective labels');
+ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_perspectives WHERE story_id = ?').get(s3.id)).n, 0, 'AI never creates perspective labels');
});
});
await check('the published product events validate as events.event-envelope@1', async () => {
await check('the published product events validate as events.event-envelope@1', async () => {
- for (const e of t.events(/^news\./)) {
+ for (const e of await t.events(/^news\./)) {
const v = contracts.validate('events.event-envelope@1', { ...e, event_id: e.event_id || 'evt_01J8Z3V9Q6N1X2Y3Z4A5B6C7D8' });
const v = contracts.validate('events.event-envelope@1', { ...e, event_id: e.event_id || 'evt_01J8Z3V9Q6N1X2Y3Z4A5B6C7D8' });
assert.ok(v.valid, `${e.event_type}: ${JSON.stringify(v.errors)}`);
assert.ok(v.valid, `${e.event_type}: ${JSON.stringify(v.errors)}`);
}
}
@@ -141,7 +141,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -141,7 +141,7 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
await check('with no AI configured, the AI seam is off and says so', async () => {
await check('with no AI configured, the AI seam is off and says so', async () => {
const r2 = launchReports(t2.sources);
const r2 = launchReports(t2.sources);
await t2.pull();
await t2.pull();
- const cid = t2.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(r2.a.id).cluster_id;
+ const cid = (await t2.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(r2.a.id)).cluster_id;
const s = (await t2.api('/stories', { json: { cluster: cid, headline: 'Written by hand' } })).json().story;
const s = (await t2.api('/stories', { json: { cluster: cid, headline: 'Written by hand' } })).json().story;
const r = await t2.api(`/stories/${s.id}/ai-drafts`, { json: { workflow: 'news.summarize_story' } });
const r = await t2.api(`/stories/${s.id}/ai-drafts`, { json: { workflow: 'news.summarize_story' } });
assert.strictEqual(r.status, 503);
assert.strictEqual(r.status, 503);
@@ -13,28 +13,30 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -13,28 +13,30 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
const other = t.sources.addItem({ source_key: 'paper-b', title: 'Clipper mission team answers questions about Jupiter cruise', url: 'https://paper-b.example/qa', published_at: '2026-09-22T10:30:00Z' });
const other = t.sources.addItem({ source_key: 'paper-b', title: 'Clipper mission team answers questions about Jupiter cruise', url: 'https://paper-b.example/qa', published_at: '2026-09-22T10:30:00Z' });
const unrelated = t.sources.addItem({ source_key: 'site-c', title: 'Local bakery wins regional sourdough prize', url: 'https://site-c.example/bakery', published_at: '2026-09-22T10:40:00Z' });
const unrelated = t.sources.addItem({ source_key: 'site-c', title: 'Local bakery wins regional sourdough prize', url: 'https://site-c.example/bakery', published_at: '2026-09-22T10:40:00Z' });
await t.pull();
await t.pull();
- const item = (itm) => t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(itm);
- const main = item(reports.a.id).cluster_id;
- const lone = item(unrelated.id).cluster_id;
+ const item = async (itm) => await t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(itm);
+ const main = (await item(reports.a.id)).cluster_id;
+ const lone = (await item(unrelated.id)).cluster_id;
await check('clustering is deterministic: rebuilding from the same items in the same order gives the same grouping', async () => {
await check('clustering is deterministic: rebuilding from the same items in the same order gives the same grouping', async () => {
const { createClusters } = require('../server/domain/clusters');
const { createClusters } = require('../server/domain/clusters');
const cfg = t.ctx.config;
const cfg = t.ctx.config;
- const { openStore } = require('../server/db');
- const store = openStore(':memory:', { now: t.ctx.store.now });
+ const { createStore } = require('../server/db');
+ const fresh = await require('./helpers/db').testDb();
+ const store = createStore(fresh.db, { now: t.ctx.store.now });
const db = store.db;
const db = store.db;
const events = [];
const events = [];
const c2 = createClusters({ store, config: cfg, outbox: { emit: (e) => events.push(e) } });
const c2 = createClusters({ store, config: cfg, outbox: { emit: (e) => events.push(e) } });
- const rows = t.db().prepare("SELECT * FROM news_source_items WHERE status <> 'duplicate' ORDER BY first_seen_at, rowid").all();
+ const rows = await t.db().prepare("SELECT * FROM news_source_items WHERE status <> 'duplicate' ORDER BY first_seen_at, seq").all();
const groups = new Map();
const groups = new Map();
for (const r of rows) {
for (const r of rows) {
- db.prepare('INSERT INTO news_source_items (id, sources_item_id, sources_revision, source_key, headline, outlet, title_key, status, published_at, summary, first_seen_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)')
+ await db.prepare('INSERT INTO news_source_items (id, sources_item_id, sources_revision, source_key, headline, outlet, title_key, status, published_at, summary, first_seen_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)')
.run(r.id, r.sources_item_id, r.sources_revision, r.source_key, r.headline, r.outlet, r.title_key, r.status, r.published_at, r.summary, r.first_seen_at, r.updated_at);
.run(r.id, r.sources_item_id, r.sources_revision, r.source_key, r.headline, r.outlet, r.title_key, r.status, r.published_at, r.summary, r.first_seen_at, r.updated_at);
- const out = c2.assign(db.prepare('SELECT * FROM news_source_items WHERE id = ?').get(r.id));
+ const out = await c2.assign(await db.prepare('SELECT * FROM news_source_items WHERE id = ?').get(r.id));
groups.set(r.id, out.cluster.id);
groups.set(r.id, out.cluster.id);
}
}
const original = new Map(rows.map((r) => [r.id, r.cluster_id]));
const original = new Map(rows.map((r) => [r.id, r.cluster_id]));
const same = (m) => { const byC = new Map(); for (const [i, c] of m) { if (!byC.has(c)) byC.set(c, []); byC.get(c).push(i); } return [...byC.values()].map((v) => v.sort().join(',')).sort(); };
const same = (m) => { const byC = new Map(); for (const [i, c] of m) { if (!byC.has(c)) byC.set(c, []); byC.get(c).push(i); } return [...byC.values()].map((v) => v.sort().join(',')).sort(); };
+ await fresh.close();
assert.deepStrictEqual(same(groups), same(original));
assert.deepStrictEqual(same(groups), same(original));
});
});
@@ -58,16 +60,16 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -58,16 +60,16 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
let mergeAudit;
let mergeAudit;
await check('merge is audited: the absorbed cluster’s items move, it is marked merged', async () => {
await check('merge is audited: the absorbed cluster’s items move, it is marked merged', async () => {
- const before = t.ctx.clusters.members(lone).map((i) => i.id);
+ const before = (await t.ctx.clusters.members(lone)).map((i) => i.id);
const r = await t.api(`/clusters/${main}/merge`, { json: { other: lone, reason: 'testing a merge' } });
const r = await t.api(`/clusters/${main}/merge`, { json: { other: lone, reason: 'testing a merge' } });
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.status, 200, r.text);
mergeAudit = r.json().audit;
mergeAudit = r.json().audit;
assert.strictEqual(mergeAudit.action, 'merge');
assert.strictEqual(mergeAudit.action, 'merge');
assert.deepStrictEqual(mergeAudit.item_ids.sort(), before.sort());
assert.deepStrictEqual(mergeAudit.item_ids.sort(), before.sort());
- assert.strictEqual(item(unrelated.id).cluster_id, main);
- assert.strictEqual(t.ctx.clusters.get(lone).status, 'merged');
- assert.strictEqual(t.ctx.clusters.get(lone).merged_into, main);
- const ev = t.events('news.cluster.updated').filter((e) => e.payload.action === 'merged');
+ assert.strictEqual((await item(unrelated.id)).cluster_id, main);
+ assert.strictEqual((await t.ctx.clusters.get(lone)).status, 'merged');
+ assert.strictEqual((await t.ctx.clusters.get(lone)).merged_into, main);
+ const ev = (await t.events('news.cluster.updated')).filter((e) => e.payload.action === 'merged');
assert.strictEqual(ev.length, 1);
assert.strictEqual(ev.length, 1);
});
});
@@ -75,9 +77,9 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -75,9 +77,9 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
const r = await t.api(`/clusters/audit/${mergeAudit.id}/reverse`, { json: { reason: 'wrong merge' } });
const r = await t.api(`/clusters/audit/${mergeAudit.id}/reverse`, { json: { reason: 'wrong merge' } });
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.json().audit.action, 'reverse_merge');
assert.strictEqual(r.json().audit.action, 'reverse_merge');
- assert.strictEqual(item(unrelated.id).cluster_id, lone);
- assert.strictEqual(item(reports.a.id).cluster_id, main);
- assert.strictEqual(t.ctx.clusters.get(lone).status, 'open');
+ assert.strictEqual((await item(unrelated.id)).cluster_id, lone);
+ assert.strictEqual((await item(reports.a.id)).cluster_id, main);
+ assert.strictEqual((await t.ctx.clusters.get(lone)).status, 'open');
const again = await t.api(`/clusters/audit/${mergeAudit.id}/reverse`, { json: {} });
const again = await t.api(`/clusters/audit/${mergeAudit.id}/reverse`, { json: {} });
assert.strictEqual(again.status, 409);
assert.strictEqual(again.status, 409);
assert.strictEqual(again.json().code, 'cluster.already_reversed');
assert.strictEqual(again.json().code, 'cluster.already_reversed');
@@ -86,26 +88,26 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -86,26 +88,26 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
let splitAudit;
let splitAudit;
let splitId;
let splitId;
await check('split is audited: chosen items move to a new cluster; at least one must stay', async () => {
await check('split is audited: chosen items move to a new cluster; at least one must stay', async () => {
- const all = t.ctx.clusters.members(main).map((i) => i.id);
+ const all = (await t.ctx.clusters.members(main)).map((i) => i.id);
let r = await t.api(`/clusters/${main}/split`, { json: { items: all } });
let r = await t.api(`/clusters/${main}/split`, { json: { items: all } });
assert.strictEqual(r.status, 422);
assert.strictEqual(r.status, 422);
assert.strictEqual(r.json().code, 'cluster.split_all');
assert.strictEqual(r.json().code, 'cluster.split_all');
- const moving = [item(other.id).id];
+ const moving = [(await item(other.id)).id];
r = await t.api(`/clusters/${main}/split`, { json: { items: moving, reason: 'a Q&A is a different story' } });
r = await t.api(`/clusters/${main}/split`, { json: { items: moving, reason: 'a Q&A is a different story' } });
assert.strictEqual(r.status, 201, r.text);
assert.strictEqual(r.status, 201, r.text);
splitAudit = r.json().audit;
splitAudit = r.json().audit;
splitId = r.json().created.id;
splitId = r.json().created.id;
- assert.strictEqual(item(other.id).cluster_id, splitId);
- assert.strictEqual(t.ctx.clusters.get(splitId).split_from, main);
+ assert.strictEqual((await item(other.id)).cluster_id, splitId);
+ assert.strictEqual((await t.ctx.clusters.get(splitId)).split_from, main);
});
});
await check('reversing the split puts the items back and dissolves the empty cluster (through the no-JS form)', async () => {
await check('reversing the split puts the items back and dissolves the empty cluster (through the no-JS form)', async () => {
const csrf = t.csrf(t.editor);
const csrf = t.csrf(t.editor);
const r = await t.get(`/clusters/audit/${splitAudit.id}/reverse`, { as: t.editor, form: { _csrf: csrf, reason: 'undo' } });
const r = await t.get(`/clusters/audit/${splitAudit.id}/reverse`, { as: t.editor, form: { _csrf: csrf, reason: 'undo' } });
assert.strictEqual(r.status, 303, r.text);
assert.strictEqual(r.status, 303, r.text);
- assert.strictEqual(item(other.id).cluster_id, main);
- assert.strictEqual(t.ctx.clusters.get(splitId).status, 'dissolved');
- const audit = t.ctx.clusters.audit(main).map((a) => a.action);
+ assert.strictEqual((await item(other.id)).cluster_id, main);
+ assert.strictEqual((await t.ctx.clusters.get(splitId)).status, 'dissolved');
+ const audit = (await t.ctx.clusters.audit(main)).map((a) => a.action);
assert.deepStrictEqual(audit, ['merge', 'reverse_merge', 'split', 'reverse_split']);
assert.deepStrictEqual(audit, ['merge', 'reverse_merge', 'split', 'reverse_split']);
});
});
@@ -16,7 +16,7 @@ const MOD = 'mod:mod_01J8ZQ4Y7N3M2K1H0G9F8E7D6C';
@@ -16,7 +16,7 @@ const MOD = 'mod:mod_01J8ZQ4Y7N3M2K1H0G9F8E7D6C';
const reader = t.network.addUser('reader');
const reader = t.network.addUser('reader');
const caps = ['news.topic.manage', 'news.story.create'];
const caps = ['news.topic.manage', 'news.story.create'];
const token = (sub, actorType, extra) => t.network.signService({ sub, actorType, aud: ['openvibe.news'], cap: caps, extra });
const token = (sub, actorType, extra) => t.network.signService({ sub, actorType, aud: ['openvibe.news'], cap: caps, extra });
- const topicsNamed = (name) => t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM news_topics WHERE name = ?').get(name).n;
+ const topicsNamed = async (name) => (await t.ctx.store.db.prepare('SELECT COUNT(*) AS n FROM news_topics WHERE name = ?').get(name)).n;
const newTopic = (as, name, headers) => t.get('/api/v1/topics', { as, headers, json: { name } });
const newTopic = (as, name, headers) => t.get('/api/v1/topics', { as, headers, json: { name } });
await check('an app or module cannot act as an editor by naming them in X-OV-Subject', async () => {
await check('an app or module cannot act as an editor by naming them in X-OV-Subject', async () => {
@@ -25,7 +25,7 @@ const MOD = 'mod:mod_01J8ZQ4Y7N3M2K1H0G9F8E7D6C';
@@ -25,7 +25,7 @@ const MOD = 'mod:mod_01J8ZQ4Y7N3M2K1H0G9F8E7D6C';
const r = await newTopic(token(sub, type, extra), `Planted by ${type}`, { 'x-ov-subject': t.editor.subject });
const r = await newTopic(token(sub, type, extra), `Planted by ${type}`, { 'x-ov-subject': t.editor.subject });
assert.strictEqual(r.status, 403, `${type} ${JSON.stringify(extra)}: ${r.text}`);
assert.strictEqual(r.status, 403, `${type} ${JSON.stringify(extra)}: ${r.text}`);
assert.strictEqual(r.json().code, 'subject.not_delegated');
assert.strictEqual(r.json().code, 'subject.not_delegated');
- assert.strictEqual(topicsNamed(`Planted by ${type}`), 0);
+ assert.strictEqual(await topicsNamed(`Planted by ${type}`), 0);
}
}
}
}
});
});
@@ -21,7 +21,7 @@ const BODY = [
@@ -21,7 +21,7 @@ const BODY = [
const t = await boot();
const t = await boot();
const reports = launchReports(t.sources);
const reports = launchReports(t.sources);
await t.pull();
await t.pull();
- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;
+ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;
const open = async (headline, body = BODY) => {
const open = async (headline, body = BODY) => {
const r = await t.api('/stories', { json: { cluster: clusterId, headline, topic: 'space', body } });
const r = await t.api('/stories', { json: { cluster: clusterId, headline, topic: 'space', body } });
assert.strictEqual(r.status, 201, r.text);
assert.strictEqual(r.status, 201, r.text);
@@ -31,9 +31,9 @@ const BODY = [
@@ -31,9 +31,9 @@ const BODY = [
const calls = (re, method) => t.community.calls.filter((c) => re.test(c.url.split('?')[0]) && (!method || c.method === method));
const calls = (re, method) => t.community.calls.filter((c) => re.test(c.url.split('?')[0]) && (!method || c.method === method));
const resolves = () => calls(/\/threads\/resolve$/, 'POST');
const resolves = () => calls(/\/threads\/resolve$/, 'POST');
const hides = () => calls(/\/visibility$/, 'PUT');
const hides = () => calls(/\/visibility$/, 'PUT');
- const settle = () => t.ctx.discussion.idle();
- const refOf = (id) => t.db().prepare('SELECT * FROM news_story_discussion_refs WHERE entity_id = ?').get(id);
- t.community.setProbe(() => t.db().inTransaction);
+ const settle = async () => await t.ctx.discussion.idle();
+ const refOf = async (id) => await t.db().prepare('SELECT * FROM news_story_discussion_refs WHERE entity_id = ?').get(id);
+ t.community.setProbe(() => t.db().stats().open > 0); // a News transaction still open when Community is called
const story = await open('Europa Clipper is on its way to Jupiter');
const story = await open('Europa Clipper is on its way to Jupiter');
const url = `/stories/${story.slug}`;
const url = `/stories/${story.slug}`;
@@ -45,7 +45,7 @@ const BODY = [
@@ -45,7 +45,7 @@ const BODY = [
assert.strictEqual(prev.status, 200, prev.text);
assert.strictEqual(prev.status, 200, prev.text);
assert.doesNotMatch(prev.text, /id="comments"/, 'the editor preview shows no thread');
assert.doesNotMatch(prev.text, /id="comments"/, 'the editor preview shows no thread');
assert.strictEqual(resolves().length, 0);
assert.strictEqual(resolves().length, 0);
- assert.strictEqual(refOf(story.id), undefined);
+ assert.strictEqual(await refOf(story.id), undefined);
});
});
await check('a published story resolves its thread once, for the story’s EntityRef, and stores only the id', async () => {
await check('a published story resolves its thread once, for the story’s EntityRef, and stores only the id', async () => {
@@ -60,7 +60,7 @@ const BODY = [
@@ -60,7 +60,7 @@ const BODY = [
const call = resolves()[0];
const call = resolves()[0];
assert.ok(call.cap.includes('community.comment.write'), 'resolved with the write capability');
assert.ok(call.cap.includes('community.comment.write'), 'resolved with the write capability');
assert.deepStrictEqual(JSON.parse(call.body).ref, { service: 'news', type: 'story', id: story.id, label: 'Europa Clipper is on its way to Jupiter' });
assert.deepStrictEqual(JSON.parse(call.body).ref, { service: 'news', type: 'story', id: story.id, label: 'Europa Clipper is on its way to Jupiter' });
- const ref = refOf(story.id);
+ const ref = await refOf(story.id);
assert.strictEqual(ref.thread_id, '1');
assert.strictEqual(ref.thread_id, '1');
assert.deepStrictEqual(Object.keys(ref).sort(), ['entity_id', 'ref', 'resolved_at', 'thread_id'], 'the reference table has no room for comments');
assert.deepStrictEqual(Object.keys(ref).sort(), ['entity_id', 'ref', 'resolved_at', 'thread_id'], 'the reference table has no room for comments');
page = await t.get(url);
page = await t.get(url);
@@ -90,8 +90,8 @@ const BODY = [
@@ -90,8 +90,8 @@ const BODY = [
const after = await t.get(url);
const after = await t.get(url);
assert.match(after.text, /<strong>Rita Reader<\/strong>/);
assert.match(after.text, /<strong>Rita Reader<\/strong>/);
assert.match(after.text, /Great summary; the 2030 arrival <b>date<\/b> matters\.<br>Thanks\./, 'escaped, read from Community');
assert.match(after.text, /Great summary; the 2030 arrival <b>date<\/b> matters\.<br>Thanks\./, 'escaped, read from Community');
- const dump = t.db().prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all()
- .map(({ name }) => JSON.stringify(t.db().prepare(`SELECT * FROM "${name}"`).all())).join('\n');
+ const names = (await t.db().prepare("SELECT table_name AS name FROM information_schema.tables WHERE table_schema = current_schema() AND table_type = 'BASE TABLE'").all()).map((r) => r.name);
+ const dump = (await Promise.all(names.map(async (name) => JSON.stringify(await t.db().prepare(`SELECT * FROM "${name}"`).all())))).join('\n');
assert.doesNotMatch(dump, /Great summary|2030 arrival/, 'no comment text is copied into News');
assert.doesNotMatch(dump, /Great summary|2030 arrival/, 'no comment text is copied into News');
});
});
@@ -111,7 +111,7 @@ const BODY = [
@@ -111,7 +111,7 @@ const BODY = [
t.community.setDown(true);
t.community.setDown(true);
const p2 = await t.get(`/stories/${fresh.slug}`);
const p2 = await t.get(`/stories/${fresh.slug}`);
assert.match(p2.text, /Comments are unavailable/);
assert.match(p2.text, /Comments are unavailable/);
- assert.strictEqual(refOf(fresh.id), undefined, 'a failed resolve stores no thread id');
+ assert.strictEqual(await refOf(fresh.id), undefined, 'a failed resolve stores no thread id');
t.community.setDown(false);
t.community.setDown(false);
});
});
@@ -136,9 +136,9 @@ const BODY = [
@@ -136,9 +136,9 @@ const BODY = [
const r = await t.get(`${url}/comments`, { as: member, form: { _csrf: t.csrf(member), message: 'Still here?' } });
const r = await t.get(`${url}/comments`, { as: member, form: { _csrf: t.csrf(member), message: 'Still here?' } });
assert.strictEqual(r.status, 409);
assert.strictEqual(r.status, 409);
- const itemA = t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).id;
+ const itemA = (await t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).id;
assert.strictEqual((await t.api(`/stories/${story.id}/sources/${itemA}`, { method: 'DELETE' })).status, 200);
assert.strictEqual((await t.api(`/stories/${story.id}/sources/${itemA}`, { method: 'DELETE' })).status, 200);
- const head = t.ctx.store.revisions.head(story.id).number;
+ const head = (await t.ctx.store.revisions.head(story.id)).number;
const fixed = BODY.split('\n\n').slice(0, 2).map((p) => p.replace('[1, 2]', '[2, 3]')).join('\n\n');
const fixed = BODY.split('\n\n').slice(0, 2).map((p) => p.replace('[1, 2]', '[2, 3]')).join('\n\n');
let rr = await t.api(`/stories/${story.id}/revisions`, { json: { body: fixed, expected_revision: head } });
let rr = await t.api(`/stories/${story.id}/revisions`, { json: { body: fixed, expected_revision: head } });
assert.strictEqual(rr.status, 201, rr.text);
assert.strictEqual(rr.status, 201, rr.text);
@@ -155,7 +155,7 @@ const BODY = [
@@ -155,7 +155,7 @@ const BODY = [
const other = await open('Europa Clipper: the launch in numbers', BODY.split('\n\n').slice(0, 2).map((p) => p.replace(/\[[\d, ]+\]$/, '[1, 2]')).join('\n\n'));
const other = await open('Europa Clipper: the launch in numbers', BODY.split('\n\n').slice(0, 2).map((p) => p.replace(/\[[\d, ]+\]$/, '[1, 2]')).join('\n\n'));
await publish(other);
await publish(other);
await t.get(`/stories/${other.slug}`);
await t.get(`/stories/${other.slug}`);
- const id = refOf(other.id).thread_id;
+ const id = (await refOf(other.id)).thread_id;
let r = await t.api(`/stories/${other.id}/unpublish`, { json: {} });
let r = await t.api(`/stories/${other.id}/unpublish`, { json: {} });
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.status, 200, r.text);
await settle();
await settle();
@@ -51,11 +51,14 @@ async function boot(opts = {}) {
@@ -51,11 +51,14 @@ async function boot(opts = {}) {
const { createApp } = require('../../server/app');
const { createApp } = require('../../server/app');
const quiet = { log() {}, warn() {}, error: (...a) => { if (process.env.VERBOSE) console.error(...a); } };
const quiet = { log() {}, warn() {}, error: (...a) => { if (process.env.VERBOSE) console.error(...a); } };
+ const { createStore } = require('../../server/db');
+ // One database per boot (PGlite, or NEWS_TEST_STORE=pg: the containers); a restart keeps it, like a file did.
+ const testdb = await require('./db').testDb();
let server = null;
let server = null;
let built = null;
let built = null;
async function start() {
async function start() {
const config = configLib.load(env);
const config = configLib.load(env);
- built = createApp({ config, now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow });
+ built = await createApp({ config, store: createStore(testdb.db, { now: clock.now }), now: clock.now, log: opts.log || quiet, limitsNow: opts.limitsNow });
await built.ctx.auth.ensureKey();
await built.ctx.auth.ensureKey();
server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
server = await new Promise((resolve) => { const s = http.createServer(built.app); s.listen(0, '127.0.0.1', () => resolve(s)); });
t.base = `http://127.0.0.1:${server.address().port}`;
t.base = `http://127.0.0.1:${server.address().port}`;
@@ -64,7 +67,7 @@ async function boot(opts = {}) {
@@ -64,7 +67,7 @@ async function boot(opts = {}) {
}
}
async function stop() {
async function stop() {
if (server) await new Promise((r) => server.close(r));
if (server) await new Promise((r) => server.close(r));
- if (built) { built.ctx.worker.stop(); await built.ctx.outbox.stop(); built.ctx.store.close(); }
+ if (built) { built.ctx.worker.stop(); await built.ctx.outbox.stop(); }
server = null; built = null;
server = null; built = null;
}
}
@@ -82,8 +85,8 @@ async function boot(opts = {}) {
@@ -82,8 +85,8 @@ async function boot(opts = {}) {
}
}
/** Rows of event_outbox as parsed envelopes. */
/** Rows of event_outbox as parsed envelopes. */
- function events(type = null) {
- return t.ctx.store.db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all().map((r) => JSON.parse(r.envelope)).filter((e) => !type || e.event_type === type || (type instanceof RegExp && type.test(e.event_type)));
+ async function events(type = null) {
+ return (await t.ctx.store.db.prepare('SELECT envelope FROM event_outbox ORDER BY id').all()).map((r) => (typeof r.envelope === 'string' ? JSON.parse(r.envelope) : r.envelope)).filter((e) => !type || e.event_type === type || (type instanceof RegExp && type.test(e.event_type)));
}
}
async function deliver(event, o = {}) {
async function deliver(event, o = {}) {
@@ -93,15 +96,15 @@ async function boot(opts = {}) {
@@ -93,15 +96,15 @@ async function boot(opts = {}) {
}
}
/** API call as the editor (browser JWT). */
/** API call as the editor (browser JWT). */
- const api = (p, o = {}) => get(`/api/v1${p}`, { as: editor, ...o });
+ const api = async (p, o = {}) => await get(`/api/v1${p}`, { as: editor, ...o });
const t = {
const t = {
network, sources, ai, community, clock, dbPath, editor, get, api, events, deliver, SECRET,
network, sources, ai, community, clock, dbPath, editor, get, api, events, deliver, SECRET,
- pull: () => t.ctx.ingest.pull(),
+ pull: async () => await t.ctx.ingest.pull(),
db: () => t.ctx.store.db,
db: () => t.ctx.store.db,
csrf: (user) => require('../../server/auth/forms').csrfToken({ formSecret: env.NEWS_FORM_SECRET }, user),
csrf: (user) => require('../../server/auth/forms').csrfToken({ formSecret: env.NEWS_FORM_SECRET }, user),
async restart() { await stop(); await start(); },
async restart() { await stop(); await start(); },
- async close() { await stop(); await network.close(); await sources.close(); await ai.close(); await community.close(); fs.rmSync(dir, { recursive: true, force: true }); },
+ async close() { await stop(); await testdb.close(); await network.close(); await sources.close(); await ai.close(); await community.close(); fs.rmSync(dir, { recursive: true, force: true }); },
};
};
await start();
await start();
return t;
return t;
new file mode 100644
new file mode 100644
@@ -0,0 +1,11 @@
@@ -0,0 +1,11 @@
+'use strict';
+/**
+ * A migrated database for one test run (ADR-035), from openvibe-sdk/testing: PGlite by default; with
+ * NEWS_TEST_STORE=pg (npm run test:pg) the PostgreSQL + PgBouncer containers, with roles and a schema of this run's own.
+ */
+const { createTestDb, pgAvailable } = require('openvibe-sdk/testing');
+const { MIGRATIONS } = require('../../server/db');
+
+const testDb = ({ store = process.env.NEWS_TEST_STORE || 'pglite', max = 4 } = {}) => createTestDb({ migrations: MIGRATIONS, store, service: 'news', max });
+
+module.exports = { testDb, pgAvailable };
@@ -46,23 +46,23 @@ async function storyOn(t, headline, items) {
@@ -46,23 +46,23 @@ async function storyOn(t, headline, items) {
const copy = S.addItem({ source_key: 'beta-daily', title: 'Water agency says two new treatment plants will open next spring', url: 'https://beta.example/b/syndicated-water' });
const copy = S.addItem({ source_key: 'beta-daily', title: 'Water agency says two new treatment plants will open next spring', url: 'https://beta.example/b/syndicated-water' });
const beta = S.addItem({ source_key: 'beta-daily', title: 'Budget vote clears way for valley water projects', url: 'https://beta.example/b/budget-vote' });
const beta = S.addItem({ source_key: 'beta-daily', title: 'Budget vote clears way for valley water projects', url: 'https://beta.example/b/budget-vote' });
await t.pull();
await t.pull();
- const row = (it) => t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(it.id);
+ const row = async (it) => await t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(it.id);
const sitemap = async () => (await t.get('/sitemaps/stories.xml')).text;
const sitemap = async () => (await t.get('/sitemaps/stories.xml')).text;
- const indexEvent = (story) => t.events('news.index_document.upserted').filter((e) => e.payload.id === story.id).pop();
+ const indexEvent = async (story) => (await t.events('news.index_document.upserted')).filter((e) => e.payload.id === story.id).pop();
await check('independence is concrete: same Sources source, same publisher domain, same outlet or a copy of one report count once', async () => {
await check('independence is concrete: same Sources source, same publisher domain, same outlet or a copy of one report count once', async () => {
assert.strictEqual(text.publisherDomain('https://tech.thepaper.example/x'), 'thepaper.example');
assert.strictEqual(text.publisherDomain('https://tech.thepaper.example/x'), 'thepaper.example');
assert.strictEqual(text.publisherDomain('https://news.bbc.co.uk/x'), 'bbc.co.uk');
assert.strictEqual(text.publisherDomain('https://news.bbc.co.uk/x'), 'bbc.co.uk');
assert.strictEqual(text.publisherDomain('https://www.bbc.co.uk/x'), 'bbc.co.uk');
assert.strictEqual(text.publisherDomain('https://www.bbc.co.uk/x'), 'bbc.co.uk');
- assert.strictEqual(row(copy).status, 'duplicate', 'the syndicated copy is a near-duplicate of the Alpha report');
- assert.strictEqual(row(copy).duplicate_of, row(wire).id);
- const n = (items) => text.independentSources(items.map(row)).length;
- assert.strictEqual(n([feed1, feed2]), 1);
- assert.strictEqual(n([world, tech]), 1);
- assert.strictEqual(n([wire, copy]), 1);
- assert.strictEqual(n([wire, copy, beta]), 1, 'the copy ties Beta Daily to the Alpha report: still one');
- assert.strictEqual(n([solo, wire]), 2);
- assert.strictEqual(n([solo, feed1, world, wire]), 4);
+ assert.strictEqual((await row(copy)).status, 'duplicate', 'the syndicated copy is a near-duplicate of the Alpha report');
+ assert.strictEqual((await row(copy)).duplicate_of, (await row(wire)).id);
+ const n = async (items) => text.independentSources(await Promise.all(items.map(row))).length;
+ assert.strictEqual(await n([feed1, feed2]), 1);
+ assert.strictEqual(await n([world, tech]), 1);
+ assert.strictEqual(await n([wire, copy]), 1);
+ assert.strictEqual(await n([wire, copy, beta]), 1, 'the copy ties Beta Daily to the Alpha report: still one');
+ assert.strictEqual(await n([solo, wire]), 2);
+ assert.strictEqual(await n([solo, feed1, world, wire]), 4);
});
});
await check('a single-source story is published and readable but noindex, and absent from the sitemap', async () => {
await check('a single-source story is published and readable but noindex, and absent from the sitemap', async () => {
@@ -76,7 +76,7 @@ async function storyOn(t, headline, items) {
@@ -76,7 +76,7 @@ async function storyOn(t, headline, items) {
assert.strictEqual(json.state, 'published');
assert.strictEqual(json.state, 'published');
assert.deepStrictEqual(json.indexability.reasons.map((r) => [r.code, r.detail]), [['unsourced', '1 of 2 sources']]);
assert.deepStrictEqual(json.indexability.reasons.map((r) => [r.code, r.detail]), [['unsourced', '1 of 2 sources']]);
assert.doesNotMatch(await sitemap(), new RegExp(story.slug));
assert.doesNotMatch(await sitemap(), new RegExp(story.slug));
- assert.strictEqual(indexEvent(story).payload.indexability.decision, 'noindex', 'Search is told it is noindex');
+ assert.strictEqual((await indexEvent(story)).payload.indexability.decision, 'noindex', 'Search is told it is noindex');
assert.match((await t.get('/')).text, new RegExp(`href="/stories/${story.slug}"`), 'still listed for readers');
assert.match((await t.get('/')).text, new RegExp(`href="/stories/${story.slug}"`), 'still listed for readers');
});
});
@@ -105,7 +105,7 @@ async function storyOn(t, headline, items) {
@@ -105,7 +105,7 @@ async function storyOn(t, headline, items) {
assert.strictEqual(page.headers.get('x-robots-tag'), null);
assert.strictEqual(page.headers.get('x-robots-tag'), null);
assert.deepStrictEqual((await t.get(`/stories/${story.slug}.json`)).json().indexability, { indexable: true, robots: 'index, follow', reasons: [] });
assert.deepStrictEqual((await t.get(`/stories/${story.slug}.json`)).json().indexability, { indexable: true, robots: 'index, follow', reasons: [] });
assert.match(await sitemap(), new RegExp(`<loc>https://openvibe.news/stories/${story.slug}</loc>`));
assert.match(await sitemap(), new RegExp(`<loc>https://openvibe.news/stories/${story.slug}</loc>`));
- assert.strictEqual(indexEvent(story).payload.indexability.decision, 'index');
+ assert.strictEqual((await indexEvent(story)).payload.indexability.decision, 'index');
});
});
await t.close();
await t.close();
@@ -9,12 +9,12 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -9,12 +9,12 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
(async () => {
(async () => {
const t = await boot();
const t = await boot();
- const count = (table) => t.db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n;
- const byItm = (id) => t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(id);
+ const count = async (table) => (await t.db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).n;
+ const byItm = async (id) => await t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(id);
await check('only topics are seeded: no stories, source items or clusters exist on boot', async () => {
await check('only topics are seeded: no stories, source items or clusters exist on boot', async () => {
- assert.ok(count('news_topics') >= 10);
- for (const table of ['news_stories', 'news_source_items', 'news_story_clusters', 'news_story_revisions', 'news_editorial_flags']) assert.strictEqual(count(table), 0, table);
+ assert.ok(await count('news_topics') >= 10);
+ for (const table of ['news_stories', 'news_source_items', 'news_story_clusters', 'news_story_revisions', 'news_editorial_flags']) assert.strictEqual(await count(table), 0, table);
const home = await t.get('/');
const home = await t.get('/');
assert.match(home.text, /No stories have been published yet/);
assert.match(home.text, /No stories have been published yet/);
});
});
@@ -26,10 +26,10 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -26,10 +26,10 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
const r = await t.pull();
const r = await t.pull();
assert.ok(r.ok, JSON.stringify(r));
assert.ok(r.ok, JSON.stringify(r));
assert.strictEqual(r.counts.created, 3);
assert.strictEqual(r.counts.created, 3);
- assert.strictEqual(count('news_source_items'), 3);
- const a = byItm(reports.a.id);
- const b = byItm(reports.b.id);
- const c = byItm(reports.c.id);
+ assert.strictEqual(await count('news_source_items'), 3);
+ const a = await byItm(reports.a.id);
+ const b = await byItm(reports.b.id);
+ const c = await byItm(reports.c.id);
assert.strictEqual(a.outlet, 'Wire A', 'outlet from the Sources registry name');
assert.strictEqual(a.outlet, 'Wire A', 'outlet from the Sources registry name');
assert.strictEqual(a.cluster_id, b.cluster_id);
assert.strictEqual(a.cluster_id, b.cluster_id);
assert.strictEqual(a.cluster_id, c.cluster_id);
assert.strictEqual(a.cluster_id, c.cluster_id);
@@ -37,88 +37,88 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -37,88 +37,88 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
assert.strictEqual(why.rule, 'shared_terms');
assert.strictEqual(why.rule, 'shared_terms');
assert.ok(why.shared_entities.includes('europa clipper'), JSON.stringify(why));
assert.ok(why.shared_entities.includes('europa clipper'), JSON.stringify(why));
assert.ok(why.score >= 3);
assert.ok(why.score >= 3);
- assert.strictEqual(t.ctx.store.getState('sources_cursor'), reports.c.change_seq);
- assert.strictEqual(t.events('news.source.ingested').length, 3);
- assert.ok(t.events('news.cluster.updated').length >= 3);
+ assert.strictEqual(await t.ctx.store.getState('sources_cursor'), reports.c.change_seq);
+ assert.strictEqual((await t.events('news.source.ingested')).length, 3);
+ assert.ok((await t.events('news.cluster.updated')).length >= 3);
});
});
await check('a second pull (and a replayed page) changes nothing and emits nothing', async () => {
await check('a second pull (and a replayed page) changes nothing and emits nothing', async () => {
- const before = t.events().length;
+ const before = (await t.events()).length;
const r = await t.pull();
const r = await t.pull();
assert.strictEqual(r.counts.created || 0, 0);
assert.strictEqual(r.counts.created || 0, 0);
- t.ctx.store.setState('sources_cursor', 0);
+ await t.ctx.store.setState('sources_cursor', 0);
const again = await t.pull();
const again = await t.pull();
assert.strictEqual(again.counts.unchanged, 3);
assert.strictEqual(again.counts.unchanged, 3);
- assert.strictEqual(t.events().length, before);
- assert.strictEqual(count('news_source_items'), 3);
+ assert.strictEqual((await t.events()).length, before);
+ assert.strictEqual(await count('news_source_items'), 3);
});
});
await check('dedupe: same canonical URL (tracking parameters and www. ignored) is a duplicate', async () => {
await check('dedupe: same canonical URL (tracking parameters and www. ignored) is a duplicate', async () => {
const dup = t.sources.addItem({ source_key: 'paper-b', title: 'Europa Clipper launch: our live coverage', url: 'https://www.wire-a.example/space/europa-clipper-launch/?utm_source=x', published_at: '2026-09-22T11:00:00Z' });
const dup = t.sources.addItem({ source_key: 'paper-b', title: 'Europa Clipper launch: our live coverage', url: 'https://www.wire-a.example/space/europa-clipper-launch/?utm_source=x', published_at: '2026-09-22T11:00:00Z' });
await t.pull();
await t.pull();
- const row = byItm(dup.id);
+ const row = await byItm(dup.id);
assert.strictEqual(row.status, 'duplicate');
assert.strictEqual(row.status, 'duplicate');
- assert.strictEqual(row.duplicate_of, byItm(reports.a.id).id);
+ assert.strictEqual(row.duplicate_of, (await byItm(reports.a.id)).id);
assert.strictEqual(JSON.parse(row.dedupe).rule, 'canonical_url');
assert.strictEqual(JSON.parse(row.dedupe).rule, 'canonical_url');
- assert.strictEqual(row.cluster_id, byItm(reports.a.id).cluster_id, 'a duplicate follows its original into the cluster');
+ assert.strictEqual(row.cluster_id, (await byItm(reports.a.id)).cluster_id, 'a duplicate follows its original into the cluster');
});
});
await check('dedupe: identical Sources content hash is a duplicate', async () => {
await check('dedupe: identical Sources content hash is a duplicate', async () => {
const dup = t.sources.addItem({ source_key: 'site-c', title: 'Syndicated copy', url: 'https://mirror.example/copy', content_hash: reports.b.provenance.content_hash });
const dup = t.sources.addItem({ source_key: 'site-c', title: 'Syndicated copy', url: 'https://mirror.example/copy', content_hash: reports.b.provenance.content_hash });
await t.pull();
await t.pull();
- const row = byItm(dup.id);
+ const row = await byItm(dup.id);
assert.strictEqual(row.status, 'duplicate');
assert.strictEqual(row.status, 'duplicate');
assert.strictEqual(JSON.parse(row.dedupe).rule, 'content_hash');
assert.strictEqual(JSON.parse(row.dedupe).rule, 'content_hash');
- assert.strictEqual(row.duplicate_of, byItm(reports.b.id).id);
+ assert.strictEqual(row.duplicate_of, (await byItm(reports.b.id)).id);
});
});
await check('dedupe: a near-identical headline (2-shingle Jaccard) within 48 h is a duplicate; a different story is not', async () => {
await check('dedupe: a near-identical headline (2-shingle Jaccard) within 48 h is a duplicate; a different story is not', async () => {
const near = t.sources.addItem({ source_key: 'wire-a', title: 'NASA launches Europa Clipper probe to Jupiter moon Europa, officials say', url: 'https://wire-a.example/space/europa-clipper-launch-updated', published_at: '2026-09-22T12:00:00Z' });
const near = t.sources.addItem({ source_key: 'wire-a', title: 'NASA launches Europa Clipper probe to Jupiter moon Europa, officials say', url: 'https://wire-a.example/space/europa-clipper-launch-updated', published_at: '2026-09-22T12:00:00Z' });
const other = t.sources.addItem({ source_key: 'paper-b', title: 'City council approves new bike lanes downtown', url: 'https://paper-b.example/bike-lanes', published_at: '2026-09-22T12:00:00Z' });
const other = t.sources.addItem({ source_key: 'paper-b', title: 'City council approves new bike lanes downtown', url: 'https://paper-b.example/bike-lanes', published_at: '2026-09-22T12:00:00Z' });
await t.pull();
await t.pull();
- const n = byItm(near.id);
+ const n = await byItm(near.id);
assert.strictEqual(n.status, 'duplicate', JSON.stringify(n));
assert.strictEqual(n.status, 'duplicate', JSON.stringify(n));
assert.strictEqual(JSON.parse(n.dedupe).rule, 'near_title');
assert.strictEqual(JSON.parse(n.dedupe).rule, 'near_title');
assert.match(JSON.parse(n.dedupe).detail, /jaccard 0\.\d\d/);
assert.match(JSON.parse(n.dedupe).detail, /jaccard 0\.\d\d/);
- const o = byItm(other.id);
+ const o = await byItm(other.id);
assert.strictEqual(o.status, 'active');
assert.strictEqual(o.status, 'active');
- assert.notStrictEqual(o.cluster_id, byItm(reports.a.id).cluster_id, 'an unrelated story starts its own cluster');
+ assert.notStrictEqual(o.cluster_id, (await byItm(reports.a.id)).cluster_id, 'an unrelated story starts its own cluster');
assert.strictEqual(JSON.parse(o.cluster_reason).rule, 'new_cluster');
assert.strictEqual(JSON.parse(o.cluster_reason).rule, 'new_cluster');
});
});
await check('a failed pull is recorded, relayed once per outage, and creates nothing', async () => {
await check('a failed pull is recorded, relayed once per outage, and creates nothing', async () => {
- const items = count('news_source_items');
- const clusters = count('news_story_clusters');
+ const items = await count('news_source_items');
+ const clusters = await count('news_story_clusters');
t.sources.addItem({ source_key: 'wire-a', title: 'Arrives while Sources is down' });
t.sources.addItem({ source_key: 'wire-a', title: 'Arrives while Sources is down' });
t.sources.setDown(true);
t.sources.setDown(true);
const r1 = await t.pull();
const r1 = await t.pull();
const r2 = await t.pull();
const r2 = await t.pull();
assert.strictEqual(r1.ok, false);
assert.strictEqual(r1.ok, false);
assert.strictEqual(r2.ok, false);
assert.strictEqual(r2.ok, false);
- assert.strictEqual(count('news_source_items'), items);
- assert.strictEqual(count('news_story_clusters'), clusters);
- assert.strictEqual(count('news_stories'), 0);
- const runs = t.db().prepare("SELECT * FROM news_ingest_runs WHERE origin = 'pull' AND state = 'failed'").all();
+ assert.strictEqual(await count('news_source_items'), items);
+ assert.strictEqual(await count('news_story_clusters'), clusters);
+ assert.strictEqual(await count('news_stories'), 0);
+ const runs = await t.db().prepare("SELECT * FROM news_ingest_runs WHERE origin = 'pull' AND state = 'failed'").all();
assert.strictEqual(runs.length, 2);
assert.strictEqual(runs.length, 2);
- assert.strictEqual(t.events('news.source.failed').filter((e) => e.payload.origin === 'pull').length, 1);
+ assert.strictEqual((await t.events('news.source.failed')).filter((e) => e.payload.origin === 'pull').length, 1);
const ready = await t.get('/api/ready');
const ready = await t.get('/api/ready');
assert.match(ready.text, /last pull failed/);
assert.match(ready.text, /last pull failed/);
t.sources.setDown(false);
t.sources.setDown(false);
const r3 = await t.pull();
const r3 = await t.pull();
assert.ok(r3.ok);
assert.ok(r3.ok);
- assert.strictEqual(count('news_source_items'), items + 1);
+ assert.strictEqual(await count('news_source_items'), items + 1);
});
});
await check('webhook: a signed sources.item.created is applied once; a redelivery changes nothing', async () => {
await check('webhook: a signed sources.item.created is applied once; a redelivery changes nothing', async () => {
const it = t.sources.addItem({ source_key: 'site-c', title: 'Europa Clipper team describes first weeks of cruise to Jupiter', url: 'https://site-c.example/cruise', published_at: '2026-09-22T12:30:00Z' });
const it = t.sources.addItem({ source_key: 'site-c', title: 'Europa Clipper team describes first weeks of cruise to Jupiter', url: 'https://site-c.example/cruise', published_at: '2026-09-22T12:30:00Z' });
const ev = { event_type: 'sources.item.created', subject: { type: 'item', id: it.id, revision: 1 }, payload: { item_id: it.id, source_key: 'site-c', category: 'news', revision: 1 } };
const ev = { event_type: 'sources.item.created', subject: { type: 'item', id: it.id, revision: 1 }, payload: { item_id: it.id, source_key: 'site-c', category: 'news', revision: 1 } };
const r = await t.deliver(ev);
const r = await t.deliver(ev);
- assert.strictEqual(r.status, 204, r.text + JSON.stringify(t.db().prepare('SELECT * FROM news_ingest_runs ORDER BY id DESC LIMIT 1').get()));
- assert.ok(byItm(it.id));
- const n = t.events('news.source.ingested').length;
+ assert.strictEqual(r.status, 204, r.text + JSON.stringify(await t.db().prepare('SELECT * FROM news_ingest_runs ORDER BY id DESC LIMIT 1').get()));
+ assert.ok(await byItm(it.id));
+ const n = (await t.events('news.source.ingested')).length;
const again = await fetch(`${t.base}/internal/events`, { method: 'POST', headers: r.headers, body: r.body });
const again = await fetch(`${t.base}/internal/events`, { method: 'POST', headers: r.headers, body: r.body });
assert.strictEqual(again.status, 204);
assert.strictEqual(again.status, 204);
- assert.strictEqual(t.events('news.source.ingested').length, n);
+ assert.strictEqual((await t.events('news.source.ingested')).length, n);
const pulled = await t.pull();
const pulled = await t.pull();
assert.strictEqual(pulled.counts.created || 0, 0, 'the pull backstop sees it as already applied');
assert.strictEqual(pulled.counts.created || 0, 0, 'the pull backstop sees it as already applied');
});
});
@@ -134,38 +134,38 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -134,38 +134,38 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
assert.strictEqual(other.status, 204);
assert.strictEqual(other.status, 204);
const forged = await t.deliver({ event_type: 'sources.item.removed', source: 'blog', payload: { item_id: reports.a.id, reason: 'forged' } });
const forged = await t.deliver({ event_type: 'sources.item.removed', source: 'blog', payload: { item_id: reports.a.id, reason: 'forged' } });
assert.strictEqual(forged.status, 204);
assert.strictEqual(forged.status, 204);
- assert.strictEqual(byItm(reports.a.id).status, 'active', 'only source "sources" may remove an item');
+ assert.strictEqual((await byItm(reports.a.id)).status, 'active', 'only source "sources" may remove an item');
});
});
await check('webhook: when Sources cannot be read the delivery is 503 (retried), recorded, relayed once, and nothing is made', async () => {
await check('webhook: when Sources cannot be read the delivery is 503 (retried), recorded, relayed once, and nothing is made', async () => {
const it = t.sources.addItem({ source_key: 'wire-a', title: 'Delivered while Sources is unreachable' });
const it = t.sources.addItem({ source_key: 'wire-a', title: 'Delivered while Sources is unreachable' });
t.sources.setDown(true);
t.sources.setDown(true);
- const before = count('news_source_items');
- const failedBefore = t.events('news.source.failed').length;
+ const before = await count('news_source_items');
+ const failedBefore = (await t.events('news.source.failed')).length;
const ev = { event_type: 'sources.item.created', payload: { item_id: it.id, source_key: 'wire-a', category: 'news', revision: 1 } };
const ev = { event_type: 'sources.item.created', payload: { item_id: it.id, source_key: 'wire-a', category: 'news', revision: 1 } };
const r1 = await t.deliver(ev, { attempt: 1 });
const r1 = await t.deliver(ev, { attempt: 1 });
const r2 = await t.deliver(ev, { attempt: 2 });
const r2 = await t.deliver(ev, { attempt: 2 });
assert.strictEqual(r1.status, 503);
assert.strictEqual(r1.status, 503);
assert.strictEqual(r2.status, 503);
assert.strictEqual(r2.status, 503);
- assert.strictEqual(count('news_source_items'), before);
- assert.strictEqual(t.events('news.source.failed').length, failedBefore + 1);
- assert.ok(t.db().prepare("SELECT COUNT(*) AS n FROM news_ingest_runs WHERE origin = 'webhook' AND state = 'failed'").get().n >= 2);
+ assert.strictEqual(await count('news_source_items'), before);
+ assert.strictEqual((await t.events('news.source.failed')).length, failedBefore + 1);
+ assert.ok((await t.db().prepare("SELECT COUNT(*) AS n FROM news_ingest_runs WHERE origin = 'webhook' AND state = 'failed'").get()).n >= 2);
t.sources.setDown(false);
t.sources.setDown(false);
});
});
await check('sources.fetch.failed for a news source is recorded and relayed; no text is fabricated', async () => {
await check('sources.fetch.failed for a news source is recorded and relayed; no text is fabricated', async () => {
- const stories = count('news_stories');
- const items = count('news_source_items');
+ const stories = await count('news_stories');
+ const items = await count('news_source_items');
const r = await t.deliver({ event_type: 'sources.fetch.failed', subject: { type: 'source', id: 'wire-a' }, payload: { source_key: 'wire-a', category: 'news', run_id: 'run_9', state: 'timeout', error_code: 'fetch.timeout', http_status: null, consecutive_failures: 3 } });
const r = await t.deliver({ event_type: 'sources.fetch.failed', subject: { type: 'source', id: 'wire-a' }, payload: { source_key: 'wire-a', category: 'news', run_id: 'run_9', state: 'timeout', error_code: 'fetch.timeout', http_status: null, consecutive_failures: 3 } });
assert.strictEqual(r.status, 204);
assert.strictEqual(r.status, 204);
- const run = t.db().prepare("SELECT * FROM news_ingest_runs WHERE origin = 'sources' ORDER BY id DESC").get();
+ const run = await t.db().prepare("SELECT * FROM news_ingest_runs WHERE origin = 'sources' ORDER BY id DESC").get();
assert.strictEqual(run.state, 'timeout');
assert.strictEqual(run.state, 'timeout');
assert.strictEqual(run.source_key, 'wire-a');
assert.strictEqual(run.source_key, 'wire-a');
- const ev = t.events('news.source.failed').pop();
+ const ev = (await t.events('news.source.failed')).pop();
assert.strictEqual(ev.payload.origin, 'sources');
assert.strictEqual(ev.payload.origin, 'sources');
assert.strictEqual(ev.payload.state, 'timeout');
assert.strictEqual(ev.payload.state, 'timeout');
- assert.strictEqual(count('news_stories'), stories);
- assert.strictEqual(count('news_source_items'), items);
+ assert.strictEqual(await count('news_stories'), stories);
+ assert.strictEqual(await count('news_source_items'), items);
const desk = await t.get('/edit', { as: t.editor });
const desk = await t.get('/edit', { as: t.editor });
assert.match(desk.text, /timeout/);
assert.match(desk.text, /timeout/);
});
});
@@ -173,8 +173,8 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
@@ -173,8 +173,8 @@ const { boot, check, done, launchReports } = require('./helpers/boot');
await check('an item without a title is rejected and recorded, never shown', async () => {
await check('an item without a title is rejected and recorded, never shown', async () => {
const it = t.sources.addItem({ source_key: 'wire-a', title: '' });
const it = t.sources.addItem({ source_key: 'wire-a', title: '' });
await t.pull();
await t.pull();
- assert.strictEqual(byItm(it.id), undefined);
- assert.ok(t.db().prepare("SELECT COUNT(*) AS n FROM news_ingest_runs WHERE state = 'rejected' AND sources_item_id = ?").get(it.id).n === 1);
+ assert.strictEqual(await byItm(it.id), undefined);
+ assert.ok((await t.db().prepare("SELECT COUNT(*) AS n FROM news_ingest_runs WHERE state = 'rejected' AND sources_item_id = ?").get(it.id)).n === 1);
});
});
await t.close();
await t.close();
@@ -48,7 +48,7 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.re
@@ -48,7 +48,7 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.re
published_at: '2026-09-22T09:00:00Z',
published_at: '2026-09-22T09:00:00Z',
});
});
await t.pull();
await t.pull();
- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(allowed.id).cluster_id;
+ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(allowed.id)).cluster_id;
let r = await t.api('/stories', { json: { cluster: clusterId, headline: 'Storms delay Europa Clipper launch', topic: 'space', body: [
let r = await t.api('/stories', { json: { cluster: clusterId, headline: 'Storms delay Europa Clipper launch', topic: 'space', body: [
'Storms over Florida delayed the launch of NASA’s Europa Clipper mission, according to reports from three outlets that covered the countdown at Kennedy Space Center. [1, 2, 3]',
'Storms over Florida delayed the launch of NASA’s Europa Clipper mission, according to reports from three outlets that covered the countdown at Kennedy Space Center. [1, 2, 3]',
'NASA said the launch would move to a later window once the weather clears, and the agency did not give a new date in the reports cited here. [2]',
'NASA said the launch would move to a later window once the weather clears, and the agency did not give a new date in the reports cited here. [2]',
@@ -59,9 +59,9 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.re
@@ -59,9 +59,9 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.re
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.status, 200, r.text);
await check('bodies are never stored: no column of News’ database holds the article text', async () => {
await check('bodies are never stored: no column of News’ database holds the article text', async () => {
- const rows = t.db().prepare('SELECT * FROM news_source_items').all();
+ const rows = await t.db().prepare('SELECT * FROM news_source_items').all();
assert.strictEqual(rows.length, 3);
assert.strictEqual(rows.length, 3);
- const dump = JSON.stringify(rows) + JSON.stringify(t.db().prepare('SELECT fields, content FROM news_story_revisions').all());
+ const dump = JSON.stringify(rows) + JSON.stringify(await t.db().prepare('SELECT fields, content FROM news_story_revisions').all());
for (const s of [BODY_SENTINEL, TAIL_SENTINEL, FORBIDDEN_SENTINEL, UNLICENSED_SENTINEL]) assert.ok(!dump.includes(s), `${s} was stored`);
for (const s of [BODY_SENTINEL, TAIL_SENTINEL, FORBIDDEN_SENTINEL, UNLICENSED_SENTINEL]) assert.ok(!dump.includes(s), `${s} was stored`);
const a = rows.find((x) => x.sources_item_id === allowed.id);
const a = rows.find((x) => x.sources_item_id === allowed.id);
assert.ok(a.summary && a.summary.length <= 280, 'the allowed summary is kept, short');
assert.ok(a.summary && a.summary.length <= 280, 'the allowed summary is kept, short');
@@ -81,13 +81,13 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.re
@@ -81,13 +81,13 @@ const longBody = `${BODY_SENTINEL} ${'The full article text goes on and on. '.re
assert.strictEqual(res.status, 200, `${p} → ${res.status}`);
assert.strictEqual(res.status, 200, `${p} → ${res.status}`);
seen += res.text;
seen += res.text;
}
}
- seen += JSON.stringify(t.events());
+ seen += JSON.stringify(await t.events());
for (const s of [BODY_SENTINEL, TAIL_SENTINEL, FORBIDDEN_SENTINEL, UNLICENSED_SENTINEL, 'The full article text goes on']) assert.ok(!seen.includes(s), `${s} leaked`);
for (const s of [BODY_SENTINEL, TAIL_SENTINEL, FORBIDDEN_SENTINEL, UNLICENSED_SENTINEL, 'The full article text goes on']) assert.ok(!seen.includes(s), `${s} leaked`);
const page = await t.get(`/stories/${story.slug}`);
const page = await t.get(`/stories/${story.slug}`);
assert.match(page.text, /Storms delayed the Europa Clipper launch\./, 'the licensed short summary is shown in the source table');
assert.match(page.text, /Storms delayed the Europa Clipper launch\./, 'the licensed short summary is shown in the source table');
const feeds = (await t.get('/feed.xml')).text + (await t.get('/feed.json')).text;
const feeds = (await t.get('/feed.xml')).text + (await t.get('/feed.json')).text;
assert.ok(!feeds.includes('Storms delayed the Europa Clipper launch.'), 'feeds carry our text only, never a source summary');
assert.ok(!feeds.includes('Storms delayed the Europa Clipper launch.'), 'feeds carry our text only, never a source summary');
- const doc = t.events('news.index_document.upserted').pop().payload;
+ const doc = (await t.events('news.index_document.upserted')).pop().payload;
assert.ok(!JSON.stringify(doc).includes('Storms delayed the Europa Clipper launch.'), 'the Search document carries our text only');
assert.ok(!JSON.stringify(doc).includes('Storms delayed the Europa Clipper launch.'), 'the Search document carries our text only');
});
});
@@ -16,7 +16,7 @@ const BODY = [
@@ -16,7 +16,7 @@ const BODY = [
const t = await boot();
const t = await boot();
const reports = launchReports(t.sources);
const reports = launchReports(t.sources);
await t.pull();
await t.pull();
- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;
+ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;
const ed = t.editor;
const ed = t.editor;
const csrf = t.csrf(ed);
const csrf = t.csrf(ed);
let storyId;
let storyId;
@@ -42,7 +42,7 @@ const BODY = [
@@ -42,7 +42,7 @@ const BODY = [
assert.match(desk.headers.get('x-robots-tag'), /noindex/);
assert.match(desk.headers.get('x-robots-tag'), /noindex/);
const r = await t.get(`/clusters/${clusterId}/stories`, { as: ed, form: { headline: 'No token' } });
const r = await t.get(`/clusters/${clusterId}/stories`, { as: ed, form: { headline: 'No token' } });
assert.strictEqual(r.status, 403);
assert.strictEqual(r.status, 403);
- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_stories').get().n, 0);
+ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_stories').get()).n, 0);
});
});
await check('open a story from the cluster page, then write paragraphs with [n] markers', async () => {
await check('open a story from the cluster page, then write paragraphs with [n] markers', async () => {
@@ -68,8 +68,8 @@ const BODY = [
@@ -68,8 +68,8 @@ const BODY = [
assert.strictEqual(r.status, 303, r.text);
assert.strictEqual(r.status, 303, r.text);
r = await t.get(`/edit/stories/${storyId}/perspective`, { as: ed, form: { _csrf: csrf, label: 'Launch coverage', description: 'Reports about the launch itself.' } });
r = await t.get(`/edit/stories/${storyId}/perspective`, { as: ed, form: { _csrf: csrf, label: 'Launch coverage', description: 'Reports about the launch itself.' } });
assert.strictEqual(r.status, 303);
assert.strictEqual(r.status, 303);
- const per = t.db().prepare('SELECT id FROM news_perspectives WHERE story_id = ?').get(storyId).id;
- const item = t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).id;
+ const per = (await t.db().prepare('SELECT id FROM news_perspectives WHERE story_id = ?').get(storyId)).id;
+ const item = (await t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).id;
r = await t.get(`/edit/stories/${storyId}/assign`, { as: ed, form: { _csrf: csrf, item, perspective: per } });
r = await t.get(`/edit/stories/${storyId}/assign`, { as: ed, form: { _csrf: csrf, item, perspective: per } });
assert.strictEqual(r.status, 303);
assert.strictEqual(r.status, 303);
r = await t.get(`/edit/stories/${storyId}/revise`, { as: ed, form: { _csrf: csrf, expectedRevision: '1', headline: 'Europa Clipper begins its trip to Jupiter', body: BODY, topic: 'space' } });
r = await t.get(`/edit/stories/${storyId}/revise`, { as: ed, form: { _csrf: csrf, expectedRevision: '1', headline: 'Europa Clipper begins its trip to Jupiter', body: BODY, topic: 'space' } });
@@ -72,14 +72,16 @@ const { boot, check, done } = require('./helpers/boot');
@@ -72,14 +72,16 @@ const { boot, check, done } = require('./helpers/boot');
});
});
await check('the seed is topics only, idempotent, and never touches an existing topic', async () => {
await check('the seed is topics only, idempotent, and never touches an existing topic', async () => {
- const n = t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get().n;
- t.db().prepare("UPDATE news_topics SET name = 'Space (renamed by an editor)' WHERE slug = 'space'").run();
+ const n = (await t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get()).n;
+ await t.db().prepare("UPDATE news_topics SET name = 'Space (renamed by an editor)' WHERE slug = 'space'").run();
await t.restart();
await t.restart();
- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get().n, n);
- assert.strictEqual(t.db().prepare("SELECT name FROM news_topics WHERE slug = 'space'").get().name, 'Space (renamed by an editor)');
- const out = execFileSync(process.execPath, [path.join(__dirname, '..', 'scripts', 'seed.js')], { env: { ...process.env, NEWS_DB_PATH: t.dbPath }, encoding: 'utf8' });
- assert.match(out, /topics: 0 added, \d+ already present/);
- for (const table of ['news_stories', 'news_source_items', 'news_story_clusters']) assert.strictEqual(t.db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n, 0, table);
+ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_topics').get()).n, n);
+ assert.strictEqual((await t.db().prepare("SELECT name FROM news_topics WHERE slug = 'space'").get()).name, 'Space (renamed by an editor)');
+ // scripts/seed.js is this call on the configured database (DATABASE_URL); here, on the test's own.
+ execFileSync(process.execPath, ['--check', path.join(__dirname, '..', 'scripts', 'seed.js')]);
+ const r = await require('../server/domain/topics').createTopics({ store: t.ctx.store }).seed();
+ assert.deepStrictEqual([r.created, r.existing], [0, n]);
+ for (const table of ['news_stories', 'news_source_items', 'news_story_clusters']) assert.strictEqual((await t.db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).n, 0, table);
});
});
await t.close();
await t.close();
@@ -18,7 +18,7 @@ const BODY = [
@@ -18,7 +18,7 @@ const BODY = [
const t = await boot();
const t = await boot();
const reports = launchReports(t.sources);
const reports = launchReports(t.sources);
await t.pull();
await t.pull();
- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;
+ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;
let r = await t.api('/stories', { json: { cluster: clusterId, headline: 'Europa Clipper heads for Jupiter', topic: 'space', body: BODY } });
let r = await t.api('/stories', { json: { cluster: clusterId, headline: 'Europa Clipper heads for Jupiter', topic: 'space', body: BODY } });
const story = r.json().story;
const story = r.json().story;
await t.api(`/stories/${story.id}/publish`, { json: {} });
await t.api(`/stories/${story.id}/publish`, { json: {} });
@@ -55,10 +55,10 @@ const BODY = [
@@ -55,10 +55,10 @@ const BODY = [
await check('out of sitemaps and Search (tombstone), labelled in listings and feeds, event emitted', async () => {
await check('out of sitemaps and Search (tombstone), labelled in listings and feeds, event emitted', async () => {
assert.doesNotMatch((await t.get('/sitemaps/stories.xml')).text, new RegExp(story.slug));
assert.doesNotMatch((await t.get('/sitemaps/stories.xml')).text, new RegExp(story.slug));
- const del = t.events('news.index_document.deleted');
+ const del = await t.events('news.index_document.deleted');
assert.strictEqual(del.length, 1);
assert.strictEqual(del.length, 1);
assert.strictEqual(del[0].payload.id, story.id);
assert.strictEqual(del[0].payload.id, story.id);
- const ev = t.events('news.story.retracted');
+ const ev = await t.events('news.story.retracted');
assert.strictEqual(ev.length, 1);
assert.strictEqual(ev.length, 1);
assert.strictEqual(ev[0].payload.publication_state, 'retracted');
assert.strictEqual(ev[0].payload.publication_state, 'retracted');
assert.ok(ev[0].payload.note.startsWith('The launch date'));
assert.ok(ev[0].payload.note.startsWith('The launch date'));
@@ -19,7 +19,7 @@ const BODY = [
@@ -19,7 +19,7 @@ const BODY = [
const t = await boot();
const t = await boot();
const reports = launchReports(t.sources);
const reports = launchReports(t.sources);
await t.pull();
await t.pull();
- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;
+ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;
const reader = t.network.addUser('reader');
const reader = t.network.addUser('reader');
let story;
let story;
@@ -42,7 +42,7 @@ const BODY = [
@@ -42,7 +42,7 @@ const BODY = [
assert.strictEqual(story.state, 'draft');
assert.strictEqual(story.state, 'draft');
assert.deepStrictEqual(story.sources.map((s) => [s.n, s.item.sources_item.id]), [[1, reports.a.id], [2, reports.b.id], [3, reports.c.id]]);
assert.deepStrictEqual(story.sources.map((s) => [s.n, s.item.sources_item.id]), [[1, reports.a.id], [2, reports.b.id], [3, reports.c.id]]);
assert.strictEqual(story.head, null);
assert.strictEqual(story.head, null);
- assert.strictEqual(t.events('news.story.created').length, 1);
+ assert.strictEqual((await t.events('news.story.created')).length, 1);
});
});
await check('a paragraph without a source, or citing a number not in the source table, is refused', async () => {
await check('a paragraph without a source, or citing a number not in the source table, is refused', async () => {
@@ -51,7 +51,7 @@ const BODY = [
@@ -51,7 +51,7 @@ const BODY = [
assert.strictEqual(r.json().code, 'story.claim_unsourced');
assert.strictEqual(r.json().code, 'story.claim_unsourced');
r = await t.api(`/stories/${story.id}/revisions`, { json: { body: 'A claim citing a source that is not attached. [9]', expected_revision: 0 } });
r = await t.api(`/stories/${story.id}/revisions`, { json: { body: 'A claim citing a source that is not attached. [9]', expected_revision: 0 } });
assert.strictEqual(r.json().code, 'story.unknown_source');
assert.strictEqual(r.json().code, 'story.unknown_source');
- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_story_revisions').get().n, 0);
+ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_story_revisions').get()).n, 0);
});
});
await check('a story with zero sources cannot be published', async () => {
await check('a story with zero sources cannot be published', async () => {
@@ -63,8 +63,8 @@ const BODY = [
@@ -63,8 +63,8 @@ const BODY = [
const pub = await t.api(`/stories/${lonely.id}/publish`, { json: {} });
const pub = await t.api(`/stories/${lonely.id}/publish`, { json: {} });
assert.strictEqual(pub.status, 422);
assert.strictEqual(pub.status, 422);
assert.strictEqual(pub.json().code, 'story.no_text');
assert.strictEqual(pub.json().code, 'story.no_text');
- assert.strictEqual(t.db().prepare("SELECT COUNT(*) AS n FROM news_stories WHERE state = 'published'").get().n, 0);
- const direct = t.ctx.stories.problems(t.ctx.stories.get(lonely.id), { number: 1, fields: { headline: 'x', paragraphs: [{ text: 'x', sources: [1] }], sources: [] }, meta: {} });
+ assert.strictEqual((await t.db().prepare("SELECT COUNT(*) AS n FROM news_stories WHERE state = 'published'").get()).n, 0);
+ const direct = await t.ctx.stories.problems(await t.ctx.stories.get(lonely.id), { number: 1, fields: { headline: 'x', paragraphs: [{ text: 'x', sources: [1] }], sources: [] }, meta: {} });
assert.ok(direct.some((p) => p.code === 'story.unsourced'));
assert.ok(direct.some((p) => p.code === 'story.unsourced'));
});
});
@@ -121,17 +121,17 @@ const BODY = [
@@ -121,17 +121,17 @@ const BODY = [
assert.deepStrictEqual(json.headline_sources, [1, 2]);
assert.deepStrictEqual(json.headline_sources, [1, 2]);
assert.strictEqual(json.indexability.indexable, true);
assert.strictEqual(json.indexability.indexable, true);
- const cites = t.db().prepare('SELECT anchor, source_item_id FROM news_story_citations WHERE entity_id = ? AND revision = 1 ORDER BY id').all(story.id);
+ const cites = await t.db().prepare('SELECT anchor, source_item_id FROM news_story_citations WHERE entity_id = ? AND revision = 1 ORDER BY id').all(story.id);
assert.deepStrictEqual(cites.filter((c) => c.anchor.startsWith('p')).map((c) => `${c.anchor}:${c.source_item_id}`), [`p1:${reports.a.id}`, `p1:${reports.b.id}`, `p2:${reports.b.id}`, `p3:${reports.c.id}`]);
assert.deepStrictEqual(cites.filter((c) => c.anchor.startsWith('p')).map((c) => `${c.anchor}:${c.source_item_id}`), [`p1:${reports.a.id}`, `p1:${reports.b.id}`, `p2:${reports.b.id}`, `p3:${reports.c.id}`]);
- const idx = t.events('news.index_document.upserted').pop();
+ const idx = (await t.events('news.index_document.upserted')).pop();
assert.ok(idx, 'Search gets the document');
assert.ok(idx, 'Search gets the document');
const doc = idx.payload;
const doc = idx.payload;
assert.ok(contracts.validate('search.index-document@1', doc).valid, JSON.stringify(contracts.validate('search.index-document@1', doc).errors));
assert.ok(contracts.validate('search.index-document@1', doc).valid, JSON.stringify(contracts.validate('search.index-document@1', doc).errors));
const prov = doc.provenance.filter((p) => p.service === 'sources').map((p) => p.id).sort();
const prov = doc.provenance.filter((p) => p.service === 'sources').map((p) => p.id).sort();
assert.deepStrictEqual(prov, [reports.a.id, reports.b.id, reports.c.id].sort());
assert.deepStrictEqual(prov, [reports.a.id, reports.b.id, reports.c.id].sort());
assert.strictEqual(doc.indexability.decision, 'index');
assert.strictEqual(doc.indexability.decision, 'index');
- const pub = t.events('news.story.published');
+ const pub = await t.events('news.story.published');
assert.strictEqual(pub.length, 1);
assert.strictEqual(pub.length, 1);
assert.strictEqual(pub[0].visibility, 'public');
assert.strictEqual(pub[0].visibility, 'public');
assert.strictEqual(pub[0].payload.canonical_url, 'https://openvibe.news/stories/europa-clipper-launches-for-jupiter');
assert.strictEqual(pub[0].payload.canonical_url, 'https://openvibe.news/stories/europa-clipper-launches-for-jupiter');
@@ -167,7 +167,7 @@ const BODY = [
@@ -167,7 +167,7 @@ const BODY = [
assert.match(page.text, /about fifty times/);
assert.match(page.text, /about fifty times/);
assert.match(page.text, /<strong>Correction<\/strong>/);
assert.match(page.text, /<strong>Correction<\/strong>/);
assert.match(page.text, /revision 2\): An earlier version said/);
assert.match(page.text, /revision 2\): An earlier version said/);
- assert.strictEqual(t.events('news.story.updated').length, 1);
+ assert.strictEqual((await t.events('news.story.updated')).length, 1);
const json = (await t.get('/stories/europa-clipper-launches-for-jupiter.json')).json();
const json = (await t.get('/stories/europa-clipper-launches-for-jupiter.json')).json();
assert.strictEqual(json.corrections.length, 1);
assert.strictEqual(json.corrections.length, 1);
assert.strictEqual(json.revision, 2);
assert.strictEqual(json.revision, 2);
@@ -195,8 +195,8 @@ const BODY = [
@@ -195,8 +195,8 @@ const BODY = [
const r = await t.api(`/stories/${story.id}/unpublish`, { json: {} });
const r = await t.api(`/stories/${story.id}/unpublish`, { json: {} });
assert.strictEqual(r.json().story.state, 'unpublished');
assert.strictEqual(r.json().story.state, 'unpublished');
assert.strictEqual((await t.get('/stories/europa-clipper-launches-for-jupiter')).status, 410);
assert.strictEqual((await t.get('/stories/europa-clipper-launches-for-jupiter')).status, 410);
- assert.strictEqual(t.events('news.index_document.deleted').length, 1);
- assert.strictEqual(t.events('news.story.unpublished').length, 1);
+ assert.strictEqual((await t.events('news.index_document.deleted')).length, 1);
+ assert.strictEqual((await t.events('news.story.unpublished')).length, 1);
assert.doesNotMatch((await t.get('/sitemaps/stories.xml')).text, /europa-clipper-launches/);
assert.doesNotMatch((await t.get('/sitemaps/stories.xml')).text, /europa-clipper-launches/);
});
});
@@ -19,7 +19,7 @@ const BODY = [
@@ -19,7 +19,7 @@ const BODY = [
const t = await boot();
const t = await boot();
const reports = launchReports(t.sources, { a: { summary: 'The Europa Clipper spacecraft lifted off on a Falcon Heavy rocket. ORIGINAL-SUMMARY-A.' } });
const reports = launchReports(t.sources, { a: { summary: 'The Europa Clipper spacecraft lifted off on a Falcon Heavy rocket. ORIGINAL-SUMMARY-A.' } });
await t.pull();
await t.pull();
- const clusterId = t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).cluster_id;
+ const clusterId = (await t.db().prepare('SELECT cluster_id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).cluster_id;
let r = await t.api('/stories', { json: { cluster: clusterId, headline: 'Europa Clipper is on its way to Jupiter', topic: 'space', body: BODY } });
let r = await t.api('/stories', { json: { cluster: clusterId, headline: 'Europa Clipper is on its way to Jupiter', topic: 'space', body: BODY } });
const story = r.json().story;
const story = r.json().story;
r = await t.api(`/stories/${story.id}/publish`, { json: {} });
r = await t.api(`/stories/${story.id}/publish`, { json: {} });
@@ -30,16 +30,16 @@ const BODY = [
@@ -30,16 +30,16 @@ const BODY = [
await check('an upstream revision of a cited source: flag + pending revision; the published text is unchanged, with a notice', async () => {
await check('an upstream revision of a cited source: flag + pending revision; the published text is unchanged, with a notice', async () => {
t.sources.updateItem(reports.b.id, { title: 'Europa Clipper lifts off; NASA now says arrival in 2031', summary: 'NASA revised its arrival estimate.' });
t.sources.updateItem(reports.b.id, { title: 'Europa Clipper lifts off; NASA now says arrival in 2031', summary: 'NASA revised its arrival estimate.' });
await t.pull();
await t.pull();
- const flags = t.db().prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND kind = 'source_updated'").all(story.id);
+ const flags = await t.db().prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND kind = 'source_updated'").all(story.id);
assert.strictEqual(flags.length, 1);
assert.strictEqual(flags.length, 1);
assert.strictEqual(flags[0].status, 'open');
assert.strictEqual(flags[0].status, 'open');
assert.strictEqual(flags[0].pending_revision, 2);
assert.strictEqual(flags[0].pending_revision, 2);
- const rev2 = t.ctx.store.revisions.get(story.id, 2);
+ const rev2 = await t.ctx.store.revisions.get(story.id, 2);
assert.strictEqual(rev2.author, 'svc:news');
assert.strictEqual(rev2.author, 'svc:news');
assert.strictEqual(rev2.meta.system.reason, 'source_updated');
assert.strictEqual(rev2.meta.system.reason, 'source_updated');
- assert.strictEqual(rev2.content, t.ctx.store.revisions.get(story.id, 1).content, 'the pending revision keeps the text for an editor to check');
+ assert.strictEqual(rev2.content, (await t.ctx.store.revisions.get(story.id, 1)).content, 'the pending revision keeps the text for an editor to check');
assert.strictEqual(rev2.fields.sources.find((s) => s.n === 2).headline, 'Europa Clipper lifts off; NASA now says arrival in 2031');
assert.strictEqual(rev2.fields.sources.find((s) => s.n === 2).headline, 'Europa Clipper lifts off; NASA now says arrival in 2031');
- const s = t.ctx.stories.get(story.id);
+ const s = await t.ctx.stories.get(story.id);
assert.strictEqual(s.published_revision, 1, 'nothing was published by the system');
assert.strictEqual(s.published_revision, 1, 'nothing was published by the system');
const page = await t.get(url);
const page = await t.get(url);
assert.match(page.text, /changed after publication\. Editors are checking/);
assert.match(page.text, /changed after publication\. Editors are checking/);
@@ -47,18 +47,18 @@ const BODY = [
@@ -47,18 +47,18 @@ const BODY = [
const body = (h) => (h.match(/<div class="story-body">[\s\S]*?<\/div>/) || [null])[0];
const body = (h) => (h.match(/<div class="story-body">[\s\S]*?<\/div>/) || [null])[0];
assert.ok(body(published.text));
assert.ok(body(published.text));
assert.strictEqual(body(page.text), body(published.text), 'the published paragraphs are byte-for-byte the same');
assert.strictEqual(body(page.text), body(published.text), 'the published paragraphs are byte-for-byte the same');
- assert.strictEqual(t.events('news.story.flagged').length, 1);
- assert.strictEqual(t.events('news.story.updated').length, 0);
+ assert.strictEqual((await t.events('news.story.flagged')).length, 1);
+ assert.strictEqual((await t.events('news.story.updated')).length, 0);
});
});
await check('an upstream removal: flag + pending revision; the removed source’s headline, summary and link leave the page; noindex for unsupported claims', async () => {
await check('an upstream removal: flag + pending revision; the removed source’s headline, summary and link leave the page; noindex for unsupported claims', async () => {
t.sources.removeItem(reports.a.id, 'licence withdrawn by the publisher');
t.sources.removeItem(reports.a.id, 'licence withdrawn by the publisher');
const d = await t.deliver({ event_type: 'sources.item.removed', subject: { type: 'item', id: reports.a.id, revision: 2 }, payload: { item_id: reports.a.id, source_key: 'wire-a', category: 'news', revision: 2, reason: 'licence withdrawn by the publisher' } });
const d = await t.deliver({ event_type: 'sources.item.removed', subject: { type: 'item', id: reports.a.id, revision: 2 }, payload: { item_id: reports.a.id, source_key: 'wire-a', category: 'news', revision: 2, reason: 'licence withdrawn by the publisher' } });
assert.strictEqual(d.status, 204, d.text);
assert.strictEqual(d.status, 204, d.text);
- const item = t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id);
+ const item = await t.db().prepare('SELECT * FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id);
assert.strictEqual(item.status, 'removed');
assert.strictEqual(item.status, 'removed');
assert.strictEqual(item.summary, null, 'the licensed summary is dropped');
assert.strictEqual(item.summary, null, 'the licensed summary is dropped');
- const flag = t.db().prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND kind = 'source_removed'").get(story.id);
+ const flag = await t.db().prepare("SELECT * FROM news_editorial_flags WHERE story_id = ? AND kind = 'source_removed'").get(story.id);
assert.strictEqual(flag.status, 'open');
assert.strictEqual(flag.status, 'open');
assert.strictEqual(flag.pending_revision, 3);
assert.strictEqual(flag.pending_revision, 3);
const page = await t.get(url);
const page = await t.get(url);
@@ -75,12 +75,12 @@ const BODY = [
@@ -75,12 +75,12 @@ const BODY = [
assert.strictEqual(json.sources.find((s) => s.n === 1).headline, undefined);
assert.strictEqual(json.sources.find((s) => s.n === 1).headline, undefined);
assert.strictEqual(json.claims[2].supported, false);
assert.strictEqual(json.claims[2].supported, false);
assert.doesNotMatch(JSON.stringify(json), /ORIGINAL-SUMMARY-A|wire-a\.example/);
assert.doesNotMatch(JSON.stringify(json), /ORIGINAL-SUMMARY-A|wire-a\.example/);
- const idx = t.events('news.index_document.upserted').pop();
+ const idx = (await t.events('news.index_document.upserted')).pop();
assert.strictEqual(idx.payload.indexability.decision, 'noindex', 'Search learns the story is noindex');
assert.strictEqual(idx.payload.indexability.decision, 'noindex', 'Search learns the story is noindex');
assert.doesNotMatch((await t.get('/sitemaps/stories.xml')).text, /europa-clipper-is-on-its-way/);
assert.doesNotMatch((await t.get('/sitemaps/stories.xml')).text, /europa-clipper-is-on-its-way/);
const replay = await t.deliver({ event_type: 'sources.item.removed', payload: { item_id: reports.a.id, reason: 'again' } });
const replay = await t.deliver({ event_type: 'sources.item.removed', payload: { item_id: reports.a.id, reason: 'again' } });
assert.strictEqual(replay.status, 204);
assert.strictEqual(replay.status, 204);
- assert.strictEqual(t.db().prepare("SELECT COUNT(*) AS n FROM news_editorial_flags WHERE kind = 'source_removed'").get().n, 1, 'a removal flags once');
+ assert.strictEqual((await t.db().prepare("SELECT COUNT(*) AS n FROM news_editorial_flags WHERE kind = 'source_removed'").get()).n, 1, 'a removal flags once');
});
});
await check('the editor desk shows the open flags, the pending revision and why it cannot be published', async () => {
await check('the editor desk shows the open flags, the pending revision and why it cannot be published', async () => {
@@ -102,13 +102,13 @@ const BODY = [
@@ -102,13 +102,13 @@ const BODY = [
r = await t.api(`/stories/${story.id}/publish`, { json: { revision: 3 } });
r = await t.api(`/stories/${story.id}/publish`, { json: { revision: 3 } });
assert.strictEqual(r.status, 409);
assert.strictEqual(r.status, 409);
assert.strictEqual(r.json().code, 'story.source_removed');
assert.strictEqual(r.json().code, 'story.source_removed');
- assert.strictEqual(t.ctx.stories.get(story.id).published_revision, 1);
+ assert.strictEqual((await t.ctx.stories.get(story.id)).published_revision, 1);
});
});
await check('the editor revises without the removed source; publishing needs a correction note, then resolves both flags', async () => {
await check('the editor revises without the removed source; publishing needs a correction note, then resolves both flags', async () => {
// [1] leaves; the first paragraph now rests on [2] and [3], two independent outlets (the index gate needs two).
// [1] leaves; the first paragraph now rests on [2] and [3], two independent outlets (the index gate needs two).
const fixed = BODY.split('\n\n').slice(0, 2).map((p) => p.replace('[1, 2]', '[2, 3]')).join('\n\n');
const fixed = BODY.split('\n\n').slice(0, 2).map((p) => p.replace('[1, 2]', '[2, 3]')).join('\n\n');
- r = await t.api(`/stories/${story.id}/sources/${t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id).id}`, { method: 'DELETE' });
+ r = await t.api(`/stories/${story.id}/sources/${(await t.db().prepare('SELECT id FROM news_source_items WHERE sources_item_id = ?').get(reports.a.id)).id}`, { method: 'DELETE' });
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.status, 200, r.text);
r = await t.api(`/stories/${story.id}/revisions`, { json: { body: fixed.replace('2030', '2031'), expected_revision: 3 } });
r = await t.api(`/stories/${story.id}/revisions`, { json: { body: fixed.replace('2030', '2031'), expected_revision: 3 } });
assert.strictEqual(r.status, 201, r.text);
assert.strictEqual(r.status, 201, r.text);
@@ -119,21 +119,21 @@ const BODY = [
@@ -119,21 +119,21 @@ const BODY = [
r = await t.api(`/stories/${story.id}/publish`, { json: { revision: n, correction: { kind: 'update', note: 'A source we cited withdrew its report, so the paragraph based only on it was removed; NASA now gives 2031 as the arrival year.' } } });
r = await t.api(`/stories/${story.id}/publish`, { json: { revision: n, correction: { kind: 'update', note: 'A source we cited withdrew its report, so the paragraph based only on it was removed; NASA now gives 2031 as the arrival year.' } } });
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.status, 200, r.text);
assert.strictEqual(r.json().resolved_flags.length, 2);
assert.strictEqual(r.json().resolved_flags.length, 2);
- const open = t.db().prepare("SELECT COUNT(*) AS n FROM news_editorial_flags WHERE story_id = ? AND status = 'open'").get(story.id).n;
+ const open = (await t.db().prepare("SELECT COUNT(*) AS n FROM news_editorial_flags WHERE story_id = ? AND status = 'open'").get(story.id)).n;
assert.strictEqual(open, 0);
assert.strictEqual(open, 0);
const page = await t.get(url);
const page = await t.get(url);
assert.doesNotMatch(page.text, /Editors are (reviewing|checking)/);
assert.doesNotMatch(page.text, /Editors are (reviewing|checking)/);
assert.match(page.text, /<strong>Update<\/strong>/);
assert.match(page.text, /<strong>Update<\/strong>/);
assert.match(page.text, /2031/);
assert.match(page.text, /2031/);
assert.match(page.text, /<meta name="robots" content="index, follow">/);
assert.match(page.text, /<meta name="robots" content="index, follow">/);
- assert.strictEqual(t.events('news.story.updated').length, 1);
+ assert.strictEqual((await t.events('news.story.updated')).length, 1);
});
});
await check('an unchanged upstream refetch (same revision) flags nothing', async () => {
await check('an unchanged upstream refetch (same revision) flags nothing', async () => {
- const before = t.db().prepare('SELECT COUNT(*) AS n FROM news_editorial_flags').get().n;
- t.ctx.store.setState('sources_cursor', 0);
+ const before = (await t.db().prepare('SELECT COUNT(*) AS n FROM news_editorial_flags').get()).n;
+ await t.ctx.store.setState('sources_cursor', 0);
await t.pull();
await t.pull();
- assert.strictEqual(t.db().prepare('SELECT COUNT(*) AS n FROM news_editorial_flags').get().n, before);
+ assert.strictEqual((await t.db().prepare('SELECT COUNT(*) AS n FROM news_editorial_flags').get()).n, before);
});
});
await t.close();
await t.close();