openvibers/openvibe.news Commits
README (purpose, owns, does not own, depends on, grants, acceptance, launch rule, threat review, deploy), STATUS alpha W17, .env.example, deploy (systemd unit, nginx vhost on 4820), CI on Node 22.22.1 (contracts check continue-on-error until release)
Changes
nnew file mode 100644@@@ -0,0 +1,69 @@++# ═══════════════════════════════════════════════════════════════++# openvibe.news — environment configuration (names only; never commit values)++# Copy to .env for development. Production: /etc/openvibe/news.env (0600).++# ═══════════════════════════════════════════════════════════════++++# Server++PORT=4820++HOST=127.0.0.1++NODE_ENV=development++# Public origin: canonical URLs, feeds, sitemaps and JSON-LD are built from it++BASE_URL=http://localhost:4820++# Proxy hops that set X-Forwarded-For (Cloudflare → nginx → Node = 2)++TRUST_PROXY=2++++# SQLite database (production unit: /var/lib/openvibe-news/news.db)++NEWS_DB_PATH=./data/news.db++++# OpenVibe.Network — SSO (OAuth2), JWKS, client-credentials tokens++OV_NETWORK_URL=https://openvibe.network++OV_NETWORK_INTERNAL_URL=http://127.0.0.1:4000++# OAuth client `news` (redirect https://openvibe.news/auth/callback). The same client is News'++# service principal svc:news for client-credentials grants (see README, "Grants").++OV_OAUTH_CLIENT_ID=news++OV_OAUTH_CLIENT_SECRET=++# Production: https://openvibe.news/auth/callback (default: BASE_URL + /auth/callback)++#OV_OAUTH_REDIRECT_URI=++# Cookies: Secure by default in production; set false for plain-http local development++#COOKIE_SECURE=false++# Signs the editor-desk form tokens (CSRF). Unset: random per process.++#NEWS_FORM_SECRET=++++# Editors: Network subjects (usr_…), comma-separated. Network admins are editors too.++NEWS_EDITORS=++++# OpenVibe.Sources — news items (sources.item.read; optionally sources.source.read for outlet names)++OV_SOURCES_INTERNAL_URL=http://127.0.0.1:4720++# Cursor pull (backstop for missed webhook deliveries); 0 turns it off++#NEWS_PULL_INTERVAL_MS=300000++#NEWS_PULL_PAGE_SIZE=100++#NEWS_PULL_MAX_PAGES=20++#NEWS_SOURCES_TIMEOUT_MS=8000++++# Licensing: the longest source summary kept (and only when the source's terms allow short summaries)++#NEWS_SUMMARY_MAX_CHARS=280++# Clustering window++#NEWS_CLUSTER_WINDOW_HOURS=72++++# OpenVibe.Events — the outbox relay runs only when EVENTS_URL and OV_OAUTH_CLIENT_SECRET are set++#EVENTS_URL=http://127.0.0.1:4300++#EVENTS_RELAY_INTERVAL_MS=2000++# Signing secret(s) of News' subscriptions (sources.item.*, sources.fetch.failed), comma-separated++# during rotation. Generate before `npm run subscribe` (e.g. `openssl rand -hex 32`).++# Unset = webhook off (503); the cursor pull still runs.++#NEWS_EVENTS_SECRET=++++# OpenVibe.AI (optional) — news.summarize_story / news.compare_perspectives drafts (ai.run.create).++# Unset: no AI; editors write every story from the source items.++#OV_AI_INTERNAL_URL=http://127.0.0.1:4700++#NEWS_AI_WAIT_MS=30000++++# Background worker (the Sources pull); off: NEWS_WORKER=off++#NEWS_WORKER=on++++# Browser origins that may call /api/v1 with a Bearer Network JWT++#API_CORS_ORIGINS=https://openvibe.network,https://openvibe.live,https://openvibe.community++++# release.json (openvibe-shared/release): the deployed commit when git is not available++#RELEASE_COMMIT=++#MIN_CLIENT_RELEASE=nnew file mode 100644@@@ -0,0 +1,20 @@++name: ci++on:++ push:++ branches: [main]++ pull_request:++jobs:++ test:++ runs-on: ubuntu-latest++ steps:++ - uses: actions/checkout@v4++ - uses: actions/setup-node@v4++ with:++ node-version: 22.22.1 # production runs Node 22++ cache: npm++ - run: npm ci --no-audit --no-fund++ - run: npm test++ # News' service manifest and capabilities are proposals (docs/) until the lead releases them++ # in openvibe-contracts; until then the check reports "no service manifest news" (placeholder).++ - run: npx openvibe-contracts-check --service news --src server++ continue-on-error: true@@@ -1,75 +1,307 @@# OpenVibe.News--> Source-backed stories: clustered coverage, cited summaries, perspectives and timelines.++> Source-backed stories: clustered coverage, cited paragraphs, perspectives, timelines and a++> public correction history.--**Status:** placeholder — planning only, no runnable code yet.--**Domain:** `openvibe.news`--**Plan:** OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §12.7.++**Status:** alpha (roadmap Wave 17, News half). The service runs and its tests pass. It is **not++deployed**, `openvibe.news` still shows its placeholder from OpenVibe.Sites, and its capabilities and++service manifest are proposals that the next openvibe-contracts release has to include. It has not++yet ingested from the running OpenVibe.Sources service (every Sources seed is disabled).++**Domain:** `openvibe.news` · **Port:** 4820 · **Service id:** `news`++**Plan:** OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §12.7; roadmap §15.13, §29, §32.**License:** AGPL-3.0 (same as every OpenVibe service).## Purpose--A publication product, not an autonomous headline generator: normalised source items, story clusters, editorial/publication state, perspective groupings and timelines with full provenance.++A publication product, not an autonomous headline generator. OpenVibe.Sources collects news items++(headline, link, outlet, dates, terms). News normalises them, removes duplicates, groups them into++explainable clusters, and gives editors the tools to write stories from them. Every paragraph an++editor publishes cites the source items it rests on. A story with no sources cannot be published.++When a source changes or is withdrawn upstream, the story is flagged and a pending revision is++prepared; published text never changes silently. Full article text from sources is never stored,++shown, fed or indexed.++++News publishes nothing until an editor does: the only seed is the topic list.## Owns--- `news_topics`, `news_source_items`, `news_story_clusters`, `news_stories`, `news_story_revisions`, `news_story_sources`, `news_perspectives`, `news_timeline_entries`, `news_editorial_flags`++The nine charter tables live in News' own SQLite (`NEWS_DB_PATH`):++++| Charter table | What it is |++|---|---|++| `news_topics` | Owned by News. Canonical topic identities (slug, name, description). The only seed (`seeds/topics.json`). |++| `news_source_items` | Owned by News. One row per Sources item (`sources_item_id` is a typed reference to `itm_…`): headline, canonical URL, outlet, authors, `published_at` as the source states it, the licence and terms notes, and a short summary **only** when those notes allow short summaries. Dedupe outcome (`status`, `duplicate_of`, `dedupe`), cluster and the stored reason for it, upstream revision and removal. No article body. |++| `news_story_clusters` | Owned by News. Clusters with their key terms and entities, time window, status (`open`, `merged`, `dissolved`) and lineage (`merged_into`, `split_from`). |++| `news_stories` | Owned by News. Publication state: slug, state (`draft`, `published`, `unpublished`, `retracted`), published revision, topic, cluster. |++| `news_story_revisions` | `openvibe-publishing/revisions` (prefix `news_story`). Immutable. Each revision holds the headline, the paragraphs with their source numbers, and a snapshot of the source table, timeline and perspectives. |++| `news_story_sources` | Owned by News. The story's source table: which items, their stable numbers `[n]` (never reused), perspective grouping, detachment. |++| `news_perspectives` | Owned by News. Editor-written labels that group a story's sources (for example an outlet group or a position a source states). Never generated. |++| `news_timeline_entries` | Owned by News. Dated entries; each date is one the source states and each entry rests on one attached source. |++| `news_editorial_flags` | Owned by News. `correction`, `update`, `retraction` (public, with notes) and `source_updated`, `source_removed` (upstream changes, with the pending revision). |++++Other tables in the same database:++++- Package companions: `news_story_citations` (one row per paragraph and timeline entry and cited++ Sources item, append-only), `news_story_reviews` (a person's approval of AI output),++ `news_story_drafts`, `news_story_revision_purges`, `news_index_revisions` (Search sequencer).++- `news_cluster_audit`: every merge, split and reversal with exactly which items moved.++- `news_ingest_runs`: what every webhook delivery, pull and upstream fetch failure did.++- `news_state` (the Sources cursor), `news_source_status` (display cache of Sources' registry++ names and health), `event_outbox` and `idempotency_receipts` (openvibe-sdk),++ `subject_projections` (Network names). None of them is authority for anything outside News.## Does not own--- source adapter policy (shared source registry)--- discussion (Community)++- **Source adapters and the source registry.** OpenVibe.Sources owns them: which sites are read,++ under which terms, how and when. News reads items with `sources.item.read` and never fetches a++ site itself.++- **Discussion.** OpenVibe.Community. (News does not attach comment threads yet.)++- **AI generation.** OpenVibe.AI. News only receives drafts.++- **Search.** OpenVibe.Search indexes what News sends through Events.++- **Identity.** OpenVibe.Network: SSO, subjects and service principals.++++## What works++++- **Ingestion** from OpenVibe.Sources, two ways with one effect:++ - an Events webhook (`POST /internal/events`, signed with `NEWS_EVENTS_SECRET`) for++ `sources.item.created|updated|removed` and `sources.fetch.failed`, each applied exactly once++ through the SDK inbox;++ - a cursor pull (`GET /api/v1/items?category=news&after=…&include_removed=1`) every++ `NEWS_PULL_INTERVAL_MS`, the backstop for missed deliveries.++ Applying an item is idempotent on (Sources item id, revision). Failures are recorded states++ (`news_ingest_runs`, `news.source.failed`, `/api/ready`, the editor desk) and never create a++ source item, a cluster or any story text. When Sources cannot be read, the delivery is answered++ 503 so Events retries it.++- **Licensing.** Only headline, URL, outlet, authors, `published_at`, the notes and — when the++ item's terms or licence note explicitly allows short summaries and nothing forbids them — a++ summary of at most `NEWS_SUMMARY_MAX_CHARS` (280) are kept. Anything else a Sources item carries++ (for example an article body an adapter mapped into `fields`) is never read. A removed item's++ summary is dropped. Source summaries appear only in a story's source table; never in feeds,++ Search documents or events.++- **Dedupe** (explainable, stored on the item): the same normalised canonical URL (tracking++ parameters, `www.`, trailing slashes ignored), the same Sources content hash, or a near-duplicate++ headline (Jaccard of word 2-shingles ≥ 0.9, or ≥ 0.75 from the same outlet, within 48 h).++ Duplicates are kept for provenance, join their original's cluster and are not attached to new++ stories by default.++- **Clusters** (deterministic): an item joins the open cluster, within `NEWS_CLUSTER_WINDOW_HOURS`++ (72), with which it shares at least one named entity and two terms, or at least three terms;++ entities score 2, terms 1; ties go to the oldest cluster. The shared entities, shared terms and++ score are stored on the item and shown on the cluster page. Editors can merge clusters and split++ items off; both are audited and reversible (`news_cluster_audit`).++- **Stories** (editorial):++ - An editor opens a story from a cluster; its live, non-duplicate items become sources++ `[1]`, `[2]`, … . Editors can attach and detach sources; numbers are never reused.++ - Paragraphs end with the source numbers they rest on (`… [1, 3]`). A revision with an uncited++ paragraph, or citing a number not in the source table, is refused.++ - Revisions are immutable with optimistic concurrency (412 on a stale base), diff and preview.++ - Publishing is refused while the revision has no sources (`story.unsourced`), cites a source++ removed upstream or detached, or is AI-generated or AI-assisted without a person's approval.++ - Timeline entries have a date the source states and one source each. Perspectives are labels++ an editor writes; readers can re-frame the source table by number, outlet or perspective++ (`?group=`), without JavaScript.++ - Corrections and updates are notes published with a revision and shown in the story's++ correction history. Retraction keeps the story at its URL with a retraction notice, noindex,++ out of sitemaps and Search, labelled in listings and feeds; it is final.++- **Upstream changes.** When a cited source is revised or removed in Sources, every story citing it++ gets an open flag and a pending revision prepared by the system (same text, refreshed source++ entry). The published revision stays exactly as it was, with a notice to readers. A removed++ source's headline, summary and link leave the public page at once; a paragraph resting only on++ it is marked, and the gate makes the story noindex (`unsupported_claims`) until an editor++ publishes a revision that no longer rests on it — with a correction or update note.++- **AI** (optional seams, never publication truth):++ - OpenVibe.AI may deliver a draft revision with `X-OV-Origin: ai` (`news.story.revise`), naming++ `news.summarize_story` or `news.compare_perspectives` and its run.++ - An editor may ask for one (`POST /api/v1/stories/:id/ai-drafts`, or the desk's button when++ `OV_AI_INTERNAL_URL` is set). Only News' stored fields are sent. Claims the model did not cite++ are dropped and listed as gaps. A failed run makes no text. AI never creates perspective labels.++ - AI-generated revisions, and AI-assisted ones (an editor's edit of AI text), cannot be published++ and are hidden from indexes until a person (`usr_`) records an approving review of that++ revision. Stub-provider output is held too. The disclosure is on the page, in the JSON and in++ the Search document.++ - With no AI configured, editors write every story from the source items.++++### Routes (server-rendered, useful without JavaScript)++++| Route | What |++|---|---|++| `/` | published stories, newest first (retracted ones stay, labelled) and the topics |++| `/topics`, `/topics/:slug` | topics and each topic's stories |++| `/stories/:slug`, `/stories/:slug.json` | a story (text with source numbers, source table, timeline, perspectives, corrections), and the same story as data |++| `/feed.xml`, `/atom.xml`, `/feed.json`, `/topics/:slug/{feed.xml,atom.xml,feed.json}` | feeds (News' own text only) |++| `/sitemap.xml` → `/sitemaps/stories.xml`, `/sitemaps/topics.xml` | indexable stories and topics only |++| `/robots.txt`, `/llms.txt` | the automated-consumer policy and machine orientation |++| `/edit`, `/edit/stories/:id`, `/clusters/:id` | the editor desk (forms, Network SSO, form tokens; editors only; private, noindex) |++| `/auth/*` | sign-in; the same session layer as OpenVibe.Community and OpenVibe.Blog |++| `/internal/events` | the Events webhook (loopback; nginx does not proxy it) |++| `/api/health`, `/api/ready`, `/release.json`, `/metrics` | health, truthful readiness, release, metrics (`/metrics` loopback only) |++++### Discoverability (roadmap §32)++++- Robots meta, canonical and `X-Robots-Tag` come from the `openvibe-publishing/seo` gate with++ explicit reasons. News' policy: at least one live cited source, at least 40 words; unsupported++ claims, retraction, an editor's noindex and unreviewed AI output are never indexable.++- JSON-LD `NewsArticle` from real fields only: headline, dates, the editors named in the++ revision's authorship (only when the Network name is known), the section (topic), the cited++ source URLs, and OpenVibe.News as publisher. Missing fields are omitted.++- Drafts, unpublished, retracted and noindex stories never appear in sitemaps or Search; feeds++ list published and (labelled) retracted stories, never a draft.++++### Caching--## Planned surfaces++HTML and JSON vary on `Cookie` and `Authorization`. Published stories and lists rendered for an++anonymous visitor are `public, max-age=60`; everything else (signed-in views, the desk, previews,++refusals, the API) is `private, no-store` with `X-Robots-Tag`. Feeds and sitemaps are public for++five minutes and are never built for a viewer.--- ingest with visible fetch/parse status, dedupe/cluster, factual fields kept separate from synthesis, cited summaries, perspective views, timelines, human review / noindex states++### Events (SDK outbox, same transaction as the change)--## Data (authority tables / families)++| Event | Notes |++|---|---|++| `news.source.ingested` | internal; per Sources item created, revised or removed, with the dedupe and cluster outcome |++| `news.source.failed` | internal; a Sources fetch failure of a news source, an unreadable item, or a failed pull (once per outage) |++| `news.cluster.updated` | internal; created, grew, merged, split, reversed |++| `news.story.created`, `news.story.flagged` | internal |++| `news.story.published`, `.updated`, `.unpublished` | `openvibe-publishing/index-hooks` `publicationEvent`; canonical URL, state, indexability, topic; never the text |++| `news.story.retracted` | public for a listable story; the retraction note |++| `news.index_document.upserted` / `.deleted` | `search.index-document@1` documents (provenance: the Sources items and the AI run) and tombstones, with a monotonic index revision |--- see above++### Capabilities (proposed: `docs/capabilities-proposal/`)--## Capabilities and events++Service tokens use audience `openvibe.news`, one capability per route; the editor the service acts++for goes in `X-OV-Subject` and must be an editor:--- `news.story.create|revise|publish`, `news.cluster.read`, `news.source.attach`, `news.timeline.update`++- `news.story.create`, `news.story.read`, `news.story.revise`, `news.story.publish`, `news.story.retract`++- `news.cluster.read`, `news.cluster.manage`++- `news.source.attach`, `news.timeline.update`, `news.perspective.update`++- `news.topic.manage`--Events: ``news.source.ingested|failed``, ``news.cluster.updated``, ``news.story.published|updated|retracted``++Browser and app user JWTs are judged as editors (`NEWS_EDITORS` subjects and Network admins).++Until the proposals are released, grants for these ids are decided locally with the contracts++library's matching rule (`server/auth/capabilities.js`). The service manifest proposal is++`docs/service-manifest-proposal.json`.## Depends on--- source registry--- OpenVibe.AI--- Search--- OpenVibe.Community--- OpenVibe.Events++- **Packages** (pinned by release tarball): `openvibe-publishing` v0.2.0 (revisions, citations,++ authorship, seo, index-hooks, ssr, taxonomy slugify), `openvibe-contracts` v0.19.0,++ `openvibe-shared` v1.3.0 (chrome, app icon, footer, legal, release, metrics, ready, seo),++ `openvibe-sdk` v0.2.2 (events outbox and inbox, webhook signatures, service tokens).++- **OpenVibe.Sources** (4720): `sources.item.read`; optionally `sources.source.read` (outlet++ names and source health; without it the outlet is the URL's host).++- **OpenVibe.Events** (4300): `events.event.publish`; `events.subscription.manage` to create the++ `sources.item.*` and `sources.fetch.failed` subscriptions (`npm run subscribe`).++- **OpenVibe.Network** (4000): SSO (OAuth client `news`, redirect `https://openvibe.news/auth/callback`),++ JWKS, `identity.subject.resolve` for editor names.++- **OpenVibe.Search**: consumes `news.index_document.*` through its `*.index_document.*`++ subscription; `news` must be in `SEARCH_EVENT_OWNERS`.++- **Optional:** OpenVibe.AI (4700) with `ai.run.create` for namespace `news`.--## Acceptance (must be true before "done")++### Grants the Network must hold--- every published synthesized claim traces to source records--- failed ingestion never fabricates replacement facts--- source correction/removal triggers a revision--- licensed/private material never leaks into public text++Each grant is `[client, capability, audience]`:--## Bootstrap / extraction source++- `[news, sources.item.read, openvibe.sources]`++- `[news, sources.source.read, openvibe.sources]` (optional: outlet names and health)++- `[news, events.event.publish, openvibe.events]`++- `[news, events.subscription.manage, openvibe.events]`++- `[news, identity.subject.resolve, openvibe.network]`++- `[news, ai.run.create, openvibe.ai]`, namespace `news` (optional: only with `OV_AI_INTERNAL_URL`)++- For OpenVibe.AI to deliver drafts: `[ai, news.story.revise, openvibe.news]`, and++ `[ai, news.story.read, openvibe.news]` if it reads stories back.--No current implementation; Wave 15 after the Sources/AI/publishing primitives exist.++## Acceptance (automated: `npm test`)++++| Charter / roadmap requirement | Test |++|---|---|++| Every published claim traces to source records: each paragraph's numbers resolve to Sources items in the page, the JSON, the citations table and the Search document's provenance. | `test/stories.test.js` |++| A story with zero sources cannot be published; an uncited paragraph is refused. | `test/stories.test.js` |++| Failed ingestion never fabricates: failed pulls, unreadable deliveries, `sources.fetch.failed` and title-less items are recorded and create no item, cluster or story text. | `test/ingest.test.js` |++| A source correction or removal triggers a flag and a pending revision; published text is unchanged; a removed source leaves the page; publishing a revision that rests on it is refused; the editor's revision resolves the flags with a note. | `test/upstream.test.js` |++| Licensed material never leaks: bodies and summaries beyond the allowance never reach the database, pages, JSON, feeds, sitemaps, the API, events or Search. | `test/licensing.test.js` |++| Dedupe by canonical URL, content hash and near-duplicate headline; idempotent replays. | `test/ingest.test.js` |++| Clusters are deterministic and explained; merges and splits are audited and reversible. | `test/clusters.test.js` |++| Retraction: visible at its URL with the notice, noindex with its reason, out of sitemaps and Search, event emitted, final. | `test/retraction.test.js` |++| Useful without JavaScript: the whole editorial journey with forms, and public pages complete in HTML. | `test/nojs.test.js` |++| AI output is a draft that needs a person's review (AI-assisted too); uncited claims dropped; failure makes no text. | `test/ai.test.js` |++| The contract proposals are valid and match the code; every emitted event type is declared. | `test/contracts.test.js` |++| Health, readiness, release, robots, llms.txt, sitemaps, legal, problems, CORS, topics-only seed. | `test/ops.test.js` |++++Not yet demonstrated: ingestion from the deployed OpenVibe.Sources (its seeds are disabled until a++person verifies their terms), delivery through the running OpenVibe.Events, and an OpenVibe.AI run++(the AI service is still being built; the seam is tested against a mock of its runs API).## Launch rule--This repository does not make the product real, and the domain keeps its placeholder page on--[OpenVibers/OpenVibe.Sites](https://github.com/OpenVibers/OpenVibe.Sites) until all of the--following exist here (plan §12.12):----1. an owning runtime with health/readiness endpoints and observability;--2. canonical identity/auth integration (OpenVibe.Network subjects, scoped service principals);--3. server-rendered or static public routes that are useful without JavaScript;--4. real persistence and end-to-end workflows;--5. capability and event registration against `OpenVibe.Contracts`;--6. a migration/seed strategy, a security/threat review, and sitemap/robots/feed behaviour;--7. acceptance tests proving the advertised functionality.----The launch release removes the domain from `OpenVibe.Sites/sites.json`, switches routing and--registers maturity in the ecosystem registry atomically. A placeholder is never counted as an--implemented service.++This repository alone doesn't make the product live. `openvibe.news` keeps its placeholder on++[OpenVibers/OpenVibe.Sites](https://github.com/OpenVibers/OpenVibe.Sites) until all of plan §12.12++exists:++++1. **Runtime, health, readiness, observability:** done.++2. **Canonical identity and auth:** done (Network SSO, subjects, service tokens).++3. **SSR public routes useful without JS:** done.++4. **Persistence and end-to-end workflows:** done in tests; not yet against live Sources items.++5. **Capability and event registration against OpenVibe.Contracts:** proposals are in `docs/`,++ waiting on the release.++6. **Migration and seed strategy, threat review, sitemap/robots/feed behaviour:** done. Nothing++ to migrate; the seed is topics only; the threat review is below.++7. **Acceptance tests:** done.++++The launch release removes `openvibe.news` from `OpenVibe.Sites/sites.json`, switches routing++(nginx vhost, DNS, TLS), flips the Network hub entry and registers maturity in the ecosystem++registry, together. A placeholder never counts as an implemented service, and this README doesn't++call the service live.++++## Security and threat review++++- **Identity:** only verified Network JWTs (offline RS256) and service tokens for audience++ `openvibe.news`. A bad service token is refused, never downgraded. Identity never comes from a++ body or query; `X-OV-*` headers are ignored for browsers. AI deliveries can only write drafts.++- **Webhook:** HMAC-SHA256 over the raw body with `NEWS_EVENTS_SECRET` (rotation: comma-separated),++ constant-time comparison, `evt_` ids, exactly-once inbox. Only events from source `sources` are++ applied. nginx never proxies `/internal/`.++- **CSRF:** SameSite=Lax session cookie plus an HMAC form token on every desk form.++- **XSS:** all HTML goes through `openvibe-publishing/ssr` auto-escaping; source links get++ `rel="noopener nofollow"`; helmet CSP.++- **SSRF:** News makes no outbound request to a URL from content. It calls only its configured++ Network, Sources, Events and AI hosts.++- **Licensing and privacy:** see "Licensing" above; removed sources are masked everywhere public.++ The editor view shows removal reasons; the public view does not.++- **Fabrication:** no seeded or generated stories, ratings or dates; dates come from sources or an++ editor citing one; empty feeds use a real time; JSON-LD omits unknown fields.++- **Abuse:** rate limits on `/auth`, `/edit`, `/clusters` and `/api/v1`, in Express and in the++ nginx reference.++- **Known gaps:** no Community comment threads on stories yet; no per-story Media attachments; the++ Sources registry name is cached and not refreshed after the first successful read.++++## Development++++```bash++fnm exec --using=22.22.1 npm install++fnm exec --using=22.22.1 npm test # temp databases and in-process mocks, no network++fnm exec --using=22.22.1 npm run dev # http://localhost:4820 (set OV_OAUTH_CLIENT_SECRET to sign in)++```++++## Deploy (for the lead)++++1. **Code and config:** put the code at `/opt/openvibe.news` and run `npm ci --omit=dev` on Node 22.++ Create `/etc/openvibe/news.env` (0600) from `.env.example` with `OV_OAUTH_CLIENT_SECRET`,++ `NEWS_EDITORS`, `NEWS_FORM_SECRET`, `NEWS_EVENTS_SECRET` (`openssl rand -hex 32`),++ `EVENTS_URL=http://127.0.0.1:4300`, `OV_SOURCES_INTERNAL_URL=http://127.0.0.1:4720`,++ `BASE_URL=https://openvibe.news`, and optionally `OV_AI_INTERNAL_URL`.++2. **Network:** create (or give a secret to) the OAuth client `news` with redirect++ `https://openvibe.news/auth/callback`, and add the grants listed above.++3. **Search:** make sure `news` is in `SEARCH_EVENT_OWNERS`.++4. **systemd:** install `deploy/systemd/openvibe-news.service` (port 4820, `StateDirectory=openvibe-news`).++5. **nginx:** install `deploy/nginx/openvibe.news.conf`. `/metrics` and `/internal/` are never proxied.++6. **Seed and subscribe:** `npm run seed` (topics only; also done on boot), then++ `npm run subscribe` (creates the `sources.item.*` and `sources.fetch.failed` subscriptions to++ `http://127.0.0.1:4820/internal/events`). The cursor pull starts on its own.++7. **Sources:** News shows nothing until a news source is enabled in OpenVibe.Sources (terms++ verified by a person) and an editor publishes a story.++8. **Contracts:** release the capability and manifest proposals in openvibe-contracts; then CI's++ contracts check can drop `continue-on-error`.++9. **Launch:** in the same release, remove `openvibe.news` from OpenVibe.Sites and flip the Network++ hub entry (see the launch rule above).---@@@ -1,7 +1,20 @@{"repository": "OpenVibers/OpenVibe.News",-- "stage": "placeholder",-- "code": false,++ "stage": "alpha",++ "code": true,++ "wave": "W17","plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026)",-- "updated": "2026-09-21"++ "runtime": "node 22, express 4, better-sqlite3 11",++ "port": 4820,++ "domain": "openvibe.news",++ "packages": ["openvibe-publishing v0.2.0", "openvibe-contracts v0.19.0", "openvibe-shared v1.3.0", "openvibe-sdk v0.2.2"],++ "capabilities": ["news.story.create", "news.story.read", "news.story.revise", "news.story.publish", "news.story.retract", "news.cluster.read", "news.cluster.manage", "news.source.attach", "news.timeline.update", "news.perspective.update", "news.topic.manage"],++ "capabilitiesRegistered": false,++ "serviceManifestRegistered": false,++ "inputs": "OpenVibe.Sources items (category news) via an Events subscription and a cursor pull; both untested against the running Sources service",++ "ai": "optional seam (news.summarize_story, news.compare_perspectives) tested against a mock; OpenVibe.AI output only ever lands as a draft that needs a person's review",++ "seed": "topics only; no stories, source items or clusters",++ "deployed": false,++ "placeholderInSites": true,++ "updated": "2026-09-22"}nnew file mode 100644@@@ -0,0 +1,110 @@++## ═══════════════════════════════════════════════════════════════++## openvibe.news — Nginx configuration (REFERENCE)++##++## Deployed to /etc/nginx/sites-available/openvibe.news.conf — keep them in sync and reload nginx++## after changing it. The Node app (port 4820, systemd unit openvibe-news) renders every page;++## nginx terminates TLS, redirects www → apex, rate-limits auth, editor and API, and compresses text.++##++## No proxy_cache here on purpose: the app decides cacheability per response (Cache-Control++## public only for public posts and lists shown to anonymous visitors; private, no-store for++## everything else, Vary: Cookie, Authorization). A cache added in front must honour those headers.++## ═══════════════════════════════════════════════════════════════++++limit_req_zone $binary_remote_addr zone=ovnews_api:10m rate=10r/s;++limit_req_zone $binary_remote_addr zone=ovnews_auth:10m rate=10r/m;++limit_req_zone $binary_remote_addr zone=ovnews_write:10m rate=5r/s;++++server {++ listen 80;++ listen [::]:80;++ server_name openvibe.news www.openvibe.news;++++ location /.well-known/acme-challenge/ { root /var/www/certbot; }++ location / { return 301 https://openvibe.news$request_uri; }++}++++server {++ listen 443 ssl;++ listen [::]:443 ssl;++ http2 on;++ server_name www.openvibe.news;++++ ssl_certificate /etc/letsencrypt/live/openvibe.news/fullchain.pem;++ ssl_certificate_key /etc/letsencrypt/live/openvibe.news/privkey.pem;++ ssl_protocols TLSv1.2 TLSv1.3;++++ return 301 https://openvibe.news$request_uri;++}++++server {++ listen 443 ssl;++ listen [::]:443 ssl;++ http2 on;++ server_name openvibe.news;++++ ssl_certificate /etc/letsencrypt/live/openvibe.news/fullchain.pem;++ ssl_certificate_key /etc/letsencrypt/live/openvibe.news/privkey.pem;++ ssl_protocols TLSv1.2 TLSv1.3;++++ access_log /var/log/nginx/openvibe.news.access.log;++ error_log /var/log/nginx/openvibe.news.error.log;++++ # Editor forms are URL-encoded text (no uploads); the Events webhook body is at most 256 KB.++ client_max_body_size 1m;++++ add_header X-Content-Type-Options nosniff always;++ add_header X-Frame-Options SAMEORIGIN always;++ add_header Referrer-Policy strict-origin-when-cross-origin always;++ add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;++++ gzip on;++ gzip_proxied any;++ gzip_vary on;++ gzip_comp_level 5;++ gzip_min_length 1024;++ gzip_types text/css application/javascript text/javascript application/json application/feed+json image/svg+xml text/plain application/xml text/xml application/rss+xml application/atom+xml;++++ proxy_http_version 1.1;++ proxy_set_header Host $host;++ proxy_set_header X-Real-IP $remote_addr;++ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;++ proxy_set_header X-Forwarded-Proto $scheme;++++ # ── Network SSO session endpoints (strict) ──────────────++ location /auth/ {++ limit_req zone=ovnews_auth burst=10 nodelay;++ proxy_pass http://127.0.0.1:4820;++ }++++ # ── The editor desk and cluster pages (forms, editors only) ──++ location /edit {++ limit_req zone=ovnews_write burst=20 nodelay;++ proxy_pass http://127.0.0.1:4820;++ }++ location /clusters/ {++ limit_req zone=ovnews_write burst=20 nodelay;++ proxy_pass http://127.0.0.1:4820;++ }++++ # ── The Events webhook: OpenVibe.Events delivers over loopback, never through the public vhost ──++ location /internal/ { return 404; }++++ # ── API ─────────────────────────────────────────────────++ location /api/ {++ limit_req zone=ovnews_api burst=30 nodelay;++ proxy_pass http://127.0.0.1:4820;++ }++++ # ── Prometheus metrics: host-local scrapers only (http://127.0.0.1:4820/metrics) ──++ location = /metrics { return 404; }++++ # ── Pages, feeds, sitemaps, robots.txt, llms.txt, static assets ──++ location / {++ proxy_pass http://127.0.0.1:4820;++ proxy_set_header CF-Connecting-IP $http_cf_connecting_ip;++ }++++ location ~ /\.(?!well-known) {++ deny all;++ }++}nnew file mode 100644@@@ -0,0 +1,32 @@++[Unit]++Description=OpenVibe.News — source-backed stories from OpenVibe.Sources items++# Network (sign-in, JWKS, service tokens), Sources (the items, over loopback) and Events (the outbox++# relay and the webhook) should be up first, but News degrades gracefully without them: pages and++# feeds serve; sign-in, ingestion and events wait and are reported by /api/ready.++After=network.target++Wants=openvibe-network.service++++[Service]++Type=simple++User=ubuntu++Group=ubuntu++WorkingDirectory=/opt/openvibe.news++EnvironmentFile=/etc/openvibe/news.env++Environment=NODE_ENV=production++Environment=PORT=4820++Environment=PATH=/usr/local/bin:/usr/bin:/bin++ExecStart=/usr/bin/env node /opt/openvibe.news/server/index.js++Restart=always++RestartSec=5++KillSignal=SIGTERM++TimeoutStopSec=10++NoNewPrivileges=true++PrivateTmp=true++ProtectSystem=strict++ProtectHome=true++# News' own database lives in /var/lib/openvibe-news; the code tree stays read-only.++StateDirectory=openvibe-news++Environment=NEWS_DB_PATH=/var/lib/openvibe-news/news.db++++[Install]++WantedBy=multi-user.target
new file mode 100644
new file mode 100644
@@ -0,0 +1,69 @@
@@ -0,0 +1,69 @@
+# ═══════════════════════════════════════════════════════════════
+# openvibe.news — environment configuration (names only; never commit values)
+# Copy to .env for development. Production: /etc/openvibe/news.env (0600).
+# ═══════════════════════════════════════════════════════════════
+
+# Server
+PORT=4820
+HOST=127.0.0.1
+NODE_ENV=development
+# Public origin: canonical URLs, feeds, sitemaps and JSON-LD are built from it
+BASE_URL=http://localhost:4820
+# Proxy hops that set X-Forwarded-For (Cloudflare → nginx → Node = 2)
+TRUST_PROXY=2
+
+# SQLite database (production unit: /var/lib/openvibe-news/news.db)
+NEWS_DB_PATH=./data/news.db
+
+# OpenVibe.Network — SSO (OAuth2), JWKS, client-credentials tokens
+OV_NETWORK_URL=https://openvibe.network
+OV_NETWORK_INTERNAL_URL=http://127.0.0.1:4000
+# OAuth client `news` (redirect https://openvibe.news/auth/callback). The same client is News'
+# service principal svc:news for client-credentials grants (see README, "Grants").
+OV_OAUTH_CLIENT_ID=news
+OV_OAUTH_CLIENT_SECRET=
+# Production: https://openvibe.news/auth/callback (default: BASE_URL + /auth/callback)
+#OV_OAUTH_REDIRECT_URI=
+# Cookies: Secure by default in production; set false for plain-http local development
+#COOKIE_SECURE=false
+# Signs the editor-desk form tokens (CSRF). Unset: random per process.
+#NEWS_FORM_SECRET=
+
+# Editors: Network subjects (usr_…), comma-separated. Network admins are editors too.
+NEWS_EDITORS=
+
+# OpenVibe.Sources — news items (sources.item.read; optionally sources.source.read for outlet names)
+OV_SOURCES_INTERNAL_URL=http://127.0.0.1:4720
+# Cursor pull (backstop for missed webhook deliveries); 0 turns it off
+#NEWS_PULL_INTERVAL_MS=300000
+#NEWS_PULL_PAGE_SIZE=100
+#NEWS_PULL_MAX_PAGES=20
+#NEWS_SOURCES_TIMEOUT_MS=8000
+
+# Licensing: the longest source summary kept (and only when the source's terms allow short summaries)
+#NEWS_SUMMARY_MAX_CHARS=280
+# Clustering window
+#NEWS_CLUSTER_WINDOW_HOURS=72
+
+# OpenVibe.Events — the outbox relay runs only when EVENTS_URL and OV_OAUTH_CLIENT_SECRET are set
+#EVENTS_URL=http://127.0.0.1:4300
+#EVENTS_RELAY_INTERVAL_MS=2000
+# Signing secret(s) of News' subscriptions (sources.item.*, sources.fetch.failed), comma-separated
+# during rotation. Generate before `npm run subscribe` (e.g. `openssl rand -hex 32`).
+# Unset = webhook off (503); the cursor pull still runs.
+#NEWS_EVENTS_SECRET=
+
+# OpenVibe.AI (optional) — news.summarize_story / news.compare_perspectives drafts (ai.run.create).
+# Unset: no AI; editors write every story from the source items.
+#OV_AI_INTERNAL_URL=http://127.0.0.1:4700
+#NEWS_AI_WAIT_MS=30000
+
+# Background worker (the Sources pull); off: NEWS_WORKER=off
+#NEWS_WORKER=on
+
+# Browser origins that may call /api/v1 with a Bearer Network JWT
+#API_CORS_ORIGINS=https://openvibe.network,https://openvibe.live,https://openvibe.community
+
+# release.json (openvibe-shared/release): the deployed commit when git is not available
+#RELEASE_COMMIT=
+#MIN_CLIENT_RELEASE=
new file mode 100644
new file mode 100644
@@ -0,0 +1,20 @@
@@ -0,0 +1,20 @@
+name: ci
+on:
+ push:
+ branches: [main]
+ pull_request:
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 22.22.1 # production runs Node 22
+ cache: npm
+ - run: npm ci --no-audit --no-fund
+ - run: npm test
+ # News' service manifest and capabilities are proposals (docs/) until the lead releases them
+ # in openvibe-contracts; until then the check reports "no service manifest news" (placeholder).
+ - run: npx openvibe-contracts-check --service news --src server
+ continue-on-error: true
@@ -1,75 +1,307 @@
@@ -1,75 +1,307 @@
# OpenVibe.News
# OpenVibe.News
-> Source-backed stories: clustered coverage, cited summaries, perspectives and timelines.
+> Source-backed stories: clustered coverage, cited paragraphs, perspectives, timelines and a
+> public correction history.
-**Status:** placeholder — planning only, no runnable code yet.
-**Domain:** `openvibe.news`
-**Plan:** OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §12.7.
+**Status:** alpha (roadmap Wave 17, News half). The service runs and its tests pass. It is **not
+deployed**, `openvibe.news` still shows its placeholder from OpenVibe.Sites, and its capabilities and
+service manifest are proposals that the next openvibe-contracts release has to include. It has not
+yet ingested from the running OpenVibe.Sources service (every Sources seed is disabled).
+**Domain:** `openvibe.news` · **Port:** 4820 · **Service id:** `news`
+**Plan:** OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026), §12.7; roadmap §15.13, §29, §32.
**License:** AGPL-3.0 (same as every OpenVibe service).
**License:** AGPL-3.0 (same as every OpenVibe service).
## Purpose
## Purpose
-A publication product, not an autonomous headline generator: normalised source items, story clusters, editorial/publication state, perspective groupings and timelines with full provenance.
+A publication product, not an autonomous headline generator. OpenVibe.Sources collects news items
+(headline, link, outlet, dates, terms). News normalises them, removes duplicates, groups them into
+explainable clusters, and gives editors the tools to write stories from them. Every paragraph an
+editor publishes cites the source items it rests on. A story with no sources cannot be published.
+When a source changes or is withdrawn upstream, the story is flagged and a pending revision is
+prepared; published text never changes silently. Full article text from sources is never stored,
+shown, fed or indexed.
+
+News publishes nothing until an editor does: the only seed is the topic list.
## Owns
## Owns
-- `news_topics`, `news_source_items`, `news_story_clusters`, `news_stories`, `news_story_revisions`, `news_story_sources`, `news_perspectives`, `news_timeline_entries`, `news_editorial_flags`
+The nine charter tables live in News' own SQLite (`NEWS_DB_PATH`):
+
+| Charter table | What it is |
+|---|---|
+| `news_topics` | Owned by News. Canonical topic identities (slug, name, description). The only seed (`seeds/topics.json`). |
+| `news_source_items` | Owned by News. One row per Sources item (`sources_item_id` is a typed reference to `itm_…`): headline, canonical URL, outlet, authors, `published_at` as the source states it, the licence and terms notes, and a short summary **only** when those notes allow short summaries. Dedupe outcome (`status`, `duplicate_of`, `dedupe`), cluster and the stored reason for it, upstream revision and removal. No article body. |
+| `news_story_clusters` | Owned by News. Clusters with their key terms and entities, time window, status (`open`, `merged`, `dissolved`) and lineage (`merged_into`, `split_from`). |
+| `news_stories` | Owned by News. Publication state: slug, state (`draft`, `published`, `unpublished`, `retracted`), published revision, topic, cluster. |
+| `news_story_revisions` | `openvibe-publishing/revisions` (prefix `news_story`). Immutable. Each revision holds the headline, the paragraphs with their source numbers, and a snapshot of the source table, timeline and perspectives. |
+| `news_story_sources` | Owned by News. The story's source table: which items, their stable numbers `[n]` (never reused), perspective grouping, detachment. |
+| `news_perspectives` | Owned by News. Editor-written labels that group a story's sources (for example an outlet group or a position a source states). Never generated. |
+| `news_timeline_entries` | Owned by News. Dated entries; each date is one the source states and each entry rests on one attached source. |
+| `news_editorial_flags` | Owned by News. `correction`, `update`, `retraction` (public, with notes) and `source_updated`, `source_removed` (upstream changes, with the pending revision). |
+
+Other tables in the same database:
+
+- Package companions: `news_story_citations` (one row per paragraph and timeline entry and cited
+ Sources item, append-only), `news_story_reviews` (a person's approval of AI output),
+ `news_story_drafts`, `news_story_revision_purges`, `news_index_revisions` (Search sequencer).
+- `news_cluster_audit`: every merge, split and reversal with exactly which items moved.
+- `news_ingest_runs`: what every webhook delivery, pull and upstream fetch failure did.
+- `news_state` (the Sources cursor), `news_source_status` (display cache of Sources' registry
+ names and health), `event_outbox` and `idempotency_receipts` (openvibe-sdk),
+ `subject_projections` (Network names). None of them is authority for anything outside News.
## Does not own
## Does not own
-- source adapter policy (shared source registry)
-- discussion (Community)
+- **Source adapters and the source registry.** OpenVibe.Sources owns them: which sites are read,
+ under which terms, how and when. News reads items with `sources.item.read` and never fetches a
+ site itself.
+- **Discussion.** OpenVibe.Community. (News does not attach comment threads yet.)
+- **AI generation.** OpenVibe.AI. News only receives drafts.
+- **Search.** OpenVibe.Search indexes what News sends through Events.
+- **Identity.** OpenVibe.Network: SSO, subjects and service principals.
+
+## What works
+
+- **Ingestion** from OpenVibe.Sources, two ways with one effect:
+ - an Events webhook (`POST /internal/events`, signed with `NEWS_EVENTS_SECRET`) for
+ `sources.item.created|updated|removed` and `sources.fetch.failed`, each applied exactly once
+ through the SDK inbox;
+ - a cursor pull (`GET /api/v1/items?category=news&after=…&include_removed=1`) every
+ `NEWS_PULL_INTERVAL_MS`, the backstop for missed deliveries.
+ Applying an item is idempotent on (Sources item id, revision). Failures are recorded states
+ (`news_ingest_runs`, `news.source.failed`, `/api/ready`, the editor desk) and never create a
+ source item, a cluster or any story text. When Sources cannot be read, the delivery is answered
+ 503 so Events retries it.
+- **Licensing.** Only headline, URL, outlet, authors, `published_at`, the notes and — when the
+ item's terms or licence note explicitly allows short summaries and nothing forbids them — a
+ summary of at most `NEWS_SUMMARY_MAX_CHARS` (280) are kept. Anything else a Sources item carries
+ (for example an article body an adapter mapped into `fields`) is never read. A removed item's
+ summary is dropped. Source summaries appear only in a story's source table; never in feeds,
+ Search documents or events.
+- **Dedupe** (explainable, stored on the item): the same normalised canonical URL (tracking
+ parameters, `www.`, trailing slashes ignored), the same Sources content hash, or a near-duplicate
+ headline (Jaccard of word 2-shingles ≥ 0.9, or ≥ 0.75 from the same outlet, within 48 h).
+ Duplicates are kept for provenance, join their original's cluster and are not attached to new
+ stories by default.
+- **Clusters** (deterministic): an item joins the open cluster, within `NEWS_CLUSTER_WINDOW_HOURS`
+ (72), with which it shares at least one named entity and two terms, or at least three terms;
+ entities score 2, terms 1; ties go to the oldest cluster. The shared entities, shared terms and
+ score are stored on the item and shown on the cluster page. Editors can merge clusters and split
+ items off; both are audited and reversible (`news_cluster_audit`).
+- **Stories** (editorial):
+ - An editor opens a story from a cluster; its live, non-duplicate items become sources
+ `[1]`, `[2]`, … . Editors can attach and detach sources; numbers are never reused.
+ - Paragraphs end with the source numbers they rest on (`… [1, 3]`). A revision with an uncited
+ paragraph, or citing a number not in the source table, is refused.
+ - Revisions are immutable with optimistic concurrency (412 on a stale base), diff and preview.
+ - Publishing is refused while the revision has no sources (`story.unsourced`), cites a source
+ removed upstream or detached, or is AI-generated or AI-assisted without a person's approval.
+ - Timeline entries have a date the source states and one source each. Perspectives are labels
+ an editor writes; readers can re-frame the source table by number, outlet or perspective
+ (`?group=`), without JavaScript.
+ - Corrections and updates are notes published with a revision and shown in the story's
+ correction history. Retraction keeps the story at its URL with a retraction notice, noindex,
+ out of sitemaps and Search, labelled in listings and feeds; it is final.
+- **Upstream changes.** When a cited source is revised or removed in Sources, every story citing it
+ gets an open flag and a pending revision prepared by the system (same text, refreshed source
+ entry). The published revision stays exactly as it was, with a notice to readers. A removed
+ source's headline, summary and link leave the public page at once; a paragraph resting only on
+ it is marked, and the gate makes the story noindex (`unsupported_claims`) until an editor
+ publishes a revision that no longer rests on it — with a correction or update note.
+- **AI** (optional seams, never publication truth):
+ - OpenVibe.AI may deliver a draft revision with `X-OV-Origin: ai` (`news.story.revise`), naming
+ `news.summarize_story` or `news.compare_perspectives` and its run.
+ - An editor may ask for one (`POST /api/v1/stories/:id/ai-drafts`, or the desk's button when
+ `OV_AI_INTERNAL_URL` is set). Only News' stored fields are sent. Claims the model did not cite
+ are dropped and listed as gaps. A failed run makes no text. AI never creates perspective labels.
+ - AI-generated revisions, and AI-assisted ones (an editor's edit of AI text), cannot be published
+ and are hidden from indexes until a person (`usr_`) records an approving review of that
+ revision. Stub-provider output is held too. The disclosure is on the page, in the JSON and in
+ the Search document.
+ - With no AI configured, editors write every story from the source items.
+
+### Routes (server-rendered, useful without JavaScript)
+
+| Route | What |
+|---|---|
+| `/` | published stories, newest first (retracted ones stay, labelled) and the topics |
+| `/topics`, `/topics/:slug` | topics and each topic's stories |
+| `/stories/:slug`, `/stories/:slug.json` | a story (text with source numbers, source table, timeline, perspectives, corrections), and the same story as data |
+| `/feed.xml`, `/atom.xml`, `/feed.json`, `/topics/:slug/{feed.xml,atom.xml,feed.json}` | feeds (News' own text only) |
+| `/sitemap.xml` → `/sitemaps/stories.xml`, `/sitemaps/topics.xml` | indexable stories and topics only |
+| `/robots.txt`, `/llms.txt` | the automated-consumer policy and machine orientation |
+| `/edit`, `/edit/stories/:id`, `/clusters/:id` | the editor desk (forms, Network SSO, form tokens; editors only; private, noindex) |
+| `/auth/*` | sign-in; the same session layer as OpenVibe.Community and OpenVibe.Blog |
+| `/internal/events` | the Events webhook (loopback; nginx does not proxy it) |
+| `/api/health`, `/api/ready`, `/release.json`, `/metrics` | health, truthful readiness, release, metrics (`/metrics` loopback only) |
+
+### Discoverability (roadmap §32)
+
+- Robots meta, canonical and `X-Robots-Tag` come from the `openvibe-publishing/seo` gate with
+ explicit reasons. News' policy: at least one live cited source, at least 40 words; unsupported
+ claims, retraction, an editor's noindex and unreviewed AI output are never indexable.
+- JSON-LD `NewsArticle` from real fields only: headline, dates, the editors named in the
+ revision's authorship (only when the Network name is known), the section (topic), the cited
+ source URLs, and OpenVibe.News as publisher. Missing fields are omitted.
+- Drafts, unpublished, retracted and noindex stories never appear in sitemaps or Search; feeds
+ list published and (labelled) retracted stories, never a draft.
+
+### Caching
-## Planned surfaces
+HTML and JSON vary on `Cookie` and `Authorization`. Published stories and lists rendered for an
+anonymous visitor are `public, max-age=60`; everything else (signed-in views, the desk, previews,
+refusals, the API) is `private, no-store` with `X-Robots-Tag`. Feeds and sitemaps are public for
+five minutes and are never built for a viewer.
-- ingest with visible fetch/parse status, dedupe/cluster, factual fields kept separate from synthesis, cited summaries, perspective views, timelines, human review / noindex states
+### Events (SDK outbox, same transaction as the change)
-## Data (authority tables / families)
+| Event | Notes |
+|---|---|
+| `news.source.ingested` | internal; per Sources item created, revised or removed, with the dedupe and cluster outcome |
+| `news.source.failed` | internal; a Sources fetch failure of a news source, an unreadable item, or a failed pull (once per outage) |
+| `news.cluster.updated` | internal; created, grew, merged, split, reversed |
+| `news.story.created`, `news.story.flagged` | internal |
+| `news.story.published`, `.updated`, `.unpublished` | `openvibe-publishing/index-hooks` `publicationEvent`; canonical URL, state, indexability, topic; never the text |
+| `news.story.retracted` | public for a listable story; the retraction note |
+| `news.index_document.upserted` / `.deleted` | `search.index-document@1` documents (provenance: the Sources items and the AI run) and tombstones, with a monotonic index revision |
-- see above
+### Capabilities (proposed: `docs/capabilities-proposal/`)
-## Capabilities and events
+Service tokens use audience `openvibe.news`, one capability per route; the editor the service acts
+for goes in `X-OV-Subject` and must be an editor:
-- `news.story.create|revise|publish`, `news.cluster.read`, `news.source.attach`, `news.timeline.update`
+- `news.story.create`, `news.story.read`, `news.story.revise`, `news.story.publish`, `news.story.retract`
+- `news.cluster.read`, `news.cluster.manage`
+- `news.source.attach`, `news.timeline.update`, `news.perspective.update`
+- `news.topic.manage`
-Events: ``news.source.ingested|failed``, ``news.cluster.updated``, ``news.story.published|updated|retracted``
+Browser and app user JWTs are judged as editors (`NEWS_EDITORS` subjects and Network admins).
+Until the proposals are released, grants for these ids are decided locally with the contracts
+library's matching rule (`server/auth/capabilities.js`). The service manifest proposal is
+`docs/service-manifest-proposal.json`.
## Depends on
## Depends on
-- source registry
-- OpenVibe.AI
-- Search
-- OpenVibe.Community
-- OpenVibe.Events
+- **Packages** (pinned by release tarball): `openvibe-publishing` v0.2.0 (revisions, citations,
+ authorship, seo, index-hooks, ssr, taxonomy slugify), `openvibe-contracts` v0.19.0,
+ `openvibe-shared` v1.3.0 (chrome, app icon, footer, legal, release, metrics, ready, seo),
+ `openvibe-sdk` v0.2.2 (events outbox and inbox, webhook signatures, service tokens).
+- **OpenVibe.Sources** (4720): `sources.item.read`; optionally `sources.source.read` (outlet
+ names and source health; without it the outlet is the URL's host).
+- **OpenVibe.Events** (4300): `events.event.publish`; `events.subscription.manage` to create the
+ `sources.item.*` and `sources.fetch.failed` subscriptions (`npm run subscribe`).
+- **OpenVibe.Network** (4000): SSO (OAuth client `news`, redirect `https://openvibe.news/auth/callback`),
+ JWKS, `identity.subject.resolve` for editor names.
+- **OpenVibe.Search**: consumes `news.index_document.*` through its `*.index_document.*`
+ subscription; `news` must be in `SEARCH_EVENT_OWNERS`.
+- **Optional:** OpenVibe.AI (4700) with `ai.run.create` for namespace `news`.
-## Acceptance (must be true before "done")
+### Grants the Network must hold
-- every published synthesized claim traces to source records
-- failed ingestion never fabricates replacement facts
-- source correction/removal triggers a revision
-- licensed/private material never leaks into public text
+Each grant is `[client, capability, audience]`:
-## Bootstrap / extraction source
+- `[news, sources.item.read, openvibe.sources]`
+- `[news, sources.source.read, openvibe.sources]` (optional: outlet names and health)
+- `[news, events.event.publish, openvibe.events]`
+- `[news, events.subscription.manage, openvibe.events]`
+- `[news, identity.subject.resolve, openvibe.network]`
+- `[news, ai.run.create, openvibe.ai]`, namespace `news` (optional: only with `OV_AI_INTERNAL_URL`)
+- For OpenVibe.AI to deliver drafts: `[ai, news.story.revise, openvibe.news]`, and
+ `[ai, news.story.read, openvibe.news]` if it reads stories back.
-No current implementation; Wave 15 after the Sources/AI/publishing primitives exist.
+## Acceptance (automated: `npm test`)
+
+| Charter / roadmap requirement | Test |
+|---|---|
+| Every published claim traces to source records: each paragraph's numbers resolve to Sources items in the page, the JSON, the citations table and the Search document's provenance. | `test/stories.test.js` |
+| A story with zero sources cannot be published; an uncited paragraph is refused. | `test/stories.test.js` |
+| Failed ingestion never fabricates: failed pulls, unreadable deliveries, `sources.fetch.failed` and title-less items are recorded and create no item, cluster or story text. | `test/ingest.test.js` |
+| A source correction or removal triggers a flag and a pending revision; published text is unchanged; a removed source leaves the page; publishing a revision that rests on it is refused; the editor's revision resolves the flags with a note. | `test/upstream.test.js` |
+| Licensed material never leaks: bodies and summaries beyond the allowance never reach the database, pages, JSON, feeds, sitemaps, the API, events or Search. | `test/licensing.test.js` |
+| Dedupe by canonical URL, content hash and near-duplicate headline; idempotent replays. | `test/ingest.test.js` |
+| Clusters are deterministic and explained; merges and splits are audited and reversible. | `test/clusters.test.js` |
+| Retraction: visible at its URL with the notice, noindex with its reason, out of sitemaps and Search, event emitted, final. | `test/retraction.test.js` |
+| Useful without JavaScript: the whole editorial journey with forms, and public pages complete in HTML. | `test/nojs.test.js` |
+| AI output is a draft that needs a person's review (AI-assisted too); uncited claims dropped; failure makes no text. | `test/ai.test.js` |
+| The contract proposals are valid and match the code; every emitted event type is declared. | `test/contracts.test.js` |
+| Health, readiness, release, robots, llms.txt, sitemaps, legal, problems, CORS, topics-only seed. | `test/ops.test.js` |
+
+Not yet demonstrated: ingestion from the deployed OpenVibe.Sources (its seeds are disabled until a
+person verifies their terms), delivery through the running OpenVibe.Events, and an OpenVibe.AI run
+(the AI service is still being built; the seam is tested against a mock of its runs API).
## Launch rule
## Launch rule
-This repository does not make the product real, and the domain keeps its placeholder page on
-[OpenVibers/OpenVibe.Sites](https://github.com/OpenVibers/OpenVibe.Sites) until all of the
-following exist here (plan §12.12):
-
-1. an owning runtime with health/readiness endpoints and observability;
-2. canonical identity/auth integration (OpenVibe.Network subjects, scoped service principals);
-3. server-rendered or static public routes that are useful without JavaScript;
-4. real persistence and end-to-end workflows;
-5. capability and event registration against `OpenVibe.Contracts`;
-6. a migration/seed strategy, a security/threat review, and sitemap/robots/feed behaviour;
-7. acceptance tests proving the advertised functionality.
-
-The launch release removes the domain from `OpenVibe.Sites/sites.json`, switches routing and
-registers maturity in the ecosystem registry atomically. A placeholder is never counted as an
-implemented service.
+This repository alone doesn't make the product live. `openvibe.news` keeps its placeholder on
+[OpenVibers/OpenVibe.Sites](https://github.com/OpenVibers/OpenVibe.Sites) until all of plan §12.12
+exists:
+
+1. **Runtime, health, readiness, observability:** done.
+2. **Canonical identity and auth:** done (Network SSO, subjects, service tokens).
+3. **SSR public routes useful without JS:** done.
+4. **Persistence and end-to-end workflows:** done in tests; not yet against live Sources items.
+5. **Capability and event registration against OpenVibe.Contracts:** proposals are in `docs/`,
+ waiting on the release.
+6. **Migration and seed strategy, threat review, sitemap/robots/feed behaviour:** done. Nothing
+ to migrate; the seed is topics only; the threat review is below.
+7. **Acceptance tests:** done.
+
+The launch release removes `openvibe.news` from `OpenVibe.Sites/sites.json`, switches routing
+(nginx vhost, DNS, TLS), flips the Network hub entry and registers maturity in the ecosystem
+registry, together. A placeholder never counts as an implemented service, and this README doesn't
+call the service live.
+
+## Security and threat review
+
+- **Identity:** only verified Network JWTs (offline RS256) and service tokens for audience
+ `openvibe.news`. A bad service token is refused, never downgraded. Identity never comes from a
+ body or query; `X-OV-*` headers are ignored for browsers. AI deliveries can only write drafts.
+- **Webhook:** HMAC-SHA256 over the raw body with `NEWS_EVENTS_SECRET` (rotation: comma-separated),
+ constant-time comparison, `evt_` ids, exactly-once inbox. Only events from source `sources` are
+ applied. nginx never proxies `/internal/`.
+- **CSRF:** SameSite=Lax session cookie plus an HMAC form token on every desk form.
+- **XSS:** all HTML goes through `openvibe-publishing/ssr` auto-escaping; source links get
+ `rel="noopener nofollow"`; helmet CSP.
+- **SSRF:** News makes no outbound request to a URL from content. It calls only its configured
+ Network, Sources, Events and AI hosts.
+- **Licensing and privacy:** see "Licensing" above; removed sources are masked everywhere public.
+ The editor view shows removal reasons; the public view does not.
+- **Fabrication:** no seeded or generated stories, ratings or dates; dates come from sources or an
+ editor citing one; empty feeds use a real time; JSON-LD omits unknown fields.
+- **Abuse:** rate limits on `/auth`, `/edit`, `/clusters` and `/api/v1`, in Express and in the
+ nginx reference.
+- **Known gaps:** no Community comment threads on stories yet; no per-story Media attachments; the
+ Sources registry name is cached and not refreshed after the first successful read.
+
+## Development
+
+```bash
+fnm exec --using=22.22.1 npm install
+fnm exec --using=22.22.1 npm test # temp databases and in-process mocks, no network
+fnm exec --using=22.22.1 npm run dev # http://localhost:4820 (set OV_OAUTH_CLIENT_SECRET to sign in)
+```
+
+## Deploy (for the lead)
+
+1. **Code and config:** put the code at `/opt/openvibe.news` and run `npm ci --omit=dev` on Node 22.
+ Create `/etc/openvibe/news.env` (0600) from `.env.example` with `OV_OAUTH_CLIENT_SECRET`,
+ `NEWS_EDITORS`, `NEWS_FORM_SECRET`, `NEWS_EVENTS_SECRET` (`openssl rand -hex 32`),
+ `EVENTS_URL=http://127.0.0.1:4300`, `OV_SOURCES_INTERNAL_URL=http://127.0.0.1:4720`,
+ `BASE_URL=https://openvibe.news`, and optionally `OV_AI_INTERNAL_URL`.
+2. **Network:** create (or give a secret to) the OAuth client `news` with redirect
+ `https://openvibe.news/auth/callback`, and add the grants listed above.
+3. **Search:** make sure `news` is in `SEARCH_EVENT_OWNERS`.
+4. **systemd:** install `deploy/systemd/openvibe-news.service` (port 4820, `StateDirectory=openvibe-news`).
+5. **nginx:** install `deploy/nginx/openvibe.news.conf`. `/metrics` and `/internal/` are never proxied.
+6. **Seed and subscribe:** `npm run seed` (topics only; also done on boot), then
+ `npm run subscribe` (creates the `sources.item.*` and `sources.fetch.failed` subscriptions to
+ `http://127.0.0.1:4820/internal/events`). The cursor pull starts on its own.
+7. **Sources:** News shows nothing until a news source is enabled in OpenVibe.Sources (terms
+ verified by a person) and an editor publishes a story.
+8. **Contracts:** release the capability and manifest proposals in openvibe-contracts; then CI's
+ contracts check can drop `continue-on-error`.
+9. **Launch:** in the same release, remove `openvibe.news` from OpenVibe.Sites and flip the Network
+ hub entry (see the launch rule above).
---
---
@@ -1,7 +1,20 @@
@@ -1,7 +1,20 @@
{
{
"repository": "OpenVibers/OpenVibe.News",
"repository": "OpenVibers/OpenVibe.News",
- "stage": "placeholder",
- "code": false,
+ "stage": "alpha",
+ "code": true,
+ "wave": "W17",
"plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026)",
"plan": "OpenVibe End-to-End Realignment & Implementation Plan, revision 3 (20 Sep 2026)",
- "updated": "2026-09-21"
+ "runtime": "node 22, express 4, better-sqlite3 11",
+ "port": 4820,
+ "domain": "openvibe.news",
+ "packages": ["openvibe-publishing v0.2.0", "openvibe-contracts v0.19.0", "openvibe-shared v1.3.0", "openvibe-sdk v0.2.2"],
+ "capabilities": ["news.story.create", "news.story.read", "news.story.revise", "news.story.publish", "news.story.retract", "news.cluster.read", "news.cluster.manage", "news.source.attach", "news.timeline.update", "news.perspective.update", "news.topic.manage"],
+ "capabilitiesRegistered": false,
+ "serviceManifestRegistered": false,
+ "inputs": "OpenVibe.Sources items (category news) via an Events subscription and a cursor pull; both untested against the running Sources service",
+ "ai": "optional seam (news.summarize_story, news.compare_perspectives) tested against a mock; OpenVibe.AI output only ever lands as a draft that needs a person's review",
+ "seed": "topics only; no stories, source items or clusters",
+ "deployed": false,
+ "placeholderInSites": true,
+ "updated": "2026-09-22"
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,110 @@
@@ -0,0 +1,110 @@
+## ═══════════════════════════════════════════════════════════════
+## openvibe.news — Nginx configuration (REFERENCE)
+##
+## Deployed to /etc/nginx/sites-available/openvibe.news.conf — keep them in sync and reload nginx
+## after changing it. The Node app (port 4820, systemd unit openvibe-news) renders every page;
+## nginx terminates TLS, redirects www → apex, rate-limits auth, editor and API, and compresses text.
+##
+## No proxy_cache here on purpose: the app decides cacheability per response (Cache-Control
+## public only for public posts and lists shown to anonymous visitors; private, no-store for
+## everything else, Vary: Cookie, Authorization). A cache added in front must honour those headers.
+## ═══════════════════════════════════════════════════════════════
+
+limit_req_zone $binary_remote_addr zone=ovnews_api:10m rate=10r/s;
+limit_req_zone $binary_remote_addr zone=ovnews_auth:10m rate=10r/m;
+limit_req_zone $binary_remote_addr zone=ovnews_write:10m rate=5r/s;
+
+server {
+ listen 80;
+ listen [::]:80;
+ server_name openvibe.news www.openvibe.news;
+
+ location /.well-known/acme-challenge/ { root /var/www/certbot; }
+ location / { return 301 https://openvibe.news$request_uri; }
+}
+
+server {
+ listen 443 ssl;
+ listen [::]:443 ssl;
+ http2 on;
+ server_name www.openvibe.news;
+
+ ssl_certificate /etc/letsencrypt/live/openvibe.news/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/openvibe.news/privkey.pem;
+ ssl_protocols TLSv1.2 TLSv1.3;
+
+ return 301 https://openvibe.news$request_uri;
+}
+
+server {
+ listen 443 ssl;
+ listen [::]:443 ssl;
+ http2 on;
+ server_name openvibe.news;
+
+ ssl_certificate /etc/letsencrypt/live/openvibe.news/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/openvibe.news/privkey.pem;
+ ssl_protocols TLSv1.2 TLSv1.3;
+
+ access_log /var/log/nginx/openvibe.news.access.log;
+ error_log /var/log/nginx/openvibe.news.error.log;
+
+ # Editor forms are URL-encoded text (no uploads); the Events webhook body is at most 256 KB.
+ client_max_body_size 1m;
+
+ add_header X-Content-Type-Options nosniff always;
+ add_header X-Frame-Options SAMEORIGIN always;
+ add_header Referrer-Policy strict-origin-when-cross-origin always;
+ add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
+
+ gzip on;
+ gzip_proxied any;
+ gzip_vary on;
+ gzip_comp_level 5;
+ gzip_min_length 1024;
+ gzip_types text/css application/javascript text/javascript application/json application/feed+json image/svg+xml text/plain application/xml text/xml application/rss+xml application/atom+xml;
+
+ proxy_http_version 1.1;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+
+ # ── Network SSO session endpoints (strict) ──────────────
+ location /auth/ {
+ limit_req zone=ovnews_auth burst=10 nodelay;
+ proxy_pass http://127.0.0.1:4820;
+ }
+
+ # ── The editor desk and cluster pages (forms, editors only) ──
+ location /edit {
+ limit_req zone=ovnews_write burst=20 nodelay;
+ proxy_pass http://127.0.0.1:4820;
+ }
+ location /clusters/ {
+ limit_req zone=ovnews_write burst=20 nodelay;
+ proxy_pass http://127.0.0.1:4820;
+ }
+
+ # ── The Events webhook: OpenVibe.Events delivers over loopback, never through the public vhost ──
+ location /internal/ { return 404; }
+
+ # ── API ─────────────────────────────────────────────────
+ location /api/ {
+ limit_req zone=ovnews_api burst=30 nodelay;
+ proxy_pass http://127.0.0.1:4820;
+ }
+
+ # ── Prometheus metrics: host-local scrapers only (http://127.0.0.1:4820/metrics) ──
+ location = /metrics { return 404; }
+
+ # ── Pages, feeds, sitemaps, robots.txt, llms.txt, static assets ──
+ location / {
+ proxy_pass http://127.0.0.1:4820;
+ proxy_set_header CF-Connecting-IP $http_cf_connecting_ip;
+ }
+
+ location ~ /\.(?!well-known) {
+ deny all;
+ }
+}
new file mode 100644
new file mode 100644
@@ -0,0 +1,32 @@
@@ -0,0 +1,32 @@
+[Unit]
+Description=OpenVibe.News — source-backed stories from OpenVibe.Sources items
+# Network (sign-in, JWKS, service tokens), Sources (the items, over loopback) and Events (the outbox
+# relay and the webhook) should be up first, but News degrades gracefully without them: pages and
+# feeds serve; sign-in, ingestion and events wait and are reported by /api/ready.
+After=network.target
+Wants=openvibe-network.service
+
+[Service]
+Type=simple
+User=ubuntu
+Group=ubuntu
+WorkingDirectory=/opt/openvibe.news
+EnvironmentFile=/etc/openvibe/news.env
+Environment=NODE_ENV=production
+Environment=PORT=4820
+Environment=PATH=/usr/local/bin:/usr/bin:/bin
+ExecStart=/usr/bin/env node /opt/openvibe.news/server/index.js
+Restart=always
+RestartSec=5
+KillSignal=SIGTERM
+TimeoutStopSec=10
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+# News' own database lives in /var/lib/openvibe-news; the code tree stays read-only.
+StateDirectory=openvibe-news
+Environment=NEWS_DB_PATH=/var/lib/openvibe-news/news.db
+
+[Install]
+WantedBy=multi-user.target