openvibers/openvibe.sources Commits

Sources: retention rules are sources.category@1 documents (contracts 0.295.0) (#212)

OP OpenVibers committed 7 days ago · bf64e0da31 · 6 files, +39 −25

* Sources: retention rules are sources.category@1 documents (contracts 0.295.0)

* Merge remote-tracking branch 'origin/main' into retention-contract

# Conflicts:
#	README.md
#	STATUS.json
#	package-lock.json
#	package.json

Changes

ddiff --git a/README.md b/README.md
iindex 54c73fe..e62823f 100644
---- a/README.md
++++ b/README.md
@@@ -264,10 +264,12 @@ An item's age is `retrieved_at` while it is live and `removed_at` once removed,
still sees a removal for the full window; `first_seen_at` is never the age, since a source that keeps listing a job or
a price keeps it current. The daily job (`server/jobs/retention.js`, started with the worker) deletes at most 2,000 rows
a statement, 50 batches a category a run, and its log line carries counts only. The rules are in code because Sources
--has no `openvibe-shared/config` namespace. At runtime a category the contract has but the rules do not is kept (never
--pruned) and warned about once a run, and a rule for a category the contract lacks is ignored with a warning, so a
--Contracts pin bump cannot stop Sources at boot; `test/retention.test.js` keeps the strict one-to-one check so the bump
--PR fails CI and names the missing rule.
++has no `openvibe-shared/config` namespace, and each one is the `sources.category@1` document for its category:
++`test/retention.test.js` validates every `{ key, retention }` against the contract (openvibe-contracts v0.295.0) and
++holds the strict one-to-one check between `sources.source@1`'s category list, `sources.category@1`'s keys and the rule
++table, so a pin bump fails CI and names the missing rule. At runtime a category the contract has but the rules do not is
++kept (never pruned) and warned about once a run, and a rule for a category the contract lacks is ignored with a warning,
++so a Contracts pin bump cannot stop Sources at boot.
## API (service tokens, one capability per route)
@@@ -599,7 +601,7 @@ in [OpenVibers/OpenVibe.Sites](https://github.com/OpenVibers/OpenVibe.Sites); th
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
<!-- versions:start -->
--- openvibe-contracts: v0.290.0
++- openvibe-contracts: v0.295.0
- openvibe-sdk: v0.91.0
- openvibe-shared: v4.115.1
<!-- versions:end -->
ddiff --git a/STATUS.json b/STATUS.json
iindex ad8e835..b3d152d 100644
---- a/STATUS.json
++++ b/STATUS.json
@@@ -7,7 +7,7 @@
"runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, fast-xml-parser",
"port": 4720,
"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, the merchant opt-in pages, health and JSON 404s; the JSON API is loopback-only; README \"Public host\")",
-- "contracts": "openvibe-contracts v0.290.0",
++ "contracts": "openvibe-contracts v0.295.0",
"sdk": "openvibe-sdk v0.91.0",
"shared": "openvibe-shared v4.115.1",
"capabilities": [
@@@ -18,7 +18,8 @@
"capabilitiesRegistered": true,
"contractsProposed": [
"sources.source@1 (released in openvibe-contracts v0.12.0)",
-- "sources.item@1 (released in openvibe-contracts v0.12.0)"
++ "sources.item@1 (released in openvibe-contracts v0.12.0)",
++ "sources.category@1 (released in openvibe-contracts v0.295.0)"
],
"adapters": [
"rss",
@@@ -46,6 +47,7 @@
"failed fetches recorded, never fabricated",
"item revisions and sticky removals with provenance",
"retire_after_sec: items a feed stopped showing are removed as gone_from_source (jobs, listings)",
++ "per-category item retention: one sources.category@1 window per category, pruned daily",
"cursor item API by capability",
"openvibe-sdk transactional outbox to Events (service_outbox)",
"staff-only index documents for Search",
ddiff --git a/package-lock.json b/package-lock.json
iindex bb2dbb8..e13297f 100644
---- a/package-lock.json
++++ b/package-lock.json
@@@ -12,7 +12,7 @@
"dotenv": "^16.4.7",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.290.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.295.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.91.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.115.1",
"pg": "^8.23.0"
@@@ -1044,9 +1044,9 @@
}
},
"node_modules/openvibe-contracts": {
-- "version": "0.290.0",
-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.290.0",
-- "integrity": "sha512-WX9p49Wyr5dPRQ9iQEJvVEPIQwkJYsJTLryU5DJJI/GrxzfMZFN1nfuM5W/KmL3DGoRTAOg/KAWjs0OkunJyeg==",
++ "version": "0.295.0",
++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.295.0",
++ "integrity": "sha512-GLE0+RK6gqeLxoMCVG9IG1sIS50+kDbI1BTo56jypcSNNojAibOxTdNfbsKDyNe1rWHM01AOkZ6dGcHH7FIGOw==",
"license": "AGPL-3.0-only",
"dependencies": {
"ajv": "^8.17.1",
ddiff --git a/package.json b/package.json
iindex 28832a2..3ebd6ba 100644
---- a/package.json
++++ b/package.json
@@@ -30,7 +30,7 @@
"dotenv": "^16.4.7",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.290.0",
++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.295.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.91.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.115.1",
"pg": "^8.23.0"
ddiff --git a/server/retention.js b/server/retention.js
iindex 3eba06e..6302c56 100644
---- a/server/retention.js
++++ b/server/retention.js
@@@ -51,7 +51,9 @@
* check lives in test/retention.test.js instead, so the pin bump PR fails CI and names the rule to
* add while production keeps serving in the meantime.
*/
--const { CATEGORIES } = require('./registry');
++const CATEGORY_CONTRACT = require('openvibe-contracts/contracts/sources/category.v1.json');
++/** The categories `sources.category@1` names: the contract this table implements. */
++const CATEGORY_KEYS = Object.freeze([...CATEGORY_CONTRACT.properties.key.enum]);
const DAY = 86_400_000;
/** Rows per DELETE; a few thousand is fast on PostgreSQL and stays well under the parameter limit. */
@@@ -74,7 +76,7 @@ const RULES = Object.freeze({
jobs: { days: 14, reason: 'boards retire unseen jobs after 7 days and Work pulls hourly' },
});
--// The rules are the contract's categories in review; the module deliberately does not throw when
++// The rules are the contract's categories in review; the runtime deliberately does not throw when
// they disagree (a pin bump must not stop Sources at boot; test/retention.test.js is strict).
/** The age expression shared by the prune statements: last current, not first seen. */
@@@ -123,12 +125,12 @@ function createRetention({ db, rules = RULES, now = () => Date.now(), log = cons
const counts = { pruned: 0, revisions: 0, batches: 0, categories: {}, capped: [] };
// A pin bump may reach production before this table does: keep an unruled category, ignore a
// rule the contract no longer has, and say so once per run. The strict check is in the test.
-- const known = new Set(CATEGORIES);
-- for (const category of CATEGORIES) {
++ const known = new Set(CATEGORY_KEYS);
++ for (const category of CATEGORY_KEYS) {
if (!rules[category]) log.warn(`[retention] no rule for category ${category}: items kept`);
}
for (const category of Object.keys(rules)) {
-- if (!known.has(category)) log.warn(`[retention] rule for ${category} ignored: not a sources.source@1 category`);
++ if (!known.has(category)) log.warn(`[retention] rule for ${category} ignored: not a sources.category@1 category`);
}
for (const [category, rule] of Object.entries(rules)) {
if (!known.has(category)) continue;
@@@ -151,4 +153,4 @@ function createRetention({ db, rules = RULES, now = () => Date.now(), log = cons
return { run, runOnce, rules };
}
--module.exports = { createRetention, RULES, AGE_SQL, OLDER_THAN_SQL, BATCH_SIZE, MAX_BATCHES, DAY };
++module.exports = { createRetention, RULES, CATEGORY_KEYS, AGE_SQL, OLDER_THAN_SQL, BATCH_SIZE, MAX_BATCHES, DAY };
ddiff --git a/test/retention.test.js b/test/retention.test.js
iindex 036f745..85c827e 100644
---- a/test/retention.test.js
++++ b/test/retention.test.js
@@@ -1,16 +1,17 @@
'use strict';
/**
-- * Item retention (server/retention.js, server/jobs/): the category windows, the age of a removed item,
-- * keep-latest per source key, batching, idempotence, and the runtime grace for a category a pin bump
-- * added before the rule table did. A real migrated database (PGlite, or the containers with
-- * SOURCES_TEST_STORE=pg), no network.
++ * Item retention (server/retention.js, server/jobs/): the category windows as sources.category@1
++ * documents, the age of a removed item, keep-latest per source key, batching, idempotence, and the
++ * runtime grace for a category a pin bump added before the rule table did. A real migrated database
++ * (PGlite, or the containers with SOURCES_TEST_STORE=pg), no network.
*
* node test/retention.test.js
*/
const assert = require('assert');
const { createDb } = require('openvibe-sdk/db');
const { testDb } = require('./db');
--const { createRetention, RULES, DAY } = require('../server/retention');
++const { validate } = require('openvibe-contracts');
++const { createRetention, RULES, CATEGORY_KEYS, DAY } = require('../server/retention');
const { createJobs } = require('../server/jobs');
const { CATEGORIES } = require('../server/registry');
@@@ -42,13 +43,20 @@ const revisions = async (id) => Number(await db.value('SELECT count(*) FROM item
try {
// STRICT BY DESIGN, TEST-ONLY: a Contracts pin that adds a category must fail here, on the
// bump PR, naming the missing rule. server/retention.js itself keeps + warns at runtime.
-- assert.deepStrictEqual([...CATEGORIES].sort(), Object.keys(RULES).sort(), 'one rule per source.v1 category');
++ assert.deepStrictEqual([...CATEGORIES].sort(), [...CATEGORY_KEYS].sort(), 'sources.source@1 and sources.category@1 name the same categories');
++ assert.deepStrictEqual([...CATEGORY_KEYS].sort(), Object.keys(RULES).sort(), 'one rule per sources.category@1 key');
for (const [category, rule] of Object.entries(RULES)) {
assert.strictEqual(typeof rule.reason, 'string');
assert.ok(rule.reason.length > 10, `${category} says why`);
assert.ok((Number.isInteger(rule.days) && rule.days >= 1) || rule.keepLatest === true, `${category} has a day window or keep-latest`);
assert.ok(!(rule.days && rule.keepLatest), `${category} is one shape`);
++ // The rule is the contract's document for its category, so a field the schema does not
++ // have (or a window outside 1..3650 days) fails here rather than in production.
++ const document = { key: category, retention: rule.keepLatest ? 'keep' : { days: rule.days, reason: rule.reason } };
++ const result = validate('sources.category@1', document);
++ assert.ok(result.valid, `${category} is a sources.category@1 document: ${JSON.stringify(result.errors)}`);
}
++ assert.strictEqual(validate('sources.category@1', { key: 'news', retention: { days: 0 } }).valid, false, 'a zero-day window is refused by sources.category@1');
assert.strictEqual(RULES.deals.days, 14);
assert.strictEqual(RULES.news.days, 30);
assert.strictEqual(RULES.housing.keepLatest, true);
@@@ -120,7 +128,7 @@ const revisions = async (id) => Number(await db.value('SELECT count(*) FROM item
assert.deepStrictEqual(injectedOut.categories, { deals: 1 });
assert.deepStrictEqual(await ids("WHERE id IN ('news-unruled', 'deals-injected')"), ['news-unruled'], 'unruled news kept');
assert.strictEqual(injectedLogs.filter((m) => m === '[retention] no rule for category news: items kept').length, 1);
-- assert.strictEqual(injectedLogs.filter((m) => m === '[retention] rule for weather ignored: not a sources.source@1 category').length, 1);
++ assert.strictEqual(injectedLogs.filter((m) => m === '[retention] rule for weather ignored: not a sources.category@1 category').length, 1);
await injected.runOnce();
assert.strictEqual(injectedLogs.filter((m) => m === '[retention] no rule for category news: items kept').length, 2, 'one warning per run');
assert.deepStrictEqual(await ids("WHERE id = 'news-unruled'"), ['news-unruled']);