openvibers/openvibe.sources Commits
Sources: retention rules are sources.category@1 documents (contracts 0.295.0) (#212)
Changes
@@@ -264,10 +264,12 @@ An item's age is `retrieved_at` while it is live and `removed_at` once removed,still sees a removal for the full window; `first_seen_at` is never the age, since a source that keeps listing a job ora price keeps it current. The daily job (`server/jobs/retention.js`, started with the worker) deletes at most 2,000 rowsa statement, 50 batches a category a run, and its log line carries counts only. The rules are in code because Sources--has no `openvibe-shared/config` namespace. At runtime a category the contract has but the rules do not is kept (never--pruned) and warned about once a run, and a rule for a category the contract lacks is ignored with a warning, so a--Contracts pin bump cannot stop Sources at boot; `test/retention.test.js` keeps the strict one-to-one check so the bump--PR fails CI and names the missing rule.++has no `openvibe-shared/config` namespace, and each one is the `sources.category@1` document for its category:++`test/retention.test.js` validates every `{ key, retention }` against the contract (openvibe-contracts v0.295.0) and++holds the strict one-to-one check between `sources.source@1`'s category list, `sources.category@1`'s keys and the rule++table, so a pin bump fails CI and names the missing rule. At runtime a category the contract has but the rules do not is++kept (never pruned) and warned about once a run, and a rule for a category the contract lacks is ignored with a warning,++so a Contracts pin bump cannot stop Sources at boot.## API (service tokens, one capability per route)@@@ -599,7 +601,7 @@ in [OpenVibers/OpenVibe.Sites](https://github.com/OpenVibers/OpenVibe.Sites); thPart of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).<!-- versions:start -->--- openvibe-contracts: v0.290.0++- openvibe-contracts: v0.295.0- openvibe-sdk: v0.91.0- openvibe-shared: v4.115.1<!-- versions:end -->@@@ -7,7 +7,7 @@"runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, fast-xml-parser","port": 4720,"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, the merchant opt-in pages, health and JSON 404s; the JSON API is loopback-only; README \"Public host\")",-- "contracts": "openvibe-contracts v0.290.0",++ "contracts": "openvibe-contracts v0.295.0","sdk": "openvibe-sdk v0.91.0","shared": "openvibe-shared v4.115.1","capabilities": [@@@ -18,7 +18,8 @@"capabilitiesRegistered": true,"contractsProposed": ["sources.source@1 (released in openvibe-contracts v0.12.0)",-- "sources.item@1 (released in openvibe-contracts v0.12.0)"++ "sources.item@1 (released in openvibe-contracts v0.12.0)",++ "sources.category@1 (released in openvibe-contracts v0.295.0)"],"adapters": ["rss",@@@ -46,6 +47,7 @@"failed fetches recorded, never fabricated","item revisions and sticky removals with provenance","retire_after_sec: items a feed stopped showing are removed as gone_from_source (jobs, listings)",++ "per-category item retention: one sources.category@1 window per category, pruned daily","cursor item API by capability","openvibe-sdk transactional outbox to Events (service_outbox)","staff-only index documents for Search",@@@ -12,7 +12,7 @@"dotenv": "^16.4.7","express": "^4.21.2","fast-xml-parser": "^4.5.3",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.290.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.295.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.91.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.115.1","pg": "^8.23.0"@@@ -1044,9 +1044,9 @@}},"node_modules/openvibe-contracts": {-- "version": "0.290.0",-- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.290.0",-- "integrity": "sha512-WX9p49Wyr5dPRQ9iQEJvVEPIQwkJYsJTLryU5DJJI/GrxzfMZFN1nfuM5W/KmL3DGoRTAOg/KAWjs0OkunJyeg==",++ "version": "0.295.0",++ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.295.0",++ "integrity": "sha512-GLE0+RK6gqeLxoMCVG9IG1sIS50+kDbI1BTo56jypcSNNojAibOxTdNfbsKDyNe1rWHM01AOkZ6dGcHH7FIGOw==","license": "AGPL-3.0-only","dependencies": {"ajv": "^8.17.1",@@@ -30,7 +30,7 @@"dotenv": "^16.4.7","express": "^4.21.2","fast-xml-parser": "^4.5.3",-- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.290.0",++ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.295.0","openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.91.0","openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.115.1","pg": "^8.23.0"@@@ -51,7 +51,9 @@* check lives in test/retention.test.js instead, so the pin bump PR fails CI and names the rule to* add while production keeps serving in the meantime.*/--const { CATEGORIES } = require('./registry');++const CATEGORY_CONTRACT = require('openvibe-contracts/contracts/sources/category.v1.json');++/** The categories `sources.category@1` names: the contract this table implements. */++const CATEGORY_KEYS = Object.freeze([...CATEGORY_CONTRACT.properties.key.enum]);const DAY = 86_400_000;/** Rows per DELETE; a few thousand is fast on PostgreSQL and stays well under the parameter limit. */@@@ -74,7 +76,7 @@ const RULES = Object.freeze({jobs: { days: 14, reason: 'boards retire unseen jobs after 7 days and Work pulls hourly' },});--// The rules are the contract's categories in review; the module deliberately does not throw when++// The rules are the contract's categories in review; the runtime deliberately does not throw when// they disagree (a pin bump must not stop Sources at boot; test/retention.test.js is strict)./** The age expression shared by the prune statements: last current, not first seen. */@@@ -123,12 +125,12 @@ function createRetention({ db, rules = RULES, now = () => Date.now(), log = consconst counts = { pruned: 0, revisions: 0, batches: 0, categories: {}, capped: [] };// A pin bump may reach production before this table does: keep an unruled category, ignore a// rule the contract no longer has, and say so once per run. The strict check is in the test.-- const known = new Set(CATEGORIES);-- for (const category of CATEGORIES) {++ const known = new Set(CATEGORY_KEYS);++ for (const category of CATEGORY_KEYS) {if (!rules[category]) log.warn(`[retention] no rule for category ${category}: items kept`);}for (const category of Object.keys(rules)) {-- if (!known.has(category)) log.warn(`[retention] rule for ${category} ignored: not a sources.source@1 category`);++ if (!known.has(category)) log.warn(`[retention] rule for ${category} ignored: not a sources.category@1 category`);}for (const [category, rule] of Object.entries(rules)) {if (!known.has(category)) continue;@@@ -151,4 +153,4 @@ function createRetention({ db, rules = RULES, now = () => Date.now(), log = consreturn { run, runOnce, rules };}--module.exports = { createRetention, RULES, AGE_SQL, OLDER_THAN_SQL, BATCH_SIZE, MAX_BATCHES, DAY };++module.exports = { createRetention, RULES, CATEGORY_KEYS, AGE_SQL, OLDER_THAN_SQL, BATCH_SIZE, MAX_BATCHES, DAY };@@@ -1,16 +1,17 @@'use strict';/**-- * Item retention (server/retention.js, server/jobs/): the category windows, the age of a removed item,-- * keep-latest per source key, batching, idempotence, and the runtime grace for a category a pin bump-- * added before the rule table did. A real migrated database (PGlite, or the containers with-- * SOURCES_TEST_STORE=pg), no network.++ * Item retention (server/retention.js, server/jobs/): the category windows as sources.category@1++ * documents, the age of a removed item, keep-latest per source key, batching, idempotence, and the++ * runtime grace for a category a pin bump added before the rule table did. A real migrated database++ * (PGlite, or the containers with SOURCES_TEST_STORE=pg), no network.** node test/retention.test.js*/const assert = require('assert');const { createDb } = require('openvibe-sdk/db');const { testDb } = require('./db');--const { createRetention, RULES, DAY } = require('../server/retention');++const { validate } = require('openvibe-contracts');++const { createRetention, RULES, CATEGORY_KEYS, DAY } = require('../server/retention');const { createJobs } = require('../server/jobs');const { CATEGORIES } = require('../server/registry');@@@ -42,13 +43,20 @@ const revisions = async (id) => Number(await db.value('SELECT count(*) FROM itemtry {// STRICT BY DESIGN, TEST-ONLY: a Contracts pin that adds a category must fail here, on the// bump PR, naming the missing rule. server/retention.js itself keeps + warns at runtime.-- assert.deepStrictEqual([...CATEGORIES].sort(), Object.keys(RULES).sort(), 'one rule per source.v1 category');++ assert.deepStrictEqual([...CATEGORIES].sort(), [...CATEGORY_KEYS].sort(), 'sources.source@1 and sources.category@1 name the same categories');++ assert.deepStrictEqual([...CATEGORY_KEYS].sort(), Object.keys(RULES).sort(), 'one rule per sources.category@1 key');for (const [category, rule] of Object.entries(RULES)) {assert.strictEqual(typeof rule.reason, 'string');assert.ok(rule.reason.length > 10, `${category} says why`);assert.ok((Number.isInteger(rule.days) && rule.days >= 1) || rule.keepLatest === true, `${category} has a day window or keep-latest`);assert.ok(!(rule.days && rule.keepLatest), `${category} is one shape`);++ // The rule is the contract's document for its category, so a field the schema does not++ // have (or a window outside 1..3650 days) fails here rather than in production.++ const document = { key: category, retention: rule.keepLatest ? 'keep' : { days: rule.days, reason: rule.reason } };++ const result = validate('sources.category@1', document);++ assert.ok(result.valid, `${category} is a sources.category@1 document: ${JSON.stringify(result.errors)}`);}++ assert.strictEqual(validate('sources.category@1', { key: 'news', retention: { days: 0 } }).valid, false, 'a zero-day window is refused by sources.category@1');assert.strictEqual(RULES.deals.days, 14);assert.strictEqual(RULES.news.days, 30);assert.strictEqual(RULES.housing.keepLatest, true);@@@ -120,7 +128,7 @@ const revisions = async (id) => Number(await db.value('SELECT count(*) FROM itemassert.deepStrictEqual(injectedOut.categories, { deals: 1 });assert.deepStrictEqual(await ids("WHERE id IN ('news-unruled', 'deals-injected')"), ['news-unruled'], 'unruled news kept');assert.strictEqual(injectedLogs.filter((m) => m === '[retention] no rule for category news: items kept').length, 1);-- assert.strictEqual(injectedLogs.filter((m) => m === '[retention] rule for weather ignored: not a sources.source@1 category').length, 1);++ assert.strictEqual(injectedLogs.filter((m) => m === '[retention] rule for weather ignored: not a sources.category@1 category').length, 1);await injected.runOnce();assert.strictEqual(injectedLogs.filter((m) => m === '[retention] no rule for category news: items kept').length, 2, 'one warning per run');assert.deepStrictEqual(await ids("WHERE id = 'news-unruled'"), ['news-unruled']);
@@ -264,10 +264,12 @@ An item's age is `retrieved_at` while it is live and `removed_at` once removed,
@@ -264,10 +264,12 @@ An item's age is `retrieved_at` while it is live and `removed_at` once removed,
still sees a removal for the full window; `first_seen_at` is never the age, since a source that keeps listing a job or
still sees a removal for the full window; `first_seen_at` is never the age, since a source that keeps listing a job or
a price keeps it current. The daily job (`server/jobs/retention.js`, started with the worker) deletes at most 2,000 rows
a price keeps it current. The daily job (`server/jobs/retention.js`, started with the worker) deletes at most 2,000 rows
a statement, 50 batches a category a run, and its log line carries counts only. The rules are in code because Sources
a statement, 50 batches a category a run, and its log line carries counts only. The rules are in code because Sources
-has no `openvibe-shared/config` namespace. At runtime a category the contract has but the rules do not is kept (never
-pruned) and warned about once a run, and a rule for a category the contract lacks is ignored with a warning, so a
-Contracts pin bump cannot stop Sources at boot; `test/retention.test.js` keeps the strict one-to-one check so the bump
-PR fails CI and names the missing rule.
+has no `openvibe-shared/config` namespace, and each one is the `sources.category@1` document for its category:
+`test/retention.test.js` validates every `{ key, retention }` against the contract (openvibe-contracts v0.295.0) and
+holds the strict one-to-one check between `sources.source@1`'s category list, `sources.category@1`'s keys and the rule
+table, so a pin bump fails CI and names the missing rule. At runtime a category the contract has but the rules do not is
+kept (never pruned) and warned about once a run, and a rule for a category the contract lacks is ignored with a warning,
+so a Contracts pin bump cannot stop Sources at boot.
## API (service tokens, one capability per route)
## API (service tokens, one capability per route)
@@ -599,7 +601,7 @@ in [OpenVibers/OpenVibe.Sites](https://github.com/OpenVibers/OpenVibe.Sites); th
@@ -599,7 +601,7 @@ in [OpenVibers/OpenVibe.Sites](https://github.com/OpenVibers/OpenVibe.Sites); th
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
Part of the [OpenVibe network](https://openvibe.network). Built in the open by [OpenVibers](https://github.com/OpenVibers).
<!-- versions:start -->
<!-- versions:start -->
-- openvibe-contracts: v0.290.0
+- openvibe-contracts: v0.295.0
- openvibe-sdk: v0.91.0
- openvibe-sdk: v0.91.0
- openvibe-shared: v4.115.1
- openvibe-shared: v4.115.1
<!-- versions:end -->
<!-- versions:end -->
@@ -7,7 +7,7 @@
@@ -7,7 +7,7 @@
"runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, fast-xml-parser",
"runtime": "node 22, express, PostgreSQL 18 (openvibe-sdk/db, async) + Valkey 9, fast-xml-parser",
"port": 4720,
"port": 4720,
"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, the merchant opt-in pages, health and JSON 404s; the JSON API is loopback-only; README \"Public host\")",
"domain": "sources.openvibe.network (vhost in deploy/nginx: an internal-service page, the merchant opt-in pages, health and JSON 404s; the JSON API is loopback-only; README \"Public host\")",
- "contracts": "openvibe-contracts v0.290.0",
+ "contracts": "openvibe-contracts v0.295.0",
"sdk": "openvibe-sdk v0.91.0",
"sdk": "openvibe-sdk v0.91.0",
"shared": "openvibe-shared v4.115.1",
"shared": "openvibe-shared v4.115.1",
"capabilities": [
"capabilities": [
@@ -18,7 +18,8 @@
@@ -18,7 +18,8 @@
"capabilitiesRegistered": true,
"capabilitiesRegistered": true,
"contractsProposed": [
"contractsProposed": [
"sources.source@1 (released in openvibe-contracts v0.12.0)",
"sources.source@1 (released in openvibe-contracts v0.12.0)",
- "sources.item@1 (released in openvibe-contracts v0.12.0)"
+ "sources.item@1 (released in openvibe-contracts v0.12.0)",
+ "sources.category@1 (released in openvibe-contracts v0.295.0)"
],
],
"adapters": [
"adapters": [
"rss",
"rss",
@@ -46,6 +47,7 @@
@@ -46,6 +47,7 @@
"failed fetches recorded, never fabricated",
"failed fetches recorded, never fabricated",
"item revisions and sticky removals with provenance",
"item revisions and sticky removals with provenance",
"retire_after_sec: items a feed stopped showing are removed as gone_from_source (jobs, listings)",
"retire_after_sec: items a feed stopped showing are removed as gone_from_source (jobs, listings)",
+ "per-category item retention: one sources.category@1 window per category, pruned daily",
"cursor item API by capability",
"cursor item API by capability",
"openvibe-sdk transactional outbox to Events (service_outbox)",
"openvibe-sdk transactional outbox to Events (service_outbox)",
"staff-only index documents for Search",
"staff-only index documents for Search",
@@ -12,7 +12,7 @@
@@ -12,7 +12,7 @@
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"fast-xml-parser": "^4.5.3",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.290.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.295.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.91.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.91.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.115.1",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.115.1",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -1044,9 +1044,9 @@
@@ -1044,9 +1044,9 @@
}
}
},
},
"node_modules/openvibe-contracts": {
"node_modules/openvibe-contracts": {
- "version": "0.290.0",
- "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.290.0",
- "integrity": "sha512-WX9p49Wyr5dPRQ9iQEJvVEPIQwkJYsJTLryU5DJJI/GrxzfMZFN1nfuM5W/KmL3DGoRTAOg/KAWjs0OkunJyeg==",
+ "version": "0.295.0",
+ "resolved": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.295.0",
+ "integrity": "sha512-GLE0+RK6gqeLxoMCVG9IG1sIS50+kDbI1BTo56jypcSNNojAibOxTdNfbsKDyNe1rWHM01AOkZ6dGcHH7FIGOw==",
"license": "AGPL-3.0-only",
"license": "AGPL-3.0-only",
"dependencies": {
"dependencies": {
"ajv": "^8.17.1",
"ajv": "^8.17.1",
@@ -30,7 +30,7 @@
@@ -30,7 +30,7 @@
"dotenv": "^16.4.7",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"express": "^4.21.2",
"fast-xml-parser": "^4.5.3",
"fast-xml-parser": "^4.5.3",
- "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.290.0",
+ "openvibe-contracts": "https://codeload.github.com/OpenVibers/OpenVibe.Contracts/tar.gz/refs/tags/v0.295.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.91.0",
"openvibe-sdk": "https://codeload.github.com/OpenVibers/OpenVibe.SDK/tar.gz/refs/tags/v0.91.0",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.115.1",
"openvibe-shared": "https://codeload.github.com/OpenVibers/OpenVibe.Shared/tar.gz/refs/tags/v4.115.1",
"pg": "^8.23.0"
"pg": "^8.23.0"
@@ -51,7 +51,9 @@
@@ -51,7 +51,9 @@
* check lives in test/retention.test.js instead, so the pin bump PR fails CI and names the rule to
* check lives in test/retention.test.js instead, so the pin bump PR fails CI and names the rule to
* add while production keeps serving in the meantime.
* add while production keeps serving in the meantime.
*/
*/
-const { CATEGORIES } = require('./registry');
+const CATEGORY_CONTRACT = require('openvibe-contracts/contracts/sources/category.v1.json');
+/** The categories `sources.category@1` names: the contract this table implements. */
+const CATEGORY_KEYS = Object.freeze([...CATEGORY_CONTRACT.properties.key.enum]);
const DAY = 86_400_000;
const DAY = 86_400_000;
/** Rows per DELETE; a few thousand is fast on PostgreSQL and stays well under the parameter limit. */
/** Rows per DELETE; a few thousand is fast on PostgreSQL and stays well under the parameter limit. */
@@ -74,7 +76,7 @@ const RULES = Object.freeze({
@@ -74,7 +76,7 @@ const RULES = Object.freeze({
jobs: { days: 14, reason: 'boards retire unseen jobs after 7 days and Work pulls hourly' },
jobs: { days: 14, reason: 'boards retire unseen jobs after 7 days and Work pulls hourly' },
});
});
-// The rules are the contract's categories in review; the module deliberately does not throw when
+// The rules are the contract's categories in review; the runtime deliberately does not throw when
// they disagree (a pin bump must not stop Sources at boot; test/retention.test.js is strict).
// they disagree (a pin bump must not stop Sources at boot; test/retention.test.js is strict).
/** The age expression shared by the prune statements: last current, not first seen. */
/** The age expression shared by the prune statements: last current, not first seen. */
@@ -123,12 +125,12 @@ function createRetention({ db, rules = RULES, now = () => Date.now(), log = cons
@@ -123,12 +125,12 @@ function createRetention({ db, rules = RULES, now = () => Date.now(), log = cons
const counts = { pruned: 0, revisions: 0, batches: 0, categories: {}, capped: [] };
const counts = { pruned: 0, revisions: 0, batches: 0, categories: {}, capped: [] };
// A pin bump may reach production before this table does: keep an unruled category, ignore a
// A pin bump may reach production before this table does: keep an unruled category, ignore a
// rule the contract no longer has, and say so once per run. The strict check is in the test.
// rule the contract no longer has, and say so once per run. The strict check is in the test.
- const known = new Set(CATEGORIES);
- for (const category of CATEGORIES) {
+ const known = new Set(CATEGORY_KEYS);
+ for (const category of CATEGORY_KEYS) {
if (!rules[category]) log.warn(`[retention] no rule for category ${category}: items kept`);
if (!rules[category]) log.warn(`[retention] no rule for category ${category}: items kept`);
}
}
for (const category of Object.keys(rules)) {
for (const category of Object.keys(rules)) {
- if (!known.has(category)) log.warn(`[retention] rule for ${category} ignored: not a sources.source@1 category`);
+ if (!known.has(category)) log.warn(`[retention] rule for ${category} ignored: not a sources.category@1 category`);
}
}
for (const [category, rule] of Object.entries(rules)) {
for (const [category, rule] of Object.entries(rules)) {
if (!known.has(category)) continue;
if (!known.has(category)) continue;
@@ -151,4 +153,4 @@ function createRetention({ db, rules = RULES, now = () => Date.now(), log = cons
@@ -151,4 +153,4 @@ function createRetention({ db, rules = RULES, now = () => Date.now(), log = cons
return { run, runOnce, rules };
return { run, runOnce, rules };
}
}
-module.exports = { createRetention, RULES, AGE_SQL, OLDER_THAN_SQL, BATCH_SIZE, MAX_BATCHES, DAY };
+module.exports = { createRetention, RULES, CATEGORY_KEYS, AGE_SQL, OLDER_THAN_SQL, BATCH_SIZE, MAX_BATCHES, DAY };
@@ -1,16 +1,17 @@
@@ -1,16 +1,17 @@
'use strict';
'use strict';
/**
/**
- * Item retention (server/retention.js, server/jobs/): the category windows, the age of a removed item,
- * keep-latest per source key, batching, idempotence, and the runtime grace for a category a pin bump
- * added before the rule table did. A real migrated database (PGlite, or the containers with
- * SOURCES_TEST_STORE=pg), no network.
+ * Item retention (server/retention.js, server/jobs/): the category windows as sources.category@1
+ * documents, the age of a removed item, keep-latest per source key, batching, idempotence, and the
+ * runtime grace for a category a pin bump added before the rule table did. A real migrated database
+ * (PGlite, or the containers with SOURCES_TEST_STORE=pg), no network.
*
*
* node test/retention.test.js
* node test/retention.test.js
*/
*/
const assert = require('assert');
const assert = require('assert');
const { createDb } = require('openvibe-sdk/db');
const { createDb } = require('openvibe-sdk/db');
const { testDb } = require('./db');
const { testDb } = require('./db');
-const { createRetention, RULES, DAY } = require('../server/retention');
+const { validate } = require('openvibe-contracts');
+const { createRetention, RULES, CATEGORY_KEYS, DAY } = require('../server/retention');
const { createJobs } = require('../server/jobs');
const { createJobs } = require('../server/jobs');
const { CATEGORIES } = require('../server/registry');
const { CATEGORIES } = require('../server/registry');
@@ -42,13 +43,20 @@ const revisions = async (id) => Number(await db.value('SELECT count(*) FROM item
@@ -42,13 +43,20 @@ const revisions = async (id) => Number(await db.value('SELECT count(*) FROM item
try {
try {
// STRICT BY DESIGN, TEST-ONLY: a Contracts pin that adds a category must fail here, on the
// STRICT BY DESIGN, TEST-ONLY: a Contracts pin that adds a category must fail here, on the
// bump PR, naming the missing rule. server/retention.js itself keeps + warns at runtime.
// bump PR, naming the missing rule. server/retention.js itself keeps + warns at runtime.
- assert.deepStrictEqual([...CATEGORIES].sort(), Object.keys(RULES).sort(), 'one rule per source.v1 category');
+ assert.deepStrictEqual([...CATEGORIES].sort(), [...CATEGORY_KEYS].sort(), 'sources.source@1 and sources.category@1 name the same categories');
+ assert.deepStrictEqual([...CATEGORY_KEYS].sort(), Object.keys(RULES).sort(), 'one rule per sources.category@1 key');
for (const [category, rule] of Object.entries(RULES)) {
for (const [category, rule] of Object.entries(RULES)) {
assert.strictEqual(typeof rule.reason, 'string');
assert.strictEqual(typeof rule.reason, 'string');
assert.ok(rule.reason.length > 10, `${category} says why`);
assert.ok(rule.reason.length > 10, `${category} says why`);
assert.ok((Number.isInteger(rule.days) && rule.days >= 1) || rule.keepLatest === true, `${category} has a day window or keep-latest`);
assert.ok((Number.isInteger(rule.days) && rule.days >= 1) || rule.keepLatest === true, `${category} has a day window or keep-latest`);
assert.ok(!(rule.days && rule.keepLatest), `${category} is one shape`);
assert.ok(!(rule.days && rule.keepLatest), `${category} is one shape`);
+ // The rule is the contract's document for its category, so a field the schema does not
+ // have (or a window outside 1..3650 days) fails here rather than in production.
+ const document = { key: category, retention: rule.keepLatest ? 'keep' : { days: rule.days, reason: rule.reason } };
+ const result = validate('sources.category@1', document);
+ assert.ok(result.valid, `${category} is a sources.category@1 document: ${JSON.stringify(result.errors)}`);
}
}
+ assert.strictEqual(validate('sources.category@1', { key: 'news', retention: { days: 0 } }).valid, false, 'a zero-day window is refused by sources.category@1');
assert.strictEqual(RULES.deals.days, 14);
assert.strictEqual(RULES.deals.days, 14);
assert.strictEqual(RULES.news.days, 30);
assert.strictEqual(RULES.news.days, 30);
assert.strictEqual(RULES.housing.keepLatest, true);
assert.strictEqual(RULES.housing.keepLatest, true);
@@ -120,7 +128,7 @@ const revisions = async (id) => Number(await db.value('SELECT count(*) FROM item
@@ -120,7 +128,7 @@ const revisions = async (id) => Number(await db.value('SELECT count(*) FROM item
assert.deepStrictEqual(injectedOut.categories, { deals: 1 });
assert.deepStrictEqual(injectedOut.categories, { deals: 1 });
assert.deepStrictEqual(await ids("WHERE id IN ('news-unruled', 'deals-injected')"), ['news-unruled'], 'unruled news kept');
assert.deepStrictEqual(await ids("WHERE id IN ('news-unruled', 'deals-injected')"), ['news-unruled'], 'unruled news kept');
assert.strictEqual(injectedLogs.filter((m) => m === '[retention] no rule for category news: items kept').length, 1);
assert.strictEqual(injectedLogs.filter((m) => m === '[retention] no rule for category news: items kept').length, 1);
- assert.strictEqual(injectedLogs.filter((m) => m === '[retention] rule for weather ignored: not a sources.source@1 category').length, 1);
+ assert.strictEqual(injectedLogs.filter((m) => m === '[retention] rule for weather ignored: not a sources.category@1 category').length, 1);
await injected.runOnce();
await injected.runOnce();
assert.strictEqual(injectedLogs.filter((m) => m === '[retention] no rule for category news: items kept').length, 2, 'one warning per run');
assert.strictEqual(injectedLogs.filter((m) => m === '[retention] no rule for category news: items kept').length, 2, 'one warning per run');
assert.deepStrictEqual(await ids("WHERE id = 'news-unruled'"), ['news-unruled']);
assert.deepStrictEqual(await ids("WHERE id = 'news-unruled'"), ['news-unruled']);