openvibers/openvibe.sources Commits

A patch that changes a source's endpoints clears their stored ETag and Last-Modified, so the next fetch parses with the new settings (#100)

OP OpenVibers committed 20 days ago · e4e55d65b6 · 2 files, +44 −1

Changes

ddiff --git a/server/registry.js b/server/registry.js
iindex 44b6b5e..848dd62 100644
---- a/server/registry.js
++++ b/server/registry.js
@@@ -168,6 +168,7 @@ function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {
search_visibility = @search_visibility, retire_after_sec = @retire_after_sec, updated_at = @updated_at, updated_by = @updated_by WHERE key = @key`),
del: db.prepare('DELETE FROM sources WHERE key = ?'),
itemCount: db.prepare('SELECT COUNT(*) AS n FROM items WHERE source_key = ?'),
++ resetValidators: db.prepare('DELETE FROM endpoint_state WHERE source_key = ?'),
};
function toRow(rec, by) {
@@@ -241,7 +242,11 @@ function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {
}
const merged = { ...fromRow(row), ...(changes || {}), key };
const rec = validateSource(merged, { maxItemsCap });
-- await st.update.run({ ...toRow(rec, by) });
++ const next = toRow(rec, by);
++ await st.update.run({ ...next });
++ // New endpoint settings must reach the next fetch: the stored ETag/Last-Modified would make it a 304 that parses
++ // nothing until the upstream file changes (bls-average-prices' duplicates: last waited on BLS's monthly file).
++ if (next.endpoints !== row.endpoints) await st.resetValidators.run(key);
return await st.get.get(key);
}
ddiff --git a/test/registry-validators.test.js b/test/registry-validators.test.js
nnew file mode 100644
iindex 0000000..055e9a5
---- /dev/null
++++ b/test/registry-validators.test.js
@@@ -0,0 +1,38 @@
++'use strict';
++/**
++ * A patch that changes a source's endpoints clears their stored ETag/Last-Modified (endpoint_state), so the next fetch
++ * downloads and parses with the new settings; any other patch keeps them. On 2026-10-24 bls-average-prices got its
++ * duplicates: last, and the next fetch was a 304 that parsed nothing: the setting would have waited for BLS's monthly
++ * file.
++ *
++ * node test/registry-validators.test.js
++ */
++const assert = require('assert');
++const { createRegistry } = require('../server/registry');
++const { suite } = require('./helpers');
++
++const t = suite('registry-validators');
++const source = {
++ key: 'tsv-test', name: 'TSV test', type: 'api', category: 'food', homepage_url: 'https://data.example',
++ endpoints: [{ url: 'https://data.example/series.tsv', format: 'tsv', items_path: '', item_kind: 'price', fields: { identity: 'series_id' }, extra: { value: 'value' } }],
++ robots_note: 'robots.txt read', terms_note: 'public domain', license_note: 'public domain',
++};
++
++t('endpoints changed: validators cleared; anything else: kept', async () => {
++ const fresh = await require('./db').testDb();
++ try {
++ const registry = createRegistry({ db: fresh.db });
++ await registry.create(source, 'test');
++ const put = () => fresh.db.prepare("INSERT INTO endpoint_state (source_key, url, etag, last_modified, last_status, last_fetch_at) VALUES ('tsv-test', 'https://data.example/series.tsv', '\"abc\"', 'Fri, 23 Oct 2026 12:00:00 GMT', 200, 1) ON CONFLICT (source_key, url) DO UPDATE SET etag = excluded.etag").run();
++ const count = async () => Number((await fresh.db.prepare("SELECT COUNT(*) AS n FROM endpoint_state WHERE source_key = 'tsv-test'").get()).n);
++ await put();
++ await registry.patch('tsv-test', { name: 'TSV test, renamed' }, 'test');
++ assert.strictEqual(await count(), 1, 'a name change keeps the validators');
++ await registry.patch('tsv-test', { endpoints: [{ ...source.endpoints[0], duplicates: 'last' }] }, 'test');
++ assert.strictEqual(await count(), 0, 'new endpoint settings: the next fetch is a full one');
++ } finally {
++ await fresh.close();
++ }
++});
++
++t.run();