openvibers/openvibe.sources Commits
A patch that changes a source's endpoints clears their stored ETag and Last-Modified, so the next fetch parses with the new settings (#100)
Changes
@@@ -168,6 +168,7 @@ function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {search_visibility = @search_visibility, retire_after_sec = @retire_after_sec, updated_at = @updated_at, updated_by = @updated_by WHERE key = @key`),del: db.prepare('DELETE FROM sources WHERE key = ?'),itemCount: db.prepare('SELECT COUNT(*) AS n FROM items WHERE source_key = ?'),++ resetValidators: db.prepare('DELETE FROM endpoint_state WHERE source_key = ?'),};function toRow(rec, by) {@@@ -241,7 +242,11 @@ function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {}const merged = { ...fromRow(row), ...(changes || {}), key };const rec = validateSource(merged, { maxItemsCap });-- await st.update.run({ ...toRow(rec, by) });++ const next = toRow(rec, by);++ await st.update.run({ ...next });++ // New endpoint settings must reach the next fetch: the stored ETag/Last-Modified would make it a 304 that parses++ // nothing until the upstream file changes (bls-average-prices' duplicates: last waited on BLS's monthly file).++ if (next.endpoints !== row.endpoints) await st.resetValidators.run(key);return await st.get.get(key);}nnew file mode 100644@@@ -0,0 +1,38 @@++'use strict';++/**++ * A patch that changes a source's endpoints clears their stored ETag/Last-Modified (endpoint_state), so the next fetch++ * downloads and parses with the new settings; any other patch keeps them. On 2026-10-24 bls-average-prices got its++ * duplicates: last, and the next fetch was a 304 that parsed nothing: the setting would have waited for BLS's monthly++ * file.++ *++ * node test/registry-validators.test.js++ */++const assert = require('assert');++const { createRegistry } = require('../server/registry');++const { suite } = require('./helpers');++++const t = suite('registry-validators');++const source = {++ key: 'tsv-test', name: 'TSV test', type: 'api', category: 'food', homepage_url: 'https://data.example',++ endpoints: [{ url: 'https://data.example/series.tsv', format: 'tsv', items_path: '', item_kind: 'price', fields: { identity: 'series_id' }, extra: { value: 'value' } }],++ robots_note: 'robots.txt read', terms_note: 'public domain', license_note: 'public domain',++};++++t('endpoints changed: validators cleared; anything else: kept', async () => {++ const fresh = await require('./db').testDb();++ try {++ const registry = createRegistry({ db: fresh.db });++ await registry.create(source, 'test');++ const put = () => fresh.db.prepare("INSERT INTO endpoint_state (source_key, url, etag, last_modified, last_status, last_fetch_at) VALUES ('tsv-test', 'https://data.example/series.tsv', '\"abc\"', 'Fri, 23 Oct 2026 12:00:00 GMT', 200, 1) ON CONFLICT (source_key, url) DO UPDATE SET etag = excluded.etag").run();++ const count = async () => Number((await fresh.db.prepare("SELECT COUNT(*) AS n FROM endpoint_state WHERE source_key = 'tsv-test'").get()).n);++ await put();++ await registry.patch('tsv-test', { name: 'TSV test, renamed' }, 'test');++ assert.strictEqual(await count(), 1, 'a name change keeps the validators');++ await registry.patch('tsv-test', { endpoints: [{ ...source.endpoints[0], duplicates: 'last' }] }, 'test');++ assert.strictEqual(await count(), 0, 'new endpoint settings: the next fetch is a full one');++ } finally {++ await fresh.close();++ }++});++++t.run();
@@ -168,6 +168,7 @@ function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {
@@ -168,6 +168,7 @@ function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {
search_visibility = @search_visibility, retire_after_sec = @retire_after_sec, updated_at = @updated_at, updated_by = @updated_by WHERE key = @key`),
search_visibility = @search_visibility, retire_after_sec = @retire_after_sec, updated_at = @updated_at, updated_by = @updated_by WHERE key = @key`),
del: db.prepare('DELETE FROM sources WHERE key = ?'),
del: db.prepare('DELETE FROM sources WHERE key = ?'),
itemCount: db.prepare('SELECT COUNT(*) AS n FROM items WHERE source_key = ?'),
itemCount: db.prepare('SELECT COUNT(*) AS n FROM items WHERE source_key = ?'),
+ resetValidators: db.prepare('DELETE FROM endpoint_state WHERE source_key = ?'),
};
};
function toRow(rec, by) {
function toRow(rec, by) {
@@ -241,7 +242,11 @@ function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {
@@ -241,7 +242,11 @@ function createRegistry({ db, now = () => Date.now(), maxItemsCap = 12000 }) {
}
}
const merged = { ...fromRow(row), ...(changes || {}), key };
const merged = { ...fromRow(row), ...(changes || {}), key };
const rec = validateSource(merged, { maxItemsCap });
const rec = validateSource(merged, { maxItemsCap });
- await st.update.run({ ...toRow(rec, by) });
+ const next = toRow(rec, by);
+ await st.update.run({ ...next });
+ // New endpoint settings must reach the next fetch: the stored ETag/Last-Modified would make it a 304 that parses
+ // nothing until the upstream file changes (bls-average-prices' duplicates: last waited on BLS's monthly file).
+ if (next.endpoints !== row.endpoints) await st.resetValidators.run(key);
return await st.get.get(key);
return await st.get.get(key);
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,38 @@
@@ -0,0 +1,38 @@
+'use strict';
+/**
+ * A patch that changes a source's endpoints clears their stored ETag/Last-Modified (endpoint_state), so the next fetch
+ * downloads and parses with the new settings; any other patch keeps them. On 2026-10-24 bls-average-prices got its
+ * duplicates: last, and the next fetch was a 304 that parsed nothing: the setting would have waited for BLS's monthly
+ * file.
+ *
+ * node test/registry-validators.test.js
+ */
+const assert = require('assert');
+const { createRegistry } = require('../server/registry');
+const { suite } = require('./helpers');
+
+const t = suite('registry-validators');
+const source = {
+ key: 'tsv-test', name: 'TSV test', type: 'api', category: 'food', homepage_url: 'https://data.example',
+ endpoints: [{ url: 'https://data.example/series.tsv', format: 'tsv', items_path: '', item_kind: 'price', fields: { identity: 'series_id' }, extra: { value: 'value' } }],
+ robots_note: 'robots.txt read', terms_note: 'public domain', license_note: 'public domain',
+};
+
+t('endpoints changed: validators cleared; anything else: kept', async () => {
+ const fresh = await require('./db').testDb();
+ try {
+ const registry = createRegistry({ db: fresh.db });
+ await registry.create(source, 'test');
+ const put = () => fresh.db.prepare("INSERT INTO endpoint_state (source_key, url, etag, last_modified, last_status, last_fetch_at) VALUES ('tsv-test', 'https://data.example/series.tsv', '\"abc\"', 'Fri, 23 Oct 2026 12:00:00 GMT', 200, 1) ON CONFLICT (source_key, url) DO UPDATE SET etag = excluded.etag").run();
+ const count = async () => Number((await fresh.db.prepare("SELECT COUNT(*) AS n FROM endpoint_state WHERE source_key = 'tsv-test'").get()).n);
+ await put();
+ await registry.patch('tsv-test', { name: 'TSV test, renamed' }, 'test');
+ assert.strictEqual(await count(), 1, 'a name change keeps the validators');
+ await registry.patch('tsv-test', { endpoints: [{ ...source.endpoints[0], duplicates: 'last' }] }, 'test');
+ assert.strictEqual(await count(), 0, 'new endpoint settings: the next fetch is a full one');
+ } finally {
+ await fresh.close();
+ }
+});
+
+t.run();