openvibers/openvibe.sources Commits
Seeds for Reviews' official records: the CFPB complaint database's count per company (weekly, CC0) and NYC Health's restaurant inspection results (every 6 hours, NYC Open Data), both disabled until enabled after review (#75)
Changes
@@@ -1334,6 +1334,65 @@"review_required": true,"sensitivity": "financial","default_indexability": "noindex"++ },++ {++ "key": "cfpb-complaints-by-company",++ "name": "CFPB Consumer Complaint Database: complaints per company",++ "type": "api",++ "category": "reviews",++ "homepage_url": "https://www.consumerfinance.gov/data-research/consumer-complaints/",++ "endpoints": [{++ "url": "https://www.consumerfinance.gov/data-research/consumer-complaints/search/api/v1/?size=0",++ "format": "json",++ "items_path": "aggregations.company.company.buckets",++ "item_kind": "complaint_tally",++ "max_bytes": 2097152,++ "fields": { "identity": "key", "title": "key" },++ "extra": { "company": "key", "complaints": "doc_count" }++ }],++ "robots_note": "robots.txt checked 2026-10-17: https://www.consumerfinance.gov/robots.txt does not disallow /data-research/consumer-complaints/search/api/.",++ "terms_note": "The CFPB publishes the database for public use through this documented API (cfpb.github.io/api/ccdb/). One request a week (the counts move slowly; a daily poll would rewrite thousands of items a day). Each company bucket (the company's name and how many complaints the database holds about it, every year since 2011) becomes one complaint_tally item. Complaints are what consumers sent, not findings: the CFPB says it does not verify them, and Reviews shows that next to the number. Narratives are never read.",++ "license_note": "CC0 (the API's _meta.license). Cite 'Consumer Financial Protection Bureau, Consumer Complaint Database'.",++ "min_interval_ms": 60000,++ "poll_interval_sec": 604800,++ "stale_after_sec": 1814400,++ "max_items": 8000,++ "enabled": false,++ "review_required": true,++ "sensitivity": "financial",++ "default_indexability": "noindex"++ },++ {++ "key": "nyc-restaurant-inspections",++ "name": "NYC Health: restaurant inspection results (DOHMH, NYC Open Data)",++ "type": "api",++ "category": "reviews",++ "homepage_url": "https://data.cityofnewyork.us/Health/DOHMH-New-York-City-Restaurant-Inspection-Results/43nn-pn8j",++ "endpoints": [{++ "url": "https://data.cityofnewyork.us/resource/43nn-pn8j.json?$select=:id,camis,dba,boro,building,street,zipcode,cuisine_description,inspection_date,action,violation_code,violation_description,critical_flag,score,grade,grade_date,inspection_type&$where=inspection_date%3E%272000-01-01%27&$order=inspection_date%20DESC&$limit=4000",++ "format": "json",++ "items_path": "",++ "item_kind": "inspection_violation",++ "max_bytes": 8388608,++ "fields": { "identity": ":id", "title": "dba" },++ "extra": {++ "camis": "camis", "dba": "dba", "boro": "boro", "building": "building", "street": "street", "zipcode": "zipcode",++ "cuisine": "cuisine_description", "inspection_date": "inspection_date", "inspection_type": "inspection_type", "action": "action",++ "violation_code": "violation_code", "violation": "violation_description", "critical_flag": "critical_flag",++ "score": "score", "grade": "grade", "grade_date": "grade_date"++ }++ }],++ "robots_note": "robots.txt checked 2026-10-17: https://data.cityofnewyork.us/robots.txt does not disallow /resource/ (Crawl-delay 1; this is one request a poll).",++ "terms_note": "NYC Open Data's SODA API, dataset 43nn-pn8j, published by the Department of Health and Mental Hygiene for public use under the NYC Open Data Terms of Use. Each row is one violation cited at one inspection (or an inspection with none), as the inspector recorded it that day; a letter grade is given only on some inspections. The newest 4,000 rows each poll; rows dated 1900-01-01 (not yet inspected) are left out.",++ "license_note": "NYC Open Data Terms of Use: free to use. Cite 'NYC Department of Health and Mental Hygiene, via NYC Open Data'.",++ "min_interval_ms": 60000,++ "poll_interval_sec": 21600,++ "stale_after_sec": 172800,++ "max_items": 4000,++ "enabled": false,++ "review_required": true,++ "sensitivity": "none",++ "default_indexability": "noindex"}]}nnew file mode 100644@@@ -0,0 +1,94 @@++'use strict';++/**++ * Official records for OpenVibe.Reviews (owner, 2026-10-17: Reviews should combine what Glassdoor, Yelp, the BBB… say;++ * the audit's fix: "aggregate permitted signals (official APIs…)"):++ *++ * cfpb-complaints-by-company the CFPB's complaint database, one request a week: its per-company buckets become one++ * complaint_tally item each (the company's name and its complaint count), CC0++ * nyc-restaurant-inspections NYC Health's inspection results on NYC Open Data, the newest 4,000 rows each poll:++ * one inspection_violation item per row, keyed by Socrata's row id, with the restaurant++ * (camis, name, borough, address, cuisine) and what the inspector recorded++ *++ * Both validate, start disabled, sit in the reviews category, and fit under the per-fetch cap.++ */++const assert = require('assert');++const api = require('../server/adapters/api');++const { validateSource } = require('../server/registry');++const { load } = require('../server/config');++const { suite } = require('./helpers');++++const t = suite('official-records-sources');++const seeds = require('../seeds/sources.json').sources;++const cfpb = seeds.find((s) => s.key === 'cfpb-complaints-by-company');++const nyc = seeds.find((s) => s.key === 'nyc-restaurant-inspections');++++const CFPB_BODY = JSON.stringify({++ took: 12, hits: { total: { value: 18486121, relation: 'eq' }, hits: [] },++ aggregations: {++ product: { doc_count: 18486121, product: { buckets: [{ key: 'Credit reporting', doc_count: 9 }] } },++ company: { doc_count: 18486121, company: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [++ { key: 'CAPITAL ONE FINANCIAL CORPORATION', doc_count: 34041 },++ { key: 'JPMORGAN CHASE & CO.', doc_count: 26433 },++ { key: 'Small Lender LLC', doc_count: 3 },++ ] } },++ },++ _meta: { license: 'CC0', total_record_count: 18486121 },++});++++const NYC_BODY = JSON.stringify([++ { ':id': 'row-8nwj_g8up-3hiv', camis: '50071318', dba: 'THE COFFEE CLUB', boro: 'Brooklyn', building: '9526', street: '4 AVENUE', zipcode: '11209', cuisine_description: 'Coffee/Tea',++ inspection_date: '2026-10-15T00:00:00.000', action: 'Violations were cited in the following area(s).', violation_code: '06D',++ violation_description: 'Food contact surface not properly washed, rinsed and sanitized after each use and following any activity when contamination may have occurred.',++ critical_flag: 'Critical', score: '20', inspection_type: 'Cycle Inspection / Initial Inspection' },++ { ':id': 'row-sfm2_r3gx-4bv7', camis: '50071318', dba: 'THE COFFEE CLUB', boro: 'Brooklyn', building: '9526', street: '4 AVENUE', zipcode: '11209', cuisine_description: 'Coffee/Tea',++ inspection_date: '2026-10-15T00:00:00.000', action: 'Violations were cited in the following area(s).', violation_code: '10F', violation_description: 'Non-food contact surface improperly constructed.',++ critical_flag: 'Not Critical', score: '20', inspection_type: 'Cycle Inspection / Initial Inspection' },++ { ':id': 'row-aaaa_bbbb-cccc', camis: '41234567', dba: "JOE'S PIZZA", boro: 'Manhattan', building: '7', street: 'CARMINE STREET', zipcode: '10014', cuisine_description: 'Pizza',++ inspection_date: '2026-10-14T00:00:00.000', action: 'No violations were recorded at the time of this inspection.', critical_flag: 'Not Applicable', score: '0', grade: 'A',++ grade_date: '2026-10-14T00:00:00.000', inspection_type: 'Cycle Inspection / Re-inspection' },++]);++++t('both seeds validate, start disabled, are reviews sources and fit under the per-fetch cap', () => {++ for (const s of [cfpb, nyc]) {++ assert.ok(s, 'seeded');++ const rec = validateSource(s);++ assert.strictEqual(rec.enabled, false);++ assert.strictEqual(rec.category, 'reviews');++ assert.ok(rec.robots_note && rec.terms_note && rec.license_note, `${s.key}: robots, terms and licence notes`);++ assert.ok(rec.max_items <= load({}).items.maxPerFetch, `${s.key} fits under the default cap`);++ }++ assert.strictEqual(cfpb.poll_interval_sec, 604800, 'one request a week');++ assert.ok(nyc.max_items >= 4000);++});++++t('CFPB: one complaint_tally item per company bucket, nothing from the other aggregations', () => {++ const r = api.parse(CFPB_BODY, { url: cfpb.endpoints[0].url, endpoint: cfpb.endpoints[0], maxItems: cfpb.max_items });++ assert.strictEqual(r.items.length, 3);++ const [capone] = r.items;++ assert.strictEqual(capone.kind, 'complaint_tally');++ assert.strictEqual(capone.identity, 'CAPITAL ONE FINANCIAL CORPORATION');++ assert.strictEqual(capone.title, 'CAPITAL ONE FINANCIAL CORPORATION');++ assert.deepStrictEqual(capone.fields, { company: 'CAPITAL ONE FINANCIAL CORPORATION', complaints: 34041 });++});++++t('NYC: one inspection_violation item per row, keyed by its row id, with the restaurant and the result', () => {++ const r = api.parse(NYC_BODY, { url: nyc.endpoints[0].url, endpoint: nyc.endpoints[0], maxItems: nyc.max_items });++ assert.strictEqual(r.items.length, 3);++ const [a, b, joe] = r.items;++ assert.strictEqual(a.kind, 'inspection_violation');++ assert.strictEqual(a.identity, 'row-8nwj_g8up-3hiv');++ assert.notStrictEqual(a.identity, b.identity, 'two violations at one inspection are two items');++ assert.strictEqual(a.title, 'THE COFFEE CLUB');++ assert.strictEqual(a.fields.camis, '50071318');++ assert.strictEqual(a.fields.critical_flag, 'Critical');++ assert.strictEqual(a.fields.grade, null, 'no grade given at that inspection stays null');++ assert.ok(a.fields.violation.length <= 500);++ assert.deepStrictEqual([joe.fields.grade, joe.fields.score, joe.fields.violation_code, joe.fields.boro], ['A', '0', null, 'Manhattan']);++ // The URL asks Socrata for its row id, leaves out not-yet-inspected rows (1900-01-01) and takes the newest first.++ const u = new URL(nyc.endpoints[0].url);++ assert.match(u.searchParams.get('$select'), /^:id,camis,dba,/);++ assert.strictEqual(u.searchParams.get('$where'), "inspection_date>'2000-01-01'");++ assert.strictEqual(u.searchParams.get('$order'), 'inspection_date DESC');++});++++t.run();
@@ -1334,6 +1334,65 @@
@@ -1334,6 +1334,65 @@
"review_required": true,
"review_required": true,
"sensitivity": "financial",
"sensitivity": "financial",
"default_indexability": "noindex"
"default_indexability": "noindex"
+ },
+ {
+ "key": "cfpb-complaints-by-company",
+ "name": "CFPB Consumer Complaint Database: complaints per company",
+ "type": "api",
+ "category": "reviews",
+ "homepage_url": "https://www.consumerfinance.gov/data-research/consumer-complaints/",
+ "endpoints": [{
+ "url": "https://www.consumerfinance.gov/data-research/consumer-complaints/search/api/v1/?size=0",
+ "format": "json",
+ "items_path": "aggregations.company.company.buckets",
+ "item_kind": "complaint_tally",
+ "max_bytes": 2097152,
+ "fields": { "identity": "key", "title": "key" },
+ "extra": { "company": "key", "complaints": "doc_count" }
+ }],
+ "robots_note": "robots.txt checked 2026-10-17: https://www.consumerfinance.gov/robots.txt does not disallow /data-research/consumer-complaints/search/api/.",
+ "terms_note": "The CFPB publishes the database for public use through this documented API (cfpb.github.io/api/ccdb/). One request a week (the counts move slowly; a daily poll would rewrite thousands of items a day). Each company bucket (the company's name and how many complaints the database holds about it, every year since 2011) becomes one complaint_tally item. Complaints are what consumers sent, not findings: the CFPB says it does not verify them, and Reviews shows that next to the number. Narratives are never read.",
+ "license_note": "CC0 (the API's _meta.license). Cite 'Consumer Financial Protection Bureau, Consumer Complaint Database'.",
+ "min_interval_ms": 60000,
+ "poll_interval_sec": 604800,
+ "stale_after_sec": 1814400,
+ "max_items": 8000,
+ "enabled": false,
+ "review_required": true,
+ "sensitivity": "financial",
+ "default_indexability": "noindex"
+ },
+ {
+ "key": "nyc-restaurant-inspections",
+ "name": "NYC Health: restaurant inspection results (DOHMH, NYC Open Data)",
+ "type": "api",
+ "category": "reviews",
+ "homepage_url": "https://data.cityofnewyork.us/Health/DOHMH-New-York-City-Restaurant-Inspection-Results/43nn-pn8j",
+ "endpoints": [{
+ "url": "https://data.cityofnewyork.us/resource/43nn-pn8j.json?$select=:id,camis,dba,boro,building,street,zipcode,cuisine_description,inspection_date,action,violation_code,violation_description,critical_flag,score,grade,grade_date,inspection_type&$where=inspection_date%3E%272000-01-01%27&$order=inspection_date%20DESC&$limit=4000",
+ "format": "json",
+ "items_path": "",
+ "item_kind": "inspection_violation",
+ "max_bytes": 8388608,
+ "fields": { "identity": ":id", "title": "dba" },
+ "extra": {
+ "camis": "camis", "dba": "dba", "boro": "boro", "building": "building", "street": "street", "zipcode": "zipcode",
+ "cuisine": "cuisine_description", "inspection_date": "inspection_date", "inspection_type": "inspection_type", "action": "action",
+ "violation_code": "violation_code", "violation": "violation_description", "critical_flag": "critical_flag",
+ "score": "score", "grade": "grade", "grade_date": "grade_date"
+ }
+ }],
+ "robots_note": "robots.txt checked 2026-10-17: https://data.cityofnewyork.us/robots.txt does not disallow /resource/ (Crawl-delay 1; this is one request a poll).",
+ "terms_note": "NYC Open Data's SODA API, dataset 43nn-pn8j, published by the Department of Health and Mental Hygiene for public use under the NYC Open Data Terms of Use. Each row is one violation cited at one inspection (or an inspection with none), as the inspector recorded it that day; a letter grade is given only on some inspections. The newest 4,000 rows each poll; rows dated 1900-01-01 (not yet inspected) are left out.",
+ "license_note": "NYC Open Data Terms of Use: free to use. Cite 'NYC Department of Health and Mental Hygiene, via NYC Open Data'.",
+ "min_interval_ms": 60000,
+ "poll_interval_sec": 21600,
+ "stale_after_sec": 172800,
+ "max_items": 4000,
+ "enabled": false,
+ "review_required": true,
+ "sensitivity": "none",
+ "default_indexability": "noindex"
}
}
]
]
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,94 @@
@@ -0,0 +1,94 @@
+'use strict';
+/**
+ * Official records for OpenVibe.Reviews (owner, 2026-10-17: Reviews should combine what Glassdoor, Yelp, the BBB… say;
+ * the audit's fix: "aggregate permitted signals (official APIs…)"):
+ *
+ * cfpb-complaints-by-company the CFPB's complaint database, one request a week: its per-company buckets become one
+ * complaint_tally item each (the company's name and its complaint count), CC0
+ * nyc-restaurant-inspections NYC Health's inspection results on NYC Open Data, the newest 4,000 rows each poll:
+ * one inspection_violation item per row, keyed by Socrata's row id, with the restaurant
+ * (camis, name, borough, address, cuisine) and what the inspector recorded
+ *
+ * Both validate, start disabled, sit in the reviews category, and fit under the per-fetch cap.
+ */
+const assert = require('assert');
+const api = require('../server/adapters/api');
+const { validateSource } = require('../server/registry');
+const { load } = require('../server/config');
+const { suite } = require('./helpers');
+
+const t = suite('official-records-sources');
+const seeds = require('../seeds/sources.json').sources;
+const cfpb = seeds.find((s) => s.key === 'cfpb-complaints-by-company');
+const nyc = seeds.find((s) => s.key === 'nyc-restaurant-inspections');
+
+const CFPB_BODY = JSON.stringify({
+ took: 12, hits: { total: { value: 18486121, relation: 'eq' }, hits: [] },
+ aggregations: {
+ product: { doc_count: 18486121, product: { buckets: [{ key: 'Credit reporting', doc_count: 9 }] } },
+ company: { doc_count: 18486121, company: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [
+ { key: 'CAPITAL ONE FINANCIAL CORPORATION', doc_count: 34041 },
+ { key: 'JPMORGAN CHASE & CO.', doc_count: 26433 },
+ { key: 'Small Lender LLC', doc_count: 3 },
+ ] } },
+ },
+ _meta: { license: 'CC0', total_record_count: 18486121 },
+});
+
+const NYC_BODY = JSON.stringify([
+ { ':id': 'row-8nwj_g8up-3hiv', camis: '50071318', dba: 'THE COFFEE CLUB', boro: 'Brooklyn', building: '9526', street: '4 AVENUE', zipcode: '11209', cuisine_description: 'Coffee/Tea',
+ inspection_date: '2026-10-15T00:00:00.000', action: 'Violations were cited in the following area(s).', violation_code: '06D',
+ violation_description: 'Food contact surface not properly washed, rinsed and sanitized after each use and following any activity when contamination may have occurred.',
+ critical_flag: 'Critical', score: '20', inspection_type: 'Cycle Inspection / Initial Inspection' },
+ { ':id': 'row-sfm2_r3gx-4bv7', camis: '50071318', dba: 'THE COFFEE CLUB', boro: 'Brooklyn', building: '9526', street: '4 AVENUE', zipcode: '11209', cuisine_description: 'Coffee/Tea',
+ inspection_date: '2026-10-15T00:00:00.000', action: 'Violations were cited in the following area(s).', violation_code: '10F', violation_description: 'Non-food contact surface improperly constructed.',
+ critical_flag: 'Not Critical', score: '20', inspection_type: 'Cycle Inspection / Initial Inspection' },
+ { ':id': 'row-aaaa_bbbb-cccc', camis: '41234567', dba: "JOE'S PIZZA", boro: 'Manhattan', building: '7', street: 'CARMINE STREET', zipcode: '10014', cuisine_description: 'Pizza',
+ inspection_date: '2026-10-14T00:00:00.000', action: 'No violations were recorded at the time of this inspection.', critical_flag: 'Not Applicable', score: '0', grade: 'A',
+ grade_date: '2026-10-14T00:00:00.000', inspection_type: 'Cycle Inspection / Re-inspection' },
+]);
+
+t('both seeds validate, start disabled, are reviews sources and fit under the per-fetch cap', () => {
+ for (const s of [cfpb, nyc]) {
+ assert.ok(s, 'seeded');
+ const rec = validateSource(s);
+ assert.strictEqual(rec.enabled, false);
+ assert.strictEqual(rec.category, 'reviews');
+ assert.ok(rec.robots_note && rec.terms_note && rec.license_note, `${s.key}: robots, terms and licence notes`);
+ assert.ok(rec.max_items <= load({}).items.maxPerFetch, `${s.key} fits under the default cap`);
+ }
+ assert.strictEqual(cfpb.poll_interval_sec, 604800, 'one request a week');
+ assert.ok(nyc.max_items >= 4000);
+});
+
+t('CFPB: one complaint_tally item per company bucket, nothing from the other aggregations', () => {
+ const r = api.parse(CFPB_BODY, { url: cfpb.endpoints[0].url, endpoint: cfpb.endpoints[0], maxItems: cfpb.max_items });
+ assert.strictEqual(r.items.length, 3);
+ const [capone] = r.items;
+ assert.strictEqual(capone.kind, 'complaint_tally');
+ assert.strictEqual(capone.identity, 'CAPITAL ONE FINANCIAL CORPORATION');
+ assert.strictEqual(capone.title, 'CAPITAL ONE FINANCIAL CORPORATION');
+ assert.deepStrictEqual(capone.fields, { company: 'CAPITAL ONE FINANCIAL CORPORATION', complaints: 34041 });
+});
+
+t('NYC: one inspection_violation item per row, keyed by its row id, with the restaurant and the result', () => {
+ const r = api.parse(NYC_BODY, { url: nyc.endpoints[0].url, endpoint: nyc.endpoints[0], maxItems: nyc.max_items });
+ assert.strictEqual(r.items.length, 3);
+ const [a, b, joe] = r.items;
+ assert.strictEqual(a.kind, 'inspection_violation');
+ assert.strictEqual(a.identity, 'row-8nwj_g8up-3hiv');
+ assert.notStrictEqual(a.identity, b.identity, 'two violations at one inspection are two items');
+ assert.strictEqual(a.title, 'THE COFFEE CLUB');
+ assert.strictEqual(a.fields.camis, '50071318');
+ assert.strictEqual(a.fields.critical_flag, 'Critical');
+ assert.strictEqual(a.fields.grade, null, 'no grade given at that inspection stays null');
+ assert.ok(a.fields.violation.length <= 500);
+ assert.deepStrictEqual([joe.fields.grade, joe.fields.score, joe.fields.violation_code, joe.fields.boro], ['A', '0', null, 'Manhattan']);
+ // The URL asks Socrata for its row id, leaves out not-yet-inspected rows (1900-01-01) and takes the newest first.
+ const u = new URL(nyc.endpoints[0].url);
+ assert.match(u.searchParams.get('$select'), /^:id,camis,dba,/);
+ assert.strictEqual(u.searchParams.get('$where'), "inspection_date>'2000-01-01'");
+ assert.strictEqual(u.searchParams.get('$order'), 'inspection_date DESC');
+});
+
+t.run();