openvibers/openvibe.tools Commits

Tools: Domain Reputation works (public signals, each finding priced) (#64)

OP OpenVibers committed 29 days ago · 41fa5ed68a · 13 files, +246 −14

* Tools: Domain Reputation works (public signals, each finding priced)

Domain Reputation works on openvibe.tools. It was the last net tool marked "not available yet", so every net tool is now built.

**What it is:** one report from public signals, never a private list. Each finding says what was seen and what it costs, and the score is 100 minus the costs (never below 0). The grade is good at 90+, fair at 70+, poor below that.

| Signal | Good | Costs |
|---|---|---|
| Blacklists (the blacklist checker's DNSBLs: the address and the domain) | not listed | 25 per listing, up to 75 |
| SPF | `-all` or `~all` | none: 15, more than one record: 15, `+all`: 25, `?all` or no `all`: 5 |
| DMARC | `p=reject` or `p=quarantine` | none: 15, `p=none`: 5, no valid `p=`: 10 |
| MX | listed | none is a note, not a cost |
| HTTPS (one TLS handshake through the guard, on the name as typed) | valid | expired or not accepted: 15, under 14 days: 5, nothing on 443 is a note |
| Domain age (RDAP registration; the last two labels when the full name has none) | over 180 days | under 30 days: 20, under 180: 5 |
| Reverse DNS, for an address | forward-confirmed | no PTR: 10, PTR not pointing back: 5 |

- DKIM is left out: it needs a selector, so the page points to the DKIM lookup.
- An IPv6 address skips the IPv4-only blacklists, with a note.
- A private address, or a name that resolves to one, is refused with 403 like every other net tool.
- Descriptor: a lookup tool (DNS questions plus one TLS handshake). Throttle 6 a minute per target, cost 4, cached 5 min, with an output schema.
- The page renders the findings table. The SEO copy now describes the report, and its FAQ explains why it is not the providers' private score.

**Tests:** `net-reputation.test.js` covers:
- a well-kept domain scoring 100;
- a neglected one losing exactly its costs (two listings 50, `+all` 25, no DMARC 15, expired certificate 15, a week old 20; the score floored at 0);
- addresses with forward-confirmed, unconfirmed and missing PTR;
- IPv6 skipping the blacklists;
- private refusals.

`net-family` confirms the 403. The sunset, run-api, net-tools and tools-api tests are updated. The full suite passes except the two failures this PR fixed (rerun green).

* Traceroute/MTR: drop the host resolver's synthetic names (_gateway) from hops

Changes

ddiff --git a/apps/gateway/public/net.html b/apps/gateway/public/net.html
iindex 96aa126..cdf0ade 100644
---- a/apps/gateway/public/net.html
++++ b/apps/gateway/public/net.html
@@@ -688,7 +688,7 @@
latency: { name: 'TCP Latency Test', icon: 'fa-gauge-high', cat: 'diag', endpoint: '/ping', placeholder: 'Host (10 timed TCP connections to port 443)', count: 10 },
// Email & reputation
blacklist: { name: 'Blacklist Checker', icon: 'fa-ban', cat: 'mail', endpoint: '/blacklist', placeholder: 'IPv4 address or domain' },
-- reputation: { name: 'Domain Reputation Checker', icon: 'fa-star', cat: 'mail', endpoint: null, unavailable: 'A combined reputation report is not available yet. The blacklist, SPF, DKIM and DMARC checkers cover its parts.', instead: 'blacklist' },
++ reputation: { name: 'Domain Reputation Checker', icon: 'fa-star', cat: 'mail', endpoint: '/reputation', placeholder: 'Domain or IPv4 address' },
smtp: { name: 'SMTP Server Test', icon: 'fa-paper-plane', cat: 'mail', endpoint: '/smtp', placeholder: 'Mail server or domain (its MX is used)', smtpPorts: true },
// Website checks
uptime: { name: 'Website Uptime Checker', icon: 'fa-heart-pulse', cat: 'web', endpoint: '/uptime', placeholder: 'URL or domain' },
@@@ -1040,6 +1040,7 @@
if (tool.endpoint === '/mtr') return renderTrace(el, data, true);
if (tool.endpoint === '/dnsprop') return renderDnsProp(el, data);
if (tool.endpoint === '/blacklist') return renderBlacklist(el, data);
++ if (tool.endpoint === '/reputation') return renderReputation(el, data);
if (tool.endpoint === '/smtp') return renderSMTP(el, data);
if (tool.endpoint === '/uptime') return renderUptime(el, data);
if (tool.endpoint === '/robots') return renderRobots(el, data);
@@@ -1364,6 +1365,28 @@
el.innerHTML = html;
}
++ // ── Reputation: the public signals, each with what it saw and what it cost ──
++ function renderReputation(el, data) {
++ const findings = data.findings || [];
++ const tone = { good: 'badge-green', warn: 'badge-yellow', bad: 'badge-red', info: '' };
++ const word = { good: 'Good', warn: 'Check', bad: 'Problem', info: 'Note' };
++ let html = `<div class="badges">
++ <span class="badge badge-cyan">${esc(data.domain || data.target)}</span>
++ <span class="badge ${data.grade === 'good' ? 'badge-green' : data.grade === 'fair' ? 'badge-yellow' : 'badge-red'}">Score ${data.score} / 100 (${esc(data.grade)})</span>
++ </div>`;
++ html += `<div class="card"><div class="card-head"><i class="fa-solid fa-star"></i> What the score is made of</div>
++ <table class="res-table"><tr><th>Check</th><th>Result</th><th>Finding</th><th>Cost</th></tr>
++ ${findings.map(f => `<tr>
++ <td class="key">${esc(f.check)}</td>
++ <td><span class="badge ${tone[f.status] || ''}">${word[f.status] || esc(f.status)}</span></td>
++ <td class="val" style="white-space:normal">${esc(f.detail)}</td>
++ <td class="val">${f.cost ? `−${f.cost}` : '—'}</td>
++ </tr>`).join('')}
++ </table>
++ <p style="color:var(--text-muted);font-size:12px;margin:10px 0 0">Public signals only: mail providers keep their own private scores. The blacklist checker shows every list, and the DKIM lookup checks a selector.</p>${from(data)}</div>`;
++ el.innerHTML = html;
++ }
++
// ── IPv4 (address or CIDR block) and IPv6 ────────────────
function renderIPv4(el, data) {
const v = data.ipv4 || {};
ddiff --git a/apps/gateway/server/net/checks.js b/apps/gateway/server/net/checks.js
iindex 355e091..99a3abe 100644
---- a/apps/gateway/server/net/checks.js
++++ b/apps/gateway/server/net/checks.js
@@@ -8,6 +8,7 @@
// blacklist DNSBL lookups for an IPv4 address or a domain
// uptime is a site up right now, from this server (status, redirects, time)
// dnsprop the same record from ten public resolvers, compared
++// reputation public signals in one report: blacklists, SPF, DMARC, MX, HTTPS, domain age; reverse DNS for an address
// Everything that connects to a visitor's target (robots, sitemap, smtp, uptime) goes through the SSRF
// guard (_shared/egress): public addresses only, checked after DNS, dialled as checked, redirects
// re-checked. blacklist and dnsprop only ask DNS servers questions; the target is never contacted.
@@@ -535,7 +536,120 @@ function createChecks({ egress, resolverFor = defaultResolver, tlsUpgrade = tls.
}, 20_000);
}
-- return { robots, sitemap, smtp, blacklist, uptime, dnsprop };
++ // ── reputation ──
++ // One report from public signals, never a private list: the DNSBLs above, how the domain protects its mail (SPF,
++ // DMARC, MX), its HTTPS certificate (one TLS handshake through the guard), how long it has been registered (RDAP,
++ // `rdap(type, name)` from the routes), and for an address its reverse DNS and whether that name points back. Each
++ // finding says what was seen and what it costs; the score is 100 minus the costs (never below 0).
++ const txtOf = async (name) => { try { return (await system().resolveTxt(name)).map(t => t.join('')); } catch { return []; } };
++ const DAY = 86400000;
++
++ async function reputation(target, { rdap = null, reverse = null } = {}) {
++ const findings = [];
++ const add = (check, status, detail, cost = 0) => { findings.push({ check, status, detail, cost }); };
++ const isIp = net.isIP(target) !== 0;
++ if (isIp && !egress.isAllowed(target)) throw new TargetRefused(`${target} is a private or reserved address; it has no public reputation`);
++ const domain = isIp ? null : target.replace(/^www\./, '');
++
++ const [bl, mx, spf, dmarc, https, reg, ptr] = await Promise.all([
++ net.isIPv6(target) ? null : blacklist(target).catch((err) => { if (err instanceof TargetRefused) throw err; return { error: err.message }; }),
++ domain ? system().resolveMx(domain).catch(() => []) : null,
++ domain ? txtOf(domain).then(t => t.filter(v => /^v=spf1(\s|$)/i.test(v))) : null,
++ domain ? txtOf(`_dmarc.${domain}`).then(t => t.filter(v => /^v=DMARC1(\s*;|$)/i.test(v))) : null,
++ domain ? (async () => {
++ let dest;
++ try { dest = await egress.resolve(target, { prefer: 4 }); } catch (err) { if (err instanceof TargetRefused) throw err; return { none: 'The name has no address' }; }
++ try {
++ const { cert } = await egress.tlsHandshake(dest, 443, { timeoutMs: 8000, detailed: false });
++ const validTo = cert && cert.valid_to ? new Date(cert.valid_to) : null;
++ return { daysLeft: validTo ? Math.ceil((validTo - Date.now()) / DAY) : null, issuer: (cert && cert.issuer && (cert.issuer.O || cert.issuer.CN)) || null };
++ } catch (err) {
++ const code = String((err && (err.code || err.message)) || '');
++ if (/ECONNREFUSED|ETIMEDOUT|timeout|EHOSTUNREACH|ECONNRESET/i.test(code)) return { none: 'Nothing answers HTTPS on port 443' };
++ return { error: code.slice(0, 160) };
++ }
++ })() : null,
++ domain && rdap ? (async () => {
++ const labels = domain.split('.');
++ const names = [domain, ...(labels.length > 2 ? [labels.slice(-2).join('.')] : [])];
++ for (const n of names) {
++ try {
++ const r = await rdap('domain', n);
++ const ev = r && r.ok && r.data && Array.isArray(r.data.events) ? r.data.events.find(e => e.eventAction === 'registration') : null;
++ if (ev && ev.eventDate) return { name: n, registered: ev.eventDate };
++ } catch { /* the next name, or unknown */ }
++ }
++ return null;
++ })() : null,
++ isIp && reverse ? (async () => {
++ let names = [];
++ try { names = await reverse(target); } catch { names = []; }
++ const name = names[0] || null;
++ if (!name) return { name: null };
++ let back = [];
++ try { back = net.isIPv6(target) ? await system().resolve6(name) : await system().resolve4(name); } catch { back = []; }
++ return { name, confirmed: back.includes(target) };
++ })() : null,
++ ]);
++
++ // Blacklists
++ if (!bl) add('Blacklists', 'info', 'Not checked: the public blacklists list IPv4 addresses and domains, not IPv6 addresses.');
++ else if (bl.error) add('Blacklists', 'info', `Not checked: ${bl.error}`);
++ else if (bl.listedCount) add('Blacklists', 'bad', `Listed on ${bl.listedCount} of ${bl.lists.length}: ${bl.lists.filter(l => l.listed === true).map(l => l.name).join(', ')}.`, Math.min(75, 25 * bl.listedCount));
++ else add('Blacklists', 'good', `Not listed on any of the ${bl.cleanCount} lists that answered${bl.unknownCount ? ` (${bl.unknownCount} could not be checked)` : ''}.`);
++
++ if (domain) {
++ // Mail: MX, SPF, DMARC
++ if (mx.length) add('Mail servers (MX)', 'good', `${mx.length} mail ${mx.length === 1 ? 'server' : 'servers'}: ${mx.sort((a, b) => a.priority - b.priority).slice(0, 3).map(m => m.exchange).join(', ')}.`);
++ else add('Mail servers (MX)', 'info', 'No MX record: this domain does not receive mail. If it never sends mail either, SPF "v=spf1 -all" and DMARC p=reject stop others sending as it.');
++ if (!spf.length) add('SPF', 'warn', 'No SPF record: anyone can send mail claiming to be from this domain, and receivers trust its mail less.', 15);
++ else if (spf.length > 1) add('SPF', 'bad', `${spf.length} SPF records: receivers treat more than one as a permanent error, as if there were none.`, 15);
++ else {
++ const all = (/(^|\s)([+?~-]?)all(\s|$)/i.exec(spf[0]) || [])[2];
++ if (all === '+' || all === '') add('SPF', 'bad', `SPF ends in "${all}all": it allows every server on the internet to send as this domain.`, 25);
++ else if (all === '?') add('SPF', 'warn', 'SPF ends in "?all" (neutral): it lists the senders but asks receivers to ignore the rest.', 5);
++ else if (all === undefined) add('SPF', 'warn', 'SPF has no "all" at the end, so mail from any other server is neither passed nor failed.', 5);
++ else add('SPF', 'good', `SPF ends in "${all}all": mail from servers it does not list ${all === '-' ? 'fails' : 'soft-fails'}.`);
++ }
++ if (!dmarc.length) add('DMARC', 'warn', 'No DMARC policy: receivers are not told what to do with mail that fails SPF and DKIM, and the owner gets no reports.', 15);
++ else {
++ const p = ((/(?:^|;)\s*p\s*=\s*(none|quarantine|reject)/i.exec(dmarc[0]) || [])[1] || '').toLowerCase();
++ if (p === 'reject' || p === 'quarantine') add('DMARC', 'good', `DMARC policy p=${p}: mail that fails is ${p === 'reject' ? 'refused' : 'sent to spam'}.`);
++ else if (p === 'none') add('DMARC', 'warn', 'DMARC policy p=none: failures are reported, not stopped (a fine first step, not an end state).', 5);
++ else add('DMARC', 'warn', 'The DMARC record has no valid p= policy, so receivers ignore it.', 10);
++ }
++ // HTTPS
++ if (https.none) add('HTTPS', 'info', `${https.none}: no website to check.`);
++ else if (https.error) add('HTTPS', 'bad', `The HTTPS certificate is not accepted: ${https.error}.`, 15);
++ else if (https.daysLeft !== null && https.daysLeft < 0) add('HTTPS', 'bad', 'The HTTPS certificate has expired.', 15);
++ else if (https.daysLeft !== null && https.daysLeft < 14) add('HTTPS', 'warn', `The HTTPS certificate expires in ${https.daysLeft} ${https.daysLeft === 1 ? 'day' : 'days'}.`, 5);
++ else add('HTTPS', 'good', `A valid HTTPS certificate${https.issuer ? ` from ${https.issuer}` : ''}${https.daysLeft !== null ? `, ${https.daysLeft} days left` : ''}.`);
++ // Age
++ if (!reg) add('Domain age', 'info', 'The registration date is not published (RDAP), so the age is unknown.');
++ else {
++ const days = Math.floor((Date.now() - Date.parse(reg.registered)) / DAY);
++ const when = new Date(reg.registered).toISOString().slice(0, 10);
++ if (!Number.isFinite(days)) add('Domain age', 'info', 'The registration date could not be read.');
++ else if (days < 30) add('Domain age', 'bad', `${reg.name} was registered ${days} ${days === 1 ? 'day' : 'days'} ago (${when}): mail filters and browsers treat new domains with suspicion.`, 20);
++ else if (days < 180) add('Domain age', 'warn', `${reg.name} was registered ${Math.round(days / 30)} months ago (${when}).`, 5);
++ else add('Domain age', 'good', `${reg.name} was registered on ${when} (${Math.floor(days / 365) >= 1 ? `${Math.floor(days / 365)} ${Math.floor(days / 365) === 1 ? 'year' : 'years'}` : `${Math.round(days / 30)} months`}).`);
++ }
++ } else if (ptr) {
++ if (!ptr.name) add('Reverse DNS', 'warn', 'No reverse DNS (PTR) name: many mail servers reject mail from addresses without one.', 10);
++ else if (!ptr.confirmed) add('Reverse DNS', 'warn', `The reverse name ${ptr.name} does not point back to ${target}, so it is not forward-confirmed.`, 5);
++ else add('Reverse DNS', 'good', `${ptr.name}, and it points back to ${target} (forward-confirmed).`);
++ }
++
++ const cost = findings.reduce((s, f) => s + f.cost, 0);
++ const score = Math.max(0, 100 - cost);
++ return {
++ target, kind: isIp ? 'ip' : 'domain', ...(domain && { domain }), score, grade: score >= 90 ? 'good' : score >= 70 ? 'fair' : 'poor',
++ findings, blacklist: bl && !bl.error ? { listedCount: bl.listedCount, cleanCount: bl.cleanCount, unknownCount: bl.unknownCount, lists: bl.lists } : null,
++ checkedFrom: FROM,
++ };
++ }
++
++ return { robots, sitemap, smtp, blacklist, uptime, dnsprop, reputation };
}
module.exports = { createChecks, parseRobots, testRobots, analyseSitemap, smtpReader, IP_LISTS, DOMAIN_LISTS, RESOLVERS, PROP_TYPES, SMTP_PORTS };
ddiff --git a/apps/gateway/server/net/config.js b/apps/gateway/server/net/config.js
iindex 241a6b5..60a7a76 100644
---- a/apps/gateway/server/net/config.js
++++ b/apps/gateway/server/net/config.js
@@@ -129,7 +129,7 @@ const NET_TOOLS = [
{ id: 'latency', subdomain: 'latency', name: 'OpenVibeLatency', icon: 'fa-gauge-high', desc: 'Latency tester — measure response times', endpoint: '/ping' },
// Reputation / mail / infra
{ id: 'blacklist', subdomain: 'blacklist', name: 'OpenVibeBlacklist', icon: 'fa-ban', desc: 'Blacklist / RBL check for IP & domain', endpoint: '/blacklist' },
-- { id: 'reputation', subdomain: 'reputation', name: 'OpenVibeReputation', icon: 'fa-star', desc: 'Domain & IP reputation score', status: 'unavailable', unavailable: 'A combined reputation report is not available yet. The blacklist, SPF, DKIM and DMARC checkers cover its parts.' },
++ { id: 'reputation', subdomain: 'reputation', name: 'OpenVibeReputation', icon: 'fa-star', desc: 'Domain & IP reputation score', endpoint: '/reputation' },
{ id: 'smtp', subdomain: 'smtp', name: 'OpenVibeSMTP', icon: 'fa-paper-plane', desc: 'SMTP server connectivity tester', endpoint: '/smtp' },
{ id: 'uptime', subdomain: 'uptime', name: 'OpenVibeUptime', icon: 'fa-heart-pulse', desc: 'Uptime & availability monitor', endpoint: '/uptime' },
{ id: 'robots', subdomain: 'robots', name: 'OpenVibeRobots', icon: 'fa-robot', desc: 'robots.txt analyzer', endpoint: '/robots' },
ddiff --git a/apps/gateway/server/net/descriptors.js b/apps/gateway/server/net/descriptors.js
iindex bdd135d..15b743b 100644
---- a/apps/gateway/server/net/descriptors.js
++++ b/apps/gateway/server/net/descriptors.js
@@@ -35,7 +35,7 @@ const CACHE_TTL = {
dns: MIN, dig: MIN, nslookup: MIN, mx: MIN, txt: MIN, ns: MIN, spf: MIN, dkim: MIN, dmarc: MIN, dnspropagation: MIN,
ip: 10 * MIN, geoip: 10 * MIN, isp: 10 * MIN, asn: 10 * MIN, ipv4: 10 * MIN, ipv6: 10 * MIN,
hostname: 5 * MIN, rdns: 5 * MIN, whois: 60 * MIN, rdap: 60 * MIN,
-- blacklist: 5 * MIN, ssl: 5 * MIN, robots: 5 * MIN, sitemap: 5 * MIN, lookup: 5 * MIN,
++ blacklist: 5 * MIN, reputation: 5 * MIN, ssl: 5 * MIN, robots: 5 * MIN, sitemap: 5 * MIN, lookup: 5 * MIN,
};
const target = (description) => ({ type: 'string', minLength: 1, maxLength: 253 + 2048, description });
const HOST = 'A domain or an IP address';
@@@ -59,6 +59,7 @@ const OUTPUT = {
'/ping': out({ target: S, ip: S, count: I, results: A, stats: out({ min: { type: 'number' }, max: { type: 'number' }, avg: { type: 'number' }, loss: { type: 'number' } }) }, ['target', 'stats']),
'/trace': out({ target: S, ip: S, family: I, reached: B, probes: I, hops: { type: 'array', items: out({ hop: I, ip: nul(S), host: nul(S), asn: nul(S), loss: nul({ type: 'number' }), sent: I, last: nul({ type: 'number' }), avg: nul({ type: 'number' }), best: nul({ type: 'number' }), worst: nul({ type: 'number' }), stdev: nul({ type: 'number' }) }) } }, ['target', 'ip', 'hops']),
'/mtr': out({ target: S, ip: S, family: I, reached: B, probes: I, hops: A }, ['target', 'ip', 'hops']),
++ '/reputation': out({ target: S, kind: { enum: ['ip', 'domain'] }, domain: S, score: I, grade: { enum: ['good', 'fair', 'poor'] }, findings: { type: 'array', items: out({ check: S, status: { enum: ['good', 'warn', 'bad', 'info'] }, detail: S, cost: I }) }, blacklist: nul(O) }, ['target', 'score', 'findings']),
'/port': out({ target: S, ip: S, ports: { type: 'array', items: out({ port: I, status: S, ms: { type: ['number', 'null'] } }) } }, ['target', 'ports']),
'/headers': out({ url: S, status: I, statusText: S, headers: O, timing: O, security: O, securityScore: S, server: nul(S), poweredBy: nul(S), contentType: nul(S) }, ['url', 'status', 'headers']),
'/redirects': out({ originalUrl: S, finalUrl: S, hops: I, chain: A, stopped: S }, ['originalUrl', 'finalUrl', 'chain']),
@@@ -100,7 +101,7 @@ const TOOLS = {
redirects: { input: obj({ target: target(URL_OR_DOMAIN) }), timeoutMs: 60000, perTarget: 10, cost: 3 },
ssl: { input: obj({ target: target('A domain or an IP address'), port: { enum: TLS_PORTS, default: 443 } }), perTarget: 10, cost: 2 },
blacklist: { input: obj({ target: target('An IPv4 address or a domain') }), timeoutMs: 20000, perTarget: 10, cost: 3 },
-- reputation: {},
++ reputation: { input: obj({ target: target('A domain or a public IP address') }), timeoutMs: 30000, perTarget: 6, cost: 4 },
smtp: { input: obj({ target: target('A mail server, or a domain (its first MX is tested)'), port: { enum: SMTP_PORTS, default: 25 } }), timeoutMs: 40000, perTarget: 3, cost: 5, anonymous: false },
uptime: { input: obj({ target: target(URL_OR_DOMAIN) }), timeoutMs: 20000, perTarget: 10, cost: 2 },
robots: { input: obj({ target: target(URL_OR_DOMAIN), path: { type: 'string', maxLength: 2048, description: 'A path (or URL) to test against the rules' }, ua: { type: 'string', maxLength: 100, default: '*', description: 'The crawler to test as' } }), perTarget: 10, cost: 2 },
ddiff --git a/apps/gateway/server/net/routes.js b/apps/gateway/server/net/routes.js
iindex 2981209..e1ba65c 100644
---- a/apps/gateway/server/net/routes.js
++++ b/apps/gateway/server/net/routes.js
@@@ -1000,6 +1000,7 @@ module.exports = function createNetRoutes(db, requireAuth, opts = {}) {
router.get('/smtp/:target?', run('SMTP test', (req) => checks.smtp(host(req), { port: req.query.port })));
router.get('/blacklist/:target?', run('Blacklist check', (req) => checks.blacklist(host(req))));
router.get('/dnsprop/:target?', run('DNS propagation check', (req) => checks.dnsprop(host(req), { type: req.query.type })));
++ router.get('/reputation/:target?', run('Reputation check', (req) => checks.reputation(host(req), { rdap: rdapLookup, reverse: reverseDns })));
return router;
};
ddiff --git a/apps/gateway/server/net/trace.js b/apps/gateway/server/net/trace.js
iindex 2c7f5db..d3b5819 100644
---- a/apps/gateway/server/net/trace.js
++++ b/apps/gateway/server/net/trace.js
@@@ -76,6 +76,9 @@ function createTracer({ exec = execMtr, reverse = null, max = RUNNING_MAX } = {}
try {
const got = await Promise.race([reverse(ip), new Promise((r) => setTimeout(() => r([]), 1500).unref())]);
name = Array.isArray(got) && typeof got[0] === 'string' ? got[0].slice(0, 253) : '';
++ // The host's own resolver answers synthetic names (systemd-resolved: "_gateway" for the default gateway):
++ // not public DNS, so not a hop's name.
++ if (name.startsWith('_') || !name.includes('.')) name = '';
} catch { name = ''; }
if (names.size > 2000) names.delete(names.keys().next().value);
names.set(ip, name);
ddiff --git a/apps/gateway/server/seo/catalog.js b/apps/gateway/server/seo/catalog.js
iindex bab73e0..9fce8e4 100644
---- a/apps/gateway/server/seo/catalog.js
++++ b/apps/gateway/server/seo/catalog.js
@@@ -387,7 +387,7 @@ const NET = {
httpstatus: { name: 'HTTP Status Codes', title: 'HTTP Status Code Checker — What Does a URL Return?', desc: 'Check the HTTP status code any URL returns from our server, with its redirect target and response headers. Instant.', kw: 'http status codes, status code checker, check status code, 404 meaning, 301 vs 302, http response codes', about: 'Requests a URL from the OpenVibe server and shows the status code it returns, where a redirect points and the response headers — the quickest way to confirm a 301, a 404 or a 5xx.', bullets: ['The status code and reason a URL returns', 'Where a 3xx redirect points', 'The response headers alongside'], faq: [['What is the difference between 301 and 302?', '301 is permanent and transfers ranking signals to the new URL. 302 is temporary and tells search engines to keep indexing the original.'], ['Should a missing page return 404 or 410?', '404 means not found and may be retried. 410 means permanently gone and gets pages dropped from an index faster.']] },
latency: { name: 'TCP Latency Test', title: 'TCP Latency Test — Connect Time to Any Host', desc: 'Measure TCP connect latency to any host from our server: how long a connection to port 443 takes, repeated ten times to show the spread. No install.', kw: 'latency test, response time test, tcp latency, connect time test, network latency check', about: 'Opens a TCP connection to the host on port 443 ten times from the OpenVibe server (one location) and reports each connect time with the minimum, average, maximum and loss. It measures the network path only, not how long the web server takes to answer.', bullets: ['Ten timed TCP connections to port 443', 'Minimum, average, maximum and loss', 'Runs from the OpenVibe server (one location)'], faq: [['Is this the same as time to first byte?', 'No. It times only the TCP connection. For how long the server takes to answer a request, use the website uptime check.'], ['Why is latency high?', 'Usually distance: the host is far from our server with no edge node in between. A CDN fixes exactly this for visitors near its nodes.']] },
blacklist: { name: 'Blacklist Checker', title: 'IP & Domain Blacklist Checker — RBL Lookup', desc: 'Check whether an IPv4 address or a domain is listed on major email blacklists and RBLs, with the reason and a delisting link. Find out why mail is rejected.', kw: 'blacklist check, rbl lookup, ip blacklist checker, email blacklist, spam list check', about: 'Queries well-known DNS blacklists to see whether an IPv4 address or a domain is listed — the usual explanation when mail is being rejected or silently dropped. A domain is checked on the domain lists and its address on the IP lists.', bullets: ['Seven IP blacklists and three domain lists in one query', 'Shows which list flagged you and the reason it gives', 'Delisting links for every list'], faq: [['How do I get removed from a blacklist?', 'Fix the cause first — an open relay, a compromised account or a bad sending pattern — then use the operator\'s delisting form. Requesting removal without fixing the cause gets you relisted.'], ['What does "refused the query" mean?', 'Some lists do not answer queries that come through large DNS resolvers. That is not a listing; check that list on its own website.']] },
-- reputation: { name: 'Domain Reputation Checker', title: 'Domain & IP Reputation Checker — Not Available Yet', desc: 'A combined reputation report is not available yet. The blacklist checker and the SPF, DKIM and DMARC checkers cover its parts today.', kw: 'domain reputation, ip reputation checker, email reputation, sender score, domain trust check', about: 'A single reputation score needs sources we do not have yet, so this report is not available. Its parts already work: the blacklist checker for DNSBL listings, and the SPF, DKIM and DMARC checkers for email authentication.', bullets: ['Not available yet', 'Blacklist status: the blacklist checker', 'Email authentication: the SPF, DKIM and DMARC checkers'], faq: [['What hurts domain reputation most?', 'Sending mail without SPF, DKIM and DMARC, appearing on blacklists, and hosting alongside abusive neighbours on a shared IP.'], ['How long does reputation take to recover?', 'Weeks of consistent, authenticated sending. There is no way to reset it instantly.']] },
++ reputation: { name: 'Domain Reputation Checker', title: 'Domain & IP Reputation Checker — Blacklists, SPF, DMARC, Age', desc: 'One reputation report for a domain or IP: blacklist listings, SPF, DMARC and MX, the HTTPS certificate, domain age, and reverse DNS for an address. Every point explained.', kw: 'domain reputation, ip reputation checker, email reputation, sender score, domain trust check, is my domain blacklisted', about: 'The report gathers the public signals mail filters and browsers weigh: whether the domain or address is on the public blacklists, how the domain protects its mail (SPF, DMARC, MX), whether its HTTPS certificate is valid, how long it has been registered, and for an address whether it has a reverse DNS name that points back. Each finding says what was seen and what it costs; the score is 100 minus those costs.', bullets: ['Blacklist listings from the same lists as the blacklist checker', 'SPF, DMARC and MX read and explained', 'HTTPS certificate, domain age (RDAP) and reverse DNS'], faq: [['Is this the score mail providers use?', 'No. Gmail, Microsoft and others keep private scores built from how people react to your mail. This report checks the public signals those scores start from, and every point it takes off is explained.'], ['What hurts domain reputation most?', 'Sending mail without SPF, DKIM and DMARC, appearing on blacklists, and hosting alongside abusive neighbours on a shared IP.'], ['Why is DKIM not in the score?', 'DKIM keys live under a selector only the sender knows. Check yours with the DKIM lookup and the selector your mail service gave you.']] },
smtp: { name: 'SMTP Server Test', title: 'SMTP Server Test — Banner, STARTTLS & Certificate Check', desc: 'Test a mail server from our server: connection, greeting banner, EHLO extensions, STARTTLS with its TLS version and certificate, and the login methods offered. Ports 25 and 587.', kw: 'smtp test, smtp server check, mail server test, starttls check, smtp connectivity', about: 'Connects to a mail server — or to the main MX host of a domain — reads its greeting, sends EHLO and, when STARTTLS is offered, upgrades the connection to report the TLS version and certificate. It never sends a message.', bullets: ['The greeting banner and every EHLO extension', 'STARTTLS support, TLS version and certificate', 'Port 25 (server to server) or 587 (submission)'], faq: [['Which SMTP port should I use?', 'Port 587 with STARTTLS for sending mail from a client, or 465 for implicit TLS. Port 25 is for server-to-server delivery and is blocked by most consumer ISPs.'], ['Does this test send an email?', 'No. It stops after the greeting, EHLO and (when offered) STARTTLS, then says QUIT.']] },
uptime: { name: 'Website Uptime Checker', title: 'Website Uptime Checker — Is a Site Down? Online Test', desc: 'Check whether a website is up right now from our server: the HTTP status, where redirects lead and how long the server took to answer. "Is it down" checker.', kw: 'uptime checker, is this site down, website down checker, site availability test, server status check', about: 'Requests the page from the OpenVibe server (one location, outside your network) and reports whether it answered, the status code, every redirect and the time until the server replied — enough to settle "is it down for everyone or just me".', bullets: ['Up or down, with the HTTP status', 'Every redirect hop and the final address', 'Time until the server answered'], faq: [['The site works for me but not for a friend — what now?', 'If it is up here, the server is fine and the trouble is between your friend and the site: their DNS, their network or a regional block.'], ['Does this monitor continuously?', 'No, this is an on-demand check. It tells you the state right now.']] },
robots: { name: 'Robots.txt Tester', title: 'Robots.txt Tester & Analyzer — Online Checker', desc: 'Fetch and analyse any site\'s robots.txt, check which URLs are blocked, and catch rules that hide your site from search engines.', kw: 'robots.txt tester, robots txt checker, crawl rules, disallow checker, seo crawl test', about: 'Read a site\'s robots.txt, see the rules that apply to each crawler, and test whether a specific URL would be allowed or blocked — the file where an accidental line can remove a whole site from search.', bullets: ['Rules grouped by user-agent', 'Test any URL against the live rules', 'Flags the accidental site-wide disallow'], faq: [['Does robots.txt remove a page from Google?', 'No. It stops crawling, not indexing — a blocked URL can still appear without a description. Use a noindex meta tag to remove a page properly.'], ['Should robots.txt list my sitemap?', 'Yes. A Sitemap line is the simplest way to point every crawler at your sitemap without registering anywhere.']] },
ddiff --git a/apps/gateway/test/net-reputation.test.js b/apps/gateway/test/net-reputation.test.js
nnew file mode 100644
iindex 0000000..cdda70a
---- /dev/null
++++ b/apps/gateway/test/net-reputation.test.js
@@@ -0,0 +1,93 @@
++'use strict';
++// Domain & IP reputation (checks.reputation): each public signal becomes a finding with what was seen and what it
++// cost, and the score is 100 minus the costs. A well-kept domain scores 100; one with a blacklist listing, a
++// "+all" SPF, no DMARC, an expired certificate and a week-old registration loses exactly those points; an address
++// gets reverse DNS (forward-confirmed or not); a private address is refused and an IPv6 address skips the
++// IPv4-only blacklists. DNS, TLS and RDAP are fakes: nothing here touches a real network.
++const assert = require('assert');
++const { TargetRefused } = require('../../_shared/egress');
++const { createChecks } = require('../server/net/checks');
++
++const A = { 'good.example': ['93.184.216.10'], 'bad.example': ['93.184.216.20'], 'www.bad.example': ['93.184.216.20'], 'mail.good.example': ['93.184.216.10'], 'host.isp.example': ['93.184.216.31'], 'lan.example': ['192.168.1.5'] };
++const LISTED = { '20.216.184.93.zen.spamhaus.org': ['127.0.0.2'], 'bad.example.dbl.spamhaus.org': ['127.0.1.2'] };
++const TXT = {
++ 'good.example': [['v=spf1 include:_spf.mail.example -all'], ['google-site-verification=abc']],
++ '_dmarc.good.example': [['v=DMARC1; p=reject; rua=mailto:[email protected]']],
++ 'bad.example': [['v=spf1 +all']],
++};
++const MX = { 'good.example': [{ exchange: 'mail.good.example', priority: 10 }] };
++const fail = (code) => Promise.reject(Object.assign(new Error(code), { code }));
++const resolverFor = () => ({
++ resolve4: (q) => (LISTED[q] ? Promise.resolve(LISTED[q]) : A[q] ? Promise.resolve(A[q]) : fail('ENOTFOUND')),
++ resolve6: () => fail('ENODATA'),
++ resolveTxt: (q) => (TXT[q] ? Promise.resolve(TXT[q]) : fail('ENODATA')),
++ resolveMx: (q) => (MX[q] ? Promise.resolve(MX[q]) : fail('ENODATA')),
++});
++const PRIVATE = /^(10\.|127\.|192\.168\.|169\.254\.|::1$|fd)/;
++const daysFromNow = (d) => new Date(Date.now() + d * 86400000).toUTCString();
++const egress = {
++ isAllowed: (ip) => !PRIVATE.test(ip),
++ async resolve(host) {
++ const a = A[host];
++ if (!a) throw Object.assign(new Error('not found'), { status: 400 });
++ if (PRIVATE.test(a[0])) throw new TargetRefused(`${host} resolves to a private address`);
++ return { host, address: a[0], family: 4 };
++ },
++ async tlsHandshake(dest) {
++ if (dest.host === 'good.example') return { cert: { valid_to: daysFromNow(60), issuer: { O: "Let's Encrypt" } }, protocol: 'TLSv1.3' };
++ return { cert: { valid_to: daysFromNow(-3), issuer: { O: 'Old CA' } }, protocol: 'TLSv1.2' };
++ },
++};
++const REG = { 'good.example': '2009-04-01T00:00:00Z', 'bad.example': new Date(Date.now() - 7 * 86400000).toISOString() };
++const rdap = async (_type, name) => (REG[name] ? { ok: true, data: { events: [{ eventAction: 'registration', eventDate: REG[name] }] } } : { ok: false, status: 404 });
++const reverse = async (ip) => ({ '93.184.216.31': ['host.isp.example'], '93.184.216.32': ['elsewhere.example'] }[ip] || []);
++
++(async () => {
++ const checks = createChecks({ egress, resolverFor });
++ const byCheck = (r) => Object.fromEntries(r.findings.map(f => [f.check, f]));
++
++ // A well-kept domain: nothing to take off.
++ let r = await checks.reputation('good.example', { rdap, reverse });
++ let f = byCheck(r);
++ assert.strictEqual(r.kind, 'domain');
++ assert.strictEqual(r.score, 100, JSON.stringify(r.findings));
++ assert.strictEqual(r.grade, 'good');
++ assert.deepStrictEqual(['Blacklists', 'Mail servers (MX)', 'SPF', 'DMARC', 'HTTPS', 'Domain age'].map(k => f[k].status), ['good', 'good', 'good', 'good', 'good', 'good']);
++ assert.match(f.SPF.detail, /"-all"/);
++ assert.match(f['Domain age'].detail, /2009-04-01/);
++
++ // A neglected one: each problem costs what it says.
++ r = await checks.reputation('www.bad.example', { rdap, reverse });
++ f = byCheck(r);
++ assert.strictEqual(r.domain, 'bad.example', 'www. is the same domain');
++ assert.strictEqual(f.Blacklists.status, 'bad');
++ assert.strictEqual(f.Blacklists.cost, 50, 'two listings (the IP on ZEN, the domain on DBL)');
++ assert.strictEqual(f.SPF.cost, 25, '+all');
++ assert.strictEqual(f.DMARC.cost, 15, 'no DMARC');
++ assert.strictEqual(f['Mail servers (MX)'].status, 'info');
++ assert.strictEqual(f.HTTPS.cost, 15, 'expired certificate');
++ assert.strictEqual(f['Domain age'].cost, 20, 'a week old');
++ assert.strictEqual(r.score, 0, 'never below zero');
++ assert.strictEqual(r.grade, 'poor');
++ assert.strictEqual(r.score, Math.max(0, 100 - r.findings.reduce((s, x) => s + x.cost, 0)));
++
++ // An address: blacklists and reverse DNS.
++ r = await checks.reputation('93.184.216.31', { rdap, reverse });
++ f = byCheck(r);
++ assert.strictEqual(r.kind, 'ip');
++ assert.strictEqual(f['Reverse DNS'].status, 'good');
++ assert.match(f['Reverse DNS'].detail, /forward-confirmed/);
++ assert.strictEqual(r.score, 100);
++ r = await checks.reputation('93.184.216.32', { rdap, reverse });
++ assert.strictEqual(byCheck(r)['Reverse DNS'].cost, 5, 'a PTR name that does not point back');
++ r = await checks.reputation('93.184.216.33', { rdap, reverse });
++ assert.strictEqual(byCheck(r)['Reverse DNS'].cost, 10, 'no PTR name');
++ r = await checks.reputation('2606:4700:4700::1111', { rdap, reverse });
++ assert.strictEqual(byCheck(r).Blacklists.status, 'info', 'IPv6: the blacklists are not asked');
++ assert.strictEqual(r.blacklist, null);
++
++ // Refused: private addresses, and names that resolve to them.
++ await assert.rejects(checks.reputation('192.168.1.1', { rdap, reverse }), (e) => e instanceof TargetRefused && /private or reserved/.test(e.message));
++ await assert.rejects(checks.reputation('lan.example', { rdap, reverse }), (e) => e instanceof TargetRefused);
++ console.log('net-reputation: ok');
++})().catch((e) => { console.error(e); process.exit(1); });
ddiff --git a/apps/gateway/test/net-tools.test.js b/apps/gateway/test/net-tools.test.js
iindex 5429af5..1797bf9 100644
---- a/apps/gateway/test/net-tools.test.js
++++ b/apps/gateway/test/net-tools.test.js
@@@ -332,7 +332,7 @@ async function fakeFetch(url) {
const probe = await fetch(`${base}/api/net${t.endpoint}?target=`);
assert.notStrictEqual(probe.status, 404, `${t.id} → ${t.endpoint} exists`);
}
-- assert.deepStrictEqual(NET_TOOLS.filter(t => t.status === 'unavailable').map(t => t.id), ['reputation']);
++ assert.deepStrictEqual(NET_TOOLS.filter(t => t.status === 'unavailable').map(t => t.id), [], 'every net tool is built');
} finally {
srv.closeAllConnections(); srv.close();
}
ddiff --git a/apps/gateway/test/net-trace.test.js b/apps/gateway/test/net-trace.test.js
iindex 37a442b..b3c3d62 100644
---- a/apps/gateway/test/net-trace.test.js
++++ b/apps/gateway/test/net-trace.test.js
@@@ -40,7 +40,7 @@ const REPORT = (dst) => JSON.stringify({ report: { mtr: { dst, tests: 3 }, hubs:
if (release) await new Promise(r => { release.push(r); });
return { stdout: REPORT(args[args.length - 1]) };
};
-- const names = { '15.204.72.1': ['vl1.example-provider.net'] };
++ const names = { '15.204.72.1': ['vl1.example-provider.net'], '93.184.216.34': ['_gateway'] };
const tracer = createTracer({ exec, reverse: async (ip) => names[ip] || [] });
let run = await tracer.run('trace', { address: '93.184.216.34', family: 4 });
assert.deepStrictEqual(calls[0].args, ['-4', '-n', '-z', '-j', '-c', '3', '-m', '30', '-U', '5', '--', '93.184.216.34']);
@@@ -48,7 +48,7 @@ const REPORT = (dst) => JSON.stringify({ report: { mtr: { dst, tests: 3 }, hubs:
assert.strictEqual(run.reached, true);
assert.strictEqual(run.probes, 3);
assert.strictEqual(run.hops[0].host, 'vl1.example-provider.net', 'a hop with a PTR name is named');
-- assert.strictEqual(run.hops[2].host, null);
++ assert.strictEqual(run.hops[2].host, null, 'a synthetic local name (_gateway) is not a hop name');
run = await tracer.run('mtr', { address: '2606:4700:4700::1111', family: 6 });
assert.deepStrictEqual(calls[1].args.slice(0, 6), ['-6', '-n', '-z', '-j', '-c', '10']);
assert.strictEqual(calls[1].args[calls[1].args.length - 1], '2606:4700:4700::1111');
ddiff --git a/apps/gateway/test/run-api.test.js b/apps/gateway/test/run-api.test.js
iindex dc419d5..cf2c9f4 100644
---- a/apps/gateway/test/run-api.test.js
++++ b/apps/gateway/test/run-api.test.js
@@@ -109,9 +109,6 @@ function checkProblem(r, status, code, where) {
checkProblem(await run('nosuchtool', {}), 404, 'tools.tool.not_found', 'unknown');
checkProblem(await run('yt', {}), 404, 'tools.tool.not_runnable', 'yt: api false');
checkProblem(await run('regex', { input: {} }), 404, 'tools.tool.not_runnable', 'regex: page only');
-- r = await run('reputation', {});
-- checkProblem(r, 503, 'tools.tool.unavailable', 'descriptor status unavailable');
-- assert.match(r.body.detail, /not available yet/);
checkProblem(await run('protectpdf', { input: { password: 'x' } }), 503, 'tools.tool.unavailable', 'qpdf missing on docs');
checkProblem(await call('/api/v1/tools/jsonminify/run', { method: 'GET' }), 405, 'method_not_allowed', 'GET');
ddiff --git a/apps/gateway/test/sunset-routes.test.js b/apps/gateway/test/sunset-routes.test.js
iindex 1529998..2fb0070 100644
---- a/apps/gateway/test/sunset-routes.test.js
++++ b/apps/gateway/test/sunset-routes.test.js
@@@ -12,7 +12,7 @@ const NET = [
'/ssl/:target?', '/headers/:target?', '/redirects/:target?',
'/port/:target?', '/ping/:target?', '/lookup/:target?',
'/robots/:target?', '/sitemap/:target?', '/uptime/:target?',
-- '/smtp/:target?', '/blacklist/:target?', '/dnsprop/:target?',
++ '/smtp/:target?', '/blacklist/:target?', '/dnsprop/:target?', '/reputation/:target?',
];
const DEV = ['/tools', '/opengraph', '/read'];
const PROCESS = ['/api/process', '/api/process/direct', '/api/process/multi', '/api/process/anything', '/api/process/anything/nested'];
ddiff --git a/apps/gateway/test/tools-api.test.js b/apps/gateway/test/tools-api.test.js
iindex ab0ef70..ec4ade1 100644
---- a/apps/gateway/test/tools-api.test.js
++++ b/apps/gateway/test/tools-api.test.js
@@@ -190,7 +190,7 @@ const get = async (url, headers = {}, method = 'GET') => {
for (const id of ['protectpdf', 'unlockpdf', 'pdf2jpg', 'heic', 'mp3', 'merge', 'yt']) assert.strictEqual((await status(G, id)).status, 'unavailable', `gateway: ${id}`);
for (const id of ['png', 'mergepdf', 'fancy', 'dns']) assert.strictEqual((await status(G, id)).status, 'stable', `gateway: ${id}`);
l = await count('status=unavailable');
-- assert.ok(l.tools.length >= 1 + 36 + 5 && contracts.tools.checkList(l).valid, 'status filter'); // reputation (net), then the tools without binaries here
++ assert.ok(l.tools.length >= 36 + 5 && contracts.tools.checkList(l).valid, 'status filter'); // the tools without binaries here
// Readiness: the registry is checked, and it passes.
const ready = (await get(`${G}/api/ready`)).body;