openvibers/openvibe.watch Commits
Watch: user regexes cannot hang the process (validated at save, 50 ms deadline per match); API obeys the cadence floor and a per-owner cap (#14)
Changes
@@@ -80,6 +80,10 @@ VALKEY_PREFIX=#WATCH_OBSERVATIONS_MAX_PER_CHECK=100#WATCH_RETENTION_DEFAULT_DAYS=90++# Registry: how many watches one owner (a person, an agent or a service) may hold. The cadence floor++# (15 minutes) is fixed in the registry.++#WATCH_MAX_PER_OWNER=50++# OpenVibe.Run — the expensive checks (browser/code), step 6. Never used until then: a run source# answers watch.carrier_unavailable and nothing is executed.#WATCH_RUN_URL=http://127.0.0.1:4920@@@ -99,8 +99,9 @@ server {location = /api/ready { proxy_pass http://127.0.0.1:4730; }location = /release.json { proxy_pass http://127.0.0.1:4730; }-- # Metrics are for host-local scrapers only (http://127.0.0.1:4730/metrics).-- location = /metrics { return 404; }++ # Metrics are for host-local scrapers only (http://127.0.0.1:4730/metrics). Case-insensitive so++ # /Metrics and friends are refused the same way.++ location ~* ^/metrics(/|$) { return 404; }# ── Sign-in: the network's own budget. /auth/me is the navbar's session probe and gets the# page budget instead, because it runs on every page view. ──@@@ -126,16 +127,19 @@ server {location = /llms-full.txt { proxy_pass http://127.0.0.1:4730; }# ── The watches pages: reads keep the page budget, writes (a POST) the strict form zone. The-- # map above gives GET/HEAD an empty key, which nginx does not count. ──++ # map above gives GET/HEAD an empty key, which nginx does not count, so watch_forms only++ # limits writes; watch_pages limits every read (POSTs count in both, which is fine). ──location ^~ /watches {limit_req zone=watch_forms burst=10 nodelay;++ limit_req zone=watch_pages burst=40 nodelay;limit_req_status 429;proxy_pass http://127.0.0.1:4730;}# ── OpenVibe.Events' delivery route: loopback only (POST /internal/events) ──# Events posts to http://127.0.0.1:4730/internal/events directly; never proxied from outside (ADR-033).-- location /internal/ { return 404; }++ # Case-insensitive so /Internal and /internal/events alike are refused.++ location ~* ^/internal(/|$) { return 404; }# ── Pages ───────────────────────────────────────────────location / {@@@ -253,6 +253,8 @@ function createCheck({ db, registry, config, carriers, extract, condition, obserconst stated = out.state === 'ok';let value = null;let snapshot = null;++ // A pattern that hit its deadline stated no match; the reason is recorded on the run.++ let patternError = null;if (stated) {try {const extracted = extract(watch.extraction, {@@@ -263,6 +265,7 @@ function createCheck({ db, registry, config, carriers, extract, condition, obser});value = extracted.value;snapshot = extracted.snapshot;++ if (extracted.error) patternError = extracted.error;} catch (err) {const unsupported = err instanceof ExtractError && err.code === 'unsupported';const run = {@@@ -310,6 +313,7 @@ function createCheck({ db, registry, config, carriers, extract, condition, obserconst lastFired = await st.lastTrigger.get(watch.id);const repeat = verdict.met && !verdict.changed && lastFired != null && verdict.since != null && lastFired.started_at >= verdict.since;const met = verdict.met && !repeat;++ if (verdict.error && !patternError) patternError = verdict.error;const retentionDays = watch.retention && Number.isInteger(watch.retention.observations_days)? watch.retention.observations_days@@@ -321,6 +325,7 @@ function createCheck({ db, registry, config, carriers, extract, condition, obserstate: met ? 'condition_met' : (stated ? (verdict.changed ? 'changed' : 'ok') : out.state),http_status: out.http_status ?? null, bytes: out.bytes ?? null, raw_body_hash: out.raw_body_hash ?? null,observations: stated ? 1 : 0, triggers: met ? 1 : 0, finished_at: finishedAt,++ detail: patternError ? String(patternError).slice(0, 500) : null,};// The observation row is built before the transaction (it needs the previous one's hash),@@@ -14,11 +14,13 @@** Numeric comparison refuses to guess: a value that is not a number (or a plain decimal string) is* never "greater than" anything, and an unknown value is never low (the same rule Deals' watches-- * use). `matches` compiles the watch's own pattern with a length cap and tests a bounded string.++ * use). `matches` runs the watch's own pattern through safe-regex: it is bounded in length and in++ * time, an invalid or unsafe pattern simply does not match, and a pattern that hits the deadline++ * answers "no match" with `error` set (the check records "pattern took too long").*/const { canonicalJson } = require('./util');++const safe = require('./safe-regex');--const MAX_PATTERN = 200;// Bounded input: the watch's own pattern only ever runs over this much text (see extract.js: a// value is capped at the same size), so a pathological pattern cannot scan a whole body.const MAX_TESTED = 65536;@@@ -58,8 +60,9 @@ function hasChanged({ comparison }, { value, previous }) {return !same(value, before);}--/** The predicate alone: does the condition hold for this value? */--function holds({ comparison, condition }, { value, previous, changed }) {++/** The predicate alone: does the condition hold for this value? `state`, when given, records a++ * regex that hit its deadline so the check can record the error (see evaluate). */++function holds({ comparison, condition }, { value, previous, changed }, state = null) {const op = (condition && condition.op) || 'changed';const target = condition ? condition.value : null;switch (op) {@@@ -87,7 +90,7 @@ function holds({ comparison, condition }, { value, previous, changed }) {const b = asNumber(target);return a != null && b != null ? a === b : same(value, target);}-- case 'ne': return !holds({ comparison, condition: { ...condition, op: 'eq' } }, { value, previous, changed });++ case 'ne': return !holds({ comparison, condition: { ...condition, op: 'eq' } }, { value, previous, changed }, state);case 'contains': {if (typeof value === 'string') return typeof target === 'string' && value.includes(target);if (Array.isArray(value)) return value.some((v) => same(v, target));@@@ -96,10 +99,11 @@ function holds({ comparison, condition }, { value, previous, changed }) {}case 'matches': {if (typeof value !== 'string' || typeof target !== 'string') return false;-- if (!target || target.length > MAX_PATTERN) return false;let re;-- try { re = new RegExp(target); } catch { return false; }-- return re.test(value.slice(0, MAX_TESTED));++ try { re = safe.compile(target); } catch { return false; } // invalid, oversize or unsafe: never matches++ const { matched, timedOut } = safe.test(re, value.slice(0, MAX_TESTED));++ if (timedOut && state) state.error = safe.TIMEOUT_MESSAGE;++ return matched;}case 'exists': return !isAbsent(value);case 'absent': return isAbsent(value);@@@ -109,22 +113,24 @@ function holds({ comparison, condition }, { value, previous, changed }) {/*** evaluate({ comparison, condition }, { value, previous, heldSince, now })-- * → { changed, holds, met, since }++ * → { changed, holds, met, since, error }** `previous` is the previous observation ({ value, condition_met, observed_at }) or null;* `heldSince` (ms) is when the predicate began holding continuously, from the observation trail —* pass it from observations.heldSince(). When the predicate holds and no start is known, it starts-- * now, so a for_sec condition never fires on the first sighting.++ * now, so a for_sec condition never fires on the first sighting. `error` is the recorded reason a++ * `matches` pattern hit its deadline (no match was taken from it), else null.*/function evaluate({ comparison = null, condition = {} } = {}, { value, previous = null, heldSince = null, now = Date.now(), changed: given } = {}) {// The caller may know better: a check that got a 304 or an unchanged body knows nothing changed,// whatever comparing the values would say.const changed = given === undefined ? hasChanged({ comparison }, { value, previous }) : Boolean(given);-- const hold = holds({ comparison, condition }, { value, previous, changed });++ const state = { error: null };++ const hold = holds({ comparison, condition }, { value, previous, changed }, state);const forSec = condition && Number.isInteger(condition.for_sec) ? Math.max(0, condition.for_sec) : 0;const since = hold ? (heldSince != null ? heldSince : now) : null;const met = hold && (forSec === 0 || (since != null && now - since >= forSec * 1000));-- return { changed, holds: hold, met, since };++ return { changed, holds: hold, met, since, error: state.error };}module.exports = { evaluate, hasChanged, holds, asNumber, isAbsent };@@@ -110,6 +110,11 @@ function load(env = process.env) {// observations_days when the watch states none; a watch may override it (watch.watch@1).defaultDays: int(env.WATCH_RETENTION_DEFAULT_DAYS, 90),},++ // How many watches one owner (a person, an agent or a service) may hold; the registry enforces++ // it on create, for the API and the form alike.++ watches: {++ maxPerOwner: Math.max(1, int(env.WATCH_MAX_PER_OWNER, 50)),++ },// OpenVibe.Run (plan T14): the expensive last rung — browser polling — and never used before// steps 5-7 exist. WATCH_RUN_URL is Run's loopback port.@@@ -17,6 +17,7 @@* `fields` (name → path) extracts an object of several values at once, each null when it is missing.*/const { toText, decodeBody } = require('./util');++const safe = require('./safe-regex');/** Longest text value one extraction stores (a 5 MB page as `text` is a mistake, not a value). */const VALUE_MAX = 64 * 1024;@@@ -205,15 +206,15 @@ function parseJson(text) {}}--/** '/<re>/flags' or a bare pattern → a RegExp; oversized or invalid patterns throw. */++/** '/<re>/flags' or a bare pattern → a RegExp; an unsafe, oversized or invalid pattern throws. */function compileRegex(pattern) {-- const s = String(pattern || '');-- if (!s || s.length > 200) throw new ExtractError('unsupported', 'regex extraction needs a pattern of at most 200 characters');-- const m = /^\/(.*)\/([gimsuy]*)$/.exec(s);try {-- return m ? new RegExp(m[1], m[2].replace('g', '')) : new RegExp(s);-- } catch {-- throw new ExtractError('parse_error', 'selector is not a valid regular expression');++ return safe.compile(pattern);++ } catch (err) {++ // An invalid pattern is a parse error; an unsafe or oversized one is one this release++ // refuses to run (the registry already refused it when the watch was saved).++ if (err instanceof safe.UnsafeRegexError && err.reason === 'invalid') throw new ExtractError('parse_error', err.message);++ throw new ExtractError('unsupported', err.message);}}@@@ -268,10 +269,13 @@ function extract(extraction = {}, { body = null, contentType = null, document =if (text == null) throw new ExtractError('parse_error', 'there is no body to extract from');const re = compileRegex(extraction.selector);// Bounded input: a watch's own pattern is compiled as-is, so it only ever runs over a-- // window of VALUE_MAX characters (a value beyond that is not a value this service keeps).-- const m = re.exec(text.slice(0, VALUE_MAX));-- if (!m) return { value: null, snapshot };-- return { value: m.length > 1 ? (m[1] === undefined ? null : m[1]) : m[0], snapshot };++ // window of VALUE_MAX characters (a value beyond that is not a value this service keeps),++ // and under a deadline (safe-regex): a pattern that still backtracks states no match and++ // records why, rather than hanging the process.++ const { match, timedOut } = safe.exec(re, text.slice(0, VALUE_MAX));++ if (timedOut) return { value: null, snapshot, error: safe.TIMEOUT_MESSAGE };++ if (!match) return { value: null, snapshot };++ return { value: match.length > 1 ? (match[1] === undefined ? null : match[1]) : match[0], snapshot };}default:throw new ExtractError('unsupported', `extraction kind "${kind}" is not supported`);@@@ -42,7 +42,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetchtokenClient,tokenOpts: config.oauth.clientSecret ? { tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret } : null,});-- const registry = createRegistry({ db, now, outbox });++ const registry = createRegistry({ db, now, outbox, maxWatchesPerOwner: config.watches.maxPerOwner });const observations = createObservations({ db, now });// The only way out to the internet: SSRF guard (ports, private hosts, every redirect hop), byte// cap, one deadline, conditional GET; per-host spacing in front of every request.@@@ -18,12 +18,15 @@* found — 404, never 403 (an id must not confirm that someone else's watch exists).*/const { validate, ids: contractIds } = require('openvibe-contracts');++const safe = require('./safe-regex');/** Source kinds that reach out on a cadence; event and webhook sources are pushed to instead. */const POLLING = new Set(['http', 'feed', 'api', 'node', 'run']);const PUSHED = new Set(['event', 'webhook']);const RUN_STATES = ['ok', 'not_modified', 'no_change', 'changed', 'condition_met', 'http_error', 'timeout', 'parse_error', 'rate_limited', 'budget_exceeded', 'skipped', 'disabled'];const STATUSES = ['active', 'paused', 'disabled', 'failed'];++/** The shortest cadence a polling watch may state (15 minutes) — the web form's floor, enforced here for the API too. */++const MIN_EVERY_SEC = 900;/** Headers the fetcher decides; a watch may never override them (SSRF and conditional-GET safety). */const FORBIDDEN_HEADERS = new Set(['host', 'cookie', 'connection', 'content-length', 'transfer-encoding', 'user-agent', 'accept-encoding', 'if-none-match', 'if-modified-since']);@@@ -75,6 +78,7 @@ function validateRequest(body, { create = false } = {}) {if (POLLING.has(kind)) {if (cadence === null || cadence === undefined) throw new RegistryError(`a ${kind} source needs a cadence (every_sec)`);if (!Number.isInteger(cadence.every_sec)) throw new RegistryError('cadence.every_sec is required for a source that polls');++ if (cadence.every_sec < MIN_EVERY_SEC) throw new RegistryError(`cadence.every_sec must be at least ${MIN_EVERY_SEC} seconds (15 minutes)`);}if (PUSHED.has(kind) && cadence) throw new RegistryError(`a ${kind} source is pushed to and never polls: cadence must be null`);// A HEAD source states no body, so only a document/record extraction can read one: refuse@@@ -83,6 +87,17 @@ function validateRequest(body, { create = false } = {}) {throw new RegistryError('a HEAD source states no body: its extraction must be json or jsonpath (status, etag, last_modified, content_type, content_length)');}}++ // A watch's own pattern is refused when the watch is saved, not when it runs: one that would++ // backtrack for minutes (safe-regex) must never reach the check worker through the API or the++ // form. The names match the form's fields, so the message lands next to the input.++ if (body.extraction && body.extraction.kind === 'regex') {++ const bad = safe.validate(body.extraction.selector);++ if (bad) throw new RegistryError(`/extraction/selector ${bad}`);++ }++ if (body.condition && body.condition.op === 'matches' && typeof body.condition.value === 'string') {++ const bad = safe.validate(body.condition.value);++ if (bad) throw new RegistryError(`/condition/value ${bad}`);++ }return body;}@@@ -106,9 +121,10 @@ function validateView(view) {return view;}--function createRegistry({ db, now = () => Date.now(), ids = contractIds, outbox = null }) {++function createRegistry({ db, now = () => Date.now(), ids = contractIds, outbox = null, maxWatchesPerOwner = 50 }) {const st = {get: db.prepare('SELECT * FROM watches WHERE id = ? AND deleted_at IS NULL'),++ countOwned: db.prepare('SELECT COUNT(*) AS n FROM watches WHERE owner_sub = ? AND deleted_at IS NULL'),insert: db.prepare(`INSERT INTO watches (id, project_id, owner_sub, name, status, source, cadence, extraction, comparison, condition,action, budget, retention, labels, next_due_at, not_before, created_at, updated_at, updated_by)VALUES (@id, @project_id, @owner_sub, @name, @status, @source, @cadence, @extraction, @comparison, @condition,@@@ -219,6 +235,10 @@ function createRegistry({ db, now = () => Date.now(), ids = contractIds, outboxconst req = validateRequest(body, { create: true });const owner = ownerOf(principal);if (!owner) throw new RegistryError('no acting subject', 'watch.invalid');++ // A per-owner cap, so one caller cannot fill the registry (the API and the form both come++ // through here). The number is a config value (WATCH_MAX_PER_OWNER).++ const owned = await st.countOwned.get(owner);++ if (Number(owned.n) >= maxWatchesPerOwner) throw new RegistryError(`an owner may hold at most ${maxWatchesPerOwner} watches`);const t = now();const rec = {name: req.name || defaultName(req.source),nnew file mode 100644@@@ -0,0 +1,179 @@++'use strict';++/**++ * Safe regular expressions for the patterns a watch states.++ *++ * A regex extraction's `selector` and a `matches` condition's `value` are the watch's own, and they++ * run in the one Node process that serves every watch. A pattern like `(a+)+$` tested against a++ * body would otherwise backtrack for minutes and hang the process (ReDoS).++ *++ * Two guards, so a bad pattern is refused when the watch is saved rather than when it runs:++ *++ * compile() refuses what is plainly catastrophic — a pattern longer than MAX_PATTERN, or a++ * quantified group that itself contains a quantifier (`(a+)+`, `(a*)*`, `(\w+\s?)*`).++ * The registry calls validate() before it stores a watch, so the API and the form both++ * answer a validation error at create time.++ * test()/exec() run the match under a deadline in a vm context: a pattern the static check let++ * through but that still backtracks is cut off at TIMEOUT_MS and reported as a timeout++ * (no match), never a hang. The caller records that as a check error.++ *++ * The static check is a scan of the pattern's shape, not a full parse: it refuses nested quantifiers++ * without judging anything else, so an unusual-but-legitimate pattern still runs under the deadline.++ */++const vm = require('vm');++++/** Longest pattern a watch may state (the 500-character rule). */++const MAX_PATTERN = 500;++/** How long one match may run before it is cut off. */++const TIMEOUT_MS = 50;++/** The message a timed-out match records (the caller stores this as the check error). */++const TIMEOUT_MESSAGE = 'pattern took too long';++/** V8's code for a vm script that hit its timeout. */++const TIMEOUT_CODE = 'ERR_SCRIPT_EXECUTION_TIMEOUT';++++// One compiled script per operation, run with a fresh sandbox per call: recompiling the one-line++// expression on every check would be waste, and the sandbox is what carries the deadline.++const TEST_SCRIPT = new vm.Script('re.test(s)');++const EXEC_SCRIPT = new vm.Script('re.exec(s)');++++class UnsafeRegexError extends Error {++ // reason: 'too_long' | 'nested' | 'invalid' (a caller maps it to its own error code).++ constructor(detail, reason) { super(detail); this.code = 'unsafe_pattern'; this.reason = reason; }++}++++/** The source and flags of '/…/flags', or of a bare pattern. */++function split(input) {++ const s = String(input == null ? '' : input);++ const m = /^\/(.*)\/([gimsuy]*)$/.exec(s);++ return m ? { source: m[1], flags: m[2] } : { source: s, flags: '' };++}++++/** The length of a quantifier at `i` (`*`, `+`, `?` or `{n[,m]}`), or 0 when there is none. */++function quantifierAt(source, i) {++ const c = source[i];++ if (c === '*' || c === '+' || c === '?') return 1;++ if (c === '{') {++ const m = /^\{\d+(?:,\d*)?\}/.exec(source.slice(i));++ return m ? m[0].length : 0;++ }++ return 0;++}++++/**++ * Does the pattern contain a quantified group whose own body contains a quantifier? Groups are read++ * as `(...)`, including non-capturing and lookaround forms; escapes and character classes are read++ * literally, so `\+` and `[a+]` are not quantifiers. A group that is quantified and already holds a++ * quantifier (itself or a group it contains) is the exponential shape this refuses.++ */++function nestedQuantifier(source) {++ const stack = [];++ const markQuantifier = () => { const top = stack[stack.length - 1]; if (top) top.hasQuant = true; };++ let i = 0;++ while (i < source.length) {++ const c = source[i];++ const top = stack[stack.length - 1];++ // The '?' that opens a group prefix ((?:, (?=, (?<name>…) is not a quantifier.++ if (top && top.prefix && c !== '?') top.prefix = false;++ if (c === '\\') { i += 2; continue; }++ if (c === '[') {++ i++;++ if (source[i] === '^') i++;++ if (source[i] === ']') i++; // a leading ']' is a literal, not the class end++ while (i < source.length && source[i] !== ']') { if (source[i] === '\\') i++; i++; }++ i++;++ continue;++ }++ if (c === '(') { stack.push({ hasQuant: false, prefix: true }); i++; continue; }++ if (c === ')') {++ const frame = stack.pop();++ i++;++ const q = quantifierAt(source, i);++ if (q) {++ i += q;++ if (source[i] === '?') i++; // a lazy marker belongs to the quantifier++ if (frame.hasQuant) return true;++ }++ if (stack.length && frame.hasQuant) stack[stack.length - 1].hasQuant = true;++ continue;++ }++ if (c === '?') {++ if (top && top.prefix) {++ // Consume the prefix so its '?' is never read as a quantifier.++ i++;++ if (source[i] === ':' || source[i] === '=' || source[i] === '!') i++;++ else if (source[i] === '<') {++ i++;++ if (source[i] === '=' || source[i] === '!') i++;++ else { while (i < source.length && source[i] !== '>') i++; if (source[i] === '>') i++; }++ }++ continue;++ }++ markQuantifier();++ i++;++ if (source[i] === '?') i++;++ continue;++ }++ const q = quantifierAt(source, i);++ if (q) {++ markQuantifier();++ i += q;++ if (source[i] === '?') i++;++ continue;++ }++ i++;++ }++ return false;++}++++/** The reason a pattern cannot be used, or null when it is accepted (for validation at save time). */++function validate(pattern, opts) {++ try {++ compile(pattern, opts);++ return null;++ } catch (err) {++ return err instanceof UnsafeRegexError ? err.message : 'selector is not a valid regular expression';++ }++}++++/** A pattern as a RegExp, or an UnsafeRegexError (reason 'too_long' | 'nested' | 'invalid'). */++function compile(pattern, { maxLength = MAX_PATTERN } = {}) {++ const { source, flags } = split(pattern);++ if (!source || source.length > maxLength) {++ throw new UnsafeRegexError(`a pattern must be 1–${maxLength} characters`, 'too_long');++ }++ if (nestedQuantifier(source)) {++ throw new UnsafeRegexError('a quantified group must not contain another quantifier (a pattern like (a+)+ can hang the check)', 'nested');++ }++ try {++ // 'g' is dropped: a lastIndex must never carry between checks.++ return new RegExp(source, flags.replace('g', ''));++ } catch {++ throw new UnsafeRegexError('selector is not a valid regular expression', 'invalid');++ }++}++++/** Run one script under the deadline → { value, timedOut }; only a timeout is caught. */++function underDeadline(script, re, s, timeout) {++ try {++ return { value: script.runInNewContext({ re, s }, { timeout }), timedOut: false };++ } catch (err) {++ if (err && err.code === TIMEOUT_CODE) return { value: null, timedOut: true };++ throw err;++ }++}++++/** re.test(s) under the deadline → { matched, timedOut }; a timeout is not a match. */++function test(re, s, { timeout = TIMEOUT_MS } = {}) {++ const out = underDeadline(TEST_SCRIPT, re, s, timeout);++ return { matched: out.timedOut ? false : Boolean(out.value), timedOut: out.timedOut };++}++++/** re.exec(s) under the deadline → { match, timedOut }; a timeout is no match. */++function exec(re, s, { timeout = TIMEOUT_MS } = {}) {++ const out = underDeadline(EXEC_SCRIPT, re, s, timeout);++ return { match: out.timedOut ? null : out.value, timedOut: out.timedOut };++}++++module.exports = {++ MAX_PATTERN, TIMEOUT_MS, TIMEOUT_MESSAGE, UnsafeRegexError,++ validate, compile, test, exec, nestedQuantifier,++};@@@ -83,6 +83,10 @@ t('an invalid body is 422 with the validator\'s detail, and nothing is written',[mine({ action: [] }), /watch.watch-request@1/],[mine({ source: { kind: 'http', url: 'http://example.org/x', headers: { cookie: 'a=b' } } }), /cookie/],[mine({ nope: true }), /watch.watch-request@1/],++ // the registry's own rules the API must obey too: the cadence floor and a safe pattern++ [mine({ cadence: { every_sec: 60 } }), /at least 900/],++ [mine({ extraction: { kind: 'regex', selector: '(a+)+$' } }), /extraction\/selector/],++ [mine({ condition: { op: 'matches', value: '(a+)+$' } }), /condition\/value/],];for (const [body, re] of bad) {const res = await api('POST', '/api/v1/watches', { token: ME, body });@@@ -99,10 +103,10 @@ t('an invalid body is 422 with the validator\'s detail, and nothing is written',t('patch, pause, resume, delete — and another owner gets 404 for every one of them', async () => {const { body: { watch } } = await api('POST', '/api/v1/watches', { token: ME, body: mine() });-- const patched = await api('PATCH', `/api/v1/watches/${watch.id}`, { token: ME, body: { name: 'renamed', cadence: { every_sec: 120 } } });++ const patched = await api('PATCH', `/api/v1/watches/${watch.id}`, { token: ME, body: { name: 'renamed', cadence: { every_sec: 1800 } } });assert.strictEqual(patched.status, 200);assert.strictEqual(patched.body.watch.name, 'renamed');-- assert.strictEqual(patched.body.watch.cadence.every_sec, 120);++ assert.strictEqual(patched.body.watch.cadence.every_sec, 1800);assert.ok(validate('watch.watch-result@1', patched.body).valid);const paused = await api('POST', `/api/v1/watches/${watch.id}/pause`, { token: ME });@@@ -16,7 +16,7 @@ let web;const watch = async (overrides = {}) => (await svc.registry.create({name: 'carrier watch',source: { kind: 'http', url: `${web.origin}/page`, format: null },-- cadence: { every_sec: 60 },++ cadence: { every_sec: 900 },extraction: { kind: 'text' },condition: { op: 'eq', value: 'never-matches' },action: [{ kind: 'notification' }],@@@ -122,7 +122,7 @@ t('a private host is refused before any request (nothing is even connected to)',const w = await alone.registry.create({name: 'loopback watch',source: { kind: 'http', url: `${aSite.origin}/page`, format: null },-- cadence: { every_sec: 60 }, extraction: { kind: 'text' }, condition: { op: 'eq', value: 'x' },++ cadence: { every_sec: 900 }, extraction: { kind: 'text' }, condition: { op: 'eq', value: 'x' },action: [{ kind: 'notification' }],}, OWNER);const out = await alone.check.run(w.id, { trigger: 'manual' });@@@ -21,7 +21,7 @@ const service = { sub: 'svc:check-test' };const def = (overrides = {}) => ({name: 'price watch',source: { kind: 'http', url: `${web.origin}/price`, format: null },-- cadence: { every_sec: 60 },++ cadence: { every_sec: 900 },extraction: { kind: 'text' },condition: { op: 'lt', value: 10 },action: [{ kind: 'notification' }],@@@ -129,13 +129,13 @@ t('a failing check is a recorded state and a watch.check.failed event, and it balet r = await row(w.id);assert.strictEqual(r.last_state, 'parse_error');assert.strictEqual(Number(r.consecutive_failures), 1);-- assert.strictEqual(r.next_due_at - r.last_check_at, 60000, 'first failure: every_sec · 2^0');++ assert.strictEqual(r.next_due_at - r.last_check_at, 900000, 'first failure: every_sec · 2^0');const second = await check(w.id);assert.strictEqual(second.failures, 2);r = await row(w.id);assert.strictEqual(Number(r.consecutive_failures), 2);-- assert.strictEqual(r.next_due_at - r.last_check_at, 120000, 'second failure: every_sec · 2^1');++ assert.strictEqual(r.next_due_at - r.last_check_at, 1800000, 'second failure: every_sec · 2^1');assert.strictEqual((await payloads('watch.check.failed')).length, 2);// a success clears the backoff and the counter (19.99 is above the watch's lt 10, so it is a@@@ -145,7 +145,7 @@ t('a failing check is a recorded state and a watch.check.failed event, and it baawait check(w.id);r = await row(w.id);assert.deepStrictEqual([Number(r.consecutive_failures), r.last_state], [0, 'changed']);-- assert.ok(r.next_due_at - r.last_check_at >= 60000 && r.next_due_at - r.last_check_at <= 60100, 'back to the cadence');++ assert.ok(r.next_due_at - r.last_check_at >= 900000 && r.next_due_at - r.last_check_at <= 900100, 'back to the cadence');});t('a timeout is its own state, and a 429 waits as long as the site asked', async () => {@@@ -162,6 +162,24 @@ t('a timeout is its own state, and a 429 waits as long as the site asked', asyncassert.ok(r.not_before - r.last_check_at >= 119000, 'Retry-After: 120 s became not_before');});++t('a pattern that hits its deadline is a recorded check error, never a hang', async () => {++ // A regex extraction whose pattern slips past the static check still runs, and the deadline++ // makes it a no-match with the reason recorded on the run (the process stays up).++ price = 'a'.repeat(40) + '!';++ const slowExtract = await create(person, { extraction: { kind: 'regex', selector: '^(\\w|\\w\\w)*$' } });++ const withExtract = await check(slowExtract.id, 'manual');++ assert.strictEqual(withExtract.run.detail, 'pattern took too long');++ assert.strictEqual(withExtract.run.state, 'changed', 'the run completed with no match');++ assert.strictEqual(withExtract.value, null);++++ // The same, for a `matches` condition: no match, and the run records why.++ const slowMatch = await create(person, { condition: { op: 'matches', value: '^(\\w|\\w\\w)*$' } });++ const withMatch = await check(slowMatch.id, 'manual');++ assert.strictEqual(withMatch.run.detail, 'pattern took too long');++ assert.strictEqual(withMatch.verdict.error, 'pattern took too long');++ price = '19.99';++});++t('an event, webhook, node or run source is accepted but never fetched (steps 5-7)', async () => {const cases = [['event', { kind: 'event', pattern: 'release.published' }, { op: 'exists' }],@@@ -173,7 +191,7 @@ t('an event, webhook, node or run source is accepted but never fetched (steps 5-for (const [kind, source, condition] of cases) {// an event or a webhook is pushed to and never polls; node and run sources poll through// OpenVibe.Node and OpenVibe.Run, so they carry a cadence — and still nothing is fetched-- const cadence = (kind === 'event' || kind === 'webhook') ? null : { every_sec: 60 };++ const cadence = (kind === 'event' || kind === 'webhook') ? null : { every_sec: 900 };const w = await create(person, { source, cadence, extraction: { kind: 'text' }, condition });assert.strictEqual((await row(w.id)).status, 'active');const out = await check(w.id, 'manual');@@@ -64,6 +64,18 @@ t('contains / matches', () => {assert.strictEqual(holds({ op: 'matches', value: null }, 'anything'), false);});++t('matches: an unsafe pattern never matches, and a slow one records the timeout', () => {++ // a nested quantifier is refused by the static check and simply does not match++ assert.strictEqual(evaluate({ condition: { op: 'matches', value: '(a+)+$' } }, { value: 'aaaa' }).holds, false);++ // an alternation slips past the static check; the deadline answers no match and records why++ const slow = evaluate({ condition: { op: 'matches', value: '^(\\w|\\w\\w)*$' } }, { value: 'a'.repeat(40) + '!' });++ assert.strictEqual(slow.holds, false);++ assert.strictEqual(slow.error, 'pattern took too long');++ // a normal pattern holds and records no error++ const ok = evaluate({ condition: { op: 'matches', value: '^v\\d+\\.\\d+' } }, { value: 'v2.3 released' });++ assert.deepStrictEqual([ok.holds, ok.error], [true, null]);++});++t('exists / absent', () => {const holds = (op, value) => evaluate({ condition: { op } }, { value }).holds;assert.strictEqual(holds('exists', 'x'), true);@@@ -87,7 +87,17 @@ t('regex: the first match, a group when the pattern has one', () => {assert.strictEqual(extract({ kind: 'regex', selector: '/(\\d+) items/i' }, { body: body('12 ITEMS left') }).value, '12');assert.strictEqual(extract({ kind: 'regex', selector: 'nothing' }, { body: body('abc') }).value, null);assert.throws(() => extract({ kind: 'regex', selector: '(' }, { body: body('x') }), (e) => e.code === 'parse_error');-- assert.throws(() => extract({ kind: 'regex', selector: 'x'.repeat(300) }, { body: body('x') }), (e) => e.code === 'unsupported');++ assert.throws(() => extract({ kind: 'regex', selector: 'x'.repeat(600) }, { body: body('x') }), (e) => e.code === 'unsupported');++});++++t('regex: an unsafe pattern is refused, and a slow one times out as no match', () => {++ // a nested quantifier is refused at compile time (the registry refuses it at save time too)++ assert.throws(() => extract({ kind: 'regex', selector: '(a+)+$' }, { body: body('aaaa') }),++ (e) => e.code === 'unsupported' && /quantified group/.test(e.message));++ // an alternation slips past the static check; the deadline makes it a no-match with the reason++ const out = extract({ kind: 'regex', selector: '^(\\w|\\w\\w)*$' }, { body: body('a'.repeat(40) + '!') });++ assert.strictEqual(out.value, null, 'a pattern that hit its deadline states no match');++ assert.strictEqual(out.error, 'pattern took too long');});t('ai is not a check: it refuses, and the refusal is `unsupported`', () => {@@@ -175,7 +175,7 @@ function watchDef(overrides = {}) {return {name: 'Stub watch',source: { kind: 'http', url: 'https://example.org/page', format: null },-- cadence: { every_sec: 60, jitter_sec: 0 },++ cadence: { every_sec: 900, jitter_sec: 0 },extraction: { kind: 'text' },condition: { op: 'changed' },action: [{ kind: 'notification', category: 'service', priority: 'normal' }],@@@ -16,7 +16,7 @@ const someoneElse = { sub: newUser() };const def = (overrides = {}) => ({name: 'a watch',source: { kind: 'http', url: 'https://example.org/page', format: null },-- cadence: { every_sec: 60 },++ cadence: { every_sec: 900 },extraction: { kind: 'text' },condition: { op: 'changed' },action: [{ kind: 'notification' }],@@@ -67,6 +67,27 @@ t('create: what a watch requires, enforced in code over the released schema', asassert.strictEqual(head.source.method, 'HEAD');});++t('create: the 15-minute cadence floor is enforced here, not only in the form', async () => {++ await invalid(def({ cadence: { every_sec: 60 } }), /15 minutes/);++ await invalid(def({ cadence: { every_sec: 899 } }), /at least 900/);++ const ok = await svc.registry.create(def({ cadence: { every_sec: 900 } }), me);++ assert.strictEqual(ok.cadence.every_sec, 900);++});++++t('create: a user regex is validated when the watch is saved, not when it runs', async () => {++ // a nested quantifier would backtrack for minutes over a 64 KiB body; it never reaches the worker++ await invalid(def({ extraction: { kind: 'regex', selector: '(a+)+$' } }), /extraction\/selector/);++ await invalid(def({ condition: { op: 'matches', value: '(a+)+$' } }), /condition\/value/);++ // an invalid or oversized pattern too++ await invalid(def({ extraction: { kind: 'regex', selector: '(' } }), /extraction\/selector/);++ await invalid(def({ condition: { op: 'matches', value: 'x'.repeat(600) } }), /condition\/value/);++ // a normal pattern is accepted++ const w = await svc.registry.create(def({ extraction: { kind: 'regex', selector: '(\\d+\\.\\d+) EUR' } }), me);++ assert.strictEqual(w.extraction.selector, '(\\d+\\.\\d+) EUR');++ const m = await svc.registry.create(def({ condition: { op: 'matches', value: '^v\\d+\\.\\d+' } }), me);++ assert.strictEqual(m.condition.value, '^v\\d+\\.\\d+');++});++t('create: an id, an owner, a name, the arm and the event', async () => {const before = Date.now();const row = await svc.registry.create(def({ name: undefined }), me);@@@ -193,6 +214,22 @@ t('all(): the keyset page is id order (a wch_ ULID is creation order)', async ()assert.strictEqual((await svc.registry.all(other, { status: 'active' })).length, 4);});++t('create: one owner holds at most WATCH_MAX_PER_OWNER watches, per owner', async () => {++ const capped = await boot({ env: { WATCH_MAX_PER_OWNER: '2' } });++ try {++ const owner = { sub: newUser() };++ await capped.registry.create(def(), owner);++ await capped.registry.create(def(), owner);++ await assert.rejects(() => capped.registry.create(def(), owner), (err) => {++ assert.strictEqual(err.code, 'watch.invalid');++ assert.match(err.message, /at most 2 watches/);++ return true;++ });++ // the cap is per owner: another subject still creates++ await capped.registry.create(def(), { sub: newUser() });++ } finally { await capped.stop(); }++});++t('done', async () => { await svc.stop(); });t.run();nnew file mode 100644@@@ -0,0 +1,44 @@++'use strict';++/** safe-regex: nested quantifiers and oversized patterns refused at compile time; a slow pattern cut++ * off by the deadline (no match, a timeout, never a hang); a normal pattern still matches. */++const assert = require('assert');++const { suite } = require('./helpers');++const safe = require('../server/safe-regex');++++const t = suite('safe-regex');++++t('a quantified group holding a quantifier is refused; a normal pattern compiles', () => {++ for (const bad of ['(a+)+$', '(a*)*', '(\\w+\\s?)*', '((a+)x)+', '(a?)+']) {++ assert.ok(safe.validate(bad), `${bad} is refused`);++ assert.throws(() => safe.compile(bad), (err) => err.code === 'unsafe_pattern' && err.reason === 'nested');++ }++ for (const ok of ['(\\d+\\.\\d+) EUR', '\\d{4}-\\d{2}-\\d{2}', '(cat|dog)', '^v\\d+\\.\\d+', 'a{2,4}', '[a+]+']) {++ assert.strictEqual(safe.validate(ok), null, `${ok} is accepted`);++ }++});++++t('an oversized, empty or invalid pattern is refused', () => {++ assert.match(safe.validate('x'.repeat(501)), /1–500/);++ assert.match(safe.validate(''), /1–500/);++ assert.match(safe.validate('('), /not a valid/);++});++++t('a slow pattern that slips past the static check is cut off by the deadline', () => {++ // alternation, no nested quantifier: the static scan accepts it, the deadline stops it++ const re = safe.compile('^(\\w|\\w\\w)*$');++ const start = Date.now();++ const out = safe.test(re, 'a'.repeat(40) + '!');++ const elapsed = Date.now() - start;++ assert.deepStrictEqual(out, { matched: false, timedOut: true }, 'the deadline was hit, so no match');++ assert.ok(elapsed < 200, `returned in ~${elapsed}ms rather than hanging`);++});++++t('a normal pattern still matches, under the deadline (test and exec)', () => {++ assert.strictEqual(safe.test(safe.compile('^v\\d+\\.\\d+'), 'v2.3 released').matched, true);++ assert.strictEqual(safe.test(safe.compile('^v\\d+\\.\\d+'), 'vX').matched, false);++ const { match, timedOut } = safe.exec(safe.compile('/(\\d+\\.\\d+) EUR/'), 'now 12.50 EUR');++ assert.strictEqual(timedOut, false);++ assert.strictEqual(match[1], '12.50');++});++++t.run();@@@ -13,7 +13,7 @@ let web;const create = async (name, overrides = {}) => await svc.registry.create({name,source: { kind: 'http', url: `${web.origin}/page`, format: null },-- cadence: { every_sec: 60 },++ cadence: { every_sec: 900 },extraction: { kind: 'text' },condition: { op: 'eq', value: 'nothing' },action: [{ kind: 'notification' }],@@@ -93,10 +93,10 @@ t('a failing watch backs off: the scheduler leaves it alone until its next due tconst out = await svc.check.run(w.id, { trigger: 'schedule' });assert.strictEqual(out.run.state, 'http_error');const row = await svc.db.prepare('SELECT * FROM watches WHERE id = ?').get(w.id);-- assert.strictEqual(row.next_due_at - row.last_check_at, 60000, 'first failure: every_sec · 2^0');++ assert.strictEqual(row.next_due_at - row.last_check_at, 900000, 'first failure: every_sec · 2^0');let due = new Set((await svc.registry.due(50)).map(d => d.id));assert.ok(!due.has(w.id), 'not due until the backoff has run');-- await dueAt(w.id, row.next_due_at - 120000);++ await dueAt(w.id, row.next_due_at - 960000);due = new Set((await svc.registry.due(50)).map(d => d.id));assert.ok(due.has(w.id));});@@@ -113,7 +113,7 @@ t2('the worker checks a due watch without anyone asking', async () => {try {const w = await svc2.registry.create({name: 'auto', source: { kind: 'http', url: `${web2.origin}/page`, format: null },-- cadence: { every_sec: 60 }, extraction: { kind: 'text' },++ cadence: { every_sec: 900 }, extraction: { kind: 'text' },condition: { op: 'eq', value: 'nothing' }, action: [{ kind: 'notification' }],}, { sub: newUser() });// a new watch is due at once (its first reading), then armed by cadence after each check;@@@ -121,7 +121,7 @@ t2('the worker checks a due watch without anyone asking', async () => {await svc2.db.prepare('UPDATE watches SET next_due_at = 0 WHERE id = ?').run(w.id);for (let i = 0; i < 100 && (await svc2.db.prepare('SELECT COUNT(*) AS n FROM check_runs WHERE watch_id = ?').get(w.id)).n === 0; i++) await sleep(50);const runs = await svc2.db.prepare('SELECT * FROM check_runs WHERE watch_id = ?').all(w.id);-- assert.strictEqual(runs.length, 1, 'the scheduler ran exactly one check (the cadence is a minute)');++ assert.strictEqual(runs.length, 1, 'the scheduler ran exactly one check (the cadence is 15 minutes)');assert.strictEqual(runs[0].trigger, 'schedule');assert.strictEqual(runs[0].state, 'changed');@@@ -160,6 +160,12 @@ t('validation errors re-render the form with the registry\'s message and the perconst noValue = await create({ template: 'price', url: 'https://example.org/p', extraction: 'css', selector: '.p', op: 'lt', value: '' });assert.strictEqual(noValue.status, 422);assert.match(noValue.text, /needs a value/);++++ // A regex the registry refuses as unsafe (a nested quantifier) re-renders next to the selector.++ const unsafe = await create({ template: 'price', name: 'Unsafe', url: 'https://example.org/p', extraction: 'regex', selector: '(a+)+$', op: 'lt', value: '20', cadence: '900' });++ assert.strictEqual(unsafe.status, 422);++ assert.match(unsafe.text, /quantified group/);++ assert.match(unsafe.text, /value="\(a\+\)\+\$"/, 'the pattern is kept');});t('another person\'s watch is 404 on every page and form', async () => {
@@ -80,6 +80,10 @@ VALKEY_PREFIX=
@@ -80,6 +80,10 @@ VALKEY_PREFIX=
#WATCH_OBSERVATIONS_MAX_PER_CHECK=100
#WATCH_OBSERVATIONS_MAX_PER_CHECK=100
#WATCH_RETENTION_DEFAULT_DAYS=90
#WATCH_RETENTION_DEFAULT_DAYS=90
+# Registry: how many watches one owner (a person, an agent or a service) may hold. The cadence floor
+# (15 minutes) is fixed in the registry.
+#WATCH_MAX_PER_OWNER=50
+
# OpenVibe.Run — the expensive checks (browser/code), step 6. Never used until then: a run source
# OpenVibe.Run — the expensive checks (browser/code), step 6. Never used until then: a run source
# answers watch.carrier_unavailable and nothing is executed.
# answers watch.carrier_unavailable and nothing is executed.
#WATCH_RUN_URL=http://127.0.0.1:4920
#WATCH_RUN_URL=http://127.0.0.1:4920
@@ -99,8 +99,9 @@ server {
@@ -99,8 +99,9 @@ server {
location = /api/ready { proxy_pass http://127.0.0.1:4730; }
location = /api/ready { proxy_pass http://127.0.0.1:4730; }
location = /release.json { proxy_pass http://127.0.0.1:4730; }
location = /release.json { proxy_pass http://127.0.0.1:4730; }
- # Metrics are for host-local scrapers only (http://127.0.0.1:4730/metrics).
- location = /metrics { return 404; }
+ # Metrics are for host-local scrapers only (http://127.0.0.1:4730/metrics). Case-insensitive so
+ # /Metrics and friends are refused the same way.
+ location ~* ^/metrics(/|$) { return 404; }
# ── Sign-in: the network's own budget. /auth/me is the navbar's session probe and gets the
# ── Sign-in: the network's own budget. /auth/me is the navbar's session probe and gets the
# page budget instead, because it runs on every page view. ──
# page budget instead, because it runs on every page view. ──
@@ -126,16 +127,19 @@ server {
@@ -126,16 +127,19 @@ server {
location = /llms-full.txt { proxy_pass http://127.0.0.1:4730; }
location = /llms-full.txt { proxy_pass http://127.0.0.1:4730; }
# ── The watches pages: reads keep the page budget, writes (a POST) the strict form zone. The
# ── The watches pages: reads keep the page budget, writes (a POST) the strict form zone. The
- # map above gives GET/HEAD an empty key, which nginx does not count. ──
+ # map above gives GET/HEAD an empty key, which nginx does not count, so watch_forms only
+ # limits writes; watch_pages limits every read (POSTs count in both, which is fine). ──
location ^~ /watches {
location ^~ /watches {
limit_req zone=watch_forms burst=10 nodelay;
limit_req zone=watch_forms burst=10 nodelay;
+ limit_req zone=watch_pages burst=40 nodelay;
limit_req_status 429;
limit_req_status 429;
proxy_pass http://127.0.0.1:4730;
proxy_pass http://127.0.0.1:4730;
}
}
# ── OpenVibe.Events' delivery route: loopback only (POST /internal/events) ──
# ── OpenVibe.Events' delivery route: loopback only (POST /internal/events) ──
# Events posts to http://127.0.0.1:4730/internal/events directly; never proxied from outside (ADR-033).
# Events posts to http://127.0.0.1:4730/internal/events directly; never proxied from outside (ADR-033).
- location /internal/ { return 404; }
+ # Case-insensitive so /Internal and /internal/events alike are refused.
+ location ~* ^/internal(/|$) { return 404; }
# ── Pages ───────────────────────────────────────────────
# ── Pages ───────────────────────────────────────────────
location / {
location / {
@@ -253,6 +253,8 @@ function createCheck({ db, registry, config, carriers, extract, condition, obser
@@ -253,6 +253,8 @@ function createCheck({ db, registry, config, carriers, extract, condition, obser
const stated = out.state === 'ok';
const stated = out.state === 'ok';
let value = null;
let value = null;
let snapshot = null;
let snapshot = null;
+ // A pattern that hit its deadline stated no match; the reason is recorded on the run.
+ let patternError = null;
if (stated) {
if (stated) {
try {
try {
const extracted = extract(watch.extraction, {
const extracted = extract(watch.extraction, {
@@ -263,6 +265,7 @@ function createCheck({ db, registry, config, carriers, extract, condition, obser
@@ -263,6 +265,7 @@ function createCheck({ db, registry, config, carriers, extract, condition, obser
});
});
value = extracted.value;
value = extracted.value;
snapshot = extracted.snapshot;
snapshot = extracted.snapshot;
+ if (extracted.error) patternError = extracted.error;
} catch (err) {
} catch (err) {
const unsupported = err instanceof ExtractError && err.code === 'unsupported';
const unsupported = err instanceof ExtractError && err.code === 'unsupported';
const run = {
const run = {
@@ -310,6 +313,7 @@ function createCheck({ db, registry, config, carriers, extract, condition, obser
@@ -310,6 +313,7 @@ function createCheck({ db, registry, config, carriers, extract, condition, obser
const lastFired = await st.lastTrigger.get(watch.id);
const lastFired = await st.lastTrigger.get(watch.id);
const repeat = verdict.met && !verdict.changed && lastFired != null && verdict.since != null && lastFired.started_at >= verdict.since;
const repeat = verdict.met && !verdict.changed && lastFired != null && verdict.since != null && lastFired.started_at >= verdict.since;
const met = verdict.met && !repeat;
const met = verdict.met && !repeat;
+ if (verdict.error && !patternError) patternError = verdict.error;
const retentionDays = watch.retention && Number.isInteger(watch.retention.observations_days)
const retentionDays = watch.retention && Number.isInteger(watch.retention.observations_days)
? watch.retention.observations_days
? watch.retention.observations_days
@@ -321,6 +325,7 @@ function createCheck({ db, registry, config, carriers, extract, condition, obser
@@ -321,6 +325,7 @@ function createCheck({ db, registry, config, carriers, extract, condition, obser
state: met ? 'condition_met' : (stated ? (verdict.changed ? 'changed' : 'ok') : out.state),
state: met ? 'condition_met' : (stated ? (verdict.changed ? 'changed' : 'ok') : out.state),
http_status: out.http_status ?? null, bytes: out.bytes ?? null, raw_body_hash: out.raw_body_hash ?? null,
http_status: out.http_status ?? null, bytes: out.bytes ?? null, raw_body_hash: out.raw_body_hash ?? null,
observations: stated ? 1 : 0, triggers: met ? 1 : 0, finished_at: finishedAt,
observations: stated ? 1 : 0, triggers: met ? 1 : 0, finished_at: finishedAt,
+ detail: patternError ? String(patternError).slice(0, 500) : null,
};
};
// The observation row is built before the transaction (it needs the previous one's hash),
// The observation row is built before the transaction (it needs the previous one's hash),
@@ -14,11 +14,13 @@
@@ -14,11 +14,13 @@
*
*
* Numeric comparison refuses to guess: a value that is not a number (or a plain decimal string) is
* Numeric comparison refuses to guess: a value that is not a number (or a plain decimal string) is
* never "greater than" anything, and an unknown value is never low (the same rule Deals' watches
* never "greater than" anything, and an unknown value is never low (the same rule Deals' watches
- * use). `matches` compiles the watch's own pattern with a length cap and tests a bounded string.
+ * use). `matches` runs the watch's own pattern through safe-regex: it is bounded in length and in
+ * time, an invalid or unsafe pattern simply does not match, and a pattern that hits the deadline
+ * answers "no match" with `error` set (the check records "pattern took too long").
*/
*/
const { canonicalJson } = require('./util');
const { canonicalJson } = require('./util');
+const safe = require('./safe-regex');
-const MAX_PATTERN = 200;
// Bounded input: the watch's own pattern only ever runs over this much text (see extract.js: a
// Bounded input: the watch's own pattern only ever runs over this much text (see extract.js: a
// value is capped at the same size), so a pathological pattern cannot scan a whole body.
// value is capped at the same size), so a pathological pattern cannot scan a whole body.
const MAX_TESTED = 65536;
const MAX_TESTED = 65536;
@@ -58,8 +60,9 @@ function hasChanged({ comparison }, { value, previous }) {
@@ -58,8 +60,9 @@ function hasChanged({ comparison }, { value, previous }) {
return !same(value, before);
return !same(value, before);
}
}
-/** The predicate alone: does the condition hold for this value? */
-function holds({ comparison, condition }, { value, previous, changed }) {
+/** The predicate alone: does the condition hold for this value? `state`, when given, records a
+ * regex that hit its deadline so the check can record the error (see evaluate). */
+function holds({ comparison, condition }, { value, previous, changed }, state = null) {
const op = (condition && condition.op) || 'changed';
const op = (condition && condition.op) || 'changed';
const target = condition ? condition.value : null;
const target = condition ? condition.value : null;
switch (op) {
switch (op) {
@@ -87,7 +90,7 @@ function holds({ comparison, condition }, { value, previous, changed }) {
@@ -87,7 +90,7 @@ function holds({ comparison, condition }, { value, previous, changed }) {
const b = asNumber(target);
const b = asNumber(target);
return a != null && b != null ? a === b : same(value, target);
return a != null && b != null ? a === b : same(value, target);
}
}
- case 'ne': return !holds({ comparison, condition: { ...condition, op: 'eq' } }, { value, previous, changed });
+ case 'ne': return !holds({ comparison, condition: { ...condition, op: 'eq' } }, { value, previous, changed }, state);
case 'contains': {
case 'contains': {
if (typeof value === 'string') return typeof target === 'string' && value.includes(target);
if (typeof value === 'string') return typeof target === 'string' && value.includes(target);
if (Array.isArray(value)) return value.some((v) => same(v, target));
if (Array.isArray(value)) return value.some((v) => same(v, target));
@@ -96,10 +99,11 @@ function holds({ comparison, condition }, { value, previous, changed }) {
@@ -96,10 +99,11 @@ function holds({ comparison, condition }, { value, previous, changed }) {
}
}
case 'matches': {
case 'matches': {
if (typeof value !== 'string' || typeof target !== 'string') return false;
if (typeof value !== 'string' || typeof target !== 'string') return false;
- if (!target || target.length > MAX_PATTERN) return false;
let re;
let re;
- try { re = new RegExp(target); } catch { return false; }
- return re.test(value.slice(0, MAX_TESTED));
+ try { re = safe.compile(target); } catch { return false; } // invalid, oversize or unsafe: never matches
+ const { matched, timedOut } = safe.test(re, value.slice(0, MAX_TESTED));
+ if (timedOut && state) state.error = safe.TIMEOUT_MESSAGE;
+ return matched;
}
}
case 'exists': return !isAbsent(value);
case 'exists': return !isAbsent(value);
case 'absent': return isAbsent(value);
case 'absent': return isAbsent(value);
@@ -109,22 +113,24 @@ function holds({ comparison, condition }, { value, previous, changed }) {
@@ -109,22 +113,24 @@ function holds({ comparison, condition }, { value, previous, changed }) {
/**
/**
* evaluate({ comparison, condition }, { value, previous, heldSince, now })
* evaluate({ comparison, condition }, { value, previous, heldSince, now })
- * → { changed, holds, met, since }
+ * → { changed, holds, met, since, error }
*
*
* `previous` is the previous observation ({ value, condition_met, observed_at }) or null;
* `previous` is the previous observation ({ value, condition_met, observed_at }) or null;
* `heldSince` (ms) is when the predicate began holding continuously, from the observation trail —
* `heldSince` (ms) is when the predicate began holding continuously, from the observation trail —
* pass it from observations.heldSince(). When the predicate holds and no start is known, it starts
* pass it from observations.heldSince(). When the predicate holds and no start is known, it starts
- * now, so a for_sec condition never fires on the first sighting.
+ * now, so a for_sec condition never fires on the first sighting. `error` is the recorded reason a
+ * `matches` pattern hit its deadline (no match was taken from it), else null.
*/
*/
function evaluate({ comparison = null, condition = {} } = {}, { value, previous = null, heldSince = null, now = Date.now(), changed: given } = {}) {
function evaluate({ comparison = null, condition = {} } = {}, { value, previous = null, heldSince = null, now = Date.now(), changed: given } = {}) {
// The caller may know better: a check that got a 304 or an unchanged body knows nothing changed,
// The caller may know better: a check that got a 304 or an unchanged body knows nothing changed,
// whatever comparing the values would say.
// whatever comparing the values would say.
const changed = given === undefined ? hasChanged({ comparison }, { value, previous }) : Boolean(given);
const changed = given === undefined ? hasChanged({ comparison }, { value, previous }) : Boolean(given);
- const hold = holds({ comparison, condition }, { value, previous, changed });
+ const state = { error: null };
+ const hold = holds({ comparison, condition }, { value, previous, changed }, state);
const forSec = condition && Number.isInteger(condition.for_sec) ? Math.max(0, condition.for_sec) : 0;
const forSec = condition && Number.isInteger(condition.for_sec) ? Math.max(0, condition.for_sec) : 0;
const since = hold ? (heldSince != null ? heldSince : now) : null;
const since = hold ? (heldSince != null ? heldSince : now) : null;
const met = hold && (forSec === 0 || (since != null && now - since >= forSec * 1000));
const met = hold && (forSec === 0 || (since != null && now - since >= forSec * 1000));
- return { changed, holds: hold, met, since };
+ return { changed, holds: hold, met, since, error: state.error };
}
}
module.exports = { evaluate, hasChanged, holds, asNumber, isAbsent };
module.exports = { evaluate, hasChanged, holds, asNumber, isAbsent };
@@ -110,6 +110,11 @@ function load(env = process.env) {
@@ -110,6 +110,11 @@ function load(env = process.env) {
// observations_days when the watch states none; a watch may override it (watch.watch@1).
// observations_days when the watch states none; a watch may override it (watch.watch@1).
defaultDays: int(env.WATCH_RETENTION_DEFAULT_DAYS, 90),
defaultDays: int(env.WATCH_RETENTION_DEFAULT_DAYS, 90),
},
},
+ // How many watches one owner (a person, an agent or a service) may hold; the registry enforces
+ // it on create, for the API and the form alike.
+ watches: {
+ maxPerOwner: Math.max(1, int(env.WATCH_MAX_PER_OWNER, 50)),
+ },
// OpenVibe.Run (plan T14): the expensive last rung — browser polling — and never used before
// OpenVibe.Run (plan T14): the expensive last rung — browser polling — and never used before
// steps 5-7 exist. WATCH_RUN_URL is Run's loopback port.
// steps 5-7 exist. WATCH_RUN_URL is Run's loopback port.
@@ -17,6 +17,7 @@
@@ -17,6 +17,7 @@
* `fields` (name → path) extracts an object of several values at once, each null when it is missing.
* `fields` (name → path) extracts an object of several values at once, each null when it is missing.
*/
*/
const { toText, decodeBody } = require('./util');
const { toText, decodeBody } = require('./util');
+const safe = require('./safe-regex');
/** Longest text value one extraction stores (a 5 MB page as `text` is a mistake, not a value). */
/** Longest text value one extraction stores (a 5 MB page as `text` is a mistake, not a value). */
const VALUE_MAX = 64 * 1024;
const VALUE_MAX = 64 * 1024;
@@ -205,15 +206,15 @@ function parseJson(text) {
@@ -205,15 +206,15 @@ function parseJson(text) {
}
}
}
}
-/** '/<re>/flags' or a bare pattern → a RegExp; oversized or invalid patterns throw. */
+/** '/<re>/flags' or a bare pattern → a RegExp; an unsafe, oversized or invalid pattern throws. */
function compileRegex(pattern) {
function compileRegex(pattern) {
- const s = String(pattern || '');
- if (!s || s.length > 200) throw new ExtractError('unsupported', 'regex extraction needs a pattern of at most 200 characters');
- const m = /^\/(.*)\/([gimsuy]*)$/.exec(s);
try {
try {
- return m ? new RegExp(m[1], m[2].replace('g', '')) : new RegExp(s);
- } catch {
- throw new ExtractError('parse_error', 'selector is not a valid regular expression');
+ return safe.compile(pattern);
+ } catch (err) {
+ // An invalid pattern is a parse error; an unsafe or oversized one is one this release
+ // refuses to run (the registry already refused it when the watch was saved).
+ if (err instanceof safe.UnsafeRegexError && err.reason === 'invalid') throw new ExtractError('parse_error', err.message);
+ throw new ExtractError('unsupported', err.message);
}
}
}
}
@@ -268,10 +269,13 @@ function extract(extraction = {}, { body = null, contentType = null, document =
@@ -268,10 +269,13 @@ function extract(extraction = {}, { body = null, contentType = null, document =
if (text == null) throw new ExtractError('parse_error', 'there is no body to extract from');
if (text == null) throw new ExtractError('parse_error', 'there is no body to extract from');
const re = compileRegex(extraction.selector);
const re = compileRegex(extraction.selector);
// Bounded input: a watch's own pattern is compiled as-is, so it only ever runs over a
// Bounded input: a watch's own pattern is compiled as-is, so it only ever runs over a
- // window of VALUE_MAX characters (a value beyond that is not a value this service keeps).
- const m = re.exec(text.slice(0, VALUE_MAX));
- if (!m) return { value: null, snapshot };
- return { value: m.length > 1 ? (m[1] === undefined ? null : m[1]) : m[0], snapshot };
+ // window of VALUE_MAX characters (a value beyond that is not a value this service keeps),
+ // and under a deadline (safe-regex): a pattern that still backtracks states no match and
+ // records why, rather than hanging the process.
+ const { match, timedOut } = safe.exec(re, text.slice(0, VALUE_MAX));
+ if (timedOut) return { value: null, snapshot, error: safe.TIMEOUT_MESSAGE };
+ if (!match) return { value: null, snapshot };
+ return { value: match.length > 1 ? (match[1] === undefined ? null : match[1]) : match[0], snapshot };
}
}
default:
default:
throw new ExtractError('unsupported', `extraction kind "${kind}" is not supported`);
throw new ExtractError('unsupported', `extraction kind "${kind}" is not supported`);
@@ -42,7 +42,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
@@ -42,7 +42,7 @@ async function start({ config, db: givenDb = null, now = () => Date.now(), fetch
tokenClient,
tokenClient,
tokenOpts: config.oauth.clientSecret ? { tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret } : null,
tokenOpts: config.oauth.clientSecret ? { tokenUrl: `${config.networkInternalUrl}/oauth/token`, clientId: config.oauth.clientId, clientSecret: config.oauth.clientSecret } : null,
});
});
- const registry = createRegistry({ db, now, outbox });
+ const registry = createRegistry({ db, now, outbox, maxWatchesPerOwner: config.watches.maxPerOwner });
const observations = createObservations({ db, now });
const observations = createObservations({ db, now });
// The only way out to the internet: SSRF guard (ports, private hosts, every redirect hop), byte
// The only way out to the internet: SSRF guard (ports, private hosts, every redirect hop), byte
// cap, one deadline, conditional GET; per-host spacing in front of every request.
// cap, one deadline, conditional GET; per-host spacing in front of every request.
@@ -18,12 +18,15 @@
@@ -18,12 +18,15 @@
* found — 404, never 403 (an id must not confirm that someone else's watch exists).
* found — 404, never 403 (an id must not confirm that someone else's watch exists).
*/
*/
const { validate, ids: contractIds } = require('openvibe-contracts');
const { validate, ids: contractIds } = require('openvibe-contracts');
+const safe = require('./safe-regex');
/** Source kinds that reach out on a cadence; event and webhook sources are pushed to instead. */
/** Source kinds that reach out on a cadence; event and webhook sources are pushed to instead. */
const POLLING = new Set(['http', 'feed', 'api', 'node', 'run']);
const POLLING = new Set(['http', 'feed', 'api', 'node', 'run']);
const PUSHED = new Set(['event', 'webhook']);
const PUSHED = new Set(['event', 'webhook']);
const RUN_STATES = ['ok', 'not_modified', 'no_change', 'changed', 'condition_met', 'http_error', 'timeout', 'parse_error', 'rate_limited', 'budget_exceeded', 'skipped', 'disabled'];
const RUN_STATES = ['ok', 'not_modified', 'no_change', 'changed', 'condition_met', 'http_error', 'timeout', 'parse_error', 'rate_limited', 'budget_exceeded', 'skipped', 'disabled'];
const STATUSES = ['active', 'paused', 'disabled', 'failed'];
const STATUSES = ['active', 'paused', 'disabled', 'failed'];
+/** The shortest cadence a polling watch may state (15 minutes) — the web form's floor, enforced here for the API too. */
+const MIN_EVERY_SEC = 900;
/** Headers the fetcher decides; a watch may never override them (SSRF and conditional-GET safety). */
/** Headers the fetcher decides; a watch may never override them (SSRF and conditional-GET safety). */
const FORBIDDEN_HEADERS = new Set(['host', 'cookie', 'connection', 'content-length', 'transfer-encoding', 'user-agent', 'accept-encoding', 'if-none-match', 'if-modified-since']);
const FORBIDDEN_HEADERS = new Set(['host', 'cookie', 'connection', 'content-length', 'transfer-encoding', 'user-agent', 'accept-encoding', 'if-none-match', 'if-modified-since']);
@@ -75,6 +78,7 @@ function validateRequest(body, { create = false } = {}) {
@@ -75,6 +78,7 @@ function validateRequest(body, { create = false } = {}) {
if (POLLING.has(kind)) {
if (POLLING.has(kind)) {
if (cadence === null || cadence === undefined) throw new RegistryError(`a ${kind} source needs a cadence (every_sec)`);
if (cadence === null || cadence === undefined) throw new RegistryError(`a ${kind} source needs a cadence (every_sec)`);
if (!Number.isInteger(cadence.every_sec)) throw new RegistryError('cadence.every_sec is required for a source that polls');
if (!Number.isInteger(cadence.every_sec)) throw new RegistryError('cadence.every_sec is required for a source that polls');
+ if (cadence.every_sec < MIN_EVERY_SEC) throw new RegistryError(`cadence.every_sec must be at least ${MIN_EVERY_SEC} seconds (15 minutes)`);
}
}
if (PUSHED.has(kind) && cadence) throw new RegistryError(`a ${kind} source is pushed to and never polls: cadence must be null`);
if (PUSHED.has(kind) && cadence) throw new RegistryError(`a ${kind} source is pushed to and never polls: cadence must be null`);
// A HEAD source states no body, so only a document/record extraction can read one: refuse
// A HEAD source states no body, so only a document/record extraction can read one: refuse
@@ -83,6 +87,17 @@ function validateRequest(body, { create = false } = {}) {
@@ -83,6 +87,17 @@ function validateRequest(body, { create = false } = {}) {
throw new RegistryError('a HEAD source states no body: its extraction must be json or jsonpath (status, etag, last_modified, content_type, content_length)');
throw new RegistryError('a HEAD source states no body: its extraction must be json or jsonpath (status, etag, last_modified, content_type, content_length)');
}
}
}
}
+ // A watch's own pattern is refused when the watch is saved, not when it runs: one that would
+ // backtrack for minutes (safe-regex) must never reach the check worker through the API or the
+ // form. The names match the form's fields, so the message lands next to the input.
+ if (body.extraction && body.extraction.kind === 'regex') {
+ const bad = safe.validate(body.extraction.selector);
+ if (bad) throw new RegistryError(`/extraction/selector ${bad}`);
+ }
+ if (body.condition && body.condition.op === 'matches' && typeof body.condition.value === 'string') {
+ const bad = safe.validate(body.condition.value);
+ if (bad) throw new RegistryError(`/condition/value ${bad}`);
+ }
return body;
return body;
}
}
@@ -106,9 +121,10 @@ function validateView(view) {
@@ -106,9 +121,10 @@ function validateView(view) {
return view;
return view;
}
}
-function createRegistry({ db, now = () => Date.now(), ids = contractIds, outbox = null }) {
+function createRegistry({ db, now = () => Date.now(), ids = contractIds, outbox = null, maxWatchesPerOwner = 50 }) {
const st = {
const st = {
get: db.prepare('SELECT * FROM watches WHERE id = ? AND deleted_at IS NULL'),
get: db.prepare('SELECT * FROM watches WHERE id = ? AND deleted_at IS NULL'),
+ countOwned: db.prepare('SELECT COUNT(*) AS n FROM watches WHERE owner_sub = ? AND deleted_at IS NULL'),
insert: db.prepare(`INSERT INTO watches (id, project_id, owner_sub, name, status, source, cadence, extraction, comparison, condition,
insert: db.prepare(`INSERT INTO watches (id, project_id, owner_sub, name, status, source, cadence, extraction, comparison, condition,
action, budget, retention, labels, next_due_at, not_before, created_at, updated_at, updated_by)
action, budget, retention, labels, next_due_at, not_before, created_at, updated_at, updated_by)
VALUES (@id, @project_id, @owner_sub, @name, @status, @source, @cadence, @extraction, @comparison, @condition,
VALUES (@id, @project_id, @owner_sub, @name, @status, @source, @cadence, @extraction, @comparison, @condition,
@@ -219,6 +235,10 @@ function createRegistry({ db, now = () => Date.now(), ids = contractIds, outbox
@@ -219,6 +235,10 @@ function createRegistry({ db, now = () => Date.now(), ids = contractIds, outbox
const req = validateRequest(body, { create: true });
const req = validateRequest(body, { create: true });
const owner = ownerOf(principal);
const owner = ownerOf(principal);
if (!owner) throw new RegistryError('no acting subject', 'watch.invalid');
if (!owner) throw new RegistryError('no acting subject', 'watch.invalid');
+ // A per-owner cap, so one caller cannot fill the registry (the API and the form both come
+ // through here). The number is a config value (WATCH_MAX_PER_OWNER).
+ const owned = await st.countOwned.get(owner);
+ if (Number(owned.n) >= maxWatchesPerOwner) throw new RegistryError(`an owner may hold at most ${maxWatchesPerOwner} watches`);
const t = now();
const t = now();
const rec = {
const rec = {
name: req.name || defaultName(req.source),
name: req.name || defaultName(req.source),
new file mode 100644
new file mode 100644
@@ -0,0 +1,179 @@
@@ -0,0 +1,179 @@
+'use strict';
+/**
+ * Safe regular expressions for the patterns a watch states.
+ *
+ * A regex extraction's `selector` and a `matches` condition's `value` are the watch's own, and they
+ * run in the one Node process that serves every watch. A pattern like `(a+)+$` tested against a
+ * body would otherwise backtrack for minutes and hang the process (ReDoS).
+ *
+ * Two guards, so a bad pattern is refused when the watch is saved rather than when it runs:
+ *
+ * compile() refuses what is plainly catastrophic — a pattern longer than MAX_PATTERN, or a
+ * quantified group that itself contains a quantifier (`(a+)+`, `(a*)*`, `(\w+\s?)*`).
+ * The registry calls validate() before it stores a watch, so the API and the form both
+ * answer a validation error at create time.
+ * test()/exec() run the match under a deadline in a vm context: a pattern the static check let
+ * through but that still backtracks is cut off at TIMEOUT_MS and reported as a timeout
+ * (no match), never a hang. The caller records that as a check error.
+ *
+ * The static check is a scan of the pattern's shape, not a full parse: it refuses nested quantifiers
+ * without judging anything else, so an unusual-but-legitimate pattern still runs under the deadline.
+ */
+const vm = require('vm');
+
+/** Longest pattern a watch may state (the 500-character rule). */
+const MAX_PATTERN = 500;
+/** How long one match may run before it is cut off. */
+const TIMEOUT_MS = 50;
+/** The message a timed-out match records (the caller stores this as the check error). */
+const TIMEOUT_MESSAGE = 'pattern took too long';
+/** V8's code for a vm script that hit its timeout. */
+const TIMEOUT_CODE = 'ERR_SCRIPT_EXECUTION_TIMEOUT';
+
+// One compiled script per operation, run with a fresh sandbox per call: recompiling the one-line
+// expression on every check would be waste, and the sandbox is what carries the deadline.
+const TEST_SCRIPT = new vm.Script('re.test(s)');
+const EXEC_SCRIPT = new vm.Script('re.exec(s)');
+
+class UnsafeRegexError extends Error {
+ // reason: 'too_long' | 'nested' | 'invalid' (a caller maps it to its own error code).
+ constructor(detail, reason) { super(detail); this.code = 'unsafe_pattern'; this.reason = reason; }
+}
+
+/** The source and flags of '/…/flags', or of a bare pattern. */
+function split(input) {
+ const s = String(input == null ? '' : input);
+ const m = /^\/(.*)\/([gimsuy]*)$/.exec(s);
+ return m ? { source: m[1], flags: m[2] } : { source: s, flags: '' };
+}
+
+/** The length of a quantifier at `i` (`*`, `+`, `?` or `{n[,m]}`), or 0 when there is none. */
+function quantifierAt(source, i) {
+ const c = source[i];
+ if (c === '*' || c === '+' || c === '?') return 1;
+ if (c === '{') {
+ const m = /^\{\d+(?:,\d*)?\}/.exec(source.slice(i));
+ return m ? m[0].length : 0;
+ }
+ return 0;
+}
+
+/**
+ * Does the pattern contain a quantified group whose own body contains a quantifier? Groups are read
+ * as `(...)`, including non-capturing and lookaround forms; escapes and character classes are read
+ * literally, so `\+` and `[a+]` are not quantifiers. A group that is quantified and already holds a
+ * quantifier (itself or a group it contains) is the exponential shape this refuses.
+ */
+function nestedQuantifier(source) {
+ const stack = [];
+ const markQuantifier = () => { const top = stack[stack.length - 1]; if (top) top.hasQuant = true; };
+ let i = 0;
+ while (i < source.length) {
+ const c = source[i];
+ const top = stack[stack.length - 1];
+ // The '?' that opens a group prefix ((?:, (?=, (?<name>…) is not a quantifier.
+ if (top && top.prefix && c !== '?') top.prefix = false;
+ if (c === '\\') { i += 2; continue; }
+ if (c === '[') {
+ i++;
+ if (source[i] === '^') i++;
+ if (source[i] === ']') i++; // a leading ']' is a literal, not the class end
+ while (i < source.length && source[i] !== ']') { if (source[i] === '\\') i++; i++; }
+ i++;
+ continue;
+ }
+ if (c === '(') { stack.push({ hasQuant: false, prefix: true }); i++; continue; }
+ if (c === ')') {
+ const frame = stack.pop();
+ i++;
+ const q = quantifierAt(source, i);
+ if (q) {
+ i += q;
+ if (source[i] === '?') i++; // a lazy marker belongs to the quantifier
+ if (frame.hasQuant) return true;
+ }
+ if (stack.length && frame.hasQuant) stack[stack.length - 1].hasQuant = true;
+ continue;
+ }
+ if (c === '?') {
+ if (top && top.prefix) {
+ // Consume the prefix so its '?' is never read as a quantifier.
+ i++;
+ if (source[i] === ':' || source[i] === '=' || source[i] === '!') i++;
+ else if (source[i] === '<') {
+ i++;
+ if (source[i] === '=' || source[i] === '!') i++;
+ else { while (i < source.length && source[i] !== '>') i++; if (source[i] === '>') i++; }
+ }
+ continue;
+ }
+ markQuantifier();
+ i++;
+ if (source[i] === '?') i++;
+ continue;
+ }
+ const q = quantifierAt(source, i);
+ if (q) {
+ markQuantifier();
+ i += q;
+ if (source[i] === '?') i++;
+ continue;
+ }
+ i++;
+ }
+ return false;
+}
+
+/** The reason a pattern cannot be used, or null when it is accepted (for validation at save time). */
+function validate(pattern, opts) {
+ try {
+ compile(pattern, opts);
+ return null;
+ } catch (err) {
+ return err instanceof UnsafeRegexError ? err.message : 'selector is not a valid regular expression';
+ }
+}
+
+/** A pattern as a RegExp, or an UnsafeRegexError (reason 'too_long' | 'nested' | 'invalid'). */
+function compile(pattern, { maxLength = MAX_PATTERN } = {}) {
+ const { source, flags } = split(pattern);
+ if (!source || source.length > maxLength) {
+ throw new UnsafeRegexError(`a pattern must be 1–${maxLength} characters`, 'too_long');
+ }
+ if (nestedQuantifier(source)) {
+ throw new UnsafeRegexError('a quantified group must not contain another quantifier (a pattern like (a+)+ can hang the check)', 'nested');
+ }
+ try {
+ // 'g' is dropped: a lastIndex must never carry between checks.
+ return new RegExp(source, flags.replace('g', ''));
+ } catch {
+ throw new UnsafeRegexError('selector is not a valid regular expression', 'invalid');
+ }
+}
+
+/** Run one script under the deadline → { value, timedOut }; only a timeout is caught. */
+function underDeadline(script, re, s, timeout) {
+ try {
+ return { value: script.runInNewContext({ re, s }, { timeout }), timedOut: false };
+ } catch (err) {
+ if (err && err.code === TIMEOUT_CODE) return { value: null, timedOut: true };
+ throw err;
+ }
+}
+
+/** re.test(s) under the deadline → { matched, timedOut }; a timeout is not a match. */
+function test(re, s, { timeout = TIMEOUT_MS } = {}) {
+ const out = underDeadline(TEST_SCRIPT, re, s, timeout);
+ return { matched: out.timedOut ? false : Boolean(out.value), timedOut: out.timedOut };
+}
+
+/** re.exec(s) under the deadline → { match, timedOut }; a timeout is no match. */
+function exec(re, s, { timeout = TIMEOUT_MS } = {}) {
+ const out = underDeadline(EXEC_SCRIPT, re, s, timeout);
+ return { match: out.timedOut ? null : out.value, timedOut: out.timedOut };
+}
+
+module.exports = {
+ MAX_PATTERN, TIMEOUT_MS, TIMEOUT_MESSAGE, UnsafeRegexError,
+ validate, compile, test, exec, nestedQuantifier,
+};
@@ -83,6 +83,10 @@ t('an invalid body is 422 with the validator\'s detail, and nothing is written',
@@ -83,6 +83,10 @@ t('an invalid body is 422 with the validator\'s detail, and nothing is written',
[mine({ action: [] }), /watch.watch-request@1/],
[mine({ action: [] }), /watch.watch-request@1/],
[mine({ source: { kind: 'http', url: 'http://example.org/x', headers: { cookie: 'a=b' } } }), /cookie/],
[mine({ source: { kind: 'http', url: 'http://example.org/x', headers: { cookie: 'a=b' } } }), /cookie/],
[mine({ nope: true }), /watch.watch-request@1/],
[mine({ nope: true }), /watch.watch-request@1/],
+ // the registry's own rules the API must obey too: the cadence floor and a safe pattern
+ [mine({ cadence: { every_sec: 60 } }), /at least 900/],
+ [mine({ extraction: { kind: 'regex', selector: '(a+)+$' } }), /extraction\/selector/],
+ [mine({ condition: { op: 'matches', value: '(a+)+$' } }), /condition\/value/],
];
];
for (const [body, re] of bad) {
for (const [body, re] of bad) {
const res = await api('POST', '/api/v1/watches', { token: ME, body });
const res = await api('POST', '/api/v1/watches', { token: ME, body });
@@ -99,10 +103,10 @@ t('an invalid body is 422 with the validator\'s detail, and nothing is written',
@@ -99,10 +103,10 @@ t('an invalid body is 422 with the validator\'s detail, and nothing is written',
t('patch, pause, resume, delete — and another owner gets 404 for every one of them', async () => {
t('patch, pause, resume, delete — and another owner gets 404 for every one of them', async () => {
const { body: { watch } } = await api('POST', '/api/v1/watches', { token: ME, body: mine() });
const { body: { watch } } = await api('POST', '/api/v1/watches', { token: ME, body: mine() });
- const patched = await api('PATCH', `/api/v1/watches/${watch.id}`, { token: ME, body: { name: 'renamed', cadence: { every_sec: 120 } } });
+ const patched = await api('PATCH', `/api/v1/watches/${watch.id}`, { token: ME, body: { name: 'renamed', cadence: { every_sec: 1800 } } });
assert.strictEqual(patched.status, 200);
assert.strictEqual(patched.status, 200);
assert.strictEqual(patched.body.watch.name, 'renamed');
assert.strictEqual(patched.body.watch.name, 'renamed');
- assert.strictEqual(patched.body.watch.cadence.every_sec, 120);
+ assert.strictEqual(patched.body.watch.cadence.every_sec, 1800);
assert.ok(validate('watch.watch-result@1', patched.body).valid);
assert.ok(validate('watch.watch-result@1', patched.body).valid);
const paused = await api('POST', `/api/v1/watches/${watch.id}/pause`, { token: ME });
const paused = await api('POST', `/api/v1/watches/${watch.id}/pause`, { token: ME });
@@ -16,7 +16,7 @@ let web;
@@ -16,7 +16,7 @@ let web;
const watch = async (overrides = {}) => (await svc.registry.create({
const watch = async (overrides = {}) => (await svc.registry.create({
name: 'carrier watch',
name: 'carrier watch',
source: { kind: 'http', url: `${web.origin}/page`, format: null },
source: { kind: 'http', url: `${web.origin}/page`, format: null },
- cadence: { every_sec: 60 },
+ cadence: { every_sec: 900 },
extraction: { kind: 'text' },
extraction: { kind: 'text' },
condition: { op: 'eq', value: 'never-matches' },
condition: { op: 'eq', value: 'never-matches' },
action: [{ kind: 'notification' }],
action: [{ kind: 'notification' }],
@@ -122,7 +122,7 @@ t('a private host is refused before any request (nothing is even connected to)',
@@ -122,7 +122,7 @@ t('a private host is refused before any request (nothing is even connected to)',
const w = await alone.registry.create({
const w = await alone.registry.create({
name: 'loopback watch',
name: 'loopback watch',
source: { kind: 'http', url: `${aSite.origin}/page`, format: null },
source: { kind: 'http', url: `${aSite.origin}/page`, format: null },
- cadence: { every_sec: 60 }, extraction: { kind: 'text' }, condition: { op: 'eq', value: 'x' },
+ cadence: { every_sec: 900 }, extraction: { kind: 'text' }, condition: { op: 'eq', value: 'x' },
action: [{ kind: 'notification' }],
action: [{ kind: 'notification' }],
}, OWNER);
}, OWNER);
const out = await alone.check.run(w.id, { trigger: 'manual' });
const out = await alone.check.run(w.id, { trigger: 'manual' });
@@ -21,7 +21,7 @@ const service = { sub: 'svc:check-test' };
@@ -21,7 +21,7 @@ const service = { sub: 'svc:check-test' };
const def = (overrides = {}) => ({
const def = (overrides = {}) => ({
name: 'price watch',
name: 'price watch',
source: { kind: 'http', url: `${web.origin}/price`, format: null },
source: { kind: 'http', url: `${web.origin}/price`, format: null },
- cadence: { every_sec: 60 },
+ cadence: { every_sec: 900 },
extraction: { kind: 'text' },
extraction: { kind: 'text' },
condition: { op: 'lt', value: 10 },
condition: { op: 'lt', value: 10 },
action: [{ kind: 'notification' }],
action: [{ kind: 'notification' }],
@@ -129,13 +129,13 @@ t('a failing check is a recorded state and a watch.check.failed event, and it ba
@@ -129,13 +129,13 @@ t('a failing check is a recorded state and a watch.check.failed event, and it ba
let r = await row(w.id);
let r = await row(w.id);
assert.strictEqual(r.last_state, 'parse_error');
assert.strictEqual(r.last_state, 'parse_error');
assert.strictEqual(Number(r.consecutive_failures), 1);
assert.strictEqual(Number(r.consecutive_failures), 1);
- assert.strictEqual(r.next_due_at - r.last_check_at, 60000, 'first failure: every_sec · 2^0');
+ assert.strictEqual(r.next_due_at - r.last_check_at, 900000, 'first failure: every_sec · 2^0');
const second = await check(w.id);
const second = await check(w.id);
assert.strictEqual(second.failures, 2);
assert.strictEqual(second.failures, 2);
r = await row(w.id);
r = await row(w.id);
assert.strictEqual(Number(r.consecutive_failures), 2);
assert.strictEqual(Number(r.consecutive_failures), 2);
- assert.strictEqual(r.next_due_at - r.last_check_at, 120000, 'second failure: every_sec · 2^1');
+ assert.strictEqual(r.next_due_at - r.last_check_at, 1800000, 'second failure: every_sec · 2^1');
assert.strictEqual((await payloads('watch.check.failed')).length, 2);
assert.strictEqual((await payloads('watch.check.failed')).length, 2);
// a success clears the backoff and the counter (19.99 is above the watch's lt 10, so it is a
// a success clears the backoff and the counter (19.99 is above the watch's lt 10, so it is a
@@ -145,7 +145,7 @@ t('a failing check is a recorded state and a watch.check.failed event, and it ba
@@ -145,7 +145,7 @@ t('a failing check is a recorded state and a watch.check.failed event, and it ba
await check(w.id);
await check(w.id);
r = await row(w.id);
r = await row(w.id);
assert.deepStrictEqual([Number(r.consecutive_failures), r.last_state], [0, 'changed']);
assert.deepStrictEqual([Number(r.consecutive_failures), r.last_state], [0, 'changed']);
- assert.ok(r.next_due_at - r.last_check_at >= 60000 && r.next_due_at - r.last_check_at <= 60100, 'back to the cadence');
+ assert.ok(r.next_due_at - r.last_check_at >= 900000 && r.next_due_at - r.last_check_at <= 900100, 'back to the cadence');
});
});
t('a timeout is its own state, and a 429 waits as long as the site asked', async () => {
t('a timeout is its own state, and a 429 waits as long as the site asked', async () => {
@@ -162,6 +162,24 @@ t('a timeout is its own state, and a 429 waits as long as the site asked', async
@@ -162,6 +162,24 @@ t('a timeout is its own state, and a 429 waits as long as the site asked', async
assert.ok(r.not_before - r.last_check_at >= 119000, 'Retry-After: 120 s became not_before');
assert.ok(r.not_before - r.last_check_at >= 119000, 'Retry-After: 120 s became not_before');
});
});
+t('a pattern that hits its deadline is a recorded check error, never a hang', async () => {
+ // A regex extraction whose pattern slips past the static check still runs, and the deadline
+ // makes it a no-match with the reason recorded on the run (the process stays up).
+ price = 'a'.repeat(40) + '!';
+ const slowExtract = await create(person, { extraction: { kind: 'regex', selector: '^(\\w|\\w\\w)*$' } });
+ const withExtract = await check(slowExtract.id, 'manual');
+ assert.strictEqual(withExtract.run.detail, 'pattern took too long');
+ assert.strictEqual(withExtract.run.state, 'changed', 'the run completed with no match');
+ assert.strictEqual(withExtract.value, null);
+
+ // The same, for a `matches` condition: no match, and the run records why.
+ const slowMatch = await create(person, { condition: { op: 'matches', value: '^(\\w|\\w\\w)*$' } });
+ const withMatch = await check(slowMatch.id, 'manual');
+ assert.strictEqual(withMatch.run.detail, 'pattern took too long');
+ assert.strictEqual(withMatch.verdict.error, 'pattern took too long');
+ price = '19.99';
+});
+
t('an event, webhook, node or run source is accepted but never fetched (steps 5-7)', async () => {
t('an event, webhook, node or run source is accepted but never fetched (steps 5-7)', async () => {
const cases = [
const cases = [
['event', { kind: 'event', pattern: 'release.published' }, { op: 'exists' }],
['event', { kind: 'event', pattern: 'release.published' }, { op: 'exists' }],
@@ -173,7 +191,7 @@ t('an event, webhook, node or run source is accepted but never fetched (steps 5-
@@ -173,7 +191,7 @@ t('an event, webhook, node or run source is accepted but never fetched (steps 5-
for (const [kind, source, condition] of cases) {
for (const [kind, source, condition] of cases) {
// an event or a webhook is pushed to and never polls; node and run sources poll through
// an event or a webhook is pushed to and never polls; node and run sources poll through
// OpenVibe.Node and OpenVibe.Run, so they carry a cadence — and still nothing is fetched
// OpenVibe.Node and OpenVibe.Run, so they carry a cadence — and still nothing is fetched
- const cadence = (kind === 'event' || kind === 'webhook') ? null : { every_sec: 60 };
+ const cadence = (kind === 'event' || kind === 'webhook') ? null : { every_sec: 900 };
const w = await create(person, { source, cadence, extraction: { kind: 'text' }, condition });
const w = await create(person, { source, cadence, extraction: { kind: 'text' }, condition });
assert.strictEqual((await row(w.id)).status, 'active');
assert.strictEqual((await row(w.id)).status, 'active');
const out = await check(w.id, 'manual');
const out = await check(w.id, 'manual');
@@ -64,6 +64,18 @@ t('contains / matches', () => {
@@ -64,6 +64,18 @@ t('contains / matches', () => {
assert.strictEqual(holds({ op: 'matches', value: null }, 'anything'), false);
assert.strictEqual(holds({ op: 'matches', value: null }, 'anything'), false);
});
});
+t('matches: an unsafe pattern never matches, and a slow one records the timeout', () => {
+ // a nested quantifier is refused by the static check and simply does not match
+ assert.strictEqual(evaluate({ condition: { op: 'matches', value: '(a+)+$' } }, { value: 'aaaa' }).holds, false);
+ // an alternation slips past the static check; the deadline answers no match and records why
+ const slow = evaluate({ condition: { op: 'matches', value: '^(\\w|\\w\\w)*$' } }, { value: 'a'.repeat(40) + '!' });
+ assert.strictEqual(slow.holds, false);
+ assert.strictEqual(slow.error, 'pattern took too long');
+ // a normal pattern holds and records no error
+ const ok = evaluate({ condition: { op: 'matches', value: '^v\\d+\\.\\d+' } }, { value: 'v2.3 released' });
+ assert.deepStrictEqual([ok.holds, ok.error], [true, null]);
+});
+
t('exists / absent', () => {
t('exists / absent', () => {
const holds = (op, value) => evaluate({ condition: { op } }, { value }).holds;
const holds = (op, value) => evaluate({ condition: { op } }, { value }).holds;
assert.strictEqual(holds('exists', 'x'), true);
assert.strictEqual(holds('exists', 'x'), true);
@@ -87,7 +87,17 @@ t('regex: the first match, a group when the pattern has one', () => {
@@ -87,7 +87,17 @@ t('regex: the first match, a group when the pattern has one', () => {
assert.strictEqual(extract({ kind: 'regex', selector: '/(\\d+) items/i' }, { body: body('12 ITEMS left') }).value, '12');
assert.strictEqual(extract({ kind: 'regex', selector: '/(\\d+) items/i' }, { body: body('12 ITEMS left') }).value, '12');
assert.strictEqual(extract({ kind: 'regex', selector: 'nothing' }, { body: body('abc') }).value, null);
assert.strictEqual(extract({ kind: 'regex', selector: 'nothing' }, { body: body('abc') }).value, null);
assert.throws(() => extract({ kind: 'regex', selector: '(' }, { body: body('x') }), (e) => e.code === 'parse_error');
assert.throws(() => extract({ kind: 'regex', selector: '(' }, { body: body('x') }), (e) => e.code === 'parse_error');
- assert.throws(() => extract({ kind: 'regex', selector: 'x'.repeat(300) }, { body: body('x') }), (e) => e.code === 'unsupported');
+ assert.throws(() => extract({ kind: 'regex', selector: 'x'.repeat(600) }, { body: body('x') }), (e) => e.code === 'unsupported');
+});
+
+t('regex: an unsafe pattern is refused, and a slow one times out as no match', () => {
+ // a nested quantifier is refused at compile time (the registry refuses it at save time too)
+ assert.throws(() => extract({ kind: 'regex', selector: '(a+)+$' }, { body: body('aaaa') }),
+ (e) => e.code === 'unsupported' && /quantified group/.test(e.message));
+ // an alternation slips past the static check; the deadline makes it a no-match with the reason
+ const out = extract({ kind: 'regex', selector: '^(\\w|\\w\\w)*$' }, { body: body('a'.repeat(40) + '!') });
+ assert.strictEqual(out.value, null, 'a pattern that hit its deadline states no match');
+ assert.strictEqual(out.error, 'pattern took too long');
});
});
t('ai is not a check: it refuses, and the refusal is `unsupported`', () => {
t('ai is not a check: it refuses, and the refusal is `unsupported`', () => {
@@ -175,7 +175,7 @@ function watchDef(overrides = {}) {
@@ -175,7 +175,7 @@ function watchDef(overrides = {}) {
return {
return {
name: 'Stub watch',
name: 'Stub watch',
source: { kind: 'http', url: 'https://example.org/page', format: null },
source: { kind: 'http', url: 'https://example.org/page', format: null },
- cadence: { every_sec: 60, jitter_sec: 0 },
+ cadence: { every_sec: 900, jitter_sec: 0 },
extraction: { kind: 'text' },
extraction: { kind: 'text' },
condition: { op: 'changed' },
condition: { op: 'changed' },
action: [{ kind: 'notification', category: 'service', priority: 'normal' }],
action: [{ kind: 'notification', category: 'service', priority: 'normal' }],
@@ -16,7 +16,7 @@ const someoneElse = { sub: newUser() };
@@ -16,7 +16,7 @@ const someoneElse = { sub: newUser() };
const def = (overrides = {}) => ({
const def = (overrides = {}) => ({
name: 'a watch',
name: 'a watch',
source: { kind: 'http', url: 'https://example.org/page', format: null },
source: { kind: 'http', url: 'https://example.org/page', format: null },
- cadence: { every_sec: 60 },
+ cadence: { every_sec: 900 },
extraction: { kind: 'text' },
extraction: { kind: 'text' },
condition: { op: 'changed' },
condition: { op: 'changed' },
action: [{ kind: 'notification' }],
action: [{ kind: 'notification' }],
@@ -67,6 +67,27 @@ t('create: what a watch requires, enforced in code over the released schema', as
@@ -67,6 +67,27 @@ t('create: what a watch requires, enforced in code over the released schema', as
assert.strictEqual(head.source.method, 'HEAD');
assert.strictEqual(head.source.method, 'HEAD');
});
});
+t('create: the 15-minute cadence floor is enforced here, not only in the form', async () => {
+ await invalid(def({ cadence: { every_sec: 60 } }), /15 minutes/);
+ await invalid(def({ cadence: { every_sec: 899 } }), /at least 900/);
+ const ok = await svc.registry.create(def({ cadence: { every_sec: 900 } }), me);
+ assert.strictEqual(ok.cadence.every_sec, 900);
+});
+
+t('create: a user regex is validated when the watch is saved, not when it runs', async () => {
+ // a nested quantifier would backtrack for minutes over a 64 KiB body; it never reaches the worker
+ await invalid(def({ extraction: { kind: 'regex', selector: '(a+)+$' } }), /extraction\/selector/);
+ await invalid(def({ condition: { op: 'matches', value: '(a+)+$' } }), /condition\/value/);
+ // an invalid or oversized pattern too
+ await invalid(def({ extraction: { kind: 'regex', selector: '(' } }), /extraction\/selector/);
+ await invalid(def({ condition: { op: 'matches', value: 'x'.repeat(600) } }), /condition\/value/);
+ // a normal pattern is accepted
+ const w = await svc.registry.create(def({ extraction: { kind: 'regex', selector: '(\\d+\\.\\d+) EUR' } }), me);
+ assert.strictEqual(w.extraction.selector, '(\\d+\\.\\d+) EUR');
+ const m = await svc.registry.create(def({ condition: { op: 'matches', value: '^v\\d+\\.\\d+' } }), me);
+ assert.strictEqual(m.condition.value, '^v\\d+\\.\\d+');
+});
+
t('create: an id, an owner, a name, the arm and the event', async () => {
t('create: an id, an owner, a name, the arm and the event', async () => {
const before = Date.now();
const before = Date.now();
const row = await svc.registry.create(def({ name: undefined }), me);
const row = await svc.registry.create(def({ name: undefined }), me);
@@ -193,6 +214,22 @@ t('all(): the keyset page is id order (a wch_ ULID is creation order)', async ()
@@ -193,6 +214,22 @@ t('all(): the keyset page is id order (a wch_ ULID is creation order)', async ()
assert.strictEqual((await svc.registry.all(other, { status: 'active' })).length, 4);
assert.strictEqual((await svc.registry.all(other, { status: 'active' })).length, 4);
});
});
+t('create: one owner holds at most WATCH_MAX_PER_OWNER watches, per owner', async () => {
+ const capped = await boot({ env: { WATCH_MAX_PER_OWNER: '2' } });
+ try {
+ const owner = { sub: newUser() };
+ await capped.registry.create(def(), owner);
+ await capped.registry.create(def(), owner);
+ await assert.rejects(() => capped.registry.create(def(), owner), (err) => {
+ assert.strictEqual(err.code, 'watch.invalid');
+ assert.match(err.message, /at most 2 watches/);
+ return true;
+ });
+ // the cap is per owner: another subject still creates
+ await capped.registry.create(def(), { sub: newUser() });
+ } finally { await capped.stop(); }
+});
+
t('done', async () => { await svc.stop(); });
t('done', async () => { await svc.stop(); });
t.run();
t.run();
new file mode 100644
new file mode 100644
@@ -0,0 +1,44 @@
@@ -0,0 +1,44 @@
+'use strict';
+/** safe-regex: nested quantifiers and oversized patterns refused at compile time; a slow pattern cut
+ * off by the deadline (no match, a timeout, never a hang); a normal pattern still matches. */
+const assert = require('assert');
+const { suite } = require('./helpers');
+const safe = require('../server/safe-regex');
+
+const t = suite('safe-regex');
+
+t('a quantified group holding a quantifier is refused; a normal pattern compiles', () => {
+ for (const bad of ['(a+)+$', '(a*)*', '(\\w+\\s?)*', '((a+)x)+', '(a?)+']) {
+ assert.ok(safe.validate(bad), `${bad} is refused`);
+ assert.throws(() => safe.compile(bad), (err) => err.code === 'unsafe_pattern' && err.reason === 'nested');
+ }
+ for (const ok of ['(\\d+\\.\\d+) EUR', '\\d{4}-\\d{2}-\\d{2}', '(cat|dog)', '^v\\d+\\.\\d+', 'a{2,4}', '[a+]+']) {
+ assert.strictEqual(safe.validate(ok), null, `${ok} is accepted`);
+ }
+});
+
+t('an oversized, empty or invalid pattern is refused', () => {
+ assert.match(safe.validate('x'.repeat(501)), /1–500/);
+ assert.match(safe.validate(''), /1–500/);
+ assert.match(safe.validate('('), /not a valid/);
+});
+
+t('a slow pattern that slips past the static check is cut off by the deadline', () => {
+ // alternation, no nested quantifier: the static scan accepts it, the deadline stops it
+ const re = safe.compile('^(\\w|\\w\\w)*$');
+ const start = Date.now();
+ const out = safe.test(re, 'a'.repeat(40) + '!');
+ const elapsed = Date.now() - start;
+ assert.deepStrictEqual(out, { matched: false, timedOut: true }, 'the deadline was hit, so no match');
+ assert.ok(elapsed < 200, `returned in ~${elapsed}ms rather than hanging`);
+});
+
+t('a normal pattern still matches, under the deadline (test and exec)', () => {
+ assert.strictEqual(safe.test(safe.compile('^v\\d+\\.\\d+'), 'v2.3 released').matched, true);
+ assert.strictEqual(safe.test(safe.compile('^v\\d+\\.\\d+'), 'vX').matched, false);
+ const { match, timedOut } = safe.exec(safe.compile('/(\\d+\\.\\d+) EUR/'), 'now 12.50 EUR');
+ assert.strictEqual(timedOut, false);
+ assert.strictEqual(match[1], '12.50');
+});
+
+t.run();
@@ -13,7 +13,7 @@ let web;
@@ -13,7 +13,7 @@ let web;
const create = async (name, overrides = {}) => await svc.registry.create({
const create = async (name, overrides = {}) => await svc.registry.create({
name,
name,
source: { kind: 'http', url: `${web.origin}/page`, format: null },
source: { kind: 'http', url: `${web.origin}/page`, format: null },
- cadence: { every_sec: 60 },
+ cadence: { every_sec: 900 },
extraction: { kind: 'text' },
extraction: { kind: 'text' },
condition: { op: 'eq', value: 'nothing' },
condition: { op: 'eq', value: 'nothing' },
action: [{ kind: 'notification' }],
action: [{ kind: 'notification' }],
@@ -93,10 +93,10 @@ t('a failing watch backs off: the scheduler leaves it alone until its next due t
@@ -93,10 +93,10 @@ t('a failing watch backs off: the scheduler leaves it alone until its next due t
const out = await svc.check.run(w.id, { trigger: 'schedule' });
const out = await svc.check.run(w.id, { trigger: 'schedule' });
assert.strictEqual(out.run.state, 'http_error');
assert.strictEqual(out.run.state, 'http_error');
const row = await svc.db.prepare('SELECT * FROM watches WHERE id = ?').get(w.id);
const row = await svc.db.prepare('SELECT * FROM watches WHERE id = ?').get(w.id);
- assert.strictEqual(row.next_due_at - row.last_check_at, 60000, 'first failure: every_sec · 2^0');
+ assert.strictEqual(row.next_due_at - row.last_check_at, 900000, 'first failure: every_sec · 2^0');
let due = new Set((await svc.registry.due(50)).map(d => d.id));
let due = new Set((await svc.registry.due(50)).map(d => d.id));
assert.ok(!due.has(w.id), 'not due until the backoff has run');
assert.ok(!due.has(w.id), 'not due until the backoff has run');
- await dueAt(w.id, row.next_due_at - 120000);
+ await dueAt(w.id, row.next_due_at - 960000);
due = new Set((await svc.registry.due(50)).map(d => d.id));
due = new Set((await svc.registry.due(50)).map(d => d.id));
assert.ok(due.has(w.id));
assert.ok(due.has(w.id));
});
});
@@ -113,7 +113,7 @@ t2('the worker checks a due watch without anyone asking', async () => {
@@ -113,7 +113,7 @@ t2('the worker checks a due watch without anyone asking', async () => {
try {
try {
const w = await svc2.registry.create({
const w = await svc2.registry.create({
name: 'auto', source: { kind: 'http', url: `${web2.origin}/page`, format: null },
name: 'auto', source: { kind: 'http', url: `${web2.origin}/page`, format: null },
- cadence: { every_sec: 60 }, extraction: { kind: 'text' },
+ cadence: { every_sec: 900 }, extraction: { kind: 'text' },
condition: { op: 'eq', value: 'nothing' }, action: [{ kind: 'notification' }],
condition: { op: 'eq', value: 'nothing' }, action: [{ kind: 'notification' }],
}, { sub: newUser() });
}, { sub: newUser() });
// a new watch is due at once (its first reading), then armed by cadence after each check;
// a new watch is due at once (its first reading), then armed by cadence after each check;
@@ -121,7 +121,7 @@ t2('the worker checks a due watch without anyone asking', async () => {
@@ -121,7 +121,7 @@ t2('the worker checks a due watch without anyone asking', async () => {
await svc2.db.prepare('UPDATE watches SET next_due_at = 0 WHERE id = ?').run(w.id);
await svc2.db.prepare('UPDATE watches SET next_due_at = 0 WHERE id = ?').run(w.id);
for (let i = 0; i < 100 && (await svc2.db.prepare('SELECT COUNT(*) AS n FROM check_runs WHERE watch_id = ?').get(w.id)).n === 0; i++) await sleep(50);
for (let i = 0; i < 100 && (await svc2.db.prepare('SELECT COUNT(*) AS n FROM check_runs WHERE watch_id = ?').get(w.id)).n === 0; i++) await sleep(50);
const runs = await svc2.db.prepare('SELECT * FROM check_runs WHERE watch_id = ?').all(w.id);
const runs = await svc2.db.prepare('SELECT * FROM check_runs WHERE watch_id = ?').all(w.id);
- assert.strictEqual(runs.length, 1, 'the scheduler ran exactly one check (the cadence is a minute)');
+ assert.strictEqual(runs.length, 1, 'the scheduler ran exactly one check (the cadence is 15 minutes)');
assert.strictEqual(runs[0].trigger, 'schedule');
assert.strictEqual(runs[0].trigger, 'schedule');
assert.strictEqual(runs[0].state, 'changed');
assert.strictEqual(runs[0].state, 'changed');
@@ -160,6 +160,12 @@ t('validation errors re-render the form with the registry\'s message and the per
@@ -160,6 +160,12 @@ t('validation errors re-render the form with the registry\'s message and the per
const noValue = await create({ template: 'price', url: 'https://example.org/p', extraction: 'css', selector: '.p', op: 'lt', value: '' });
const noValue = await create({ template: 'price', url: 'https://example.org/p', extraction: 'css', selector: '.p', op: 'lt', value: '' });
assert.strictEqual(noValue.status, 422);
assert.strictEqual(noValue.status, 422);
assert.match(noValue.text, /needs a value/);
assert.match(noValue.text, /needs a value/);
+
+ // A regex the registry refuses as unsafe (a nested quantifier) re-renders next to the selector.
+ const unsafe = await create({ template: 'price', name: 'Unsafe', url: 'https://example.org/p', extraction: 'regex', selector: '(a+)+$', op: 'lt', value: '20', cadence: '900' });
+ assert.strictEqual(unsafe.status, 422);
+ assert.match(unsafe.text, /quantified group/);
+ assert.match(unsafe.text, /value="\(a\+\)\+\$"/, 'the pattern is kept');
});
});
t('another person\'s watch is 404 on every page and form', async () => {
t('another person\'s watch is 404 on every page and form', async () => {