openvibers/openvibe.codes Commits

engine: one finding, in the agent.finding@1 shape (#370)

OP OpenVibers committed 3 days ago · f1636fd51c · 5 files, +369 −0

Adds engine/finding.js: findingId() is the stable file + normalised-text
identity, validate() holds the three verdicts (confirmed needs the split
severity and a confidence; needs_validation is a lead with blockers and a plan
and no severity; rejected says why) and severity cannot exceed impact. The
transcript schema publishes the same finding shape under findings[]. The
finding module is exported from engine/index.js.

Changes

ddiff --git a/engine/README.md b/engine/README.md
iindex f1f50a3..5de6c7d 100644
---- a/engine/README.md
++++ b/engine/README.md
@@@ -218,6 +218,23 @@ failureError('{"type":"result","is_error":true,"result":"API Error: usage limit
// 'API Error: usage limit reached'
```
++### `finding.js` — one finding, and the evidence bar it must meet
++
++The `agent.finding@1` shape (OpenVibe.Contracts), validated here because Codes is the producer. The verdict is the
++evidence bar, not the severity: `confirmed` needs the split severity (likelihood, impact, overall never above impact)
++and a confidence with its reason; `needs_validation` is a lead with a claimed cause, exact blockers and a local or
++deployment plan, and no severity; `rejected` carries its reason. `findingId(file, text)` is the stable identity from the
++repository-relative file and the whitespace-normalised, lowercased text, so re-wrapping keeps the id.
++
++```js
++const { findingId, validate, isLead } = require('./engine/finding');
++
++findingId('server/x.js', 'The cache NEVER expires'); // 'server/x.js:the cache never expires'
++validate({ verdict: 'confirmed', /* … */ });
++// { valid: false, errors: ['a confirmed finding needs severity', …] }
++isLead({ verdict: 'needs_validation' }); // true
++```
++
## Tests
`test/engine-*.test.js` in the Codes repo, run by the repo's own runner (`npm test`).
ddiff --git a/engine/finding.js b/engine/finding.js
nnew file mode 100644
iindex 0000000..1b41cd6
---- /dev/null
++++ b/engine/finding.js
@@@ -0,0 +1,156 @@
++'use strict';
++
++/**
++ * One review's finding, in the shape of `agent.finding@1` (OpenVibe.Contracts).
++ *
++ * The verdict is the evidence bar, not the severity. A `confirmed` finding was reproduced or re-read and carries a
++ * split severity (likelihood and impact, with `overall_severity` never above `impact`) and a confidence with its
++ * reason. A `needs_validation` finding is an unprovable claim turned into a lead: a claimed root cause, the exact
++ * blockers, a validation plan, and no severity at all — the honest middle between noise and a silent pass. A
++ * `rejected` claim says why it does not stand.
++ *
++ * findingId(file, text) the stable identity: repository-relative file + normalised text
++ * validate(finding) { valid, errors } against agent.finding@1 (all three branches)
++ * isLead(finding) a needs_validation finding
++ * severityRank(level) 0 informational … 4 critical
++ * FINDING_VERDICTS, LEVELS the closed lists
++ *
++ * Codes is the producer of these records (the review, the hunt), so the rules are held here as data: a re-wrapped
++ * finding keeps its id, and a claim without evidence cannot become a severity.
++ */
++
++const FINDING_VERDICTS = ['confirmed', 'needs_validation', 'rejected'];
++const LEVELS = ['informational', 'low', 'medium', 'high', 'critical'];
++const EVIDENCE_KINDS = ['source', 'command', 'response', 'screenshot', 'log', 'test'];
++const CONFIDENCE_LEVELS = ['low', 'medium', 'high'];
++const MAX_TEXT = 4000;
++
++/**
++ * The stable identity of a finding: the repository-relative file and the whitespace-normalised, lowercased words.
++ * Re-wrapping the same sentence keeps the id; a genuine rewording creates a new one (which is why the prior record
++ * is fed back to the reviewer too, not only the id).
++ *
++ * @param {string} file a repository-relative path
++ * @param {string} text the finding's body
++ * @returns {string} the fingerprint, `file:normalised words`
++ */
++function findingId(file, text) {
++ const f = String(file || '').trim().replace(/^\.\//, '').replace(/^\/+/, '');
++ const t = String(text || '').toLowerCase().replace(/\s+/g, ' ').trim();
++ return `${f}:${t}`.slice(0, 200);
++}
++
++/** The rank of a severity level, for the "overall may not exceed impact" rule. */
++function severityRank(level) {
++ return LEVELS.indexOf(String(level || '').toLowerCase());
++}
++
++const isObject = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
++const isText = (v, max = MAX_TEXT) => typeof v === 'string' && v.trim().length > 0 && v.length <= max;
++const isRelative = (p) => typeof p === 'string' && p.trim().length > 0 && p.length <= 500 && !p.startsWith('/') && !p.split('/').includes('..');
++const unknownKeys = (v, allowed) => Object.keys(v || {}).filter((k) => !allowed.includes(k));
++
++/** One location: a repository-relative file, an optional line and symbol. */
++function checkLocation(loc, where, errors) {
++ if (!isObject(loc)) { errors.push(`${where} must be an object`); return; }
++ for (const k of unknownKeys(loc, ['file', 'line', 'symbol'])) errors.push(`${where}.${k} is not a location field`);
++ if (!isRelative(loc.file)) errors.push(`${where}.file must be a repository-relative path`);
++ if (loc.line !== undefined && (!Number.isInteger(loc.line) || loc.line < 1)) errors.push(`${where}.line must be an integer of at least 1`);
++ if (loc.symbol !== undefined && !isText(loc.symbol, 200)) errors.push(`${where}.symbol must be a short non-empty string`);
++}
++
++/** One piece of evidence: what kind, what it shows, and the command/answer/location that shows it. */
++function checkEvidence(ev, where, errors) {
++ if (!isObject(ev)) { errors.push(`${where} must be an object`); return; }
++ for (const k of unknownKeys(ev, ['kind', 'summary', 'location', 'command', 'output'])) errors.push(`${where}.${k} is not an evidence field`);
++ if (!EVIDENCE_KINDS.includes(ev.kind)) errors.push(`${where}.kind must be one of ${EVIDENCE_KINDS.join(', ')}`);
++ if (!isText(ev.summary, 1000)) errors.push(`${where}.summary must be a non-empty sentence`);
++ if (ev.location !== undefined) checkLocation(ev.location, `${where}.location`, errors);
++ if (ev.command !== undefined && !isText(ev.command, 2000)) errors.push(`${where}.command must be a non-empty command`);
++ if (ev.output !== undefined && !isText(ev.output, MAX_TEXT)) errors.push(`${where}.output must be non-empty text`);
++}
++
++/** The split severity, with the rule the schema cannot state: overall_severity never exceeds impact. */
++function checkSeverity(sev, errors) {
++ if (!isObject(sev)) { errors.push('severity must be an object'); return; }
++ for (const k of unknownKeys(sev, ['likelihood', 'impact', 'overall_severity'])) errors.push(`severity.${k} is not a severity field`);
++ for (const k of ['likelihood', 'impact', 'overall_severity']) if (!LEVELS.includes(sev[k])) errors.push(`severity.${k} must be one of ${LEVELS.join(', ')}`);
++ if (LEVELS.includes(sev.impact) && LEVELS.includes(sev.overall_severity) && severityRank(sev.overall_severity) > severityRank(sev.impact)) {
++ errors.push('severity.overall_severity cannot exceed severity.impact (severity cannot exceed demonstrated impact)');
++ }
++}
++
++function checkConfidence(conf, errors) {
++ if (!isObject(conf)) { errors.push('confidence must be an object'); return; }
++ for (const k of unknownKeys(conf, ['level', 'reason'])) errors.push(`confidence.${k} is not a confidence field`);
++ if (!CONFIDENCE_LEVELS.includes(conf.level)) errors.push(`confidence.level must be one of ${CONFIDENCE_LEVELS.join(', ')}`);
++ if (!isText(conf.reason, 1000)) errors.push('confidence.reason must say why');
++}
++
++function checkPlan(plan, errors) {
++ if (!isObject(plan)) { errors.push('validation_plan must be an object'); return; }
++ for (const k of unknownKeys(plan, ['local', 'deployment'])) errors.push(`validation_plan.${k} is not a plan field`);
++ if (!isText(plan.local, 2000) && !isText(plan.deployment, 2000)) errors.push('validation_plan needs a local or a deployment step');
++ for (const k of ['local', 'deployment']) if (plan[k] !== undefined && !isText(plan[k], 2000)) errors.push(`validation_plan.${k} must be non-empty text`);
++}
++
++/**
++ * Validate one finding against `agent.finding@1`.
++ *
++ * @param {object} finding the record
++ * @returns {{ valid: boolean, errors: string[] }}
++ */
++function validate(finding) {
++ const errors = [];
++ if (!isObject(finding)) return { valid: false, errors: ['a finding must be an object'] };
++ for (const k of unknownKeys(finding, [
++ 'verdict', 'fingerprint', 'title', 'description', 'trace', 'evidence',
++ 'severity', 'confidence', 'remediation', 'claimed_root_cause', 'blockers', 'validation_plan', 'reason',
++ ])) errors.push(`${k} is not a finding field`);
++ if (!FINDING_VERDICTS.includes(finding.verdict)) errors.push(`verdict must be one of ${FINDING_VERDICTS.join(', ')}`);
++ if (!isText(finding.fingerprint, 200)) errors.push('fingerprint must be the stable file:normalised-text id');
++ if (!isText(finding.title, 200)) errors.push('title must be non-empty');
++ if (!isText(finding.description)) errors.push('description must be non-empty');
++ if (!Array.isArray(finding.trace) || !finding.trace.length) errors.push('trace must name at least one location');
++ else if (finding.trace.length > 50) errors.push('trace holds at most 50 locations');
++ else finding.trace.forEach((loc, i) => checkLocation(loc, `trace[${i}]`, errors));
++ if (!Array.isArray(finding.evidence) || !finding.evidence.length) errors.push('evidence must hold at least one record');
++ else if (finding.evidence.length > 50) errors.push('evidence holds at most 50 records');
++ else finding.evidence.forEach((ev, i) => checkEvidence(ev, `evidence[${i}]`, errors));
++ if (finding.remediation !== undefined) {
++ const r = finding.remediation;
++ if (!isObject(r)) errors.push('remediation must be an object');
++ else {
++ for (const k of unknownKeys(r, ['strategy', 'code_changes'])) errors.push(`remediation.${k} is not a remediation field`);
++ if (!isText(r.strategy, 2000)) errors.push('remediation.strategy must say what to change');
++ }
++ }
++ if (finding.verdict === 'confirmed') {
++ if (finding.severity === undefined) errors.push('a confirmed finding needs severity');
++ else checkSeverity(finding.severity, errors);
++ if (finding.confidence === undefined) errors.push('a confirmed finding needs confidence');
++ else checkConfidence(finding.confidence, errors);
++ } else if (finding.verdict === 'needs_validation') {
++ if (finding.severity !== undefined) errors.push('a needs_validation lead carries no severity');
++ if (!isText(finding.claimed_root_cause, 2000)) errors.push('a needs_validation lead needs claimed_root_cause');
++ if (!Array.isArray(finding.blockers) || !finding.blockers.length) errors.push('a needs_validation lead needs at least one blocker');
++ else if (finding.blockers.length > 20) errors.push('blockers holds at most 20 entries');
++ else finding.blockers.forEach((b, i) => { if (!isText(b, 500)) errors.push(`blockers[${i}] must name one exact blocker`); });
++ if (finding.validation_plan === undefined) errors.push('a needs_validation lead needs a validation_plan');
++ else checkPlan(finding.validation_plan, errors);
++ } else if (finding.verdict === 'rejected') {
++ if (finding.severity !== undefined) errors.push('a rejected claim carries no severity');
++ if (!isText(finding.reason, 2000)) errors.push('a rejected claim needs a reason');
++ }
++ return { valid: errors.length === 0, errors };
++}
++
++/** A needs_validation finding: the honest middle state. */
++function isLead(finding) {
++ return !!finding && finding.verdict === 'needs_validation';
++}
++
++module.exports = {
++ FINDING_VERDICTS, LEVELS, EVIDENCE_KINDS, CONFIDENCE_LEVELS,
++ findingId, severityRank, validate, isLead,
++};
ddiff --git a/engine/index.js b/engine/index.js
iindex b448d0d..966ee09 100644
---- a/engine/index.js
++++ b/engine/index.js
@@@ -19,6 +19,7 @@
* engine.redact the secrets that never reach an event stream
* engine.proof the proof of a finished attempt: its tests, its pages, its diff, and a second opinion
* engine.review the second opinion itself: a model of another family, through OpenVibe.AI
++ * engine.finding one finding in the agent.finding@1 shape: confirmed, a lead that needs validation, or rejected
*
* The modules do not depend on each other in a cycle, and none of them reads the portal's log folder.
*/
@@@ -39,6 +40,7 @@ const logins = require('./logins');
const redact = require('./redact');
const proof = require('./proof');
const review = require('./review');
++const finding = require('./finding');
const goal = require('./goal');
module.exports = {
@@@ -58,6 +60,7 @@ module.exports = {
redact,
proof,
review,
++ finding,
goal,
// The names a caller usually wants, in one place.
classify: router.classify,
@@@ -78,4 +81,6 @@ module.exports = {
jobLines: joblines.lines,
probeLogins: logins.probe,
scrub: redact.scrub,
++ findingId: finding.findingId,
++ validateFinding: finding.validate,
};
ddiff --git a/engine/transcript.schema.json b/engine/transcript.schema.json
iindex a92797e..397e392 100644
---- a/engine/transcript.schema.json
++++ b/engine/transcript.schema.json
@@@ -41,6 +41,11 @@
"type": "integer",
"minimum": 0,
"description": "The number of feed lines that were read."
++ },
++ "findings": {
++ "type": "array",
++ "description": "The structured findings of the task's review, in the agent.finding@1 shape (OpenVibe.Contracts): a confirmed finding carries severity and confidence; a needs_validation lead carries a claimed cause, exact blockers and a validation plan and no severity; a rejected claim carries its reason. Empty or absent when no review produced a record.",
++ "items": { "$ref": "#/$defs/finding" }
}
},
"additionalProperties": true,
@@@ -154,6 +159,112 @@
}
},
"additionalProperties": true
++ },
++ "finding": {
++ "type": "object",
++ "description": "One review finding (agent.finding@1): the verdict is the evidence bar; a needs_validation lead has no severity.",
++ "additionalProperties": false,
++ "required": ["verdict", "fingerprint", "title", "description", "trace", "evidence"],
++ "properties": {
++ "verdict": { "type": "string", "enum": ["confirmed", "needs_validation", "rejected"] },
++ "fingerprint": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Repository-relative file + whitespace-normalised, lowercased text; stable across re-wrapping." },
++ "title": { "type": "string", "minLength": 1, "maxLength": 200 },
++ "description": { "type": "string", "minLength": 1, "maxLength": 4000 },
++ "trace": { "type": "array", "minItems": 1, "maxItems": 50, "items": { "$ref": "#/$defs/findingLocation" } },
++ "evidence": { "type": "array", "minItems": 1, "maxItems": 50, "items": { "$ref": "#/$defs/findingEvidence" } },
++ "severity": {
++ "type": "object",
++ "additionalProperties": false,
++ "required": ["likelihood", "impact", "overall_severity"],
++ "properties": {
++ "likelihood": { "$ref": "#/$defs/findingLevel" },
++ "impact": { "$ref": "#/$defs/findingLevel" },
++ "overall_severity": { "$ref": "#/$defs/findingLevel" }
++ }
++ },
++ "confidence": {
++ "type": "object",
++ "additionalProperties": false,
++ "required": ["level", "reason"],
++ "properties": {
++ "level": { "type": "string", "enum": ["low", "medium", "high"] },
++ "reason": { "type": "string", "minLength": 1, "maxLength": 1000 }
++ }
++ },
++ "remediation": {
++ "type": "object",
++ "additionalProperties": false,
++ "required": ["strategy"],
++ "properties": {
++ "strategy": { "type": "string", "minLength": 1, "maxLength": 2000 },
++ "code_changes": {
++ "type": "array",
++ "maxItems": 20,
++ "items": {
++ "type": "object",
++ "additionalProperties": false,
++ "required": ["file", "change"],
++ "properties": {
++ "file": { "type": "string", "minLength": 1, "maxLength": 500 },
++ "change": { "type": "string", "minLength": 1, "maxLength": 1000 }
++ }
++ }
++ }
++ }
++ },
++ "claimed_root_cause": { "type": "string", "minLength": 1, "maxLength": 2000 },
++ "blockers": { "type": "array", "minItems": 1, "maxItems": 20, "items": { "type": "string", "minLength": 1, "maxLength": 500 } },
++ "validation_plan": {
++ "type": "object",
++ "additionalProperties": false,
++ "properties": {
++ "local": { "type": "string", "minLength": 1, "maxLength": 2000 },
++ "deployment": { "type": "string", "minLength": 1, "maxLength": 2000 }
++ },
++ "anyOf": [ { "required": ["local"] }, { "required": ["deployment"] } ]
++ },
++ "reason": { "type": "string", "minLength": 1, "maxLength": 2000 }
++ },
++ "allOf": [
++ {
++ "if": { "properties": { "verdict": { "const": "confirmed" } }, "required": ["verdict"] },
++ "then": { "required": ["severity", "confidence"] }
++ },
++ {
++ "if": { "properties": { "verdict": { "const": "needs_validation" } }, "required": ["verdict"] },
++ "then": { "required": ["claimed_root_cause", "blockers", "validation_plan"], "not": { "required": ["severity"] } }
++ },
++ {
++ "if": { "properties": { "verdict": { "const": "rejected" } }, "required": ["verdict"] },
++ "then": { "required": ["reason"], "not": { "required": ["severity"] } }
++ }
++ ]
++ },
++ "findingLocation": {
++ "type": "object",
++ "additionalProperties": false,
++ "required": ["file"],
++ "properties": {
++ "file": { "type": "string", "minLength": 1, "maxLength": 500 },
++ "line": { "type": "integer", "minimum": 1 },
++ "symbol": { "type": "string", "minLength": 1, "maxLength": 200 }
++ }
++ },
++ "findingEvidence": {
++ "type": "object",
++ "additionalProperties": false,
++ "required": ["kind", "summary"],
++ "properties": {
++ "kind": { "type": "string", "enum": ["source", "command", "response", "screenshot", "log", "test"] },
++ "summary": { "type": "string", "minLength": 1, "maxLength": 1000 },
++ "location": { "$ref": "#/$defs/findingLocation" },
++ "command": { "type": "string", "minLength": 1, "maxLength": 2000 },
++ "output": { "type": "string", "minLength": 1, "maxLength": 4000 }
++ }
++ },
++ "findingLevel": {
++ "type": "string",
++ "enum": ["informational", "low", "medium", "high", "critical"]
}
}
}
ddiff --git a/test/engine-finding.test.js b/test/engine-finding.test.js
nnew file mode 100644
iindex 0000000..4a2f1e6
---- /dev/null
++++ b/test/engine-finding.test.js
@@@ -0,0 +1,80 @@
++'use strict';
++// The agent.finding@1 shape: one review finding, the evidence bar of each verdict, and the stable id.
++
++const assert = require('node:assert/strict');
++const fs = require('node:fs');
++const path = require('node:path');
++const { check, done } = require('./helpers/boot');
++const finding = require('../engine/finding');
++
++const base = (over = {}) => ({
++ verdict: 'confirmed',
++ fingerprint: 'server/x.js:the cache never expires',
++ title: 'The cache never expires',
++ description: 'The entry has no TTL.',
++ trace: [{ file: 'server/x.js', line: 31, symbol: 'put' }],
++ evidence: [{ kind: 'source', summary: 'The options object has no ttl.' }],
++ severity: { likelihood: 'low', impact: 'high', overall_severity: 'high' },
++ confidence: { level: 'high', reason: 'Read in the source.' },
++ ...over,
++});
++
++const invalid = (f) => {
++ const r = finding.validate(f);
++ assert.equal(r.valid, false, `expected invalid: ${JSON.stringify(f.verdict)}`);
++ return r.errors.join('; ');
++};
++
++(async () => {
++ await check('a confirmed finding needs the split severity, a confidence and evidence', () => {
++ assert.deepEqual(finding.validate(base()), { valid: true, errors: [] });
++ assert.match(invalid(base({ severity: undefined })), /confirmed finding needs severity/);
++ assert.match(invalid(base({ confidence: undefined })), /confirmed finding needs confidence/);
++ assert.match(invalid(base({ evidence: [] })), /evidence must hold at least one record/);
++ assert.match(invalid(base({ severity: { likelihood: 'low', impact: 'low', overall_severity: 'critical' } })), /cannot exceed severity\.impact/);
++ });
++
++ await check('a needs_validation lead is exact, has no severity and is never a silent pass', () => {
++ const lead = {
++ verdict: 'needs_validation', fingerprint: 'server/billing.js:the retry may debit twice',
++ title: 'A retried debit may charge twice', description: 'The retry does not read the receipt.',
++ trace: [{ file: 'server/billing.js', line: 118 }], evidence: [{ kind: 'source', summary: 'The retry calls debit().' }],
++ claimed_root_cause: 'The idempotency key is not reused.', blockers: ['The provider sandbox credential is not here.'],
++ validation_plan: { local: 'Post a retried payment to the sandbox and read the ledger.' },
++ };
++ assert.deepEqual(finding.validate(lead), { valid: true, errors: [] });
++ assert.equal(finding.isLead(lead), true);
++ assert.match(invalid({ ...lead, severity: { likelihood: 'low', impact: 'low', overall_severity: 'low' } }), /carries no severity/);
++ assert.match(invalid({ ...lead, blockers: [] }), /at least one blocker/);
++ assert.match(invalid({ ...lead, validation_plan: {} }), /local or a deployment step/);
++ });
++
++ await check('a rejected claim says why; unknown fields and unsafe paths are refused', () => {
++ const rejected = {
++ verdict: 'rejected', fingerprint: 'server/cache.js:the cache never expires', title: 'The cache never expires',
++ description: 'The review missed the TTL.', trace: [{ file: 'server/cache.js', line: 31 }],
++ evidence: [{ kind: 'source', summary: 'The TTL is passed on line 34.' }], reason: 'Line 34 sets the TTL.',
++ };
++ assert.deepEqual(finding.validate(rejected), { valid: true, errors: [] });
++ assert.match(invalid({ ...rejected, reason: undefined }), /rejected claim needs a reason/);
++ assert.match(invalid(base({ surprise: true })), /surprise is not a finding field/);
++ assert.match(invalid(base({ trace: [{ file: '/etc/passwd' }] })), /repository-relative/);
++ assert.match(invalid(base({ trace: [{ file: '../outside' }] })), /repository-relative/);
++ });
++
++ await check('the identity survives re-wrapping and lowercasing, not a rewording', () => {
++ assert.equal(finding.findingId('./server/x.js', 'The cache\nNEVER expires'), 'server/x.js:the cache never expires');
++ assert.equal(finding.findingId('server/x.js', 'the cache never expires '), finding.findingId('server/x.js', 'The cache never expires'));
++ assert.notEqual(finding.findingId('server/x.js', 'the cache expires slowly'), finding.findingId('server/x.js', 'the cache never expires'));
++ });
++
++ await check('the published transcript schema carries the same three verdicts', () => {
++ const schema = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'engine', 'transcript.schema.json'), 'utf8'));
++ assert.deepEqual(schema.properties.findings.items.$ref, '#/$defs/finding');
++ assert.deepEqual(schema.$defs.finding.properties.verdict.enum, ['confirmed', 'needs_validation', 'rejected']);
++ assert.equal(schema.$defs.finding.properties.severity.type, 'object');
++ assert.ok(schema.$defs.finding.allOf.some((r) => r.then && r.then.not), 'a lead and a rejection exclude severity');
++ assert.ok(schema.$defs.finding.properties.validation_plan, 'a lead has its plan');
++ });
++ done();
++})();