openvibers/openvibe.tools Commits
Tools: a new "read" tool turns any public web page into clean text or Markdown (title, description, main text, links) through the SSRF guard and the run API. (#22)
Changes
@@@ -193,7 +193,8 @@ lorem:{n:'Lorem Ipsum Generator',i:'fa-paragraph',c:'qual',d:'Lorem ipsum generacurlconvert:{n:'Online curl',i:'fa-terminal',c:'http',d:'Parse curl \u2192 fetch, Python, Node.js',p:"curl -X POST https://api.example.com/users -H 'Content-Type: application/json' -H 'Authorization: Bearer tok123' -d '{\"name\":\"openvibe\"}'",m:['To fetch','To Python','To Node','Parsed']},webhook:{n:'Webhook Tester',i:'fa-satellite-dish',c:'http',d:'Webhook request inspector bin',p:'',m:['Create Bin']},color:{n:'Color Converter',i:'fa-palette',c:'fe',d:'Color picker & HEX/RGB/HSL converter',p:'#a78bfa',m:['Convert']},--opengraph:{n:'Open Graph Preview',i:'fa-share-nodes',c:'fe',d:'Open Graph & Twitter Card preview',p:'https://openvibe.live',m:['Preview']}++opengraph:{n:'Open Graph Preview',i:'fa-share-nodes',c:'fe',d:'Open Graph & Twitter Card preview',p:'https://openvibe.live',m:['Preview']},++read:{n:'Web Page Reader',i:'fa-book-open',c:'fe',d:'Read any web page as clean text or Markdown',p:'https://openvibe.live',m:['Markdown','Text']}};const CATS={data:{l:'Data & Formats',i:'fa-database'},enc:{l:'Encoding & Crypto',i:'fa-lock'},time:{l:'Time & Scheduling',i:'fa-clock'},qual:{l:'Code Quality',i:'fa-wand-magic-sparkles'},http:{l:'HTTP & API',i:'fa-terminal'},fe:{l:'Frontend & SEO',i:'fa-palette'}};@@@ -409,6 +410,13 @@ P.opengraph=async(s)=>{+(d.seo?'<div style="margin-top:14px"><div class="jl">SEO Recommendations</div>'+d.seo.map(x=>'<div style="color:'+(x.type==='error'?'#f87171':x.type==='warning'?'#fbbf24':'#34d399')+'">'+(x.type==='error'?'\u2717':x.type==='warning'?'\u26a0':'\u2713')+' '+H(x.message)+'</div>').join('')+'</div>':'')};};++P.read=async(s,m)=>{++ const r=await fetch('/api/v1/tools/read/run',{method:'POST',headers:{'Content-Type':'application/json'},credentials:'same-origin',body:JSON.stringify({input:{url:s.trim(),format:m==='Text'?'text':'markdown'}})});++ const b=await r.json();if(b.state!=='succeeded'||!b.result)throw new Error((b.error&&(b.error.detail||b.error.title))||b.detail||'Failed to read the page');++ const d=b.result.data;++ return (d.title?d.title+'\n':'')+(d.description?d.description+'\n':'')+(d.title||d.description?'\n':'')+d.text+(d.truncated?'\n\n[truncated]':'')+(d.links.length?'\n\nLinks\n'+d.links.map(l=>'- '+l.text+' \u2192 '+l.href).join('\n'):'');++};++// ═══════════════ UTILITY ═══════════════function copyOut(){navigator.clipboard.writeText(($('out').innerText||'')).then(()=>toast('Copied!')).catch(()=>{})}// The output becomes an unlisted OpenVibe.Community paste (through this gateway's /api/pastes proxy, as@@@ -56,6 +56,8 @@ const DEV_TOOLS = [{ id: 'color', subdomain: 'color', name: 'OpenVibeColor', icon: 'fa-palette', desc: 'Color picker, HEX/RGB/HSL converter & palette generator', category: 'frontend' },{ id: 'opengraph', subdomain: 'opengraph', name: 'OpenVibeOpenGraph', icon: 'fa-share-nodes', desc: 'Open Graph & Twitter Card preview & validator', category: 'frontend' },++ { id: 'read', subdomain: 'read', name: 'OpenVibeReader', icon: 'fa-book-open', desc: 'Read any web page as clean text or Markdown: title, description, main text and links', category: 'frontend' },++// Single-purpose tools: one job per address (search engines and people both look for the exact job).{ id: 'jsminify', subdomain: 'jsminify', name: "JavaScript Minifier", icon: 'fa-compress', desc: "Shrink JavaScript with a real parser, not regex", category: 'quality' },{ id: 'jsformat', subdomain: 'jsformat', name: "JavaScript Beautifier", icon: 'fa-wand-magic-sparkles', desc: "Turn minified or messy JavaScript into readable code", category: 'quality' },@@@ -86,6 +88,8 @@ const DEV_ALIASES = {colours: 'color',colors: 'color',og: 'opengraph',++ reader: 'read',++ readability: 'read',guid: 'uuid',unix: 'timestamp',unixtime: 'timestamp',@@@ -87,6 +87,33 @@ const SPECS = [route: { method: 'GET', path: '/api/dev/opengraph', query: { url: '{url}' } },example: { input: { url: 'https://openvibe.tools' } },},++ {++ id: 'read', execution: 'sync', api: true,++ input: obj({++ url: { type: 'string', minLength: 1, maxLength: 2048, description: 'A page URL (https:// is assumed when missing)' },++ format: { enum: ['markdown', 'text'], default: 'markdown', description: 'markdown keeps headings, list items and [links](href); text is plain' },++ max_chars: { type: 'integer', minimum: 1000, maximum: 50000, default: 20000, description: 'Longest text to return; longer pages are cut and truncated is true' },++ }, ['url']),++ files: null,++ output: {++ kind: 'json', schema: {++ type: 'object', required: ['url', 'status', 'title', 'text', 'truncated', 'chars', 'links'],++ properties: {++ url: { type: 'string', description: 'The final URL after redirects' },++ status: { type: 'integer' }, content_type: { type: 'string' },++ title: { type: 'string' }, description: { type: 'string' }, lang: { type: 'string' },++ text: { type: 'string', description: 'The main text of the page' },++ truncated: { type: 'boolean', description: 'The page or the text was cut at a limit' },++ chars: { type: 'integer', description: 'Length of text' },++ links: { type: 'array', maxItems: 50, items: { type: 'object', required: ['text', 'href'], properties: { text: { type: 'string' }, href: { type: 'string' } } } },++ },++ },++ },++ limits: { timeoutMs: 14000, maxInputBytes: 4 * KiB, perTargetPerMinute: 10 },++ auth: ANYONE, quotaClass: 'tools-fetch', cost: 2, egress: true,++ route: { method: 'GET', path: '/api/dev/read', query: { url: '{url}' } },++ example: { input: { url: 'https://openvibe.tools' } },++ },page('jsminify'), page('jsformat'),engine('cssminify', obj({ text: text() }), TEXT, { text: '.a { color: #aabbcc; margin: 0px; }' }),engine('cssformat', obj({ text: text() }), TEXT, { text: '.a{color:red}' }),@@@ -103,11 +130,11 @@ const SPECS = [];// The run API reuses an engine's answer for 10 minutes (same tool, same input): not for these, whose--// output is random or depends on the time. Open Graph (a page's tags) may be reused for 10 minutes.++// output is random or depends on the time. Open Graph (a page's tags) and the page reader may be reused for 10 minutes.const FRESH = new Set(['jwt', 'uuid', 'timestamp', 'cron', 'lorem']);for (const spec of SPECS) {if (FRESH.has(spec.id)) spec.cacheTtlMs = 0;-- if (spec.id === 'opengraph') spec.cacheTtlMs = 10 * 60 * 1000;++ if (spec.id === 'opengraph' || spec.id === 'read') spec.cacheTtlMs = 10 * 60 * 1000;}module.exports = { SPECS };nnew file mode 100644@@@ -0,0 +1,271 @@++'use strict';++++// ═══════════════════════════════════════════════════════════════++// Reader: HTML → readable text, with no dependencies.++//++// readHtml(html, { baseUrl, format, maxChars }) → { title, description, lang, text, truncated,++// links }. A forgiving tokenizer builds a small tree (void elements, raw-text elements, implied++// </p> and </li>); the walk keeps <main> or <article> when there is one, drops navigation and++// page furniture, and writes headings as `#` lines, list items as `- `, links as [text](href).++// ═══════════════════════════════════════════════════════════════++++const VOID = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr']);++const RAW = new Set(['script', 'style', 'textarea', 'title']);++const SKIP = new Set(['head', 'nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript', 'svg', 'form', 'iframe', 'template', 'select', 'textarea', 'button', 'object', 'canvas', 'math']);++const BLOCK = new Set(['p', 'div', 'section', 'article', 'main', 'ul', 'ol', 'li', 'blockquote', 'pre', 'table', 'tr', 'figure', 'figcaption', 'dl', 'dt', 'dd', 'address', 'details', 'summary', 'fieldset', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'caption', 'thead', 'tbody', 'tfoot']);++const MAX_DEPTH = 256; // deeper nesting is flattened into the parent: no recursion blow-ups++const MAX_LINKS = 50;++const MAX_ATTR_TEXT = 300;++++const ENTITIES = {++ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ', shy: '',++ mdash: '—', ndash: '–', hellip: '…', copy: '©', reg: '®', trade: '™', laquo: '«', raquo: '»',++ lsquo: '‘', rsquo: '’', sbquo: '‚', ldquo: '“', rdquo: '”', bdquo: '„', bull: '•', middot: '·',++ times: '×', divide: '÷', euro: '€', pound: '£', yen: '¥', cent: '¢', sect: '§', para: '¶',++ deg: '°', plusmn: '±', frac12: '½', frac14: '¼', frac34: '¾', larr: '←', rarr: '→', uarr: '↑',++ darr: '↓', hearts: '♥', check: '✓', iexcl: '¡', iquest: '¿', eacute: 'é', egrave: 'è', agrave: 'à',++ aacute: 'á', uuml: 'ü', ouml: 'ö', auml: 'ä', szlig: 'ß', ntilde: 'ñ', ccedil: 'ç',++};++++function decodeEntities(s) {++ if (!s || s.indexOf('&') === -1) return s || '';++ return s.replace(/&(#x[0-9a-f]{1,6}|#[0-9]{1,7}|[a-z][a-z0-9]{1,9});/gi, (m, e) => {++ if (e[0] === '#') {++ const cp = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);++ if (!cp || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff)) return '';++ return cp === 0xa0 ? ' ' : String.fromCodePoint(cp);++ }++ const v = ENTITIES[e] !== undefined ? ENTITIES[e] : ENTITIES[e.toLowerCase()];++ return v === undefined ? m : v;++ });++}++++// ── Tokenizer + tree ─────────────────────────────────────────++function parseAttrs(src) {++ const attrs = {};++ const re = /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;++ let m;++ while ((m = re.exec(src))) {++ const k = m[1].toLowerCase();++ if (attrs[k] === undefined) attrs[k] = decodeEntities(m[2] !== undefined ? m[2] : m[3] !== undefined ? m[3] : m[4] || '');++ }++ return attrs;++}++++function parse(html) {++ const root = { tag: '#root', attrs: {}, children: [], parent: null };++ let cur = root, depth = 0;++ const open = []; // element stack (cur is its top)++ const pushText = (t) => { if (t) cur.children.push({ text: decodeEntities(t) }); };++ const tokens = /<!--[\s\S]*?(?:-->|$)|<![^>]*>|<\?[^>]*>|<\/([a-zA-Z][^\s>\/]*)[^>]*>|<([a-zA-Z][^\s>\/]*)((?:"[^"]*"|'[^']*'|[^>"'])*)>/g;++ let last = 0, m;++ while ((m = tokens.exec(html))) {++ if (m.index > last) pushText(html.slice(last, m.index));++ last = tokens.lastIndex;++ if (m[1]) { // </tag>++ const name = m[1].toLowerCase();++ for (let i = open.length - 1; i >= 0; i--) {++ if (open[i].tag === name) { open.length = i; cur = i ? open[i - 1] : root; depth = open.length; break; }++ }++ } else if (m[2]) { // <tag ...>++ const name = m[2].toLowerCase();++ const attrs = parseAttrs(m[3] || '');++ // Implied end tags: a new <li> closes the open one in the same list, a new block closes an open <p>.++ if (name === 'li') {++ for (let i = open.length - 1; i >= 0; i--) {++ if (/^(ul|ol|menu)$/.test(open[i].tag)) break;++ if (open[i].tag === 'li') { open.length = i; cur = i ? open[i - 1] : root; depth = open.length; break; }++ }++ } else if (cur.tag === 'p' && BLOCK.has(name)) {++ open.pop(); cur = open.length ? open[open.length - 1] : root; depth = open.length;++ }++ const node = { tag: name, attrs, children: [], parent: cur };++ cur.children.push(node);++ const selfClose = /\/\s*$/.test(m[3] || '');++ if (VOID.has(name) || (selfClose && !RAW.has(name))) continue;++ if (RAW.has(name)) { // everything up to the matching close tag is text++ const close = new RegExp('</' + name + '\\s*>', 'ig');++ close.lastIndex = tokens.lastIndex;++ const c = close.exec(html);++ const end = c ? c.index : html.length;++ const body = html.slice(tokens.lastIndex, end);++ if (body && (name === 'title' || name === 'textarea')) node.children.push({ text: body });++ tokens.lastIndex = last = c ? close.lastIndex : html.length;++ continue;++ }++ if (depth >= MAX_DEPTH) continue; // too deep: its children stay with the parent++ open.push(node); cur = node; depth++;++ }++ }++ if (last < html.length) pushText(html.slice(last));++ return root;++}++++function find(node, pred) {++ const stack = [node];++ while (stack.length) {++ const n = stack.pop();++ if (n.tag && pred(n)) return n;++ if (n.children) for (let i = n.children.length - 1; i >= 0; i--) stack.push(n.children[i]);++ }++ return null;++}++function rawText(n) {++ if (n.text !== undefined) return n.text;++ return (n.children || []).map(rawText).join('');++}++const collapse = (s) => s.replace(/[\s ]+/g, ' ').trim();++++function metaContent(root, ...names) {++ const wanted = new Set(names);++ const found = {};++ const stack = [root];++ while (stack.length) {++ const n = stack.pop();++ if (n.tag === 'meta') {++ const k = (n.attrs.name || n.attrs.property || '').toLowerCase();++ if (wanted.has(k) && n.attrs.content && !found[k]) found[k] = collapse(decodeEntities(n.attrs.content));++ }++ if (n.children) for (let i = n.children.length - 1; i >= 0; i--) stack.push(n.children[i]);++ }++ for (const k of names) if (found[k]) return found[k];++ return '';++}++++// ── Output ───────────────────────────────────────────────────++function absolute(href, baseUrl) {++ if (!href || /^\s*(javascript|data|mailto|tel|blob):/i.test(href)) return null;++ try {++ const u = new URL(decodeEntities(href).trim(), baseUrl);++ if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;++ if (u.username || u.password) return null;++ return u.href;++ } catch { return null; }++}++++function render(rootNode, { baseUrl, markdown }) {++ const blocks = []; // { text, list }++ const links = [];++ const seen = new Set();++ const pageNoHash = (() => { try { const u = new URL(baseUrl); u.hash = ''; return u.href; } catch { return ''; } })();++ let buf = '', prefix = '', flushes = 0, listDepth = 0, quote = 0, pre = 0, listItem = false;++++ function flush() {++ const lines = buf.split('\n').map(collapse).filter(Boolean);++ buf = '';++ if (!lines.length) { prefix = ''; listItem = false; return; }++ const pad = '> '.repeat(quote);++ blocks.push({ text: lines.map((l, i) => (i === 0 ? pad + prefix : pad + (prefix ? ' '.repeat(prefix.length) : '')) + l).join('\n'), list: listItem });++ prefix = ''; listItem = false; flushes++;++ }++ function addLink(text, href) {++ if (links.length >= MAX_LINKS || seen.has(href)) return;++ const noHash = href.replace(/#.*$/, '');++ if (noHash === pageNoHash) return;++ seen.add(href);++ links.push({ text: collapse(text).slice(0, MAX_ATTR_TEXT) || href, href });++ }++++ function walk(n) {++ if (n.text !== undefined) { buf += pre ? n.text : n.text.replace(/\s+/g, ' '); return; }++ const t = n.tag;++ if (SKIP.has(t) || n.attrs.hidden !== undefined || /^true$/i.test(n.attrs['aria-hidden'] || '')) return;++ if (t === 'br') { buf += '\n'; return; }++ if (t === 'hr') { flush(); if (markdown) blocks.push({ text: '---', list: false }); return; }++ if (t === 'img') { return; }++ if (t === 'pre') {++ flush();++ const body = rawText(n).replace(/\r/g, '').replace(/^\n+|\s+$/g, '');++ if (body) blocks.push({ text: markdown ? '```\n' + body + '\n```' : body, list: false, raw: true });++ return;++ }++ if (/^h[1-6]$/.test(t)) {++ flush();++ if (markdown) prefix = '#'.repeat(Number(t[1])) + ' ';++ kids(n); flush(); return;++ }++ if (t === 'ul' || t === 'ol' || t === 'menu') { flush(); listDepth++; kids(n); flush(); listDepth--; return; }++ if (t === 'li') {++ flush();++ prefix = ' '.repeat(Math.max(0, listDepth - 1)) + '- ';++ listItem = true;++ // A nested list ends this item's own text first, so it is flushed by the child list.++ kids(n); flush(); return;++ }++ if (t === 'blockquote') { flush(); quote++; kids(n); flush(); quote--; return; }++ if (t === 'td' || t === 'th') { if (collapse(buf)) buf += ' | '; kids(n); return; }++ if (t === 'tr') { flush(); kids(n); buf = buf.replace(/\s*\|\s*$/, ''); flush(); return; }++ if (t === 'a') {++ const href = absolute(n.attrs.href, baseUrl);++ const start = buf.length, before = flushes;++ kids(n);++ if (!href || href.replace(/#.*$/, '') === pageNoHash) return; // not a link, or the page itself++ const inner = collapse(buf.slice(start)) || collapse(n.attrs['aria-label'] || n.attrs.title || '');++ addLink(inner, href);++ if (markdown && flushes === before && inner && start <= buf.length) {++ const lead = /^\s/.test(buf.slice(start)) ? ' ' : '', trail = /\s$/.test(buf.slice(start)) ? ' ' : '';++ buf = buf.slice(0, start) + lead + '[' + inner.replace(/([\[\]])/g, '\\$1') + '](' + href.replace(/\)/g, '%29').replace(/\s/g, '%20') + ')' + trail;++ }++ return;++ }++ if (t === 'code' && markdown && !pre) {++ const c = collapse(rawText(n));++ if (c) buf += '`' + c.replace(/`/g, "'") + '`';++ return;++ }++ if (BLOCK.has(t)) { flush(); kids(n); flush(); return; }++ kids(n);++ }++ function kids(n) { for (const c of n.children) walk(c); }++++ walk(rootNode);++ flush();++ let text = '';++ blocks.forEach((b, i) => {++ if (i) text += (b.list && blocks[i - 1].list) ? '\n' : '\n\n';++ text += b.text;++ });++ return { text, links };++}++++function readHtml(html, { baseUrl, format = 'markdown', maxChars = 20000 } = {}) {++ const doc = parse(html);++ const markdown = format !== 'text';++ const htmlEl = find(doc, n => n.tag === 'html');++ const titleEl = find(doc, n => n.tag === 'title');++ let title = titleEl ? collapse(decodeEntities(rawText(titleEl))) : '';++ if (!title) title = metaContent(doc, 'og:title', 'twitter:title');++ const description = metaContent(doc, 'description', 'og:description', 'twitter:description');++ const lang = ((htmlEl && htmlEl.attrs.lang) || '').trim().slice(0, 35);++++ const body = find(doc, n => n.tag === 'body') || doc;++ const main = find(body, n => n.tag === 'main' || (n.attrs.role || '').toLowerCase() === 'main') || find(body, n => n.tag === 'article');++ let out = render(main || body, { baseUrl, markdown });++ // A <main>/<article> that holds almost nothing (or whose header was dropped) is not the page: use the body.++ if (main && out.text.length < 200) {++ const whole = render(body, { baseUrl, markdown });++ if (whole.text.length > out.text.length) out = whole;++ }++ if (!title) { const h1 = find(body, n => n.tag === 'h1'); if (h1) title = collapse(decodeEntities(rawText(h1))); }++++ let text = out.text;++ let truncated = false;++ if (text.length > maxChars) {++ text = text.slice(0, maxChars);++ const cut = Math.max(text.lastIndexOf('\n'), text.lastIndexOf('. '));++ if (cut > maxChars * 0.8) text = text.slice(0, cut + 1);++ text = text.trimEnd();++ truncated = true;++ }++ return { title: title.slice(0, 500), description: description.slice(0, 1000), lang, text, truncated, links: out.links };++}++++/** Decode a response body using the header's charset, then a <meta charset>, then UTF-8. */++function decodeBody(buf, contentType) {++ let cs = /charset\s*=\s*"?([\w.:-]+)/i.exec(contentType || '');++ if (!cs) cs = /<meta[^>]+charset\s*=\s*["']?\s*([\w.:-]+)/i.exec(buf.subarray(0, 4096).toString('latin1'));++ const label = cs ? cs[1].toLowerCase() : 'utf-8';++ try { return new TextDecoder(label).decode(buf); } catch { return buf.toString('utf8'); }++}++++module.exports = { readHtml, decodeBody, decodeEntities };@@@ -17,6 +17,7 @@ const { DEV_TOOLS } = require('./config');const { createEgress, TargetRefused } = require('../../../_shared/egress');const { createCallerResolver } = require('../../../_shared/guard/caller');const guardLimits = require('../../../_shared/guard/limits');++const { readHtml, decodeBody } = require('./reader');// ── In-memory webhook bin storage ────────────────────────────const webhookBins = new Map(); // binId → { created, requests, owner, ipKey }@@@ -33,6 +34,8 @@ setInterval(cleanupBins, 5 * 60 * 1000).unref();// ── Helpers ──────────────────────────────────────────────────const OG_MAX_BYTES = 2 * 1024 * 1024; // Open Graph tags live in <head>; never buffer more than this++const READ_MAX_BYTES = 2 * 1024 * 1024; // the reader never buffers more than this of a page++const READ_USER_AGENT = 'OpenVibeReader/1.0 (+https://read.openvibe.tools)';function extractOGTags(html) {const tags = {};@@@ -164,7 +167,7 @@ module.exports = function createDevRoutes(db, requireAuth, opts = {}) {const guard = opts.guard || null;// Every dev API call (and the Open Graph fetcher's own tools-fetch quota, below).-- if (guard) router.use(guard.toolQuota((req) => (req.method === 'GET' && req.path === '/opengraph' ? 'opengraph' : null)));++ if (guard) router.use(guard.toolQuota((req) => (req.method === 'GET' && (req.path === '/opengraph' || req.path === '/read') ? req.path.slice(1) : null)));// ── List all dev tools ───────────────────────────────────router.get('/tools', (_req, res) => {@@@ -246,6 +249,78 @@ module.exports = function createDevRoutes(db, requireAuth, opts = {}) {}});++ // ── Web page reader ──────────────────────────────────────++ // A page's readable text (title, description, main text as light Markdown, outgoing links),++ // fetched through the same SSRF guard as Open Graph.++ router.get('/read', async (req, res) => {++ let targetUrl = String(req.query.url || req.query.target || '').trim();++ if (!targetUrl) return res.status(400).json({ error: 'Missing ?url= parameter' });++ const format = req.query.format === undefined || req.query.format === '' ? 'markdown' : String(req.query.format);++ if (format !== 'markdown' && format !== 'text') return res.status(400).json({ error: 'format must be "markdown" or "text"' });++ let maxChars = 20000;++ if (req.query.max_chars !== undefined && req.query.max_chars !== '') {++ maxChars = Number(req.query.max_chars);++ if (!Number.isInteger(maxChars) || maxChars < 1000 || maxChars > 50000) return res.status(400).json({ error: 'max_chars must be an integer from 1000 to 50000' });++ }++ if (targetUrl.length > 2048) return res.status(400).json({ error: 'URL is too long' });++ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(targetUrl) && !/^https?:\/\//i.test(targetUrl)) return res.status(400).json({ error: 'Only http and https URLs are supported' });++ if (!/^https?:\/\//i.test(targetUrl)) targetUrl = 'https://' + targetUrl;++ try {++ new URL(targetUrl);++ } catch {++ return res.status(400).json({ error: 'Invalid URL' });++ }++ // Per-target throttle (descriptor read: limits.perTargetPerMinute), across every caller.++ if (guard && !guard.target(req, res, { tool: 'read', target: targetUrl })) return;++++ try {++ const response = await egress.follow(targetUrl, {++ method: 'GET',++ headers: {++ 'User-Agent': READ_USER_AGENT,++ 'Accept': 'text/html,application/xhtml+xml,text/plain;q=0.9',++ },++ timeoutMs: 10000,++ maxBytes: READ_MAX_BYTES,++ maxRedirects: 5,++ });++ const finalUrl = response.url || targetUrl;++ const contentType = String(response.headers['content-type'] || '').split(';')[0].trim().toLowerCase();++ const raw = response.body;++ const sniffedHtml = !contentType && /^\s*(<!doctype html|<html|<head|<body)/i.test(raw.subarray(0, 512).toString('latin1'));++ const isHtml = /^(text\/html|application\/xhtml\+xml)$/.test(contentType) || sniffedHtml;++ const isText = contentType === 'text/plain';++ if (!isHtml && !isText) {++ return res.status(415).json({ error: `Not a readable page (${contentType || 'unknown content type'}); the reader handles HTML and plain text`, code: 'tools.read.unsupported_content_type', content_type: contentType || null });++ }++ const decoded = decodeBody(raw, response.headers['content-type']);++ let out;++ if (isHtml) {++ out = readHtml(decoded, { baseUrl: finalUrl, format, maxChars });++ } else {++ const plain = decoded.replace(/\r\n?/g, '\n').trim();++ out = { title: '', description: '', lang: '', text: plain.slice(0, maxChars), truncated: plain.length > maxChars, links: [] };++ }++ res.json({++ ok: true,++ url: finalUrl,++ status: response.status,++ content_type: contentType || (isHtml ? 'text/html' : 'text/plain'),++ title: out.title,++ description: out.description,++ lang: out.lang,++ text: out.text,++ truncated: out.truncated || Boolean(response.truncated),++ chars: out.text.length,++ links: out.links,++ });++ } catch (err) {++ if (err instanceof TargetRefused) return res.status(403).json({ error: err.message, code: err.code });++ if (err && err.status === 400) return res.status(400).json({ error: err.message });++ res.status(502).json({ error: `Failed to fetch: ${err.message}` });++ }++ });++// ── Webhook Bins ─────────────────────────────────────────// Their own router (above): the bins' API, reachable at /api/dev/webhook/* on the gateway.router.use('/webhook', createWebhookRouter(opts));@@@ -238,6 +238,14 @@ const DEV = {bullets: ['Live preview of the social card for any public URL', 'Lists Open Graph, Twitter Card and standard meta tags', 'Flags missing titles, descriptions and images'],faq: [['Why does my link preview show no image?', 'Usually a missing og:image tag, or an image URL that is relative rather than absolute. Social platforms need a full https URL.'], ['Why has my preview not updated after I fixed the tags?', 'Platforms cache previews aggressively. Most offer a debugger that forces a re-scrape; this tool always fetches live so you can confirm the tags themselves are correct.']],},++ read: {++ name: 'Web Page Reader', title: 'Website to Text & URL to Markdown Converter — Web Page Reader',++ desc: 'Read any web page as clean text or Markdown: title, description, main text and links. A reader mode for any URL, and an API for scripts and AI agents.',++ kw: 'website to text, url to markdown, reader mode, extract text from webpage, web page to markdown, webpage reader, scrape article text',++ about: 'Enter a URL and the reader fetches the page and returns what a person would read: its title and description, the main text as light Markdown (headings, lists and links kept) and the page\'s outgoing links. Navigation, headers, footers, scripts and styles are left out. The same result is available as JSON from the tools API, which is how scripts and AI agents read pages.',++ bullets: ['Title, description, main text and up to 50 outgoing links', 'Markdown with headings, lists and links, or plain text', 'Only public web addresses: private and internal targets are refused'],++ faq: [['Does it run JavaScript?', 'No. It reads the HTML the server sends, which covers articles, documentation and most blogs. A page that builds its content in the browser will come back mostly empty.'], ['Is there a size limit?', 'The page is read up to about 2 MB and the text is cut at max_chars (20,000 by default, up to 50,000); the result says when it was truncated.'], ['Can I call it from code?', 'Yes: POST /api/v1/tools/read/run with {"input":{"url":"https://example.com"}}. No account is needed within the free limits.']],++ },jsminify: {name: "JavaScript Minifier", title: "JavaScript Minifier \u2014 Minify JS Online",desc: "Shrink JavaScript with a real parser, not regex. Paste JavaScript and get a smaller file that behaves the same.",@@@ -192,7 +192,7 @@ for (const m of ['jsonfmt', 'md', 'codediff', 'slugify', 'entities']) { assert.o// ── Counts ──const tally = (f) => snap.tools.reduce((o, d) => { const k = f(d); o[k] = (o[k] || 0) + 1; return o; }, {});const fam = tally(d => d.family);--assert.deepStrictEqual(fam, { dev: 33, net: 38, img: 15, audio: 36, docs: 10, text: 33, media: 1, places: 2, pastes: 1 });++assert.deepStrictEqual(fam, { dev: 34, net: 38, img: 15, audio: 36, docs: 10, text: 33, media: 1, places: 2, pastes: 1 });fs.rmSync(process.env.DATA_DIR, { recursive: true, force: true });console.log(`descriptors: ${snap.tools.length} tools valid (${Object.entries(tally(d => `${d.execution}${d.api ? '+api' : ''}`)).map(([k, n]) => `${k} ${n}`).join(', ')}); ${jobs} job tools pass their app's validation; ${syncApi} sync tools run through served routes`);nnew file mode 100644@@@ -0,0 +1,206 @@++'use strict';++// The "read" tool (web page reader): HTML → title, description, readable Markdown/text, links; the++// shared SSRF guard (private targets 403, redirects re-checked); the raw size cap; content types;++// and the run API (POST /api/v1/tools/read/run) answering JSON that matches the descriptor's output++// schema. Mock resolver and transports: nothing touches a real network.++const assert = require('assert');++const express = require('express');++const { EventEmitter } = require('events');++const { Readable } = require('stream');++const contracts = require('openvibe-contracts');++const { createEgress } = require('../../_shared/egress');++const { createGuard } = require('../../_shared/guard');++const { ajvFrom } = require('../../_shared/tools/run');++const createDevRoutes = require('../server/dev/routes');++const { readHtml } = require('../server/dev/reader');++const { SPECS } = require('../server/dev/descriptors');++++const PAGE = `<!doctype html>++<html lang="en"><head><meta charset="utf-8"><title>Hello & welcome</title>++<meta name="description" content="A page about "things"">++<style>body{color:red}</style><script>var hidden = "script text";</script></head>++<body>++<header><a href="/home">HEADER-LINK</a> HEADER TEXT</header>++<nav><ul><li><a href="/nav">NAV-ITEM</a></li></ul></nav>++<main>++<h1>Big title</h1>++<p>First paragraph with a <a href="/docs/intro?x=1&y=2">relative link</a> and © 2026 text.</p>++<h2>Things</h2>++<ul><li>one<li>two with <a href="https://other.example/page">other</a></ul>++<p>Duplicate <a href="/docs/intro?x=1&y=2">again</a>, <a href="javascript:alert(1)">js</a>, <a href="mailto:[email protected]">mail</a>, <a href="#top">top</a>.</p>++<pre>line 1++ line 2</pre>++<form><input name="q"> FORM-TEXT</form>++<svg><text>SVG-TEXT</text></svg>++<noscript>NOSCRIPT-TEXT</noscript>++</main>++<aside>ASIDE-TEXT</aside>++<footer>FOOTER TEXT</footer>++</body></html>`;++++const DNS = { 'public.example': ['93.184.216.34'], 'redirector.example': ['93.184.216.35'], 'rebind.example': ['127.0.0.1'], 'lan.example': ['192.168.1.20'] };++const lookup = (host, _o, cb) => { const a = DNS[host]; if (!a) return cb(Object.assign(new Error('nf'), { code: 'ENOTFOUND' })); cb(null, a.map(address => ({ address, family: 4 }))); };++const dialled = [];++const seenReq = [];++function fakeSocket(address) {++ const s = new EventEmitter();++ s.setTimeout = () => {}; s.destroy = () => {}; s.end = () => {};++ process.nextTick(() => s.emit('connect'));++ dialled.push(address);++ return s;++}++// path → { status, type, body (string|Buffer), location }++const SITE = {++ '/': { type: 'text/html; charset=utf-8', body: PAGE },++ '/plain': { type: 'text/plain', body: 'just some text\r\nsecond line\r\n' },++ '/image': { type: 'image/png', body: Buffer.from([0x89, 0x50, 0x4e, 0x47]) },++ '/json': { type: 'application/json', body: '{"a":1}' },++ '/big': { type: 'text/html', body: '<html><body><main><p>' + 'word '.repeat(600) + '</p></main></body></html>' },++ '/huge': { type: 'text/html', body: '<html><body>' + '<p>' + 'x'.repeat(1024) + '</p>' + '<p>padding</p>'.repeat(300000) + '</body></html>' },++ '/notfound': { status: 404, type: 'text/html', body: '<html><head><title>Gone</title></head><body><p>Nothing here.</p></body></html>' },++ '/latin1': { type: 'text/html; charset=iso-8859-1', body: Buffer.from('<html><body><p>caf\xe9</p></body></html>', 'latin1') },++};++function fakeHttp(opts, cb) {++ const req = new EventEmitter();++ req.destroy = () => {};++ req.end = () => {++ const answer = (address) => {++ dialled.push(address);++ seenReq.push({ host: opts.hostname, path: opts.path, headers: opts.headers });++ let page;++ if (opts.hostname === 'redirector.example') {++ const to = { '/to-loopback': 'http://127.0.0.1:3000/admin', '/to-public': 'https://public.example/plain' }[opts.path];++ page = to ? { status: 302, location: to, body: '' } : SITE['/'];++ } else page = SITE[opts.path] || SITE['/'];++ const res = Readable.from(page.body.length ? [Buffer.isBuffer(page.body) ? page.body : Buffer.from(page.body)] : []);++ Object.assign(res, { statusCode: page.status || 200, statusMessage: 'x', headers: { 'content-type': page.type || 'text/html', ...(page.location && { location: page.location }) } });++ cb(res);++ };++ if (opts.lookup) opts.lookup(opts.hostname, {}, (err, address) => err ? req.emit('error', err) : answer(address)); else answer(opts.hostname);++ };++ return req;++}++const egress = createEgress({++ lookup, tcpConnect: ({ host }) => fakeSocket(host),++ tlsConnect: (o) => { const s = fakeSocket(o.host); s.getPeerCertificate = () => ({}); return s; },++ httpRequest: fakeHttp, httpsRequest: fakeHttp,++});++++(async () => {++ // ── The extraction itself ──++ const r = readHtml(PAGE, { baseUrl: 'https://public.example/a/b', format: 'markdown', maxChars: 20000 });++ assert.equal(r.title, 'Hello & welcome');++ assert.equal(r.description, 'A page about "things"');++ assert.equal(r.lang, 'en');++ for (const gone of ['HEADER', 'NAV-ITEM', 'FOOTER', 'ASIDE', 'FORM-TEXT', 'SVG-TEXT', 'NOSCRIPT', 'script text', 'color:red']) assert.ok(!r.text.includes(gone), `dropped: ${gone}`);++ assert.ok(r.text.startsWith('# Big title\n\nFirst paragraph with a [relative link](https://public.example/docs/intro?x=1&y=2) and © 2026 text.'), r.text);++ assert.ok(r.text.includes('## Things\n\n- one\n- two with [other](https://other.example/page)\n\n'), r.text);++ assert.ok(r.text.includes('```\nline 1\n line 2\n```'));++ assert.deepEqual(r.links, [{ text: 'relative link', href: 'https://public.example/docs/intro?x=1&y=2' }, { text: 'other', href: 'https://other.example/page' }], 'absolute http(s) only, de-duplicated, no javascript:/mailto:/#fragment');++ assert.equal(r.truncated, false);++ const plain = readHtml(PAGE, { baseUrl: 'https://public.example/', format: 'text', maxChars: 20000 });++ assert.ok(plain.text.startsWith('Big title\n\nFirst paragraph with a relative link and'), plain.text);++ assert.ok(!plain.text.includes('](') && !plain.text.includes('```'));++ assert.equal(plain.links.length, 2, 'links are still listed in text mode');++ // No <main>: body minus furniture; many links are capped at 50.++ const many = readHtml('<body><nav>N</nav><p>' + Array.from({ length: 80 }, (_, i) => `<a href="/p${i}">p${i}</a>`).join(' ') + '</p></body>', { baseUrl: 'https://x.example/' });++ assert.equal(many.links.length, 50);++ assert.ok(!many.text.includes('N\n'));++ // Hostile / broken markup does not throw or hang.++ readHtml('<div>'.repeat(5000) + 'deep' + '<a href="' + 'a'.repeat(100000), { baseUrl: 'https://x.example/' });++ readHtml('<<<>>><p <b>>��&bogus;', { baseUrl: 'https://x.example/' });++ const cut = readHtml('<main><p>' + 'sentence one. '.repeat(500) + '</p></main>', { baseUrl: 'https://x.example/', maxChars: 1000 });++ assert.ok(cut.truncated && cut.text.length <= 1000);++ console.log('extraction: ok');++++ // ── Route: SSRF, caps, content types ──++ const app = express();++ const devRouter = createDevRoutes(null, null, { egress });++ app.use('/api/dev', devRouter);++ const srv = await new Promise(res => { const s = app.listen(0, '127.0.0.1', () => res(s)); });++ const base = `http://127.0.0.1:${srv.address().port}`;++ const q = encodeURIComponent;++ const get = async (p) => { const x = await fetch(base + p); return { status: x.status, body: await x.json() }; };++++ for (const u of ['http://localhost:3000/', 'http://127.0.0.1:4000/', 'http://[::1]/', 'http://10.1.1.1/', 'http://192.168.0.1/', 'http://169.254.169.254/latest/meta-data/', 'https://rebind.example/', 'https://lan.example/', 'localhost:4100']) {++ dialled.length = 0;++ const x = await get(`/api/dev/read?url=${q(u)}`);++ assert.equal(x.status, 403, `${u} → ${x.status}`);++ assert.ok(x.body.code && /not a public internet address/.test(x.body.error));++ assert.deepEqual(dialled, [], `${u}: no connection`);++ }++ dialled.length = 0;++ assert.equal((await get(`/api/dev/read?url=${q('https://redirector.example/to-loopback')}`)).status, 403, 'a redirect to loopback is refused');++ assert.deepEqual(dialled, ['93.184.216.35'], 'only the public hop was dialled');++ assert.equal((await get('/api/dev/read')).status, 400);++ assert.equal((await get(`/api/dev/read?url=${q('ftp://public.example/')}`)).status, 400);++ assert.equal((await get('/api/dev/read?url=public.example&format=pdf')).status, 400);++ assert.equal((await get('/api/dev/read?url=public.example&max_chars=10')).status, 400);++ assert.equal((await get('/api/dev/read?url=public.example&max_chars=99999')).status, 400);++ assert.equal((await get('/api/dev/read?url=nowhere.example')).status, 400, 'an unresolvable name is a bad URL');++++ let x = await get('/api/dev/read?url=public.example');++ assert.equal(x.status, 200);++ assert.deepEqual([x.body.url, x.body.status, x.body.content_type, x.body.title, x.body.lang, x.body.truncated], ['https://public.example/', 200, 'text/html', 'Hello & welcome', 'en', false]);++ assert.equal(x.body.chars, x.body.text.length);++ const sent = seenReq[seenReq.length - 1].headers;++ assert.equal(sent['User-Agent'], 'OpenVibeReader/1.0 (+https://read.openvibe.tools)');++ assert.equal(sent.Accept, 'text/html,application/xhtml+xml,text/plain;q=0.9');++++ x = await get(`/api/dev/read?url=${q('https://redirector.example/to-public')}`);++ assert.equal(x.status, 200); assert.equal(x.body.url, 'https://public.example/plain', 'the final URL after redirects');++ assert.deepEqual([x.body.content_type, x.body.text, x.body.title, x.body.links], ['text/plain', 'just some text\nsecond line', '', []], 'plain text is returned as is');++++ x = await get('/api/dev/read?url=public.example/image');++ assert.equal(x.status, 415); assert.equal(x.body.code, 'tools.read.unsupported_content_type');++ assert.equal((await get('/api/dev/read?url=public.example/json')).status, 415);++++ x = await get('/api/dev/read?url=public.example/notfound');++ assert.deepEqual([x.status, x.body.status, x.body.title, x.body.text], [200, 404, 'Gone', 'Nothing here.'], 'the upstream status is reported');++ assert.equal((await get('/api/dev/read?url=public.example/latin1')).body.text, 'café', 'the declared charset is honoured');++++ x = await get('/api/dev/read?url=public.example/big&max_chars=1000');++ assert.equal(x.body.truncated, true); assert.ok(x.body.text.length <= 1000 && x.body.chars === x.body.text.length);++ x = await get('/api/dev/read?url=public.example/huge');++ assert.equal(x.status, 200); assert.equal(x.body.truncated, true, 'the 2 MB raw cap is reported');++ assert.ok(x.body.chars <= 20000, 'and the default max_chars still applies');++ console.log('route: ok');++++ // ── Through the run API ──++ const guard = createGuard({ app: 'gateway', contracts, issuer: 'https://openvibe.network', keys: { get: () => null }, specs: SPECS, env: { TOOLS_GUARD: 'report' }, log: { log() {}, warn() {}, error() {} }, pruneIntervalMs: 0 });++ const runApp = express();++ runApp.use(guard.identify);++ const { createGatewayRun } = require('../server/run');++ const runDev = createDevRoutes(null, null, { egress, guard });++ const toolRegistry = require('../server/registry/descriptors');++ const run = createGatewayRun({++ guard, contracts, toolRegistry, routers: { net: null, dev: runDev }, ports: () => ({}),++ parseJson: express.json({ limit: '64kb' }), jobIndex: new Map(), ...ajvFrom(require),++ });++ runApp.use(run.handle);++ const rsrv = await new Promise(res => { const s = runApp.listen(0, '127.0.0.1', () => res(s)); });++ const rbase = `http://127.0.0.1:${rsrv.address().port}`;++ const post = async (id, body) => { const y = await fetch(`${rbase}/api/v1/tools/${id}/run`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); return { status: y.status, body: await y.json() }; };++++ let y = await post('read', { input: { url: 'public.example', format: 'text', max_chars: 5000 } });++ assert.equal(y.status, 200, JSON.stringify(y.body));++ assert.equal(y.body.state, 'succeeded');++ const data = y.body.result.data;++ const spec = SPECS.find(s => s.id === 'read');++ const { Ajv, addFormats } = ajvFrom(require);++ const ajv = new Ajv({ strict: false }); if (addFormats) addFormats(ajv);++ assert.ok(ajv.validate(spec.output.schema, data), JSON.stringify(ajv.errors));++ assert.equal(data.title, 'Hello & welcome');++ assert.ok(data.text.startsWith('Big title'));++ y = await post('read', { input: { url: 'http://127.0.0.1:3000/' } });++ assert.ok(y.status === 403 || (y.body.error && /public internet/.test(JSON.stringify(y.body))), `run API refuses loopback: ${y.status} ${JSON.stringify(y.body)}`);++ y = await post('read', { input: { url: 'public.example', max_chars: 5 } });++ assert.equal(y.status, 422, 'input validated against the descriptor');++ y = await post('read', { input: {} });++ assert.equal(y.status, 422);++ console.log('run API: ok');++++ srv.closeAllConnections(); srv.close(); rsrv.closeAllConnections(); rsrv.close();++ console.log('read tool: all checks passed');++ process.exit(0);++})().catch(e => { console.error(e); process.exit(1); });@@@ -14,7 +14,7 @@ const NET = ['/robots/:target?', '/sitemap/:target?', '/uptime/:target?','/smtp/:target?', '/blacklist/:target?', '/dnsprop/:target?',];--const DEV = ['/tools', '/opengraph'];++const DEV = ['/tools', '/opengraph', '/read'];const PROCESS = ['/api/process', '/api/process/direct', '/api/process/multi', '/api/process/anything', '/api/process/anything/nested'];const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-sunset-routes-'));@@@ -66,7 +66,7 @@ const get = async (url, headers = {}, method = 'GET') => {const list = r.body;const lc = contracts.tools.checkList(list);assert.ok(lc.valid, JSON.stringify(lc.errors.slice(0, 5)));-- assert.strictEqual(list.count, 169);++ assert.strictEqual(list.count, 170);assert.ok(list.tools.every(d => !d.input || d.input.$ref), 'inputs are $refs in the list');assert.ok(!Number.isNaN(Date.parse(list.updated_at)));assert.deepStrictEqual(list.families.map(f => f.id).sort(), ['audio', 'dev', 'docs', 'img', 'media', 'net', 'pastes', 'places', 'text']);@@@ -150,7 +150,7 @@ const get = async (url, headers = {}, method = 'GET') => {// ── /api/catalog.json is unchanged (Network reads it) ──const catalog = (await get(`${G}/api/catalog.json`)).body;-- assert.strictEqual(catalog.tools.length, 169);++ assert.strictEqual(catalog.tools.length, 170);assert.ok(catalog.tools.every(t => ['available', 'unavailable'].includes(t.status) && t.hosts && t.hosts.canonical));assert.ok(!('api' in catalog.tools[0]) && !('execution' in catalog.tools[0]), 'no descriptor fields leak into the catalogue');
@@ -193,7 +193,8 @@ lorem:{n:'Lorem Ipsum Generator',i:'fa-paragraph',c:'qual',d:'Lorem ipsum genera
@@ -193,7 +193,8 @@ lorem:{n:'Lorem Ipsum Generator',i:'fa-paragraph',c:'qual',d:'Lorem ipsum genera
curlconvert:{n:'Online curl',i:'fa-terminal',c:'http',d:'Parse curl \u2192 fetch, Python, Node.js',p:"curl -X POST https://api.example.com/users -H 'Content-Type: application/json' -H 'Authorization: Bearer tok123' -d '{\"name\":\"openvibe\"}'",m:['To fetch','To Python','To Node','Parsed']},
curlconvert:{n:'Online curl',i:'fa-terminal',c:'http',d:'Parse curl \u2192 fetch, Python, Node.js',p:"curl -X POST https://api.example.com/users -H 'Content-Type: application/json' -H 'Authorization: Bearer tok123' -d '{\"name\":\"openvibe\"}'",m:['To fetch','To Python','To Node','Parsed']},
webhook:{n:'Webhook Tester',i:'fa-satellite-dish',c:'http',d:'Webhook request inspector bin',p:'',m:['Create Bin']},
webhook:{n:'Webhook Tester',i:'fa-satellite-dish',c:'http',d:'Webhook request inspector bin',p:'',m:['Create Bin']},
color:{n:'Color Converter',i:'fa-palette',c:'fe',d:'Color picker & HEX/RGB/HSL converter',p:'#a78bfa',m:['Convert']},
color:{n:'Color Converter',i:'fa-palette',c:'fe',d:'Color picker & HEX/RGB/HSL converter',p:'#a78bfa',m:['Convert']},
-opengraph:{n:'Open Graph Preview',i:'fa-share-nodes',c:'fe',d:'Open Graph & Twitter Card preview',p:'https://openvibe.live',m:['Preview']}
+opengraph:{n:'Open Graph Preview',i:'fa-share-nodes',c:'fe',d:'Open Graph & Twitter Card preview',p:'https://openvibe.live',m:['Preview']},
+read:{n:'Web Page Reader',i:'fa-book-open',c:'fe',d:'Read any web page as clean text or Markdown',p:'https://openvibe.live',m:['Markdown','Text']}
};
};
const CATS={data:{l:'Data & Formats',i:'fa-database'},enc:{l:'Encoding & Crypto',i:'fa-lock'},time:{l:'Time & Scheduling',i:'fa-clock'},qual:{l:'Code Quality',i:'fa-wand-magic-sparkles'},http:{l:'HTTP & API',i:'fa-terminal'},fe:{l:'Frontend & SEO',i:'fa-palette'}};
const CATS={data:{l:'Data & Formats',i:'fa-database'},enc:{l:'Encoding & Crypto',i:'fa-lock'},time:{l:'Time & Scheduling',i:'fa-clock'},qual:{l:'Code Quality',i:'fa-wand-magic-sparkles'},http:{l:'HTTP & API',i:'fa-terminal'},fe:{l:'Frontend & SEO',i:'fa-palette'}};
@@ -409,6 +410,13 @@ P.opengraph=async(s)=>{
@@ -409,6 +410,13 @@ P.opengraph=async(s)=>{
+(d.seo?'<div style="margin-top:14px"><div class="jl">SEO Recommendations</div>'+d.seo.map(x=>'<div style="color:'+(x.type==='error'?'#f87171':x.type==='warning'?'#fbbf24':'#34d399')+'">'+(x.type==='error'?'\u2717':x.type==='warning'?'\u26a0':'\u2713')+' '+H(x.message)+'</div>').join('')+'</div>':'')};
+(d.seo?'<div style="margin-top:14px"><div class="jl">SEO Recommendations</div>'+d.seo.map(x=>'<div style="color:'+(x.type==='error'?'#f87171':x.type==='warning'?'#fbbf24':'#34d399')+'">'+(x.type==='error'?'\u2717':x.type==='warning'?'\u26a0':'\u2713')+' '+H(x.message)+'</div>').join('')+'</div>':'')};
};
};
+P.read=async(s,m)=>{
+ const r=await fetch('/api/v1/tools/read/run',{method:'POST',headers:{'Content-Type':'application/json'},credentials:'same-origin',body:JSON.stringify({input:{url:s.trim(),format:m==='Text'?'text':'markdown'}})});
+ const b=await r.json();if(b.state!=='succeeded'||!b.result)throw new Error((b.error&&(b.error.detail||b.error.title))||b.detail||'Failed to read the page');
+ const d=b.result.data;
+ return (d.title?d.title+'\n':'')+(d.description?d.description+'\n':'')+(d.title||d.description?'\n':'')+d.text+(d.truncated?'\n\n[truncated]':'')+(d.links.length?'\n\nLinks\n'+d.links.map(l=>'- '+l.text+' \u2192 '+l.href).join('\n'):'');
+};
+
// ═══════════════ UTILITY ═══════════════
// ═══════════════ UTILITY ═══════════════
function copyOut(){navigator.clipboard.writeText(($('out').innerText||'')).then(()=>toast('Copied!')).catch(()=>{})}
function copyOut(){navigator.clipboard.writeText(($('out').innerText||'')).then(()=>toast('Copied!')).catch(()=>{})}
// The output becomes an unlisted OpenVibe.Community paste (through this gateway's /api/pastes proxy, as
// The output becomes an unlisted OpenVibe.Community paste (through this gateway's /api/pastes proxy, as
@@ -56,6 +56,8 @@ const DEV_TOOLS = [
@@ -56,6 +56,8 @@ const DEV_TOOLS = [
{ id: 'color', subdomain: 'color', name: 'OpenVibeColor', icon: 'fa-palette', desc: 'Color picker, HEX/RGB/HSL converter & palette generator', category: 'frontend' },
{ id: 'color', subdomain: 'color', name: 'OpenVibeColor', icon: 'fa-palette', desc: 'Color picker, HEX/RGB/HSL converter & palette generator', category: 'frontend' },
{ id: 'opengraph', subdomain: 'opengraph', name: 'OpenVibeOpenGraph', icon: 'fa-share-nodes', desc: 'Open Graph & Twitter Card preview & validator', category: 'frontend' },
{ id: 'opengraph', subdomain: 'opengraph', name: 'OpenVibeOpenGraph', icon: 'fa-share-nodes', desc: 'Open Graph & Twitter Card preview & validator', category: 'frontend' },
+ { id: 'read', subdomain: 'read', name: 'OpenVibeReader', icon: 'fa-book-open', desc: 'Read any web page as clean text or Markdown: title, description, main text and links', category: 'frontend' },
+
// Single-purpose tools: one job per address (search engines and people both look for the exact job).
// Single-purpose tools: one job per address (search engines and people both look for the exact job).
{ id: 'jsminify', subdomain: 'jsminify', name: "JavaScript Minifier", icon: 'fa-compress', desc: "Shrink JavaScript with a real parser, not regex", category: 'quality' },
{ id: 'jsminify', subdomain: 'jsminify', name: "JavaScript Minifier", icon: 'fa-compress', desc: "Shrink JavaScript with a real parser, not regex", category: 'quality' },
{ id: 'jsformat', subdomain: 'jsformat', name: "JavaScript Beautifier", icon: 'fa-wand-magic-sparkles', desc: "Turn minified or messy JavaScript into readable code", category: 'quality' },
{ id: 'jsformat', subdomain: 'jsformat', name: "JavaScript Beautifier", icon: 'fa-wand-magic-sparkles', desc: "Turn minified or messy JavaScript into readable code", category: 'quality' },
@@ -86,6 +88,8 @@ const DEV_ALIASES = {
@@ -86,6 +88,8 @@ const DEV_ALIASES = {
colours: 'color',
colours: 'color',
colors: 'color',
colors: 'color',
og: 'opengraph',
og: 'opengraph',
+ reader: 'read',
+ readability: 'read',
guid: 'uuid',
guid: 'uuid',
unix: 'timestamp',
unix: 'timestamp',
unixtime: 'timestamp',
unixtime: 'timestamp',
@@ -87,6 +87,33 @@ const SPECS = [
@@ -87,6 +87,33 @@ const SPECS = [
route: { method: 'GET', path: '/api/dev/opengraph', query: { url: '{url}' } },
route: { method: 'GET', path: '/api/dev/opengraph', query: { url: '{url}' } },
example: { input: { url: 'https://openvibe.tools' } },
example: { input: { url: 'https://openvibe.tools' } },
},
},
+ {
+ id: 'read', execution: 'sync', api: true,
+ input: obj({
+ url: { type: 'string', minLength: 1, maxLength: 2048, description: 'A page URL (https:// is assumed when missing)' },
+ format: { enum: ['markdown', 'text'], default: 'markdown', description: 'markdown keeps headings, list items and [links](href); text is plain' },
+ max_chars: { type: 'integer', minimum: 1000, maximum: 50000, default: 20000, description: 'Longest text to return; longer pages are cut and truncated is true' },
+ }, ['url']),
+ files: null,
+ output: {
+ kind: 'json', schema: {
+ type: 'object', required: ['url', 'status', 'title', 'text', 'truncated', 'chars', 'links'],
+ properties: {
+ url: { type: 'string', description: 'The final URL after redirects' },
+ status: { type: 'integer' }, content_type: { type: 'string' },
+ title: { type: 'string' }, description: { type: 'string' }, lang: { type: 'string' },
+ text: { type: 'string', description: 'The main text of the page' },
+ truncated: { type: 'boolean', description: 'The page or the text was cut at a limit' },
+ chars: { type: 'integer', description: 'Length of text' },
+ links: { type: 'array', maxItems: 50, items: { type: 'object', required: ['text', 'href'], properties: { text: { type: 'string' }, href: { type: 'string' } } } },
+ },
+ },
+ },
+ limits: { timeoutMs: 14000, maxInputBytes: 4 * KiB, perTargetPerMinute: 10 },
+ auth: ANYONE, quotaClass: 'tools-fetch', cost: 2, egress: true,
+ route: { method: 'GET', path: '/api/dev/read', query: { url: '{url}' } },
+ example: { input: { url: 'https://openvibe.tools' } },
+ },
page('jsminify'), page('jsformat'),
page('jsminify'), page('jsformat'),
engine('cssminify', obj({ text: text() }), TEXT, { text: '.a { color: #aabbcc; margin: 0px; }' }),
engine('cssminify', obj({ text: text() }), TEXT, { text: '.a { color: #aabbcc; margin: 0px; }' }),
engine('cssformat', obj({ text: text() }), TEXT, { text: '.a{color:red}' }),
engine('cssformat', obj({ text: text() }), TEXT, { text: '.a{color:red}' }),
@@ -103,11 +130,11 @@ const SPECS = [
@@ -103,11 +130,11 @@ const SPECS = [
];
];
// The run API reuses an engine's answer for 10 minutes (same tool, same input): not for these, whose
// The run API reuses an engine's answer for 10 minutes (same tool, same input): not for these, whose
-// output is random or depends on the time. Open Graph (a page's tags) may be reused for 10 minutes.
+// output is random or depends on the time. Open Graph (a page's tags) and the page reader may be reused for 10 minutes.
const FRESH = new Set(['jwt', 'uuid', 'timestamp', 'cron', 'lorem']);
const FRESH = new Set(['jwt', 'uuid', 'timestamp', 'cron', 'lorem']);
for (const spec of SPECS) {
for (const spec of SPECS) {
if (FRESH.has(spec.id)) spec.cacheTtlMs = 0;
if (FRESH.has(spec.id)) spec.cacheTtlMs = 0;
- if (spec.id === 'opengraph') spec.cacheTtlMs = 10 * 60 * 1000;
+ if (spec.id === 'opengraph' || spec.id === 'read') spec.cacheTtlMs = 10 * 60 * 1000;
}
}
module.exports = { SPECS };
module.exports = { SPECS };
new file mode 100644
new file mode 100644
@@ -0,0 +1,271 @@
@@ -0,0 +1,271 @@
+'use strict';
+
+// ═══════════════════════════════════════════════════════════════
+// Reader: HTML → readable text, with no dependencies.
+//
+// readHtml(html, { baseUrl, format, maxChars }) → { title, description, lang, text, truncated,
+// links }. A forgiving tokenizer builds a small tree (void elements, raw-text elements, implied
+// </p> and </li>); the walk keeps <main> or <article> when there is one, drops navigation and
+// page furniture, and writes headings as `#` lines, list items as `- `, links as [text](href).
+// ═══════════════════════════════════════════════════════════════
+
+const VOID = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr']);
+const RAW = new Set(['script', 'style', 'textarea', 'title']);
+const SKIP = new Set(['head', 'nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript', 'svg', 'form', 'iframe', 'template', 'select', 'textarea', 'button', 'object', 'canvas', 'math']);
+const BLOCK = new Set(['p', 'div', 'section', 'article', 'main', 'ul', 'ol', 'li', 'blockquote', 'pre', 'table', 'tr', 'figure', 'figcaption', 'dl', 'dt', 'dd', 'address', 'details', 'summary', 'fieldset', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'caption', 'thead', 'tbody', 'tfoot']);
+const MAX_DEPTH = 256; // deeper nesting is flattened into the parent: no recursion blow-ups
+const MAX_LINKS = 50;
+const MAX_ATTR_TEXT = 300;
+
+const ENTITIES = {
+ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ', shy: '',
+ mdash: '—', ndash: '–', hellip: '…', copy: '©', reg: '®', trade: '™', laquo: '«', raquo: '»',
+ lsquo: '‘', rsquo: '’', sbquo: '‚', ldquo: '“', rdquo: '”', bdquo: '„', bull: '•', middot: '·',
+ times: '×', divide: '÷', euro: '€', pound: '£', yen: '¥', cent: '¢', sect: '§', para: '¶',
+ deg: '°', plusmn: '±', frac12: '½', frac14: '¼', frac34: '¾', larr: '←', rarr: '→', uarr: '↑',
+ darr: '↓', hearts: '♥', check: '✓', iexcl: '¡', iquest: '¿', eacute: 'é', egrave: 'è', agrave: 'à',
+ aacute: 'á', uuml: 'ü', ouml: 'ö', auml: 'ä', szlig: 'ß', ntilde: 'ñ', ccedil: 'ç',
+};
+
+function decodeEntities(s) {
+ if (!s || s.indexOf('&') === -1) return s || '';
+ return s.replace(/&(#x[0-9a-f]{1,6}|#[0-9]{1,7}|[a-z][a-z0-9]{1,9});/gi, (m, e) => {
+ if (e[0] === '#') {
+ const cp = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
+ if (!cp || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff)) return '';
+ return cp === 0xa0 ? ' ' : String.fromCodePoint(cp);
+ }
+ const v = ENTITIES[e] !== undefined ? ENTITIES[e] : ENTITIES[e.toLowerCase()];
+ return v === undefined ? m : v;
+ });
+}
+
+// ── Tokenizer + tree ─────────────────────────────────────────
+function parseAttrs(src) {
+ const attrs = {};
+ const re = /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
+ let m;
+ while ((m = re.exec(src))) {
+ const k = m[1].toLowerCase();
+ if (attrs[k] === undefined) attrs[k] = decodeEntities(m[2] !== undefined ? m[2] : m[3] !== undefined ? m[3] : m[4] || '');
+ }
+ return attrs;
+}
+
+function parse(html) {
+ const root = { tag: '#root', attrs: {}, children: [], parent: null };
+ let cur = root, depth = 0;
+ const open = []; // element stack (cur is its top)
+ const pushText = (t) => { if (t) cur.children.push({ text: decodeEntities(t) }); };
+ const tokens = /<!--[\s\S]*?(?:-->|$)|<![^>]*>|<\?[^>]*>|<\/([a-zA-Z][^\s>\/]*)[^>]*>|<([a-zA-Z][^\s>\/]*)((?:"[^"]*"|'[^']*'|[^>"'])*)>/g;
+ let last = 0, m;
+ while ((m = tokens.exec(html))) {
+ if (m.index > last) pushText(html.slice(last, m.index));
+ last = tokens.lastIndex;
+ if (m[1]) { // </tag>
+ const name = m[1].toLowerCase();
+ for (let i = open.length - 1; i >= 0; i--) {
+ if (open[i].tag === name) { open.length = i; cur = i ? open[i - 1] : root; depth = open.length; break; }
+ }
+ } else if (m[2]) { // <tag ...>
+ const name = m[2].toLowerCase();
+ const attrs = parseAttrs(m[3] || '');
+ // Implied end tags: a new <li> closes the open one in the same list, a new block closes an open <p>.
+ if (name === 'li') {
+ for (let i = open.length - 1; i >= 0; i--) {
+ if (/^(ul|ol|menu)$/.test(open[i].tag)) break;
+ if (open[i].tag === 'li') { open.length = i; cur = i ? open[i - 1] : root; depth = open.length; break; }
+ }
+ } else if (cur.tag === 'p' && BLOCK.has(name)) {
+ open.pop(); cur = open.length ? open[open.length - 1] : root; depth = open.length;
+ }
+ const node = { tag: name, attrs, children: [], parent: cur };
+ cur.children.push(node);
+ const selfClose = /\/\s*$/.test(m[3] || '');
+ if (VOID.has(name) || (selfClose && !RAW.has(name))) continue;
+ if (RAW.has(name)) { // everything up to the matching close tag is text
+ const close = new RegExp('</' + name + '\\s*>', 'ig');
+ close.lastIndex = tokens.lastIndex;
+ const c = close.exec(html);
+ const end = c ? c.index : html.length;
+ const body = html.slice(tokens.lastIndex, end);
+ if (body && (name === 'title' || name === 'textarea')) node.children.push({ text: body });
+ tokens.lastIndex = last = c ? close.lastIndex : html.length;
+ continue;
+ }
+ if (depth >= MAX_DEPTH) continue; // too deep: its children stay with the parent
+ open.push(node); cur = node; depth++;
+ }
+ }
+ if (last < html.length) pushText(html.slice(last));
+ return root;
+}
+
+function find(node, pred) {
+ const stack = [node];
+ while (stack.length) {
+ const n = stack.pop();
+ if (n.tag && pred(n)) return n;
+ if (n.children) for (let i = n.children.length - 1; i >= 0; i--) stack.push(n.children[i]);
+ }
+ return null;
+}
+function rawText(n) {
+ if (n.text !== undefined) return n.text;
+ return (n.children || []).map(rawText).join('');
+}
+const collapse = (s) => s.replace(/[\s ]+/g, ' ').trim();
+
+function metaContent(root, ...names) {
+ const wanted = new Set(names);
+ const found = {};
+ const stack = [root];
+ while (stack.length) {
+ const n = stack.pop();
+ if (n.tag === 'meta') {
+ const k = (n.attrs.name || n.attrs.property || '').toLowerCase();
+ if (wanted.has(k) && n.attrs.content && !found[k]) found[k] = collapse(decodeEntities(n.attrs.content));
+ }
+ if (n.children) for (let i = n.children.length - 1; i >= 0; i--) stack.push(n.children[i]);
+ }
+ for (const k of names) if (found[k]) return found[k];
+ return '';
+}
+
+// ── Output ───────────────────────────────────────────────────
+function absolute(href, baseUrl) {
+ if (!href || /^\s*(javascript|data|mailto|tel|blob):/i.test(href)) return null;
+ try {
+ const u = new URL(decodeEntities(href).trim(), baseUrl);
+ if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
+ if (u.username || u.password) return null;
+ return u.href;
+ } catch { return null; }
+}
+
+function render(rootNode, { baseUrl, markdown }) {
+ const blocks = []; // { text, list }
+ const links = [];
+ const seen = new Set();
+ const pageNoHash = (() => { try { const u = new URL(baseUrl); u.hash = ''; return u.href; } catch { return ''; } })();
+ let buf = '', prefix = '', flushes = 0, listDepth = 0, quote = 0, pre = 0, listItem = false;
+
+ function flush() {
+ const lines = buf.split('\n').map(collapse).filter(Boolean);
+ buf = '';
+ if (!lines.length) { prefix = ''; listItem = false; return; }
+ const pad = '> '.repeat(quote);
+ blocks.push({ text: lines.map((l, i) => (i === 0 ? pad + prefix : pad + (prefix ? ' '.repeat(prefix.length) : '')) + l).join('\n'), list: listItem });
+ prefix = ''; listItem = false; flushes++;
+ }
+ function addLink(text, href) {
+ if (links.length >= MAX_LINKS || seen.has(href)) return;
+ const noHash = href.replace(/#.*$/, '');
+ if (noHash === pageNoHash) return;
+ seen.add(href);
+ links.push({ text: collapse(text).slice(0, MAX_ATTR_TEXT) || href, href });
+ }
+
+ function walk(n) {
+ if (n.text !== undefined) { buf += pre ? n.text : n.text.replace(/\s+/g, ' '); return; }
+ const t = n.tag;
+ if (SKIP.has(t) || n.attrs.hidden !== undefined || /^true$/i.test(n.attrs['aria-hidden'] || '')) return;
+ if (t === 'br') { buf += '\n'; return; }
+ if (t === 'hr') { flush(); if (markdown) blocks.push({ text: '---', list: false }); return; }
+ if (t === 'img') { return; }
+ if (t === 'pre') {
+ flush();
+ const body = rawText(n).replace(/\r/g, '').replace(/^\n+|\s+$/g, '');
+ if (body) blocks.push({ text: markdown ? '```\n' + body + '\n```' : body, list: false, raw: true });
+ return;
+ }
+ if (/^h[1-6]$/.test(t)) {
+ flush();
+ if (markdown) prefix = '#'.repeat(Number(t[1])) + ' ';
+ kids(n); flush(); return;
+ }
+ if (t === 'ul' || t === 'ol' || t === 'menu') { flush(); listDepth++; kids(n); flush(); listDepth--; return; }
+ if (t === 'li') {
+ flush();
+ prefix = ' '.repeat(Math.max(0, listDepth - 1)) + '- ';
+ listItem = true;
+ // A nested list ends this item's own text first, so it is flushed by the child list.
+ kids(n); flush(); return;
+ }
+ if (t === 'blockquote') { flush(); quote++; kids(n); flush(); quote--; return; }
+ if (t === 'td' || t === 'th') { if (collapse(buf)) buf += ' | '; kids(n); return; }
+ if (t === 'tr') { flush(); kids(n); buf = buf.replace(/\s*\|\s*$/, ''); flush(); return; }
+ if (t === 'a') {
+ const href = absolute(n.attrs.href, baseUrl);
+ const start = buf.length, before = flushes;
+ kids(n);
+ if (!href || href.replace(/#.*$/, '') === pageNoHash) return; // not a link, or the page itself
+ const inner = collapse(buf.slice(start)) || collapse(n.attrs['aria-label'] || n.attrs.title || '');
+ addLink(inner, href);
+ if (markdown && flushes === before && inner && start <= buf.length) {
+ const lead = /^\s/.test(buf.slice(start)) ? ' ' : '', trail = /\s$/.test(buf.slice(start)) ? ' ' : '';
+ buf = buf.slice(0, start) + lead + '[' + inner.replace(/([\[\]])/g, '\\$1') + '](' + href.replace(/\)/g, '%29').replace(/\s/g, '%20') + ')' + trail;
+ }
+ return;
+ }
+ if (t === 'code' && markdown && !pre) {
+ const c = collapse(rawText(n));
+ if (c) buf += '`' + c.replace(/`/g, "'") + '`';
+ return;
+ }
+ if (BLOCK.has(t)) { flush(); kids(n); flush(); return; }
+ kids(n);
+ }
+ function kids(n) { for (const c of n.children) walk(c); }
+
+ walk(rootNode);
+ flush();
+ let text = '';
+ blocks.forEach((b, i) => {
+ if (i) text += (b.list && blocks[i - 1].list) ? '\n' : '\n\n';
+ text += b.text;
+ });
+ return { text, links };
+}
+
+function readHtml(html, { baseUrl, format = 'markdown', maxChars = 20000 } = {}) {
+ const doc = parse(html);
+ const markdown = format !== 'text';
+ const htmlEl = find(doc, n => n.tag === 'html');
+ const titleEl = find(doc, n => n.tag === 'title');
+ let title = titleEl ? collapse(decodeEntities(rawText(titleEl))) : '';
+ if (!title) title = metaContent(doc, 'og:title', 'twitter:title');
+ const description = metaContent(doc, 'description', 'og:description', 'twitter:description');
+ const lang = ((htmlEl && htmlEl.attrs.lang) || '').trim().slice(0, 35);
+
+ const body = find(doc, n => n.tag === 'body') || doc;
+ const main = find(body, n => n.tag === 'main' || (n.attrs.role || '').toLowerCase() === 'main') || find(body, n => n.tag === 'article');
+ let out = render(main || body, { baseUrl, markdown });
+ // A <main>/<article> that holds almost nothing (or whose header was dropped) is not the page: use the body.
+ if (main && out.text.length < 200) {
+ const whole = render(body, { baseUrl, markdown });
+ if (whole.text.length > out.text.length) out = whole;
+ }
+ if (!title) { const h1 = find(body, n => n.tag === 'h1'); if (h1) title = collapse(decodeEntities(rawText(h1))); }
+
+ let text = out.text;
+ let truncated = false;
+ if (text.length > maxChars) {
+ text = text.slice(0, maxChars);
+ const cut = Math.max(text.lastIndexOf('\n'), text.lastIndexOf('. '));
+ if (cut > maxChars * 0.8) text = text.slice(0, cut + 1);
+ text = text.trimEnd();
+ truncated = true;
+ }
+ return { title: title.slice(0, 500), description: description.slice(0, 1000), lang, text, truncated, links: out.links };
+}
+
+/** Decode a response body using the header's charset, then a <meta charset>, then UTF-8. */
+function decodeBody(buf, contentType) {
+ let cs = /charset\s*=\s*"?([\w.:-]+)/i.exec(contentType || '');
+ if (!cs) cs = /<meta[^>]+charset\s*=\s*["']?\s*([\w.:-]+)/i.exec(buf.subarray(0, 4096).toString('latin1'));
+ const label = cs ? cs[1].toLowerCase() : 'utf-8';
+ try { return new TextDecoder(label).decode(buf); } catch { return buf.toString('utf8'); }
+}
+
+module.exports = { readHtml, decodeBody, decodeEntities };
@@ -17,6 +17,7 @@ const { DEV_TOOLS } = require('./config');
@@ -17,6 +17,7 @@ const { DEV_TOOLS } = require('./config');
const { createEgress, TargetRefused } = require('../../../_shared/egress');
const { createEgress, TargetRefused } = require('../../../_shared/egress');
const { createCallerResolver } = require('../../../_shared/guard/caller');
const { createCallerResolver } = require('../../../_shared/guard/caller');
const guardLimits = require('../../../_shared/guard/limits');
const guardLimits = require('../../../_shared/guard/limits');
+const { readHtml, decodeBody } = require('./reader');
// ── In-memory webhook bin storage ────────────────────────────
// ── In-memory webhook bin storage ────────────────────────────
const webhookBins = new Map(); // binId → { created, requests, owner, ipKey }
const webhookBins = new Map(); // binId → { created, requests, owner, ipKey }
@@ -33,6 +34,8 @@ setInterval(cleanupBins, 5 * 60 * 1000).unref();
@@ -33,6 +34,8 @@ setInterval(cleanupBins, 5 * 60 * 1000).unref();
// ── Helpers ──────────────────────────────────────────────────
// ── Helpers ──────────────────────────────────────────────────
const OG_MAX_BYTES = 2 * 1024 * 1024; // Open Graph tags live in <head>; never buffer more than this
const OG_MAX_BYTES = 2 * 1024 * 1024; // Open Graph tags live in <head>; never buffer more than this
+const READ_MAX_BYTES = 2 * 1024 * 1024; // the reader never buffers more than this of a page
+const READ_USER_AGENT = 'OpenVibeReader/1.0 (+https://read.openvibe.tools)';
function extractOGTags(html) {
function extractOGTags(html) {
const tags = {};
const tags = {};
@@ -164,7 +167,7 @@ module.exports = function createDevRoutes(db, requireAuth, opts = {}) {
@@ -164,7 +167,7 @@ module.exports = function createDevRoutes(db, requireAuth, opts = {}) {
const guard = opts.guard || null;
const guard = opts.guard || null;
// Every dev API call (and the Open Graph fetcher's own tools-fetch quota, below).
// Every dev API call (and the Open Graph fetcher's own tools-fetch quota, below).
- if (guard) router.use(guard.toolQuota((req) => (req.method === 'GET' && req.path === '/opengraph' ? 'opengraph' : null)));
+ if (guard) router.use(guard.toolQuota((req) => (req.method === 'GET' && (req.path === '/opengraph' || req.path === '/read') ? req.path.slice(1) : null)));
// ── List all dev tools ───────────────────────────────────
// ── List all dev tools ───────────────────────────────────
router.get('/tools', (_req, res) => {
router.get('/tools', (_req, res) => {
@@ -246,6 +249,78 @@ module.exports = function createDevRoutes(db, requireAuth, opts = {}) {
@@ -246,6 +249,78 @@ module.exports = function createDevRoutes(db, requireAuth, opts = {}) {
}
}
});
});
+ // ── Web page reader ──────────────────────────────────────
+ // A page's readable text (title, description, main text as light Markdown, outgoing links),
+ // fetched through the same SSRF guard as Open Graph.
+ router.get('/read', async (req, res) => {
+ let targetUrl = String(req.query.url || req.query.target || '').trim();
+ if (!targetUrl) return res.status(400).json({ error: 'Missing ?url= parameter' });
+ const format = req.query.format === undefined || req.query.format === '' ? 'markdown' : String(req.query.format);
+ if (format !== 'markdown' && format !== 'text') return res.status(400).json({ error: 'format must be "markdown" or "text"' });
+ let maxChars = 20000;
+ if (req.query.max_chars !== undefined && req.query.max_chars !== '') {
+ maxChars = Number(req.query.max_chars);
+ if (!Number.isInteger(maxChars) || maxChars < 1000 || maxChars > 50000) return res.status(400).json({ error: 'max_chars must be an integer from 1000 to 50000' });
+ }
+ if (targetUrl.length > 2048) return res.status(400).json({ error: 'URL is too long' });
+ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(targetUrl) && !/^https?:\/\//i.test(targetUrl)) return res.status(400).json({ error: 'Only http and https URLs are supported' });
+ if (!/^https?:\/\//i.test(targetUrl)) targetUrl = 'https://' + targetUrl;
+ try {
+ new URL(targetUrl);
+ } catch {
+ return res.status(400).json({ error: 'Invalid URL' });
+ }
+ // Per-target throttle (descriptor read: limits.perTargetPerMinute), across every caller.
+ if (guard && !guard.target(req, res, { tool: 'read', target: targetUrl })) return;
+
+ try {
+ const response = await egress.follow(targetUrl, {
+ method: 'GET',
+ headers: {
+ 'User-Agent': READ_USER_AGENT,
+ 'Accept': 'text/html,application/xhtml+xml,text/plain;q=0.9',
+ },
+ timeoutMs: 10000,
+ maxBytes: READ_MAX_BYTES,
+ maxRedirects: 5,
+ });
+ const finalUrl = response.url || targetUrl;
+ const contentType = String(response.headers['content-type'] || '').split(';')[0].trim().toLowerCase();
+ const raw = response.body;
+ const sniffedHtml = !contentType && /^\s*(<!doctype html|<html|<head|<body)/i.test(raw.subarray(0, 512).toString('latin1'));
+ const isHtml = /^(text\/html|application\/xhtml\+xml)$/.test(contentType) || sniffedHtml;
+ const isText = contentType === 'text/plain';
+ if (!isHtml && !isText) {
+ return res.status(415).json({ error: `Not a readable page (${contentType || 'unknown content type'}); the reader handles HTML and plain text`, code: 'tools.read.unsupported_content_type', content_type: contentType || null });
+ }
+ const decoded = decodeBody(raw, response.headers['content-type']);
+ let out;
+ if (isHtml) {
+ out = readHtml(decoded, { baseUrl: finalUrl, format, maxChars });
+ } else {
+ const plain = decoded.replace(/\r\n?/g, '\n').trim();
+ out = { title: '', description: '', lang: '', text: plain.slice(0, maxChars), truncated: plain.length > maxChars, links: [] };
+ }
+ res.json({
+ ok: true,
+ url: finalUrl,
+ status: response.status,
+ content_type: contentType || (isHtml ? 'text/html' : 'text/plain'),
+ title: out.title,
+ description: out.description,
+ lang: out.lang,
+ text: out.text,
+ truncated: out.truncated || Boolean(response.truncated),
+ chars: out.text.length,
+ links: out.links,
+ });
+ } catch (err) {
+ if (err instanceof TargetRefused) return res.status(403).json({ error: err.message, code: err.code });
+ if (err && err.status === 400) return res.status(400).json({ error: err.message });
+ res.status(502).json({ error: `Failed to fetch: ${err.message}` });
+ }
+ });
+
// ── Webhook Bins ─────────────────────────────────────────
// ── Webhook Bins ─────────────────────────────────────────
// Their own router (above): the bins' API, reachable at /api/dev/webhook/* on the gateway.
// Their own router (above): the bins' API, reachable at /api/dev/webhook/* on the gateway.
router.use('/webhook', createWebhookRouter(opts));
router.use('/webhook', createWebhookRouter(opts));
@@ -238,6 +238,14 @@ const DEV = {
@@ -238,6 +238,14 @@ const DEV = {
bullets: ['Live preview of the social card for any public URL', 'Lists Open Graph, Twitter Card and standard meta tags', 'Flags missing titles, descriptions and images'],
bullets: ['Live preview of the social card for any public URL', 'Lists Open Graph, Twitter Card and standard meta tags', 'Flags missing titles, descriptions and images'],
faq: [['Why does my link preview show no image?', 'Usually a missing og:image tag, or an image URL that is relative rather than absolute. Social platforms need a full https URL.'], ['Why has my preview not updated after I fixed the tags?', 'Platforms cache previews aggressively. Most offer a debugger that forces a re-scrape; this tool always fetches live so you can confirm the tags themselves are correct.']],
faq: [['Why does my link preview show no image?', 'Usually a missing og:image tag, or an image URL that is relative rather than absolute. Social platforms need a full https URL.'], ['Why has my preview not updated after I fixed the tags?', 'Platforms cache previews aggressively. Most offer a debugger that forces a re-scrape; this tool always fetches live so you can confirm the tags themselves are correct.']],
},
},
+ read: {
+ name: 'Web Page Reader', title: 'Website to Text & URL to Markdown Converter — Web Page Reader',
+ desc: 'Read any web page as clean text or Markdown: title, description, main text and links. A reader mode for any URL, and an API for scripts and AI agents.',
+ kw: 'website to text, url to markdown, reader mode, extract text from webpage, web page to markdown, webpage reader, scrape article text',
+ about: 'Enter a URL and the reader fetches the page and returns what a person would read: its title and description, the main text as light Markdown (headings, lists and links kept) and the page\'s outgoing links. Navigation, headers, footers, scripts and styles are left out. The same result is available as JSON from the tools API, which is how scripts and AI agents read pages.',
+ bullets: ['Title, description, main text and up to 50 outgoing links', 'Markdown with headings, lists and links, or plain text', 'Only public web addresses: private and internal targets are refused'],
+ faq: [['Does it run JavaScript?', 'No. It reads the HTML the server sends, which covers articles, documentation and most blogs. A page that builds its content in the browser will come back mostly empty.'], ['Is there a size limit?', 'The page is read up to about 2 MB and the text is cut at max_chars (20,000 by default, up to 50,000); the result says when it was truncated.'], ['Can I call it from code?', 'Yes: POST /api/v1/tools/read/run with {"input":{"url":"https://example.com"}}. No account is needed within the free limits.']],
+ },
jsminify: {
jsminify: {
name: "JavaScript Minifier", title: "JavaScript Minifier \u2014 Minify JS Online",
name: "JavaScript Minifier", title: "JavaScript Minifier \u2014 Minify JS Online",
desc: "Shrink JavaScript with a real parser, not regex. Paste JavaScript and get a smaller file that behaves the same.",
desc: "Shrink JavaScript with a real parser, not regex. Paste JavaScript and get a smaller file that behaves the same.",
@@ -192,7 +192,7 @@ for (const m of ['jsonfmt', 'md', 'codediff', 'slugify', 'entities']) { assert.o
@@ -192,7 +192,7 @@ for (const m of ['jsonfmt', 'md', 'codediff', 'slugify', 'entities']) { assert.o
// ── Counts ──
// ── Counts ──
const tally = (f) => snap.tools.reduce((o, d) => { const k = f(d); o[k] = (o[k] || 0) + 1; return o; }, {});
const tally = (f) => snap.tools.reduce((o, d) => { const k = f(d); o[k] = (o[k] || 0) + 1; return o; }, {});
const fam = tally(d => d.family);
const fam = tally(d => d.family);
-assert.deepStrictEqual(fam, { dev: 33, net: 38, img: 15, audio: 36, docs: 10, text: 33, media: 1, places: 2, pastes: 1 });
+assert.deepStrictEqual(fam, { dev: 34, net: 38, img: 15, audio: 36, docs: 10, text: 33, media: 1, places: 2, pastes: 1 });
fs.rmSync(process.env.DATA_DIR, { recursive: true, force: true });
fs.rmSync(process.env.DATA_DIR, { recursive: true, force: true });
console.log(`descriptors: ${snap.tools.length} tools valid (${Object.entries(tally(d => `${d.execution}${d.api ? '+api' : ''}`)).map(([k, n]) => `${k} ${n}`).join(', ')}); ${jobs} job tools pass their app's validation; ${syncApi} sync tools run through served routes`);
console.log(`descriptors: ${snap.tools.length} tools valid (${Object.entries(tally(d => `${d.execution}${d.api ? '+api' : ''}`)).map(([k, n]) => `${k} ${n}`).join(', ')}); ${jobs} job tools pass their app's validation; ${syncApi} sync tools run through served routes`);
new file mode 100644
new file mode 100644
@@ -0,0 +1,206 @@
@@ -0,0 +1,206 @@
+'use strict';
+// The "read" tool (web page reader): HTML → title, description, readable Markdown/text, links; the
+// shared SSRF guard (private targets 403, redirects re-checked); the raw size cap; content types;
+// and the run API (POST /api/v1/tools/read/run) answering JSON that matches the descriptor's output
+// schema. Mock resolver and transports: nothing touches a real network.
+const assert = require('assert');
+const express = require('express');
+const { EventEmitter } = require('events');
+const { Readable } = require('stream');
+const contracts = require('openvibe-contracts');
+const { createEgress } = require('../../_shared/egress');
+const { createGuard } = require('../../_shared/guard');
+const { ajvFrom } = require('../../_shared/tools/run');
+const createDevRoutes = require('../server/dev/routes');
+const { readHtml } = require('../server/dev/reader');
+const { SPECS } = require('../server/dev/descriptors');
+
+const PAGE = `<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><title>Hello & welcome</title>
+<meta name="description" content="A page about "things"">
+<style>body{color:red}</style><script>var hidden = "script text";</script></head>
+<body>
+<header><a href="/home">HEADER-LINK</a> HEADER TEXT</header>
+<nav><ul><li><a href="/nav">NAV-ITEM</a></li></ul></nav>
+<main>
+<h1>Big title</h1>
+<p>First paragraph with a <a href="/docs/intro?x=1&y=2">relative link</a> and © 2026 text.</p>
+<h2>Things</h2>
+<ul><li>one<li>two with <a href="https://other.example/page">other</a></ul>
+<p>Duplicate <a href="/docs/intro?x=1&y=2">again</a>, <a href="javascript:alert(1)">js</a>, <a href="mailto:[email protected]">mail</a>, <a href="#top">top</a>.</p>
+<pre>line 1
+ line 2</pre>
+<form><input name="q"> FORM-TEXT</form>
+<svg><text>SVG-TEXT</text></svg>
+<noscript>NOSCRIPT-TEXT</noscript>
+</main>
+<aside>ASIDE-TEXT</aside>
+<footer>FOOTER TEXT</footer>
+</body></html>`;
+
+const DNS = { 'public.example': ['93.184.216.34'], 'redirector.example': ['93.184.216.35'], 'rebind.example': ['127.0.0.1'], 'lan.example': ['192.168.1.20'] };
+const lookup = (host, _o, cb) => { const a = DNS[host]; if (!a) return cb(Object.assign(new Error('nf'), { code: 'ENOTFOUND' })); cb(null, a.map(address => ({ address, family: 4 }))); };
+const dialled = [];
+const seenReq = [];
+function fakeSocket(address) {
+ const s = new EventEmitter();
+ s.setTimeout = () => {}; s.destroy = () => {}; s.end = () => {};
+ process.nextTick(() => s.emit('connect'));
+ dialled.push(address);
+ return s;
+}
+// path → { status, type, body (string|Buffer), location }
+const SITE = {
+ '/': { type: 'text/html; charset=utf-8', body: PAGE },
+ '/plain': { type: 'text/plain', body: 'just some text\r\nsecond line\r\n' },
+ '/image': { type: 'image/png', body: Buffer.from([0x89, 0x50, 0x4e, 0x47]) },
+ '/json': { type: 'application/json', body: '{"a":1}' },
+ '/big': { type: 'text/html', body: '<html><body><main><p>' + 'word '.repeat(600) + '</p></main></body></html>' },
+ '/huge': { type: 'text/html', body: '<html><body>' + '<p>' + 'x'.repeat(1024) + '</p>' + '<p>padding</p>'.repeat(300000) + '</body></html>' },
+ '/notfound': { status: 404, type: 'text/html', body: '<html><head><title>Gone</title></head><body><p>Nothing here.</p></body></html>' },
+ '/latin1': { type: 'text/html; charset=iso-8859-1', body: Buffer.from('<html><body><p>caf\xe9</p></body></html>', 'latin1') },
+};
+function fakeHttp(opts, cb) {
+ const req = new EventEmitter();
+ req.destroy = () => {};
+ req.end = () => {
+ const answer = (address) => {
+ dialled.push(address);
+ seenReq.push({ host: opts.hostname, path: opts.path, headers: opts.headers });
+ let page;
+ if (opts.hostname === 'redirector.example') {
+ const to = { '/to-loopback': 'http://127.0.0.1:3000/admin', '/to-public': 'https://public.example/plain' }[opts.path];
+ page = to ? { status: 302, location: to, body: '' } : SITE['/'];
+ } else page = SITE[opts.path] || SITE['/'];
+ const res = Readable.from(page.body.length ? [Buffer.isBuffer(page.body) ? page.body : Buffer.from(page.body)] : []);
+ Object.assign(res, { statusCode: page.status || 200, statusMessage: 'x', headers: { 'content-type': page.type || 'text/html', ...(page.location && { location: page.location }) } });
+ cb(res);
+ };
+ if (opts.lookup) opts.lookup(opts.hostname, {}, (err, address) => err ? req.emit('error', err) : answer(address)); else answer(opts.hostname);
+ };
+ return req;
+}
+const egress = createEgress({
+ lookup, tcpConnect: ({ host }) => fakeSocket(host),
+ tlsConnect: (o) => { const s = fakeSocket(o.host); s.getPeerCertificate = () => ({}); return s; },
+ httpRequest: fakeHttp, httpsRequest: fakeHttp,
+});
+
+(async () => {
+ // ── The extraction itself ──
+ const r = readHtml(PAGE, { baseUrl: 'https://public.example/a/b', format: 'markdown', maxChars: 20000 });
+ assert.equal(r.title, 'Hello & welcome');
+ assert.equal(r.description, 'A page about "things"');
+ assert.equal(r.lang, 'en');
+ for (const gone of ['HEADER', 'NAV-ITEM', 'FOOTER', 'ASIDE', 'FORM-TEXT', 'SVG-TEXT', 'NOSCRIPT', 'script text', 'color:red']) assert.ok(!r.text.includes(gone), `dropped: ${gone}`);
+ assert.ok(r.text.startsWith('# Big title\n\nFirst paragraph with a [relative link](https://public.example/docs/intro?x=1&y=2) and © 2026 text.'), r.text);
+ assert.ok(r.text.includes('## Things\n\n- one\n- two with [other](https://other.example/page)\n\n'), r.text);
+ assert.ok(r.text.includes('```\nline 1\n line 2\n```'));
+ assert.deepEqual(r.links, [{ text: 'relative link', href: 'https://public.example/docs/intro?x=1&y=2' }, { text: 'other', href: 'https://other.example/page' }], 'absolute http(s) only, de-duplicated, no javascript:/mailto:/#fragment');
+ assert.equal(r.truncated, false);
+ const plain = readHtml(PAGE, { baseUrl: 'https://public.example/', format: 'text', maxChars: 20000 });
+ assert.ok(plain.text.startsWith('Big title\n\nFirst paragraph with a relative link and'), plain.text);
+ assert.ok(!plain.text.includes('](') && !plain.text.includes('```'));
+ assert.equal(plain.links.length, 2, 'links are still listed in text mode');
+ // No <main>: body minus furniture; many links are capped at 50.
+ const many = readHtml('<body><nav>N</nav><p>' + Array.from({ length: 80 }, (_, i) => `<a href="/p${i}">p${i}</a>`).join(' ') + '</p></body>', { baseUrl: 'https://x.example/' });
+ assert.equal(many.links.length, 50);
+ assert.ok(!many.text.includes('N\n'));
+ // Hostile / broken markup does not throw or hang.
+ readHtml('<div>'.repeat(5000) + 'deep' + '<a href="' + 'a'.repeat(100000), { baseUrl: 'https://x.example/' });
+ readHtml('<<<>>><p <b>>��&bogus;', { baseUrl: 'https://x.example/' });
+ const cut = readHtml('<main><p>' + 'sentence one. '.repeat(500) + '</p></main>', { baseUrl: 'https://x.example/', maxChars: 1000 });
+ assert.ok(cut.truncated && cut.text.length <= 1000);
+ console.log('extraction: ok');
+
+ // ── Route: SSRF, caps, content types ──
+ const app = express();
+ const devRouter = createDevRoutes(null, null, { egress });
+ app.use('/api/dev', devRouter);
+ const srv = await new Promise(res => { const s = app.listen(0, '127.0.0.1', () => res(s)); });
+ const base = `http://127.0.0.1:${srv.address().port}`;
+ const q = encodeURIComponent;
+ const get = async (p) => { const x = await fetch(base + p); return { status: x.status, body: await x.json() }; };
+
+ for (const u of ['http://localhost:3000/', 'http://127.0.0.1:4000/', 'http://[::1]/', 'http://10.1.1.1/', 'http://192.168.0.1/', 'http://169.254.169.254/latest/meta-data/', 'https://rebind.example/', 'https://lan.example/', 'localhost:4100']) {
+ dialled.length = 0;
+ const x = await get(`/api/dev/read?url=${q(u)}`);
+ assert.equal(x.status, 403, `${u} → ${x.status}`);
+ assert.ok(x.body.code && /not a public internet address/.test(x.body.error));
+ assert.deepEqual(dialled, [], `${u}: no connection`);
+ }
+ dialled.length = 0;
+ assert.equal((await get(`/api/dev/read?url=${q('https://redirector.example/to-loopback')}`)).status, 403, 'a redirect to loopback is refused');
+ assert.deepEqual(dialled, ['93.184.216.35'], 'only the public hop was dialled');
+ assert.equal((await get('/api/dev/read')).status, 400);
+ assert.equal((await get(`/api/dev/read?url=${q('ftp://public.example/')}`)).status, 400);
+ assert.equal((await get('/api/dev/read?url=public.example&format=pdf')).status, 400);
+ assert.equal((await get('/api/dev/read?url=public.example&max_chars=10')).status, 400);
+ assert.equal((await get('/api/dev/read?url=public.example&max_chars=99999')).status, 400);
+ assert.equal((await get('/api/dev/read?url=nowhere.example')).status, 400, 'an unresolvable name is a bad URL');
+
+ let x = await get('/api/dev/read?url=public.example');
+ assert.equal(x.status, 200);
+ assert.deepEqual([x.body.url, x.body.status, x.body.content_type, x.body.title, x.body.lang, x.body.truncated], ['https://public.example/', 200, 'text/html', 'Hello & welcome', 'en', false]);
+ assert.equal(x.body.chars, x.body.text.length);
+ const sent = seenReq[seenReq.length - 1].headers;
+ assert.equal(sent['User-Agent'], 'OpenVibeReader/1.0 (+https://read.openvibe.tools)');
+ assert.equal(sent.Accept, 'text/html,application/xhtml+xml,text/plain;q=0.9');
+
+ x = await get(`/api/dev/read?url=${q('https://redirector.example/to-public')}`);
+ assert.equal(x.status, 200); assert.equal(x.body.url, 'https://public.example/plain', 'the final URL after redirects');
+ assert.deepEqual([x.body.content_type, x.body.text, x.body.title, x.body.links], ['text/plain', 'just some text\nsecond line', '', []], 'plain text is returned as is');
+
+ x = await get('/api/dev/read?url=public.example/image');
+ assert.equal(x.status, 415); assert.equal(x.body.code, 'tools.read.unsupported_content_type');
+ assert.equal((await get('/api/dev/read?url=public.example/json')).status, 415);
+
+ x = await get('/api/dev/read?url=public.example/notfound');
+ assert.deepEqual([x.status, x.body.status, x.body.title, x.body.text], [200, 404, 'Gone', 'Nothing here.'], 'the upstream status is reported');
+ assert.equal((await get('/api/dev/read?url=public.example/latin1')).body.text, 'café', 'the declared charset is honoured');
+
+ x = await get('/api/dev/read?url=public.example/big&max_chars=1000');
+ assert.equal(x.body.truncated, true); assert.ok(x.body.text.length <= 1000 && x.body.chars === x.body.text.length);
+ x = await get('/api/dev/read?url=public.example/huge');
+ assert.equal(x.status, 200); assert.equal(x.body.truncated, true, 'the 2 MB raw cap is reported');
+ assert.ok(x.body.chars <= 20000, 'and the default max_chars still applies');
+ console.log('route: ok');
+
+ // ── Through the run API ──
+ const guard = createGuard({ app: 'gateway', contracts, issuer: 'https://openvibe.network', keys: { get: () => null }, specs: SPECS, env: { TOOLS_GUARD: 'report' }, log: { log() {}, warn() {}, error() {} }, pruneIntervalMs: 0 });
+ const runApp = express();
+ runApp.use(guard.identify);
+ const { createGatewayRun } = require('../server/run');
+ const runDev = createDevRoutes(null, null, { egress, guard });
+ const toolRegistry = require('../server/registry/descriptors');
+ const run = createGatewayRun({
+ guard, contracts, toolRegistry, routers: { net: null, dev: runDev }, ports: () => ({}),
+ parseJson: express.json({ limit: '64kb' }), jobIndex: new Map(), ...ajvFrom(require),
+ });
+ runApp.use(run.handle);
+ const rsrv = await new Promise(res => { const s = runApp.listen(0, '127.0.0.1', () => res(s)); });
+ const rbase = `http://127.0.0.1:${rsrv.address().port}`;
+ const post = async (id, body) => { const y = await fetch(`${rbase}/api/v1/tools/${id}/run`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); return { status: y.status, body: await y.json() }; };
+
+ let y = await post('read', { input: { url: 'public.example', format: 'text', max_chars: 5000 } });
+ assert.equal(y.status, 200, JSON.stringify(y.body));
+ assert.equal(y.body.state, 'succeeded');
+ const data = y.body.result.data;
+ const spec = SPECS.find(s => s.id === 'read');
+ const { Ajv, addFormats } = ajvFrom(require);
+ const ajv = new Ajv({ strict: false }); if (addFormats) addFormats(ajv);
+ assert.ok(ajv.validate(spec.output.schema, data), JSON.stringify(ajv.errors));
+ assert.equal(data.title, 'Hello & welcome');
+ assert.ok(data.text.startsWith('Big title'));
+ y = await post('read', { input: { url: 'http://127.0.0.1:3000/' } });
+ assert.ok(y.status === 403 || (y.body.error && /public internet/.test(JSON.stringify(y.body))), `run API refuses loopback: ${y.status} ${JSON.stringify(y.body)}`);
+ y = await post('read', { input: { url: 'public.example', max_chars: 5 } });
+ assert.equal(y.status, 422, 'input validated against the descriptor');
+ y = await post('read', { input: {} });
+ assert.equal(y.status, 422);
+ console.log('run API: ok');
+
+ srv.closeAllConnections(); srv.close(); rsrv.closeAllConnections(); rsrv.close();
+ console.log('read tool: all checks passed');
+ process.exit(0);
+})().catch(e => { console.error(e); process.exit(1); });
@@ -14,7 +14,7 @@ const NET = [
@@ -14,7 +14,7 @@ const NET = [
'/robots/:target?', '/sitemap/:target?', '/uptime/:target?',
'/robots/:target?', '/sitemap/:target?', '/uptime/:target?',
'/smtp/:target?', '/blacklist/:target?', '/dnsprop/:target?',
'/smtp/:target?', '/blacklist/:target?', '/dnsprop/:target?',
];
];
-const DEV = ['/tools', '/opengraph'];
+const DEV = ['/tools', '/opengraph', '/read'];
const PROCESS = ['/api/process', '/api/process/direct', '/api/process/multi', '/api/process/anything', '/api/process/anything/nested'];
const PROCESS = ['/api/process', '/api/process/direct', '/api/process/multi', '/api/process/anything', '/api/process/anything/nested'];
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-sunset-routes-'));
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ov-sunset-routes-'));
@@ -66,7 +66,7 @@ const get = async (url, headers = {}, method = 'GET') => {
@@ -66,7 +66,7 @@ const get = async (url, headers = {}, method = 'GET') => {
const list = r.body;
const list = r.body;
const lc = contracts.tools.checkList(list);
const lc = contracts.tools.checkList(list);
assert.ok(lc.valid, JSON.stringify(lc.errors.slice(0, 5)));
assert.ok(lc.valid, JSON.stringify(lc.errors.slice(0, 5)));
- assert.strictEqual(list.count, 169);
+ assert.strictEqual(list.count, 170);
assert.ok(list.tools.every(d => !d.input || d.input.$ref), 'inputs are $refs in the list');
assert.ok(list.tools.every(d => !d.input || d.input.$ref), 'inputs are $refs in the list');
assert.ok(!Number.isNaN(Date.parse(list.updated_at)));
assert.ok(!Number.isNaN(Date.parse(list.updated_at)));
assert.deepStrictEqual(list.families.map(f => f.id).sort(), ['audio', 'dev', 'docs', 'img', 'media', 'net', 'pastes', 'places', 'text']);
assert.deepStrictEqual(list.families.map(f => f.id).sort(), ['audio', 'dev', 'docs', 'img', 'media', 'net', 'pastes', 'places', 'text']);
@@ -150,7 +150,7 @@ const get = async (url, headers = {}, method = 'GET') => {
@@ -150,7 +150,7 @@ const get = async (url, headers = {}, method = 'GET') => {
// ── /api/catalog.json is unchanged (Network reads it) ──
// ── /api/catalog.json is unchanged (Network reads it) ──
const catalog = (await get(`${G}/api/catalog.json`)).body;
const catalog = (await get(`${G}/api/catalog.json`)).body;
- assert.strictEqual(catalog.tools.length, 169);
+ assert.strictEqual(catalog.tools.length, 170);
assert.ok(catalog.tools.every(t => ['available', 'unavailable'].includes(t.status) && t.hosts && t.hosts.canonical));
assert.ok(catalog.tools.every(t => ['available', 'unavailable'].includes(t.status) && t.hosts && t.hosts.canonical));
assert.ok(!('api' in catalog.tools[0]) && !('execution' in catalog.tools[0]), 'no descriptor fields leak into the catalogue');
assert.ok(!('api' in catalog.tools[0]) && !('execution' in catalog.tools[0]), 'no descriptor fields leak into the catalogue');