openvibers/openvibe.chat Commits
Chat: preview the addresses in a message (#481)
Changes
nnew file mode 100644@@@ -0,0 +1,29 @@++-- phase: expand++-- Address previews for messages (server/chat/link-previews.js). A message with one or more https addresses gets up++-- to three previews, fetched by a background job after the message is stored — never by the request. The rows live++-- apart from the message so the message itself is never rewritten by a lookup:++--++-- link_previews one row per address on a message: the address's place in the message, its URL, and the++-- preview as JSON text. The fields are data only (text and numbers), never markup.++-- link_preview_cache what one address resolved to, kept a day: a second message with the same address shows the++-- same preview without another fetch. `data` is NULL for "fetched, no preview".++--++-- Expand only: two new tables; older releases never read or write them and every existing query keeps working.++CREATE TABLE IF NOT EXISTS link_previews (++ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,++ message_scope text COLLATE "C" NOT NULL, -- 'room' (chat and dm scopes join it later)++ message_id bigint NOT NULL,++ position smallint NOT NULL, -- 0..2, the address's order in the message++ url text COLLATE "C" NOT NULL, -- the address the preview is for++ data text COLLATE "C" NOT NULL, -- { url, site_name, title, description, image, kind, width, height }++ created_at text COLLATE "C" NOT NULL DEFAULT ov_now(),++ UNIQUE (message_scope, message_id, position)++);++-- A message's previews are read together, in the order the addresses appeared.++CREATE INDEX IF NOT EXISTS idx_link_previews_message ON link_previews (message_scope, message_id, position);++++CREATE TABLE IF NOT EXISTS link_preview_cache (++ url text COLLATE "C" PRIMARY KEY,++ data text COLLATE "C", -- the preview as JSON text; NULL = fetched, no preview++ fetched_at bigint NOT NULL -- epoch milliseconds; a day makes it stale++);@@@ -34,6 +34,7 @@ const soundboard = require('./soundboard-service');const audioQueue = require('./audio-queue');const dm = require('./dm');const linkCards = require('./link-cards');++const linkPreviews = require('./link-previews');const networkBlocks = require('./network-blocks');const presence = require('../presence');const vipBadges = require('../vip/badges');@@@ -1189,6 +1190,16 @@ class ChatServer {});if (result.lastInsertRowid) chatMsg.id = Number(result.lastInsertRowid);if (pending && result.lastInsertRowid) linkCards.fillLater('chat_messages', Number(result.lastInsertRowid), pending);++ // General address previews run after the line is out (server/chat/link-previews.js): the saved message++ // carries embeds: [] and a later frame carries the filled list. Channel/global viewers get the same frame.++ if (result.lastInsertRowid) {++ const id = Number(result.lastInsertRowid);++ linkPreviews.schedule({ scope: 'chat', messageId: id, text, onReady: (embeds) => {++ const frame = { type: 'chat_embeds', id, embeds };++ if (client.streamId || client.channelUserId) this.broadcastToChannelRoom(client.channelUserId, client.streamId, frame).catch(() => {});++ else this.broadcastGlobal(frame).catch(() => {});++ } });++ }firstChat = !!result.first_chat;} catch (err) {// Not saved means not sent: a line everyone saw but history, moderation and replays never@@@ -20,6 +20,7 @@*/const db = require('../db/database');const linkCards = require('./link-cards');++const linkPreviews = require('./link-previews');const PAGE_MEMO_MS = 2000;const MAX_LIMIT = 500;@@@ -133,6 +134,7 @@ async function page(room, { limit, before, channelUsername, decorate } = {}) {const rows = page.rows;const messages = decorate ? decorate(rows.map((x) => ({ ...x }))) : rows.map((x) => ({ ...x }));await linkCards.hydrate(messages, 'chat_messages');++ await linkPreviews.attach(messages, 'chat');return { messages, latest_id: latestIdOf(rows), more: page.more };}@@@ -150,6 +152,7 @@ async function delta(room, { afterId, limit, channelUsername, decorate } = {}) {if (!complete) rows = rows.slice(0, lim);const messages = decorate ? decorate(rows.map((x) => ({ ...x }))) : rows.map((x) => ({ ...x }));await linkCards.hydrate(messages, 'chat_messages');++ await linkPreviews.attach(messages, 'chat');return { messages, latest_id: latestIdOf(rows) || after, complete, deleted_ids: await deletedIds(room, after) };}nnew file mode 100644@@@ -0,0 +1,443 @@++'use strict';++/**++ * Address previews in a message (plan "embeds", part 1): a message with one or more https addresses gets up to three++ * previews, fetched by a background job after the message is stored — never by the request that sent it, and never++ * for an address the egress module refuses.++ *++ * { url, site_name, title, description, image, kind: 'link' | 'video' | 'image', width, height }++ *++ * The job reads the page through openvibe-shared/egress: public addresses only (DNS is checked again at connect, so++ * a name that answers a private address is refused), https only, at most two redirects each checked on its own,++ * 4 s and 512 KiB per request, no cookies, a plain user agent. YouTube and Vimeo are read from their oEmbed endpoint++ * on a fixed origin instead of their page. A picture is only kept when it already lives on openvibe.media: a page's++ * own image is not hot-linked (a Media proxy path can add those later).++ *++ * Previews live apart from the message (link_previews) and are cached by address for a day (link_preview_cache), so++ * sending, and every later send of the same address, never waits on another site. The fields are data, never markup:++ * control characters are gone, whitespace folded, sizes capped, and the API/renderer escape them as text.++ *++ * The room wiring (server/rooms/rooms.js) attaches the rows on every read as `embeds` ([] before the job is done),++ * and the job announces `room_message_embeds` when it has something.++ */++const { assertPublicUrl, safeLookup } = require('openvibe-shared/egress');++const db = require('../db/database');++++const MAX_URLS = 3;++const MAX_URL = 2048;++const MAX_BYTES = 512 * 1024;++const FETCH_TIMEOUT_MS = 4000;++const MAX_REDIRECTS = 2;++const CACHE_MS = 86_400_000; // a found preview is reused a day++const MISS_MS = 10 * 60_000; // a miss is retried after ten minutes, not hammered++const MAX_TITLE = 120;++const MAX_DESCRIPTION = 300;++const MAX_SITE = 80;++const MAX_JOBS = 2;++const MAX_QUEUE = 200;++const KINDS = ['link', 'video', 'image'];++const USER_AGENT = 'OpenVibe-Chat-LinkPreview/1 (+https://openvibe.chat)';++const PREVIEWS_ON = String(process.env.CHAT_LINK_PREVIEWS || '').toLowerCase() !== 'off';++++const cache = new Map(); // url → { at, value } (in-process fast path in front of the table)++let _request = null; // tests: (url) => { status, headers, body }++let _lookup = null; // tests: dns.lookup stand-in for the egress check++const queue = [];++let running = 0;++let idleWaiters = [];++++// ── addresses ─────────────────────────────────────────────────────────────────────────────────────────────────────++++/** An https address as a normalized href, or null (http, credentials, a fragment or a very long address is no preview). */++function normalizeUrl(raw) {++ let u;++ try { u = new URL(String(raw || '')); } catch { return null; }++ if (u.protocol !== 'https:' || u.username || u.password) return null;++ u.hash = '';++ const href = u.href;++ return href.length > MAX_URL ? null : href;++}++++/**++ * The https addresses in a message body, in order, at most MAX_URLS: trailing sentence punctuation a writer leaves++ * on a URL is not part of it, and the same address twice is one preview.++ */++function addresses(text) {++ const out = [];++ const seen = new Set();++ const re = /https:\/\/[^\s<>"']+/g;++ let m;++ while ((m = re.exec(String(text || ''))) && out.length < MAX_URLS) {++ const href = normalizeUrl(m[0].replace(/[.,;:!?]+$/, '').replace(/\)+$/, ''));++ if (!href || seen.has(href)) continue;++ seen.add(href);++ out.push(href);++ }++ return out;++}++++// ── text from a page ─────────────────────────────────────────────────────────────────────────────────────────────++++/** Plain text from a page field: controls gone, whitespace folded, a known entity decoded, cut to size. */++function textOf(value, max) {++ const text = String(value == null ? '' : value)++ .replace(/[\u0000-\u001f\u007f]+/g, ' ')++ .replace(/\s+/g, ' ')++ .trim();++ return Array.from(text).slice(0, max).join('');++}++++const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", '#39': "'", nbsp: ' ' };++function decodeEntities(value) {++ return String(value == null ? '' : value)++ .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Math.min(Number(n), 0x10ffff)))++ .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(Math.min(parseInt(n, 16), 0x10ffff)))++ .replace(/&([a-z]+|#39);/gi, (all, name) => (Object.prototype.hasOwnProperty.call(ENTITIES, name.toLowerCase()) ? ENTITIES[name.toLowerCase()] : all));++}++++/** A meta tag's attributes as an object, names lower-cased. */++function attributesOf(tag) {++ const out = {};++ const re = /([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*("([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g;++ let m;++ while ((m = re.exec(tag))) out[m[1].toLowerCase()] = decodeEntities(m[3] != null ? m[3] : m[4] != null ? m[4] : m[5]);++ return out;++}++++/** The Open Graph and Twitter fields a page states, plus its <title>. Only the first answer of each name counts. */++function pageFields(html) {++ const fields = {};++ const metas = String(html || '').match(/<meta\b[^>]*>/gi) || [];++ for (const tag of metas) {++ const a = attributesOf(tag);++ const key = String(a.property || a.name || '').toLowerCase();++ if (!key || a.content == null) continue;++ if (key.startsWith('og:') || key.startsWith('twitter:')) {++ if (fields[key] == null) fields[key] = a.content;++ }++ }++ const title = /<title\b[^>]*>([\s\S]*?)<\/title>/i.exec(String(html || ''));++ const docTitle = title ? decodeEntities(title[1]) : '';++ fields['html:title'] = docTitle;++ return fields;++}++++/** A picture is kept only when it is already on OpenVibe.Media (https): never a page's own hot-linked image. */++function imageOf(value) {++ if (typeof value !== 'string' || !value) return null;++ try {++ const u = new URL(value);++ if (u.protocol !== 'https:' || u.port || u.username || u.password) return null;++ return u.hostname.toLowerCase() === 'openvibe.media' ? u.href : null;++ } catch { return null; }++}++++const intOf = (value) => {++ const n = parseInt(value, 10);++ return Number.isFinite(n) && n > 0 && n <= 100000 ? n : 0;++};++const siteOf = (url, fields) => textOf(fields['og:site_name'] || fields['twitter:site'] || url.hostname.replace(/^www\./, ''), MAX_SITE);++++/** An oEmbed answer as a preview, or null when it has no title. */++function previewFromOembed(url, item) {++ if (!item || typeof item !== 'object') return null;++ const title = textOf(item.title, MAX_TITLE);++ if (!title) return null;++ const author = textOf(item.author_name, 80);++ return {++ url,++ site_name: textOf(item.provider_name, MAX_SITE) || siteOf(new URL(url), {}),++ title,++ description: author ? `Video by ${author}` : '',++ image: imageOf(item.thumbnail_url),++ kind: 'video',++ width: intOf(item.width || item.thumbnail_width),++ height: intOf(item.height || item.thumbnail_height),++ };++}++++/** The preview a page states, or null when the response is not a page/picture we can preview. */++function previewFromResponse(url, res) {++ const type = String((res.headers && (res.headers['content-type'] || res.headers['Content-Type'])) || '').toLowerCase();++ if (type.startsWith('image/')) {++ let name = '';++ try { name = decodeURIComponent(new URL(url).pathname.split('/').filter(Boolean).pop() || ''); } catch { /* host */ }++ return { url, site_name: siteOf(new URL(url), {}), title: textOf(name, MAX_TITLE) || siteOf(new URL(url), {}), description: '', image: null, kind: 'image', width: 0, height: 0 };++ }++ if (type && !type.includes('html')) return null;++ const html = String(res.body || '');++ const f = pageFields(html);++ const title = textOf(f['og:title'] || f['twitter:title'] || f['html:title'], MAX_TITLE);++ const description = textOf(f['og:description'] || f['twitter:description'], MAX_DESCRIPTION);++ const image = imageOf(f['og:image'] || f['og:image:url'] || f['twitter:image']);++ const ogType = String(f['og:type'] || '').toLowerCase();++ const card = String(f['twitter:card'] || '').toLowerCase();++ const kind = ogType.startsWith('video') || card === 'player' ? 'video' : ogType.startsWith('image') ? 'image' : 'link';++ const host = siteOf(new URL(url), f);++ return {++ url,++ site_name: host,++ title: title || host,++ description,++ image,++ kind,++ width: intOf(f['og:image:width'] || f['twitter:image:width']),++ height: intOf(f['og:image:height'] || f['twitter:image:height']),++ };++}++++// ── the guarded fetch ───────────────────────────────────────────────────────────────────────────────────────────++++/** A fixed oEmbed endpoint for the addresses on the short allow-list, or null. */++function oembedFor(url) {++ const host = url.hostname.toLowerCase().replace(/^www\./, '');++ const target = encodeURIComponent(url.href);++ if (host === 'youtu.be' || host === 'youtube.com' || host === 'm.youtube.com' || host === 'music.youtube.com') {++ return `https://www.youtube.com/oembed?format=json&url=${target}`;++ }++ if (host === 'vimeo.com' || host === 'player.vimeo.com') return `https://vimeo.com/api/oembed.json?url=${target}`;++ return null;++}++++/** One GET, https only, through the egress lookup, no redirects followed here, at most FETCH_TIMEOUT_MS and MAX_BYTES. */++function requestOnce(url) {++ if (_request) return Promise.resolve(_request(url));++ return new Promise((resolve, reject) => {++ const https = require('https');++ let settled = false;++ const done = (err, value) => {++ if (settled) return;++ settled = true;++ if (err) reject(err); else resolve(value);++ };++ const req = https.request(url, {++ method: 'GET',++ lookup: safeLookup,++ timeout: FETCH_TIMEOUT_MS,++ headers: {++ accept: 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.5',++ 'accept-language': 'en',++ 'user-agent': USER_AGENT,++ },++ }, (res) => {++ const chunks = [];++ let size = 0;++ res.on('data', (chunk) => {++ const room = MAX_BYTES - size;++ if (room <= 0) { res.destroy(); done(null, { status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks).toString('utf8') }); return; }++ chunks.push(chunk.length > room ? chunk.subarray(0, room) : chunk);++ size += Math.min(chunk.length, room);++ if (size >= MAX_BYTES) { res.destroy(); done(null, { status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks).toString('utf8') }); }++ });++ res.on('end', () => done(null, { status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks).toString('utf8') }));++ res.on('error', (err) => done(err));++ });++ req.on('timeout', () => { req.destroy(new Error('link-previews: timed out')); });++ req.on('error', (err) => done(err));++ req.end();++ });++}++++const REDIRECTS = new Set([301, 302, 303, 307, 308]);++++/** A URL through its redirects (at most MAX_REDIRECTS, each hop checked public and https), then one response. */++async function fetchFollowing(url) {++ let current = new URL(url);++ for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {++ if (current.protocol !== 'https:' || current.username || current.password) return null;++ await assertPublicUrl(current, { protocols: ['https:'], ...(_lookup ? { lookup: _lookup } : {}) });++ const res = await requestOnce(current.href);++ if (!res) return null;++ const location = res.headers && (res.headers.location || res.headers.Location);++ if (REDIRECTS.has(Number(res.status)) && location) {++ try { current = new URL(String(location), current); } catch { return null; }++ continue;++ }++ return res.status === 200 ? res : null;++ }++ return null;++}++++/** Read one address: oEmbed when it is on the short allow-list, the page otherwise. A failure is no preview. */++async function readUrl(url) {++ const oembed = oembedFor(new URL(url));++ if (oembed) {++ try {++ const res = await fetchFollowing(oembed);++ if (res) {++ const item = JSON.parse(String(res.body || ''));++ const preview = previewFromOembed(url, item);++ if (preview) return preview;++ }++ } catch { /* fall through to the page */ }++ }++ const res = await fetchFollowing(url);++ return res ? previewFromResponse(url, res) : null;++}++++// ── cache and storage ─────────────────────────────────────────────────────────────────────────────────────────++++function parsePreview(value) {++ let p = value;++ if (typeof p === 'string') { try { p = JSON.parse(p); } catch { return null; } }++ if (!p || typeof p !== 'object' || Array.isArray(p)) return null;++ if (!KINDS.includes(p.kind) || typeof p.title !== 'string' || !p.title || typeof p.url !== 'string') return null;++ return {++ url: p.url,++ site_name: textOf(p.site_name, MAX_SITE),++ title: textOf(p.title, MAX_TITLE),++ description: textOf(p.description, MAX_DESCRIPTION),++ image: imageOf(p.image),++ kind: p.kind,++ width: intOf(p.width),++ height: intOf(p.height),++ };++}++++async function cached(value) {++ const url = normalizeUrl(value);++ if (!url) return { hit: true, value: null };++ const mem = cache.get(url);++ if (mem && Date.now() - mem.at < (mem.value ? CACHE_MS : MISS_MS)) return { hit: true, value: mem.value };++ let row;++ try { row = await db.get('SELECT data, fetched_at FROM link_preview_cache WHERE url = ?', [url]); } catch { return { hit: false, value: null }; }++ if (!row) return { hit: false, value: null };++ const age = Date.now() - Number(row.fetched_at || 0);++ if (!(age >= 0 && age < (row.data ? CACHE_MS : MISS_MS))) return { hit: false, value: null };++ const preview = row.data ? parsePreview(row.data) : null;++ cache.set(url, { at: Date.now(), value: preview });++ return { hit: true, value: preview };++}++++async function remember(url, preview) {++ cache.set(url, { at: Date.now(), value: preview });++ if (cache.size > 1000) cache.delete(cache.keys().next().value);++ try {++ await db.run(`INSERT INTO link_preview_cache (url, data, fetched_at) VALUES (?, ?, ?)++ ON CONFLICT (url) DO UPDATE SET data = EXCLUDED.data, fetched_at = EXCLUDED.fetched_at`,++ [url, preview ? JSON.stringify(preview) : null, Date.now()]);++ } catch { /* the cache is an optimization; the preview still shows */ }++}++++/** The preview for an address, from the cache or once through the guarded fetch. */++async function previewOf(value) {++ const url = normalizeUrl(value);++ if (!url) return null;++ const hit = await cached(url);++ if (hit.hit) return hit.value;++ let preview = null;++ try { preview = await readUrl(url); } catch { preview = null; }++ const checked = preview ? parsePreview({ ...preview, url }) : null;++ await remember(url, checked);++ return checked;++}++++// ── the background job ───────────────────────────────────────────────────────────────────────────────────────++++function idle() {++ if (!running && !queue.length) return Promise.resolve();++ return new Promise((resolve) => idleWaiters.push(resolve));++}++function settled() {++ if (running || queue.length) return;++ const waiters = idleWaiters;++ idleWaiters = [];++ for (const done of waiters) done();++}++++async function store(job, position, preview) {++ if (!preview) {++ await db.run('DELETE FROM link_previews WHERE message_scope = ? AND message_id = ? AND position = ?', [job.scope, job.messageId, position]).catch(() => {});++ return;++ }++ await db.run(`INSERT INTO link_previews (message_scope, message_id, position, url, data) VALUES (?, ?, ?, ?, ?)++ ON CONFLICT (message_scope, message_id, position)++ DO UPDATE SET url = EXCLUDED.url, data = EXCLUDED.data`,++ [job.scope, job.messageId, position, preview.url, JSON.stringify(preview)]);++}++++async function announce(job) {++ const embeds = await embedsOf(job.scope, job.messageId);++ if (typeof job.onReady === 'function') {++ try { await job.onReady(embeds); } catch { /* the line is already out */ }++ }++ if (!embeds.length || !job.room) return;++ try {++ const chatServer = require('./chat-server');++ chatServer.broadcastToRoom(job.room.id, { type: 'room_message_embeds', room: job.room.slug, id: job.messageId, embeds });++ } catch { /* a preview that lands after everyone left needs no event */ }++}++++async function run(job) {++ for (let i = 0; i < job.urls.length; i++) {++ let preview = null;++ try { preview = await previewOf(job.urls[i]); } catch { preview = null; }++ await store(job, i, preview).catch(() => {});++ }++ await announce(job).catch(() => {});++}++++function pump() {++ while (running < MAX_JOBS && queue.length) {++ const job = queue.shift();++ running++;++ run(job).catch(() => {}).finally(() => { running--; setImmediate(pump); });++ }++ settled();++}++++/**++ * Queue a message for previews, after it is stored and its response is out. Returns the job (tests await idle()), or++ * null when there is nothing to do. Never awaited by the request.++ */++function schedule({ scope = 'room', messageId, text, room = null, onReady = null } = {}) {++ if (!PREVIEWS_ON) return null;++ // A test that does not stub the network never makes one (the suite's messages carry all sorts of addresses).++ if (process.env.NODE_ENV === 'test' && !_request) return null;++ const id = Number(messageId);++ if (!(id > 0) || (scope !== 'room' && scope !== 'dm' && scope !== 'chat')) return null;++ const urls = addresses(text);++ if (!urls.length) return null;++ if (queue.length >= MAX_QUEUE) queue.shift();++ const job = { scope, messageId: id, urls, room, onReady: typeof onReady === 'function' ? onReady : null };++ queue.push(job);++ setImmediate(pump);++ return job;++}++++/** The stored previews for one message, in the order their addresses appeared. */++async function embedsOf(scope, messageId) {++ const rows = await db.all('SELECT data FROM link_previews WHERE message_scope = ? AND message_id = ? ORDER BY position', [scope, Number(messageId) || 0]).catch(() => []);++ return rows.map((r) => parsePreview(r.data)).filter(Boolean);++}++++/** Put `embeds` on every row ([] when there are none yet), for the read the client makes. */++async function attach(rows, scope = 'room') {++ if (!Array.isArray(rows)) return rows;++ const list = rows.filter(Boolean);++ for (const row of list) row.embeds = [];++ if (!list.length) return rows;++ const ids = [...new Set(list.map((r) => Number(r.id)).filter((n) => n > 0))];++ if (!ids.length) return rows;++ const marks = ids.map(() => '?').join(',');++ const records = await db.all(`SELECT message_id, data FROM link_previews WHERE message_scope = ? AND message_id IN (${marks}) ORDER BY message_id, position`,++ [scope, ...ids]).catch(() => []);++ const byId = new Map();++ for (const rec of records) {++ const preview = parsePreview(rec.data);++ if (!preview) continue;++ const key = Number(rec.message_id);++ if (!byId.has(key)) byId.set(key, []);++ byId.get(key).push(preview);++ }++ for (const row of list) row.embeds = byId.get(Number(row.id)) || [];++ return rows;++}++++module.exports = {++ addresses, normalizeUrl, textOf, decodeEntities, pageFields, previewFromResponse, previewFromOembed, imageOf,++ previewOf, schedule, embedsOf, attach, idle,++ CACHE_MS, MISS_MS, MAX_URLS, MAX_BYTES, FETCH_TIMEOUT_MS, MAX_REDIRECTS, PREVIEWS_ON,++ _setRequest(fn) { _request = fn || null; },++ _setLookup(fn) { _lookup = fn || null; },++ _reset() { _request = null; _lookup = null; cache.clear(); },++};@@@ -49,6 +49,7 @@ const ctx = require('../live-context');const wordFilter = require('../chat/word-filter');const permissions = require('../auth/permissions');const linkCards = require('../chat/link-cards');++const linkPreviews = require('../chat/link-previews');const SLUG_RE = /^[a-z0-9](?:[a-z0-9-]{1,30}[a-z0-9])$/;const RESERVED = new Set(['global', 'new', 'rooms', 'settings', 'messages', 'updates', 'auth', 'api', 'admin', 'staff', 'help', 'system', 'ice-servers']);@@@ -319,6 +320,8 @@ async function post(room, user, text, { now = Date.now(), ai = true, timezone =});const row = await messageById(id);if (pending) linkCards.fillLater('room_messages', id, pending);++ // General address previews are a background job: the message and its response never wait on another site.++ linkPreviews.schedule({ scope: 'room', messageId: id, text: message, room: { id: room.id, slug: room.slug } });// Activity: the people this names, and the person it answers (server/chat/activity.js). A failure here never// takes a message back.const activity = require('../chat/activity');@@@ -350,6 +353,7 @@ async function postAsAi(room, member, text, { now = Date.now(), how = null } = {await announceMessage(room, { subject_id: null, username: member.handle, display_name: member.name }, Number(r.lastInsertRowid), message, now);return Number(r.lastInsertRowid);});++ linkPreviews.schedule({ scope: 'room', messageId: id, text: message, room: { id: room.id, slug: room.slug } });return await messageById(id);}@@@ -365,6 +369,7 @@ async function messageById(id) {if (row) {row.attachments = messageFiles.attachmentRefs(messageFiles.filesOf(row));await linkCards.hydrate([row], 'room_messages');++ await linkPreviews.attach([row], 'room');}return row;}@@@ -380,6 +385,7 @@ async function historyPage(room, { before = null, limit = 50 } = {}) {const more = rows.length > lim;rows.length = Math.min(rows.length, lim);await linkCards.hydrate(rows, 'room_messages');++ await linkPreviews.attach(rows, 'room');return { messages: rows, more };}@@@ -393,6 +399,7 @@ async function history(room, { before = null, after = null, limit = 60 } = {}) {: await db.all(`${MSG_SELECT} WHERE m.room_id = ? AND m.is_deleted = 0 ORDER BY m.id DESC LIMIT ?`, [room.id, lim]);for (const row of rows) row.attachments = messageFiles.attachmentRefs(messageFiles.filesOf(row));await linkCards.hydrate(rows, 'room_messages');++ await linkPreviews.attach(rows, 'room');return after != null ? rows : rows.reverse();}nnew file mode 100644@@@ -0,0 +1,140 @@++'use strict';++/**++ * Address previews (part 1): the addresses a message carries, the guarded page read, the day cache, and the rows a++ * read gets back. The fetches are stubbed with _setRequest; the address check is the real openvibe-shared/egress one++ * (a private literal or a name answering a private address is refused before the stub is called).++ */++const assert = require('assert');++const { ids } = require('openvibe-contracts');++const { boot, suite } = require('./helpers');++let previews;++++const t = suite('link-previews');++let h, alice, room;++++const page = (head) => `<html><head>${head}</head><body>body</body></html>`;++const type = (body) => ({ status: 200, headers: { 'content-type': 'text/html; charset=utf-8' }, body });++++t('boot', async () => {++ h = await boot();++ previews = require('../server/chat/link-previews');++ alice = h.addUser('alice', { subject: ids.newId('user') });++ await h.ctx.upsertUser(h.live.users.get(alice.id));++ h.netModules.subjects.add(alice.subject_id);++ await h.ctx.sync();++ const created = await h.http('POST', '/api/chat/rooms', { token: alice.token, body: { name: 'Preview room' } });++ assert.strictEqual(created.status, 201, created.text);++ room = created.body.room;++});++++t('addresses: https only, in order, three at most, sentence punctuation dropped', async () => {++ const found = previews.addresses('see https://one.example/a, and https://two.example/b. also http://plain.example and https://one.example/a https://three.example/c https://four.example/d');++ assert.deepStrictEqual(found, ['https://one.example/a', 'https://two.example/b', 'https://three.example/c']);++ assert.deepStrictEqual(previews.addresses('https://user:[email protected]/a'), []);++ assert.deepStrictEqual(previews.addresses('nothing to preview'), []);++});++++t('a message gets embeds: [] at once, then its previews from the background job', async () => {++ const calls = [];++ previews._setRequest(async (url) => {++ calls.push(url);++ if (url === 'https://93.184.216.34/story') {++ return type(page('<meta property="og:title" content="A story & its title"><meta property="og:site_name" content="Example"><meta property="og:description" content="What happened"><meta property="og:image" content="https://openvibe.media/pic.jpg"><meta property="og:image:width" content="640"><meta property="og:image:height" content="360">'));++ }++ return { status: 404, headers: { 'content-type': 'text/html' }, body: '' };++ });++ const sent = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'look at https://93.184.216.34/story' } });++ assert.strictEqual(sent.status, 201, sent.text);++ assert.deepStrictEqual(sent.body.message.embeds, [], 'the message goes out before the preview');++ await previews.idle();++ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });++ const message = got.body.messages.find((m) => m.id === sent.body.message.id);++ assert.strictEqual(message.embeds.length, 1, got.text);++ assert.deepStrictEqual(message.embeds[0], {++ url: 'https://93.184.216.34/story',++ site_name: 'Example',++ title: 'A story & its title',++ description: 'What happened',++ image: 'https://openvibe.media/pic.jpg',++ kind: 'link',++ width: 640,++ height: 360,++ });++ assert.deepStrictEqual(calls, ['https://93.184.216.34/story']);++});++++t('the same address is fetched once a day', async () => {++ let calls = 0;++ previews._setRequest(async () => { calls++; return type(page('<meta property="og:title" content="Cached">')); });++ const one = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'https://93.184.216.34/cached' } });++ await previews.idle();++ const two = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'again https://93.184.216.34/cached' } });++ await previews.idle();++ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });++ const titles = got.body.messages.filter((m) => [one.body.message.id, two.body.message.id].includes(m.id)).map((m) => m.embeds[0] && m.embeds[0].title);++ assert.deepStrictEqual(titles, ['Cached', 'Cached']);++ assert.strictEqual(calls, 1, 'the second message reads the cache');++});++++t('a private address is never fetched and gets no preview', async () => {++ let called = false;++ previews._setLookup((host, options, callback) => callback(null, [{ address: '10.0.0.1', family: 4 }]));++ previews._setRequest(async () => { called = true; return type(page('<meta property="og:title" content="Never">')); });++ const sent = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'https://private.example/secret and https://10.0.0.1/also' } });++ await previews.idle();++ assert.strictEqual(called, false, 'no request for a private address');++ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });++ assert.deepStrictEqual(got.body.messages.find((m) => m.id === sent.body.message.id).embeds, []);++ previews._setLookup(null);++});++++t('YouTube is read from its oEmbed endpoint and the response is video', async () => {++ let calls = [];++ previews._setLookup((host, options, callback) => callback(null, [{ address: '93.184.216.34', family: 4 }]));++ previews._setRequest(async (url) => {++ calls.push(url);++ if (url.startsWith('https://www.youtube.com/oembed?')) {++ return { status: 200, headers: { 'content-type': 'application/json' }, body: JSON.stringify({ title: 'A video', author_name: 'Someone', provider_name: 'YouTube', width: 480, height: 270 }) };++ }++ return { status: 404, headers: {}, body: '' };++ });++ const sent = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'https://www.youtube.com/watch?v=abc12345678' } });++ await previews.idle();++ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });++ const embed = got.body.messages.find((m) => m.id === sent.body.message.id).embeds[0];++ assert.strictEqual(embed.kind, 'video');++ assert.strictEqual(embed.title, 'A video');++ assert.strictEqual(embed.site_name, 'YouTube');++ assert.deepStrictEqual(calls, ['https://www.youtube.com/oembed?format=json&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3Dabc12345678']);++});++++t('markup in a page title stays text, and an outside picture is left out', async () => {++ previews._setRequest(async () => type(page('<meta property="og:title" content="<b>not bold</b>"><meta name="twitter:image" content="https://hotlink.example/x.jpg">')));++ const sent = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'https://93.184.216.34/markup' } });++ await previews.idle();++ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });++ const embed = got.body.messages.find((m) => m.id === sent.body.message.id).embeds[0];++ assert.strictEqual(embed.title, '<b>not bold</b>', 'the words are text; a renderer escapes the angle brackets');++ assert.strictEqual(embed.image, null, 'a picture is never hot-linked');++});++++t('three addresses at most, and a redirect is followed only once it is judged public', async () => {++ const calls = [];++ previews._setRequest(async (url) => {++ calls.push(url);++ if (url === 'https://93.184.216.34/redirect') return { status: 302, headers: { location: '/target' }, body: '' };++ return type(page(`<meta property="og:title" content="${url.endsWith('target') ? 'Redirected' : 'One'}">`));++ });++ const made = await h.http('POST', '/api/chat/rooms', { token: alice.token, body: { name: 'Redirect room' } });++ assert.strictEqual(made.status, 201, made.text);++ const sent = await h.http('POST', `/api/chat/rooms/${made.body.room.slug}/messages`, { token: alice.token, body: { message: 'one https://93.184.216.34/redirect and two https://93.184.216.34/b and three https://93.184.216.34/c here' } });++ assert.strictEqual(sent.status, 201, sent.text);++ await previews.idle();++ const got = await h.http('GET', `/api/chat/rooms/${made.body.room.slug}/messages`, { token: alice.token });++ const embeds = got.body.messages.find((m) => m.id === sent.body.message.id).embeds;++ assert.strictEqual(embeds.length, 3);++ assert.strictEqual(new URL(embeds[0].url).pathname, '/redirect', 'the preview keeps the address the writer sent');++ assert.ok(calls.includes('https://93.184.216.34/target'));++});++++t.run(async () => { previews._reset(); await h.close(); });@@@ -37,6 +37,7 @@ const TABLES = ['room_message_reactions', 'room_member_timeouts', 'config_snapshots', 'config_keys', 'chat_activity','agent_play_requests', 'agent_play_votes', 'agent_play_log','party_pool_settings', 'party_pool_machines', 'party_pool_holds', // 0064: OpenVibe's own computers for parties++ 'link_previews', 'link_preview_cache', // 0065: address previews beside the message, and the address cache];const INDEXES = ['idx_chat_ts_deleted', 'idx_chat_page_live', 'idx_chat_channel_user_ts', 'idx_chat_stream_ts', 'idx_emotes_channel_code', 'idx_chat_tl_dedup',@@@ -44,6 +45,7 @@ const INDEXES = ['idx_room_messages_room', 'idx_room_members_user', 'idx_network_blocks_blocked', 'idx_mod_actions_created','idx_dm_contacts_user', 'idx_rooms_forum_space','chat_app_link_tokens_by_person', 'chat_app_link_tokens_expiry', 'chat_app_callbacks_by_chat', 'chat_app_callbacks_expiry',++ 'idx_link_previews_message',];const MIRRORED = ['chat_messages', 'dm_conversations', 'dm_participants', 'dm_messages', 'dm_blocks', 'tts_voice_overrides','channel_sounds', 'relay_users', 'hidden_relay_users', 'pending_ip_messages', 'stream_first_chats', 'moderation_actions'];@@@ -131,6 +131,7 @@ t('ratchet: every file that makes an outbound request itself is reviewed', () =>'server/web/party-pages.js': 'OpenVibe.Video (fixed: https://openvibe.video, or a test\'s CHAT_PARTY_VIDEO_URL), fixed /api/v1/video/watchable paths; a pasted link is only a query parameter, never fetched; redirects refused, 2.5 s, cached a minute','server/rooms/radio.js': 'OpenVibe.Download Radio catalogue (fixed https origin, or a test\'s CHAT_RADIO_URL); track ids match rad_ ULIDs, redirects refused, 2.5 s','server/chat/link-cards.js': 'OpenVibe.Video and OpenVibe.Actor (fixed https origins, or a test\'s CHAT_LINK_VIDEO_URL / CHAT_LINK_ACTOR_URL), fixed paths (a watchable lookup, /voices/<id>.json, /skills/<slug>.json); a pasted link is a query parameter or a path segment only, never fetched; redirects refused, 2.5 s, cached a minute',++ 'server/chat/link-previews.js': 'an address a person wrote: https only, assertPublicUrl on every hop and safeLookup on the connection, at most two redirects each judged, 4 s and 512 KiB caps, no cookies; YouTube/Vimeo oEmbed on fixed origins; OpenVibe\'s own sites resolve through fixed public APIs and are never page-fetched','server/web/room-embed-pages.js': 'not the server: the sign-in window\'s own script posts to /embed/bind on this same site',};const root = path.join(__dirname, '..');
new file mode 100644
new file mode 100644
@@ -0,0 +1,29 @@
@@ -0,0 +1,29 @@
+-- phase: expand
+-- Address previews for messages (server/chat/link-previews.js). A message with one or more https addresses gets up
+-- to three previews, fetched by a background job after the message is stored — never by the request. The rows live
+-- apart from the message so the message itself is never rewritten by a lookup:
+--
+-- link_previews one row per address on a message: the address's place in the message, its URL, and the
+-- preview as JSON text. The fields are data only (text and numbers), never markup.
+-- link_preview_cache what one address resolved to, kept a day: a second message with the same address shows the
+-- same preview without another fetch. `data` is NULL for "fetched, no preview".
+--
+-- Expand only: two new tables; older releases never read or write them and every existing query keeps working.
+CREATE TABLE IF NOT EXISTS link_previews (
+ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+ message_scope text COLLATE "C" NOT NULL, -- 'room' (chat and dm scopes join it later)
+ message_id bigint NOT NULL,
+ position smallint NOT NULL, -- 0..2, the address's order in the message
+ url text COLLATE "C" NOT NULL, -- the address the preview is for
+ data text COLLATE "C" NOT NULL, -- { url, site_name, title, description, image, kind, width, height }
+ created_at text COLLATE "C" NOT NULL DEFAULT ov_now(),
+ UNIQUE (message_scope, message_id, position)
+);
+-- A message's previews are read together, in the order the addresses appeared.
+CREATE INDEX IF NOT EXISTS idx_link_previews_message ON link_previews (message_scope, message_id, position);
+
+CREATE TABLE IF NOT EXISTS link_preview_cache (
+ url text COLLATE "C" PRIMARY KEY,
+ data text COLLATE "C", -- the preview as JSON text; NULL = fetched, no preview
+ fetched_at bigint NOT NULL -- epoch milliseconds; a day makes it stale
+);
@@ -34,6 +34,7 @@ const soundboard = require('./soundboard-service');
@@ -34,6 +34,7 @@ const soundboard = require('./soundboard-service');
const audioQueue = require('./audio-queue');
const audioQueue = require('./audio-queue');
const dm = require('./dm');
const dm = require('./dm');
const linkCards = require('./link-cards');
const linkCards = require('./link-cards');
+const linkPreviews = require('./link-previews');
const networkBlocks = require('./network-blocks');
const networkBlocks = require('./network-blocks');
const presence = require('../presence');
const presence = require('../presence');
const vipBadges = require('../vip/badges');
const vipBadges = require('../vip/badges');
@@ -1189,6 +1190,16 @@ class ChatServer {
@@ -1189,6 +1190,16 @@ class ChatServer {
});
});
if (result.lastInsertRowid) chatMsg.id = Number(result.lastInsertRowid);
if (result.lastInsertRowid) chatMsg.id = Number(result.lastInsertRowid);
if (pending && result.lastInsertRowid) linkCards.fillLater('chat_messages', Number(result.lastInsertRowid), pending);
if (pending && result.lastInsertRowid) linkCards.fillLater('chat_messages', Number(result.lastInsertRowid), pending);
+ // General address previews run after the line is out (server/chat/link-previews.js): the saved message
+ // carries embeds: [] and a later frame carries the filled list. Channel/global viewers get the same frame.
+ if (result.lastInsertRowid) {
+ const id = Number(result.lastInsertRowid);
+ linkPreviews.schedule({ scope: 'chat', messageId: id, text, onReady: (embeds) => {
+ const frame = { type: 'chat_embeds', id, embeds };
+ if (client.streamId || client.channelUserId) this.broadcastToChannelRoom(client.channelUserId, client.streamId, frame).catch(() => {});
+ else this.broadcastGlobal(frame).catch(() => {});
+ } });
+ }
firstChat = !!result.first_chat;
firstChat = !!result.first_chat;
} catch (err) {
} catch (err) {
// Not saved means not sent: a line everyone saw but history, moderation and replays never
// Not saved means not sent: a line everyone saw but history, moderation and replays never
@@ -20,6 +20,7 @@
@@ -20,6 +20,7 @@
*/
*/
const db = require('../db/database');
const db = require('../db/database');
const linkCards = require('./link-cards');
const linkCards = require('./link-cards');
+const linkPreviews = require('./link-previews');
const PAGE_MEMO_MS = 2000;
const PAGE_MEMO_MS = 2000;
const MAX_LIMIT = 500;
const MAX_LIMIT = 500;
@@ -133,6 +134,7 @@ async function page(room, { limit, before, channelUsername, decorate } = {}) {
@@ -133,6 +134,7 @@ async function page(room, { limit, before, channelUsername, decorate } = {}) {
const rows = page.rows;
const rows = page.rows;
const messages = decorate ? decorate(rows.map((x) => ({ ...x }))) : rows.map((x) => ({ ...x }));
const messages = decorate ? decorate(rows.map((x) => ({ ...x }))) : rows.map((x) => ({ ...x }));
await linkCards.hydrate(messages, 'chat_messages');
await linkCards.hydrate(messages, 'chat_messages');
+ await linkPreviews.attach(messages, 'chat');
return { messages, latest_id: latestIdOf(rows), more: page.more };
return { messages, latest_id: latestIdOf(rows), more: page.more };
}
}
@@ -150,6 +152,7 @@ async function delta(room, { afterId, limit, channelUsername, decorate } = {}) {
@@ -150,6 +152,7 @@ async function delta(room, { afterId, limit, channelUsername, decorate } = {}) {
if (!complete) rows = rows.slice(0, lim);
if (!complete) rows = rows.slice(0, lim);
const messages = decorate ? decorate(rows.map((x) => ({ ...x }))) : rows.map((x) => ({ ...x }));
const messages = decorate ? decorate(rows.map((x) => ({ ...x }))) : rows.map((x) => ({ ...x }));
await linkCards.hydrate(messages, 'chat_messages');
await linkCards.hydrate(messages, 'chat_messages');
+ await linkPreviews.attach(messages, 'chat');
return { messages, latest_id: latestIdOf(rows) || after, complete, deleted_ids: await deletedIds(room, after) };
return { messages, latest_id: latestIdOf(rows) || after, complete, deleted_ids: await deletedIds(room, after) };
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,443 @@
@@ -0,0 +1,443 @@
+'use strict';
+/**
+ * Address previews in a message (plan "embeds", part 1): a message with one or more https addresses gets up to three
+ * previews, fetched by a background job after the message is stored — never by the request that sent it, and never
+ * for an address the egress module refuses.
+ *
+ * { url, site_name, title, description, image, kind: 'link' | 'video' | 'image', width, height }
+ *
+ * The job reads the page through openvibe-shared/egress: public addresses only (DNS is checked again at connect, so
+ * a name that answers a private address is refused), https only, at most two redirects each checked on its own,
+ * 4 s and 512 KiB per request, no cookies, a plain user agent. YouTube and Vimeo are read from their oEmbed endpoint
+ * on a fixed origin instead of their page. A picture is only kept when it already lives on openvibe.media: a page's
+ * own image is not hot-linked (a Media proxy path can add those later).
+ *
+ * Previews live apart from the message (link_previews) and are cached by address for a day (link_preview_cache), so
+ * sending, and every later send of the same address, never waits on another site. The fields are data, never markup:
+ * control characters are gone, whitespace folded, sizes capped, and the API/renderer escape them as text.
+ *
+ * The room wiring (server/rooms/rooms.js) attaches the rows on every read as `embeds` ([] before the job is done),
+ * and the job announces `room_message_embeds` when it has something.
+ */
+const { assertPublicUrl, safeLookup } = require('openvibe-shared/egress');
+const db = require('../db/database');
+
+const MAX_URLS = 3;
+const MAX_URL = 2048;
+const MAX_BYTES = 512 * 1024;
+const FETCH_TIMEOUT_MS = 4000;
+const MAX_REDIRECTS = 2;
+const CACHE_MS = 86_400_000; // a found preview is reused a day
+const MISS_MS = 10 * 60_000; // a miss is retried after ten minutes, not hammered
+const MAX_TITLE = 120;
+const MAX_DESCRIPTION = 300;
+const MAX_SITE = 80;
+const MAX_JOBS = 2;
+const MAX_QUEUE = 200;
+const KINDS = ['link', 'video', 'image'];
+const USER_AGENT = 'OpenVibe-Chat-LinkPreview/1 (+https://openvibe.chat)';
+const PREVIEWS_ON = String(process.env.CHAT_LINK_PREVIEWS || '').toLowerCase() !== 'off';
+
+const cache = new Map(); // url → { at, value } (in-process fast path in front of the table)
+let _request = null; // tests: (url) => { status, headers, body }
+let _lookup = null; // tests: dns.lookup stand-in for the egress check
+const queue = [];
+let running = 0;
+let idleWaiters = [];
+
+// ── addresses ─────────────────────────────────────────────────────────────────────────────────────────────────────
+
+/** An https address as a normalized href, or null (http, credentials, a fragment or a very long address is no preview). */
+function normalizeUrl(raw) {
+ let u;
+ try { u = new URL(String(raw || '')); } catch { return null; }
+ if (u.protocol !== 'https:' || u.username || u.password) return null;
+ u.hash = '';
+ const href = u.href;
+ return href.length > MAX_URL ? null : href;
+}
+
+/**
+ * The https addresses in a message body, in order, at most MAX_URLS: trailing sentence punctuation a writer leaves
+ * on a URL is not part of it, and the same address twice is one preview.
+ */
+function addresses(text) {
+ const out = [];
+ const seen = new Set();
+ const re = /https:\/\/[^\s<>"']+/g;
+ let m;
+ while ((m = re.exec(String(text || ''))) && out.length < MAX_URLS) {
+ const href = normalizeUrl(m[0].replace(/[.,;:!?]+$/, '').replace(/\)+$/, ''));
+ if (!href || seen.has(href)) continue;
+ seen.add(href);
+ out.push(href);
+ }
+ return out;
+}
+
+// ── text from a page ─────────────────────────────────────────────────────────────────────────────────────────────
+
+/** Plain text from a page field: controls gone, whitespace folded, a known entity decoded, cut to size. */
+function textOf(value, max) {
+ const text = String(value == null ? '' : value)
+ .replace(/[\u0000-\u001f\u007f]+/g, ' ')
+ .replace(/\s+/g, ' ')
+ .trim();
+ return Array.from(text).slice(0, max).join('');
+}
+
+const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", '#39': "'", nbsp: ' ' };
+function decodeEntities(value) {
+ return String(value == null ? '' : value)
+ .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Math.min(Number(n), 0x10ffff)))
+ .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(Math.min(parseInt(n, 16), 0x10ffff)))
+ .replace(/&([a-z]+|#39);/gi, (all, name) => (Object.prototype.hasOwnProperty.call(ENTITIES, name.toLowerCase()) ? ENTITIES[name.toLowerCase()] : all));
+}
+
+/** A meta tag's attributes as an object, names lower-cased. */
+function attributesOf(tag) {
+ const out = {};
+ const re = /([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*("([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g;
+ let m;
+ while ((m = re.exec(tag))) out[m[1].toLowerCase()] = decodeEntities(m[3] != null ? m[3] : m[4] != null ? m[4] : m[5]);
+ return out;
+}
+
+/** The Open Graph and Twitter fields a page states, plus its <title>. Only the first answer of each name counts. */
+function pageFields(html) {
+ const fields = {};
+ const metas = String(html || '').match(/<meta\b[^>]*>/gi) || [];
+ for (const tag of metas) {
+ const a = attributesOf(tag);
+ const key = String(a.property || a.name || '').toLowerCase();
+ if (!key || a.content == null) continue;
+ if (key.startsWith('og:') || key.startsWith('twitter:')) {
+ if (fields[key] == null) fields[key] = a.content;
+ }
+ }
+ const title = /<title\b[^>]*>([\s\S]*?)<\/title>/i.exec(String(html || ''));
+ const docTitle = title ? decodeEntities(title[1]) : '';
+ fields['html:title'] = docTitle;
+ return fields;
+}
+
+/** A picture is kept only when it is already on OpenVibe.Media (https): never a page's own hot-linked image. */
+function imageOf(value) {
+ if (typeof value !== 'string' || !value) return null;
+ try {
+ const u = new URL(value);
+ if (u.protocol !== 'https:' || u.port || u.username || u.password) return null;
+ return u.hostname.toLowerCase() === 'openvibe.media' ? u.href : null;
+ } catch { return null; }
+}
+
+const intOf = (value) => {
+ const n = parseInt(value, 10);
+ return Number.isFinite(n) && n > 0 && n <= 100000 ? n : 0;
+};
+const siteOf = (url, fields) => textOf(fields['og:site_name'] || fields['twitter:site'] || url.hostname.replace(/^www\./, ''), MAX_SITE);
+
+/** An oEmbed answer as a preview, or null when it has no title. */
+function previewFromOembed(url, item) {
+ if (!item || typeof item !== 'object') return null;
+ const title = textOf(item.title, MAX_TITLE);
+ if (!title) return null;
+ const author = textOf(item.author_name, 80);
+ return {
+ url,
+ site_name: textOf(item.provider_name, MAX_SITE) || siteOf(new URL(url), {}),
+ title,
+ description: author ? `Video by ${author}` : '',
+ image: imageOf(item.thumbnail_url),
+ kind: 'video',
+ width: intOf(item.width || item.thumbnail_width),
+ height: intOf(item.height || item.thumbnail_height),
+ };
+}
+
+/** The preview a page states, or null when the response is not a page/picture we can preview. */
+function previewFromResponse(url, res) {
+ const type = String((res.headers && (res.headers['content-type'] || res.headers['Content-Type'])) || '').toLowerCase();
+ if (type.startsWith('image/')) {
+ let name = '';
+ try { name = decodeURIComponent(new URL(url).pathname.split('/').filter(Boolean).pop() || ''); } catch { /* host */ }
+ return { url, site_name: siteOf(new URL(url), {}), title: textOf(name, MAX_TITLE) || siteOf(new URL(url), {}), description: '', image: null, kind: 'image', width: 0, height: 0 };
+ }
+ if (type && !type.includes('html')) return null;
+ const html = String(res.body || '');
+ const f = pageFields(html);
+ const title = textOf(f['og:title'] || f['twitter:title'] || f['html:title'], MAX_TITLE);
+ const description = textOf(f['og:description'] || f['twitter:description'], MAX_DESCRIPTION);
+ const image = imageOf(f['og:image'] || f['og:image:url'] || f['twitter:image']);
+ const ogType = String(f['og:type'] || '').toLowerCase();
+ const card = String(f['twitter:card'] || '').toLowerCase();
+ const kind = ogType.startsWith('video') || card === 'player' ? 'video' : ogType.startsWith('image') ? 'image' : 'link';
+ const host = siteOf(new URL(url), f);
+ return {
+ url,
+ site_name: host,
+ title: title || host,
+ description,
+ image,
+ kind,
+ width: intOf(f['og:image:width'] || f['twitter:image:width']),
+ height: intOf(f['og:image:height'] || f['twitter:image:height']),
+ };
+}
+
+// ── the guarded fetch ───────────────────────────────────────────────────────────────────────────────────────────
+
+/** A fixed oEmbed endpoint for the addresses on the short allow-list, or null. */
+function oembedFor(url) {
+ const host = url.hostname.toLowerCase().replace(/^www\./, '');
+ const target = encodeURIComponent(url.href);
+ if (host === 'youtu.be' || host === 'youtube.com' || host === 'm.youtube.com' || host === 'music.youtube.com') {
+ return `https://www.youtube.com/oembed?format=json&url=${target}`;
+ }
+ if (host === 'vimeo.com' || host === 'player.vimeo.com') return `https://vimeo.com/api/oembed.json?url=${target}`;
+ return null;
+}
+
+/** One GET, https only, through the egress lookup, no redirects followed here, at most FETCH_TIMEOUT_MS and MAX_BYTES. */
+function requestOnce(url) {
+ if (_request) return Promise.resolve(_request(url));
+ return new Promise((resolve, reject) => {
+ const https = require('https');
+ let settled = false;
+ const done = (err, value) => {
+ if (settled) return;
+ settled = true;
+ if (err) reject(err); else resolve(value);
+ };
+ const req = https.request(url, {
+ method: 'GET',
+ lookup: safeLookup,
+ timeout: FETCH_TIMEOUT_MS,
+ headers: {
+ accept: 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.5',
+ 'accept-language': 'en',
+ 'user-agent': USER_AGENT,
+ },
+ }, (res) => {
+ const chunks = [];
+ let size = 0;
+ res.on('data', (chunk) => {
+ const room = MAX_BYTES - size;
+ if (room <= 0) { res.destroy(); done(null, { status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks).toString('utf8') }); return; }
+ chunks.push(chunk.length > room ? chunk.subarray(0, room) : chunk);
+ size += Math.min(chunk.length, room);
+ if (size >= MAX_BYTES) { res.destroy(); done(null, { status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks).toString('utf8') }); }
+ });
+ res.on('end', () => done(null, { status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks).toString('utf8') }));
+ res.on('error', (err) => done(err));
+ });
+ req.on('timeout', () => { req.destroy(new Error('link-previews: timed out')); });
+ req.on('error', (err) => done(err));
+ req.end();
+ });
+}
+
+const REDIRECTS = new Set([301, 302, 303, 307, 308]);
+
+/** A URL through its redirects (at most MAX_REDIRECTS, each hop checked public and https), then one response. */
+async function fetchFollowing(url) {
+ let current = new URL(url);
+ for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+ if (current.protocol !== 'https:' || current.username || current.password) return null;
+ await assertPublicUrl(current, { protocols: ['https:'], ...(_lookup ? { lookup: _lookup } : {}) });
+ const res = await requestOnce(current.href);
+ if (!res) return null;
+ const location = res.headers && (res.headers.location || res.headers.Location);
+ if (REDIRECTS.has(Number(res.status)) && location) {
+ try { current = new URL(String(location), current); } catch { return null; }
+ continue;
+ }
+ return res.status === 200 ? res : null;
+ }
+ return null;
+}
+
+/** Read one address: oEmbed when it is on the short allow-list, the page otherwise. A failure is no preview. */
+async function readUrl(url) {
+ const oembed = oembedFor(new URL(url));
+ if (oembed) {
+ try {
+ const res = await fetchFollowing(oembed);
+ if (res) {
+ const item = JSON.parse(String(res.body || ''));
+ const preview = previewFromOembed(url, item);
+ if (preview) return preview;
+ }
+ } catch { /* fall through to the page */ }
+ }
+ const res = await fetchFollowing(url);
+ return res ? previewFromResponse(url, res) : null;
+}
+
+// ── cache and storage ─────────────────────────────────────────────────────────────────────────────────────────
+
+function parsePreview(value) {
+ let p = value;
+ if (typeof p === 'string') { try { p = JSON.parse(p); } catch { return null; } }
+ if (!p || typeof p !== 'object' || Array.isArray(p)) return null;
+ if (!KINDS.includes(p.kind) || typeof p.title !== 'string' || !p.title || typeof p.url !== 'string') return null;
+ return {
+ url: p.url,
+ site_name: textOf(p.site_name, MAX_SITE),
+ title: textOf(p.title, MAX_TITLE),
+ description: textOf(p.description, MAX_DESCRIPTION),
+ image: imageOf(p.image),
+ kind: p.kind,
+ width: intOf(p.width),
+ height: intOf(p.height),
+ };
+}
+
+async function cached(value) {
+ const url = normalizeUrl(value);
+ if (!url) return { hit: true, value: null };
+ const mem = cache.get(url);
+ if (mem && Date.now() - mem.at < (mem.value ? CACHE_MS : MISS_MS)) return { hit: true, value: mem.value };
+ let row;
+ try { row = await db.get('SELECT data, fetched_at FROM link_preview_cache WHERE url = ?', [url]); } catch { return { hit: false, value: null }; }
+ if (!row) return { hit: false, value: null };
+ const age = Date.now() - Number(row.fetched_at || 0);
+ if (!(age >= 0 && age < (row.data ? CACHE_MS : MISS_MS))) return { hit: false, value: null };
+ const preview = row.data ? parsePreview(row.data) : null;
+ cache.set(url, { at: Date.now(), value: preview });
+ return { hit: true, value: preview };
+}
+
+async function remember(url, preview) {
+ cache.set(url, { at: Date.now(), value: preview });
+ if (cache.size > 1000) cache.delete(cache.keys().next().value);
+ try {
+ await db.run(`INSERT INTO link_preview_cache (url, data, fetched_at) VALUES (?, ?, ?)
+ ON CONFLICT (url) DO UPDATE SET data = EXCLUDED.data, fetched_at = EXCLUDED.fetched_at`,
+ [url, preview ? JSON.stringify(preview) : null, Date.now()]);
+ } catch { /* the cache is an optimization; the preview still shows */ }
+}
+
+/** The preview for an address, from the cache or once through the guarded fetch. */
+async function previewOf(value) {
+ const url = normalizeUrl(value);
+ if (!url) return null;
+ const hit = await cached(url);
+ if (hit.hit) return hit.value;
+ let preview = null;
+ try { preview = await readUrl(url); } catch { preview = null; }
+ const checked = preview ? parsePreview({ ...preview, url }) : null;
+ await remember(url, checked);
+ return checked;
+}
+
+// ── the background job ───────────────────────────────────────────────────────────────────────────────────────
+
+function idle() {
+ if (!running && !queue.length) return Promise.resolve();
+ return new Promise((resolve) => idleWaiters.push(resolve));
+}
+function settled() {
+ if (running || queue.length) return;
+ const waiters = idleWaiters;
+ idleWaiters = [];
+ for (const done of waiters) done();
+}
+
+async function store(job, position, preview) {
+ if (!preview) {
+ await db.run('DELETE FROM link_previews WHERE message_scope = ? AND message_id = ? AND position = ?', [job.scope, job.messageId, position]).catch(() => {});
+ return;
+ }
+ await db.run(`INSERT INTO link_previews (message_scope, message_id, position, url, data) VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT (message_scope, message_id, position)
+ DO UPDATE SET url = EXCLUDED.url, data = EXCLUDED.data`,
+ [job.scope, job.messageId, position, preview.url, JSON.stringify(preview)]);
+}
+
+async function announce(job) {
+ const embeds = await embedsOf(job.scope, job.messageId);
+ if (typeof job.onReady === 'function') {
+ try { await job.onReady(embeds); } catch { /* the line is already out */ }
+ }
+ if (!embeds.length || !job.room) return;
+ try {
+ const chatServer = require('./chat-server');
+ chatServer.broadcastToRoom(job.room.id, { type: 'room_message_embeds', room: job.room.slug, id: job.messageId, embeds });
+ } catch { /* a preview that lands after everyone left needs no event */ }
+}
+
+async function run(job) {
+ for (let i = 0; i < job.urls.length; i++) {
+ let preview = null;
+ try { preview = await previewOf(job.urls[i]); } catch { preview = null; }
+ await store(job, i, preview).catch(() => {});
+ }
+ await announce(job).catch(() => {});
+}
+
+function pump() {
+ while (running < MAX_JOBS && queue.length) {
+ const job = queue.shift();
+ running++;
+ run(job).catch(() => {}).finally(() => { running--; setImmediate(pump); });
+ }
+ settled();
+}
+
+/**
+ * Queue a message for previews, after it is stored and its response is out. Returns the job (tests await idle()), or
+ * null when there is nothing to do. Never awaited by the request.
+ */
+function schedule({ scope = 'room', messageId, text, room = null, onReady = null } = {}) {
+ if (!PREVIEWS_ON) return null;
+ // A test that does not stub the network never makes one (the suite's messages carry all sorts of addresses).
+ if (process.env.NODE_ENV === 'test' && !_request) return null;
+ const id = Number(messageId);
+ if (!(id > 0) || (scope !== 'room' && scope !== 'dm' && scope !== 'chat')) return null;
+ const urls = addresses(text);
+ if (!urls.length) return null;
+ if (queue.length >= MAX_QUEUE) queue.shift();
+ const job = { scope, messageId: id, urls, room, onReady: typeof onReady === 'function' ? onReady : null };
+ queue.push(job);
+ setImmediate(pump);
+ return job;
+}
+
+/** The stored previews for one message, in the order their addresses appeared. */
+async function embedsOf(scope, messageId) {
+ const rows = await db.all('SELECT data FROM link_previews WHERE message_scope = ? AND message_id = ? ORDER BY position', [scope, Number(messageId) || 0]).catch(() => []);
+ return rows.map((r) => parsePreview(r.data)).filter(Boolean);
+}
+
+/** Put `embeds` on every row ([] when there are none yet), for the read the client makes. */
+async function attach(rows, scope = 'room') {
+ if (!Array.isArray(rows)) return rows;
+ const list = rows.filter(Boolean);
+ for (const row of list) row.embeds = [];
+ if (!list.length) return rows;
+ const ids = [...new Set(list.map((r) => Number(r.id)).filter((n) => n > 0))];
+ if (!ids.length) return rows;
+ const marks = ids.map(() => '?').join(',');
+ const records = await db.all(`SELECT message_id, data FROM link_previews WHERE message_scope = ? AND message_id IN (${marks}) ORDER BY message_id, position`,
+ [scope, ...ids]).catch(() => []);
+ const byId = new Map();
+ for (const rec of records) {
+ const preview = parsePreview(rec.data);
+ if (!preview) continue;
+ const key = Number(rec.message_id);
+ if (!byId.has(key)) byId.set(key, []);
+ byId.get(key).push(preview);
+ }
+ for (const row of list) row.embeds = byId.get(Number(row.id)) || [];
+ return rows;
+}
+
+module.exports = {
+ addresses, normalizeUrl, textOf, decodeEntities, pageFields, previewFromResponse, previewFromOembed, imageOf,
+ previewOf, schedule, embedsOf, attach, idle,
+ CACHE_MS, MISS_MS, MAX_URLS, MAX_BYTES, FETCH_TIMEOUT_MS, MAX_REDIRECTS, PREVIEWS_ON,
+ _setRequest(fn) { _request = fn || null; },
+ _setLookup(fn) { _lookup = fn || null; },
+ _reset() { _request = null; _lookup = null; cache.clear(); },
+};
@@ -49,6 +49,7 @@ const ctx = require('../live-context');
@@ -49,6 +49,7 @@ const ctx = require('../live-context');
const wordFilter = require('../chat/word-filter');
const wordFilter = require('../chat/word-filter');
const permissions = require('../auth/permissions');
const permissions = require('../auth/permissions');
const linkCards = require('../chat/link-cards');
const linkCards = require('../chat/link-cards');
+const linkPreviews = require('../chat/link-previews');
const SLUG_RE = /^[a-z0-9](?:[a-z0-9-]{1,30}[a-z0-9])$/;
const SLUG_RE = /^[a-z0-9](?:[a-z0-9-]{1,30}[a-z0-9])$/;
const RESERVED = new Set(['global', 'new', 'rooms', 'settings', 'messages', 'updates', 'auth', 'api', 'admin', 'staff', 'help', 'system', 'ice-servers']);
const RESERVED = new Set(['global', 'new', 'rooms', 'settings', 'messages', 'updates', 'auth', 'api', 'admin', 'staff', 'help', 'system', 'ice-servers']);
@@ -319,6 +320,8 @@ async function post(room, user, text, { now = Date.now(), ai = true, timezone =
@@ -319,6 +320,8 @@ async function post(room, user, text, { now = Date.now(), ai = true, timezone =
});
});
const row = await messageById(id);
const row = await messageById(id);
if (pending) linkCards.fillLater('room_messages', id, pending);
if (pending) linkCards.fillLater('room_messages', id, pending);
+ // General address previews are a background job: the message and its response never wait on another site.
+ linkPreviews.schedule({ scope: 'room', messageId: id, text: message, room: { id: room.id, slug: room.slug } });
// Activity: the people this names, and the person it answers (server/chat/activity.js). A failure here never
// Activity: the people this names, and the person it answers (server/chat/activity.js). A failure here never
// takes a message back.
// takes a message back.
const activity = require('../chat/activity');
const activity = require('../chat/activity');
@@ -350,6 +353,7 @@ async function postAsAi(room, member, text, { now = Date.now(), how = null } = {
@@ -350,6 +353,7 @@ async function postAsAi(room, member, text, { now = Date.now(), how = null } = {
await announceMessage(room, { subject_id: null, username: member.handle, display_name: member.name }, Number(r.lastInsertRowid), message, now);
await announceMessage(room, { subject_id: null, username: member.handle, display_name: member.name }, Number(r.lastInsertRowid), message, now);
return Number(r.lastInsertRowid);
return Number(r.lastInsertRowid);
});
});
+ linkPreviews.schedule({ scope: 'room', messageId: id, text: message, room: { id: room.id, slug: room.slug } });
return await messageById(id);
return await messageById(id);
}
}
@@ -365,6 +369,7 @@ async function messageById(id) {
@@ -365,6 +369,7 @@ async function messageById(id) {
if (row) {
if (row) {
row.attachments = messageFiles.attachmentRefs(messageFiles.filesOf(row));
row.attachments = messageFiles.attachmentRefs(messageFiles.filesOf(row));
await linkCards.hydrate([row], 'room_messages');
await linkCards.hydrate([row], 'room_messages');
+ await linkPreviews.attach([row], 'room');
}
}
return row;
return row;
}
}
@@ -380,6 +385,7 @@ async function historyPage(room, { before = null, limit = 50 } = {}) {
@@ -380,6 +385,7 @@ async function historyPage(room, { before = null, limit = 50 } = {}) {
const more = rows.length > lim;
const more = rows.length > lim;
rows.length = Math.min(rows.length, lim);
rows.length = Math.min(rows.length, lim);
await linkCards.hydrate(rows, 'room_messages');
await linkCards.hydrate(rows, 'room_messages');
+ await linkPreviews.attach(rows, 'room');
return { messages: rows, more };
return { messages: rows, more };
}
}
@@ -393,6 +399,7 @@ async function history(room, { before = null, after = null, limit = 60 } = {}) {
@@ -393,6 +399,7 @@ async function history(room, { before = null, after = null, limit = 60 } = {}) {
: await db.all(`${MSG_SELECT} WHERE m.room_id = ? AND m.is_deleted = 0 ORDER BY m.id DESC LIMIT ?`, [room.id, lim]);
: await db.all(`${MSG_SELECT} WHERE m.room_id = ? AND m.is_deleted = 0 ORDER BY m.id DESC LIMIT ?`, [room.id, lim]);
for (const row of rows) row.attachments = messageFiles.attachmentRefs(messageFiles.filesOf(row));
for (const row of rows) row.attachments = messageFiles.attachmentRefs(messageFiles.filesOf(row));
await linkCards.hydrate(rows, 'room_messages');
await linkCards.hydrate(rows, 'room_messages');
+ await linkPreviews.attach(rows, 'room');
return after != null ? rows : rows.reverse();
return after != null ? rows : rows.reverse();
}
}
new file mode 100644
new file mode 100644
@@ -0,0 +1,140 @@
@@ -0,0 +1,140 @@
+'use strict';
+/**
+ * Address previews (part 1): the addresses a message carries, the guarded page read, the day cache, and the rows a
+ * read gets back. The fetches are stubbed with _setRequest; the address check is the real openvibe-shared/egress one
+ * (a private literal or a name answering a private address is refused before the stub is called).
+ */
+const assert = require('assert');
+const { ids } = require('openvibe-contracts');
+const { boot, suite } = require('./helpers');
+let previews;
+
+const t = suite('link-previews');
+let h, alice, room;
+
+const page = (head) => `<html><head>${head}</head><body>body</body></html>`;
+const type = (body) => ({ status: 200, headers: { 'content-type': 'text/html; charset=utf-8' }, body });
+
+t('boot', async () => {
+ h = await boot();
+ previews = require('../server/chat/link-previews');
+ alice = h.addUser('alice', { subject: ids.newId('user') });
+ await h.ctx.upsertUser(h.live.users.get(alice.id));
+ h.netModules.subjects.add(alice.subject_id);
+ await h.ctx.sync();
+ const created = await h.http('POST', '/api/chat/rooms', { token: alice.token, body: { name: 'Preview room' } });
+ assert.strictEqual(created.status, 201, created.text);
+ room = created.body.room;
+});
+
+t('addresses: https only, in order, three at most, sentence punctuation dropped', async () => {
+ const found = previews.addresses('see https://one.example/a, and https://two.example/b. also http://plain.example and https://one.example/a https://three.example/c https://four.example/d');
+ assert.deepStrictEqual(found, ['https://one.example/a', 'https://two.example/b', 'https://three.example/c']);
+ assert.deepStrictEqual(previews.addresses('https://user:[email protected]/a'), []);
+ assert.deepStrictEqual(previews.addresses('nothing to preview'), []);
+});
+
+t('a message gets embeds: [] at once, then its previews from the background job', async () => {
+ const calls = [];
+ previews._setRequest(async (url) => {
+ calls.push(url);
+ if (url === 'https://93.184.216.34/story') {
+ return type(page('<meta property="og:title" content="A story & its title"><meta property="og:site_name" content="Example"><meta property="og:description" content="What happened"><meta property="og:image" content="https://openvibe.media/pic.jpg"><meta property="og:image:width" content="640"><meta property="og:image:height" content="360">'));
+ }
+ return { status: 404, headers: { 'content-type': 'text/html' }, body: '' };
+ });
+ const sent = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'look at https://93.184.216.34/story' } });
+ assert.strictEqual(sent.status, 201, sent.text);
+ assert.deepStrictEqual(sent.body.message.embeds, [], 'the message goes out before the preview');
+ await previews.idle();
+ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });
+ const message = got.body.messages.find((m) => m.id === sent.body.message.id);
+ assert.strictEqual(message.embeds.length, 1, got.text);
+ assert.deepStrictEqual(message.embeds[0], {
+ url: 'https://93.184.216.34/story',
+ site_name: 'Example',
+ title: 'A story & its title',
+ description: 'What happened',
+ image: 'https://openvibe.media/pic.jpg',
+ kind: 'link',
+ width: 640,
+ height: 360,
+ });
+ assert.deepStrictEqual(calls, ['https://93.184.216.34/story']);
+});
+
+t('the same address is fetched once a day', async () => {
+ let calls = 0;
+ previews._setRequest(async () => { calls++; return type(page('<meta property="og:title" content="Cached">')); });
+ const one = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'https://93.184.216.34/cached' } });
+ await previews.idle();
+ const two = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'again https://93.184.216.34/cached' } });
+ await previews.idle();
+ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });
+ const titles = got.body.messages.filter((m) => [one.body.message.id, two.body.message.id].includes(m.id)).map((m) => m.embeds[0] && m.embeds[0].title);
+ assert.deepStrictEqual(titles, ['Cached', 'Cached']);
+ assert.strictEqual(calls, 1, 'the second message reads the cache');
+});
+
+t('a private address is never fetched and gets no preview', async () => {
+ let called = false;
+ previews._setLookup((host, options, callback) => callback(null, [{ address: '10.0.0.1', family: 4 }]));
+ previews._setRequest(async () => { called = true; return type(page('<meta property="og:title" content="Never">')); });
+ const sent = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'https://private.example/secret and https://10.0.0.1/also' } });
+ await previews.idle();
+ assert.strictEqual(called, false, 'no request for a private address');
+ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });
+ assert.deepStrictEqual(got.body.messages.find((m) => m.id === sent.body.message.id).embeds, []);
+ previews._setLookup(null);
+});
+
+t('YouTube is read from its oEmbed endpoint and the response is video', async () => {
+ let calls = [];
+ previews._setLookup((host, options, callback) => callback(null, [{ address: '93.184.216.34', family: 4 }]));
+ previews._setRequest(async (url) => {
+ calls.push(url);
+ if (url.startsWith('https://www.youtube.com/oembed?')) {
+ return { status: 200, headers: { 'content-type': 'application/json' }, body: JSON.stringify({ title: 'A video', author_name: 'Someone', provider_name: 'YouTube', width: 480, height: 270 }) };
+ }
+ return { status: 404, headers: {}, body: '' };
+ });
+ const sent = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'https://www.youtube.com/watch?v=abc12345678' } });
+ await previews.idle();
+ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });
+ const embed = got.body.messages.find((m) => m.id === sent.body.message.id).embeds[0];
+ assert.strictEqual(embed.kind, 'video');
+ assert.strictEqual(embed.title, 'A video');
+ assert.strictEqual(embed.site_name, 'YouTube');
+ assert.deepStrictEqual(calls, ['https://www.youtube.com/oembed?format=json&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3Dabc12345678']);
+});
+
+t('markup in a page title stays text, and an outside picture is left out', async () => {
+ previews._setRequest(async () => type(page('<meta property="og:title" content="<b>not bold</b>"><meta name="twitter:image" content="https://hotlink.example/x.jpg">')));
+ const sent = await h.http('POST', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token, body: { message: 'https://93.184.216.34/markup' } });
+ await previews.idle();
+ const got = await h.http('GET', `/api/chat/rooms/${room.slug}/messages`, { token: alice.token });
+ const embed = got.body.messages.find((m) => m.id === sent.body.message.id).embeds[0];
+ assert.strictEqual(embed.title, '<b>not bold</b>', 'the words are text; a renderer escapes the angle brackets');
+ assert.strictEqual(embed.image, null, 'a picture is never hot-linked');
+});
+
+t('three addresses at most, and a redirect is followed only once it is judged public', async () => {
+ const calls = [];
+ previews._setRequest(async (url) => {
+ calls.push(url);
+ if (url === 'https://93.184.216.34/redirect') return { status: 302, headers: { location: '/target' }, body: '' };
+ return type(page(`<meta property="og:title" content="${url.endsWith('target') ? 'Redirected' : 'One'}">`));
+ });
+ const made = await h.http('POST', '/api/chat/rooms', { token: alice.token, body: { name: 'Redirect room' } });
+ assert.strictEqual(made.status, 201, made.text);
+ const sent = await h.http('POST', `/api/chat/rooms/${made.body.room.slug}/messages`, { token: alice.token, body: { message: 'one https://93.184.216.34/redirect and two https://93.184.216.34/b and three https://93.184.216.34/c here' } });
+ assert.strictEqual(sent.status, 201, sent.text);
+ await previews.idle();
+ const got = await h.http('GET', `/api/chat/rooms/${made.body.room.slug}/messages`, { token: alice.token });
+ const embeds = got.body.messages.find((m) => m.id === sent.body.message.id).embeds;
+ assert.strictEqual(embeds.length, 3);
+ assert.strictEqual(new URL(embeds[0].url).pathname, '/redirect', 'the preview keeps the address the writer sent');
+ assert.ok(calls.includes('https://93.184.216.34/target'));
+});
+
+t.run(async () => { previews._reset(); await h.close(); });
@@ -37,6 +37,7 @@ const TABLES = [
@@ -37,6 +37,7 @@ const TABLES = [
'room_message_reactions', 'room_member_timeouts', 'config_snapshots', 'config_keys', 'chat_activity',
'room_message_reactions', 'room_member_timeouts', 'config_snapshots', 'config_keys', 'chat_activity',
'agent_play_requests', 'agent_play_votes', 'agent_play_log',
'agent_play_requests', 'agent_play_votes', 'agent_play_log',
'party_pool_settings', 'party_pool_machines', 'party_pool_holds', // 0064: OpenVibe's own computers for parties
'party_pool_settings', 'party_pool_machines', 'party_pool_holds', // 0064: OpenVibe's own computers for parties
+ 'link_previews', 'link_preview_cache', // 0065: address previews beside the message, and the address cache
];
];
const INDEXES = [
const INDEXES = [
'idx_chat_ts_deleted', 'idx_chat_page_live', 'idx_chat_channel_user_ts', 'idx_chat_stream_ts', 'idx_emotes_channel_code', 'idx_chat_tl_dedup',
'idx_chat_ts_deleted', 'idx_chat_page_live', 'idx_chat_channel_user_ts', 'idx_chat_stream_ts', 'idx_emotes_channel_code', 'idx_chat_tl_dedup',
@@ -44,6 +45,7 @@ const INDEXES = [
@@ -44,6 +45,7 @@ const INDEXES = [
'idx_room_messages_room', 'idx_room_members_user', 'idx_network_blocks_blocked', 'idx_mod_actions_created',
'idx_room_messages_room', 'idx_room_members_user', 'idx_network_blocks_blocked', 'idx_mod_actions_created',
'idx_dm_contacts_user', 'idx_rooms_forum_space',
'idx_dm_contacts_user', 'idx_rooms_forum_space',
'chat_app_link_tokens_by_person', 'chat_app_link_tokens_expiry', 'chat_app_callbacks_by_chat', 'chat_app_callbacks_expiry',
'chat_app_link_tokens_by_person', 'chat_app_link_tokens_expiry', 'chat_app_callbacks_by_chat', 'chat_app_callbacks_expiry',
+ 'idx_link_previews_message',
];
];
const MIRRORED = ['chat_messages', 'dm_conversations', 'dm_participants', 'dm_messages', 'dm_blocks', 'tts_voice_overrides',
const MIRRORED = ['chat_messages', 'dm_conversations', 'dm_participants', 'dm_messages', 'dm_blocks', 'tts_voice_overrides',
'channel_sounds', 'relay_users', 'hidden_relay_users', 'pending_ip_messages', 'stream_first_chats', 'moderation_actions'];
'channel_sounds', 'relay_users', 'hidden_relay_users', 'pending_ip_messages', 'stream_first_chats', 'moderation_actions'];
@@ -131,6 +131,7 @@ t('ratchet: every file that makes an outbound request itself is reviewed', () =>
@@ -131,6 +131,7 @@ t('ratchet: every file that makes an outbound request itself is reviewed', () =>
'server/web/party-pages.js': 'OpenVibe.Video (fixed: https://openvibe.video, or a test\'s CHAT_PARTY_VIDEO_URL), fixed /api/v1/video/watchable paths; a pasted link is only a query parameter, never fetched; redirects refused, 2.5 s, cached a minute',
'server/web/party-pages.js': 'OpenVibe.Video (fixed: https://openvibe.video, or a test\'s CHAT_PARTY_VIDEO_URL), fixed /api/v1/video/watchable paths; a pasted link is only a query parameter, never fetched; redirects refused, 2.5 s, cached a minute',
'server/rooms/radio.js': 'OpenVibe.Download Radio catalogue (fixed https origin, or a test\'s CHAT_RADIO_URL); track ids match rad_ ULIDs, redirects refused, 2.5 s',
'server/rooms/radio.js': 'OpenVibe.Download Radio catalogue (fixed https origin, or a test\'s CHAT_RADIO_URL); track ids match rad_ ULIDs, redirects refused, 2.5 s',
'server/chat/link-cards.js': 'OpenVibe.Video and OpenVibe.Actor (fixed https origins, or a test\'s CHAT_LINK_VIDEO_URL / CHAT_LINK_ACTOR_URL), fixed paths (a watchable lookup, /voices/<id>.json, /skills/<slug>.json); a pasted link is a query parameter or a path segment only, never fetched; redirects refused, 2.5 s, cached a minute',
'server/chat/link-cards.js': 'OpenVibe.Video and OpenVibe.Actor (fixed https origins, or a test\'s CHAT_LINK_VIDEO_URL / CHAT_LINK_ACTOR_URL), fixed paths (a watchable lookup, /voices/<id>.json, /skills/<slug>.json); a pasted link is a query parameter or a path segment only, never fetched; redirects refused, 2.5 s, cached a minute',
+ 'server/chat/link-previews.js': 'an address a person wrote: https only, assertPublicUrl on every hop and safeLookup on the connection, at most two redirects each judged, 4 s and 512 KiB caps, no cookies; YouTube/Vimeo oEmbed on fixed origins; OpenVibe\'s own sites resolve through fixed public APIs and are never page-fetched',
'server/web/room-embed-pages.js': 'not the server: the sign-in window\'s own script posts to /embed/bind on this same site',
'server/web/room-embed-pages.js': 'not the server: the sign-in window\'s own script posts to /embed/bind on this same site',
};
};
const root = path.join(__dirname, '..');
const root = path.join(__dirname, '..');