openvibers/openvibe.chat Commits
Support inboxes: Vibe sees the pictures a signed-in visitor sends (as images, with their words) (#161)
Changes
@@@ -518,7 +518,8 @@ for people and for companies, with "direct live support with an agent", "fully minbox show inline (only openvibe.pics, at most four a message). **A visitor signed in to OpenVibe** sends one thesame way from the panel's picture button (`POST /api/chat/widget/inbox/:id/picture`, the body is the picture, theirtoken in `X-Visitor-Token`, a line in `X-Caption`): it goes to OpenVibe.Pics as them, and their messages carry-- `account: true`, so only their Pics links show as pictures. Anonymous visitors are asked to sign in (403++ `account: true`, so only their Pics links show as pictures. **Vibe sees them too:** the newest picture in a++ signed-in visitor's latest messages (two at most) goes to the model as an image with their words. Anonymous visitors are asked to sign in (403`inbox.sign_in_to_attach`) until Pics scans anonymous uploads, and a link one pastes stays a link. Wherever amessage's pictures show, their links are not repeated as text, and lists say "📷 Picture".- **Translation** (`server/inbox/translate.js`): "Translate" under a visitor's message puts it in the reader's@@@ -567,6 +567,24 @@ async function inboxTools(inbox, conv) {return { defs, run };}++/**++ * Vibe sees the pictures a signed-in visitor sent (their OpenVibe.Pics links; anyone else's stay links): in the visitor's++ * latest messages, the newest that has any, at most two, given to the model with that message (rooms/pictures forModel).++ */++async function showPictures(said, transcript) {++ const pictures = require('../rooms/pictures');++ for (let i = said.length - 1; i >= 0 && said[i].author === 'visitor'; i--) {++ if (said[i].author_user_id == null) continue;++ const links = pictures.linksIn(said[i].body).slice(0, 2);++ if (!links.length) continue;++ const images = (await Promise.all(links.map((l) => pictures.forModel(l).catch(() => null)))).filter(Boolean);++ if (images.length) {++ transcript[i] = { role: 'user', content: [{ type: 'text', text: `${transcript[i].content}\n\n(${images.length === 1 ? 'The picture they sent is' : 'The pictures they sent are'} attached.)` }, ...images.map((url) => ({ type: 'image_url', image_url: { url } }))] };++ }++ return;++ }++}++async function aiAnswer(inbox, convId, deps = {}) {if (!Number(inbox.ai_enabled) || inFlight.has(convId) || !config.oauth.clientSecret) return null;const conv = await db.get('SELECT * FROM inbox_conversations WHERE id = ?', [convId]);@@@ -585,8 +603,9 @@ async function aiAnswer(inbox, convId, deps = {}) {try {broadcaster(deps)(inbox.id, conv.id, { type: 'inbox_typing', inbox: inbox.id, conversation: conv.id, who: 'Vibe' });const history = await db.all('SELECT * FROM inbox_messages WHERE conversation_id = ? ORDER BY id DESC LIMIT 20', [conv.id]);-- const transcript = history.reverse().filter((m) => m.author !== 'system')-- .map((m) => ({ role: m.author === 'visitor' ? 'user' : 'assistant', content: `${m.author === 'agent' ? `(${m.author_name || 'a person'} from ${inbox.name}) ` : ''}${String(m.body).slice(0, 2000)}` }));++ const said = history.reverse().filter((m) => m.author !== 'system');++ const transcript = said.map((m) => ({ role: m.author === 'visitor' ? 'user' : 'assistant', content: `${m.author === 'agent' ? `(${m.author_name || 'a person'} from ${inbox.name}) ` : ''}${String(m.body).slice(0, 2000)}` }));++ await showPictures(said, transcript);const tools = await inboxTools(inbox, conv);const aiClient = deps.aiClient || require('../ai/client');// What Vibe may know about where the visitor is and who they are (a verified name only: anyone can type a name).@@@ -32,6 +32,8 @@ function startPics() {let n = 0;picsServer = require('http').createServer((req, res) => {req.resume();++ // A picture itself (what Vibe is shown): a few PNG bytes.++ if (req.method === 'GET') { res.writeHead(200, { 'content-type': 'image/png' }); return res.end(Buffer.from('89504e470d0a1a0a', 'hex')); }req.on('end', () => {const slug = `Pk${String(++n).padStart(6, '2')}`.replace(/[01]/g, '3');res.writeHead(201, { 'content-type': 'application/json' });@@@ -437,7 +439,7 @@ t('a signed-in visitor sends a picture from the panel (through OpenVibe.Pics as.then(async (r) => ({ status: r.status, body: await r.json().catch(() => null) }));const anon = (await widget('POST', '/visit', {})).body.token;assert.deepStrictEqual([(await picture({ 'x-visitor-token': anon })).status, (await picture({ 'x-visitor-token': anon })).body.code], [403, 'inbox.sign_in_to_attach']);-- const pasted = await widget('POST', '/messages', { token: anon, message: 'Look https://openvibe.pics/i/Pk222222.png' });++ const pasted = await widget('POST', '/messages', { token: anon, message: 'Look https://openvibe.pics/i/PkZZZZZZ.png' });assert.strictEqual(pasted.body.message.account, undefined, 'an anonymous visitor\'s link is only a link');const eve = h.addUser('eve', { subject: ids.newId('user') });await h.ctx.upsertUser(h.live.users.get(eve.id));@@@ -449,6 +451,11 @@ t('a signed-in visitor sends a picture from the panel (through OpenVibe.Pics asassert.ok(/^The crack, here\nhttps:\/\/openvibe\.pics\/i\/[2-9a-km-zA-HJ-NP-Z]{8}\.png$/.test(sent.body.message.body), sent.body.message.body);assert.deepStrictEqual([sent.body.message.author, sent.body.message.account], ['visitor', true]);const cid = sent.body.conversation.id;++ const shown = await until(async () => turns.find((x) => x.input.messages.some((m) => Array.isArray(m.content) && m.content.some((c) => c.type === 'image_url'))), 'Vibe is shown the picture');++ const withPic = shown.input.messages.find((m) => Array.isArray(m.content));++ assert.ok(withPic.content[0].text.includes('The crack, here') && withPic.content[0].text.includes('The picture they sent is attached.'));++ assert.ok(/^data:image\/png;base64,/.test(withPic.content[1].image_url.url), 'as an image, not a link');++ assert.ok(!turns.some((x) => x.input.messages.some((m) => Array.isArray(m.content) && m.content[0].text.includes('PkZZZZZZ'))), 'an anonymous visitor\'s link stays text');const typed = await widget('POST', '/messages', { token, message: 'And a second line' }, as);assert.strictEqual(typed.body.message.account, true, 'everything they write signed in is marked');const page = (path, u) => fetch(`${h.base}${path}`, { headers: { cookie: `ov_token=${u.token}` } }).then((x) => x.text());
@@ -518,7 +518,8 @@ for people and for companies, with "direct live support with an agent", "fully m
@@ -518,7 +518,8 @@ for people and for companies, with "direct live support with an agent", "fully m
inbox show inline (only openvibe.pics, at most four a message). **A visitor signed in to OpenVibe** sends one the
inbox show inline (only openvibe.pics, at most four a message). **A visitor signed in to OpenVibe** sends one the
same way from the panel's picture button (`POST /api/chat/widget/inbox/:id/picture`, the body is the picture, their
same way from the panel's picture button (`POST /api/chat/widget/inbox/:id/picture`, the body is the picture, their
token in `X-Visitor-Token`, a line in `X-Caption`): it goes to OpenVibe.Pics as them, and their messages carry
token in `X-Visitor-Token`, a line in `X-Caption`): it goes to OpenVibe.Pics as them, and their messages carry
- `account: true`, so only their Pics links show as pictures. Anonymous visitors are asked to sign in (403
+ `account: true`, so only their Pics links show as pictures. **Vibe sees them too:** the newest picture in a
+ signed-in visitor's latest messages (two at most) goes to the model as an image with their words. Anonymous visitors are asked to sign in (403
`inbox.sign_in_to_attach`) until Pics scans anonymous uploads, and a link one pastes stays a link. Wherever a
`inbox.sign_in_to_attach`) until Pics scans anonymous uploads, and a link one pastes stays a link. Wherever a
message's pictures show, their links are not repeated as text, and lists say "📷 Picture".
message's pictures show, their links are not repeated as text, and lists say "📷 Picture".
- **Translation** (`server/inbox/translate.js`): "Translate" under a visitor's message puts it in the reader's
- **Translation** (`server/inbox/translate.js`): "Translate" under a visitor's message puts it in the reader's
@@ -567,6 +567,24 @@ async function inboxTools(inbox, conv) {
@@ -567,6 +567,24 @@ async function inboxTools(inbox, conv) {
return { defs, run };
return { defs, run };
}
}
+/**
+ * Vibe sees the pictures a signed-in visitor sent (their OpenVibe.Pics links; anyone else's stay links): in the visitor's
+ * latest messages, the newest that has any, at most two, given to the model with that message (rooms/pictures forModel).
+ */
+async function showPictures(said, transcript) {
+ const pictures = require('../rooms/pictures');
+ for (let i = said.length - 1; i >= 0 && said[i].author === 'visitor'; i--) {
+ if (said[i].author_user_id == null) continue;
+ const links = pictures.linksIn(said[i].body).slice(0, 2);
+ if (!links.length) continue;
+ const images = (await Promise.all(links.map((l) => pictures.forModel(l).catch(() => null)))).filter(Boolean);
+ if (images.length) {
+ transcript[i] = { role: 'user', content: [{ type: 'text', text: `${transcript[i].content}\n\n(${images.length === 1 ? 'The picture they sent is' : 'The pictures they sent are'} attached.)` }, ...images.map((url) => ({ type: 'image_url', image_url: { url } }))] };
+ }
+ return;
+ }
+}
+
async function aiAnswer(inbox, convId, deps = {}) {
async function aiAnswer(inbox, convId, deps = {}) {
if (!Number(inbox.ai_enabled) || inFlight.has(convId) || !config.oauth.clientSecret) return null;
if (!Number(inbox.ai_enabled) || inFlight.has(convId) || !config.oauth.clientSecret) return null;
const conv = await db.get('SELECT * FROM inbox_conversations WHERE id = ?', [convId]);
const conv = await db.get('SELECT * FROM inbox_conversations WHERE id = ?', [convId]);
@@ -585,8 +603,9 @@ async function aiAnswer(inbox, convId, deps = {}) {
@@ -585,8 +603,9 @@ async function aiAnswer(inbox, convId, deps = {}) {
try {
try {
broadcaster(deps)(inbox.id, conv.id, { type: 'inbox_typing', inbox: inbox.id, conversation: conv.id, who: 'Vibe' });
broadcaster(deps)(inbox.id, conv.id, { type: 'inbox_typing', inbox: inbox.id, conversation: conv.id, who: 'Vibe' });
const history = await db.all('SELECT * FROM inbox_messages WHERE conversation_id = ? ORDER BY id DESC LIMIT 20', [conv.id]);
const history = await db.all('SELECT * FROM inbox_messages WHERE conversation_id = ? ORDER BY id DESC LIMIT 20', [conv.id]);
- const transcript = history.reverse().filter((m) => m.author !== 'system')
- .map((m) => ({ role: m.author === 'visitor' ? 'user' : 'assistant', content: `${m.author === 'agent' ? `(${m.author_name || 'a person'} from ${inbox.name}) ` : ''}${String(m.body).slice(0, 2000)}` }));
+ const said = history.reverse().filter((m) => m.author !== 'system');
+ const transcript = said.map((m) => ({ role: m.author === 'visitor' ? 'user' : 'assistant', content: `${m.author === 'agent' ? `(${m.author_name || 'a person'} from ${inbox.name}) ` : ''}${String(m.body).slice(0, 2000)}` }));
+ await showPictures(said, transcript);
const tools = await inboxTools(inbox, conv);
const tools = await inboxTools(inbox, conv);
const aiClient = deps.aiClient || require('../ai/client');
const aiClient = deps.aiClient || require('../ai/client');
// What Vibe may know about where the visitor is and who they are (a verified name only: anyone can type a name).
// What Vibe may know about where the visitor is and who they are (a verified name only: anyone can type a name).
@@ -32,6 +32,8 @@ function startPics() {
@@ -32,6 +32,8 @@ function startPics() {
let n = 0;
let n = 0;
picsServer = require('http').createServer((req, res) => {
picsServer = require('http').createServer((req, res) => {
req.resume();
req.resume();
+ // A picture itself (what Vibe is shown): a few PNG bytes.
+ if (req.method === 'GET') { res.writeHead(200, { 'content-type': 'image/png' }); return res.end(Buffer.from('89504e470d0a1a0a', 'hex')); }
req.on('end', () => {
req.on('end', () => {
const slug = `Pk${String(++n).padStart(6, '2')}`.replace(/[01]/g, '3');
const slug = `Pk${String(++n).padStart(6, '2')}`.replace(/[01]/g, '3');
res.writeHead(201, { 'content-type': 'application/json' });
res.writeHead(201, { 'content-type': 'application/json' });
@@ -437,7 +439,7 @@ t('a signed-in visitor sends a picture from the panel (through OpenVibe.Pics as
@@ -437,7 +439,7 @@ t('a signed-in visitor sends a picture from the panel (through OpenVibe.Pics as
.then(async (r) => ({ status: r.status, body: await r.json().catch(() => null) }));
.then(async (r) => ({ status: r.status, body: await r.json().catch(() => null) }));
const anon = (await widget('POST', '/visit', {})).body.token;
const anon = (await widget('POST', '/visit', {})).body.token;
assert.deepStrictEqual([(await picture({ 'x-visitor-token': anon })).status, (await picture({ 'x-visitor-token': anon })).body.code], [403, 'inbox.sign_in_to_attach']);
assert.deepStrictEqual([(await picture({ 'x-visitor-token': anon })).status, (await picture({ 'x-visitor-token': anon })).body.code], [403, 'inbox.sign_in_to_attach']);
- const pasted = await widget('POST', '/messages', { token: anon, message: 'Look https://openvibe.pics/i/Pk222222.png' });
+ const pasted = await widget('POST', '/messages', { token: anon, message: 'Look https://openvibe.pics/i/PkZZZZZZ.png' });
assert.strictEqual(pasted.body.message.account, undefined, 'an anonymous visitor\'s link is only a link');
assert.strictEqual(pasted.body.message.account, undefined, 'an anonymous visitor\'s link is only a link');
const eve = h.addUser('eve', { subject: ids.newId('user') });
const eve = h.addUser('eve', { subject: ids.newId('user') });
await h.ctx.upsertUser(h.live.users.get(eve.id));
await h.ctx.upsertUser(h.live.users.get(eve.id));
@@ -449,6 +451,11 @@ t('a signed-in visitor sends a picture from the panel (through OpenVibe.Pics as
@@ -449,6 +451,11 @@ t('a signed-in visitor sends a picture from the panel (through OpenVibe.Pics as
assert.ok(/^The crack, here\nhttps:\/\/openvibe\.pics\/i\/[2-9a-km-zA-HJ-NP-Z]{8}\.png$/.test(sent.body.message.body), sent.body.message.body);
assert.ok(/^The crack, here\nhttps:\/\/openvibe\.pics\/i\/[2-9a-km-zA-HJ-NP-Z]{8}\.png$/.test(sent.body.message.body), sent.body.message.body);
assert.deepStrictEqual([sent.body.message.author, sent.body.message.account], ['visitor', true]);
assert.deepStrictEqual([sent.body.message.author, sent.body.message.account], ['visitor', true]);
const cid = sent.body.conversation.id;
const cid = sent.body.conversation.id;
+ const shown = await until(async () => turns.find((x) => x.input.messages.some((m) => Array.isArray(m.content) && m.content.some((c) => c.type === 'image_url'))), 'Vibe is shown the picture');
+ const withPic = shown.input.messages.find((m) => Array.isArray(m.content));
+ assert.ok(withPic.content[0].text.includes('The crack, here') && withPic.content[0].text.includes('The picture they sent is attached.'));
+ assert.ok(/^data:image\/png;base64,/.test(withPic.content[1].image_url.url), 'as an image, not a link');
+ assert.ok(!turns.some((x) => x.input.messages.some((m) => Array.isArray(m.content) && m.content[0].text.includes('PkZZZZZZ'))), 'an anonymous visitor\'s link stays text');
const typed = await widget('POST', '/messages', { token, message: 'And a second line' }, as);
const typed = await widget('POST', '/messages', { token, message: 'And a second line' }, as);
assert.strictEqual(typed.body.message.account, true, 'everything they write signed in is marked');
assert.strictEqual(typed.body.message.account, true, 'everything they write signed in is marked');
const page = (path, u) => fetch(`${h.base}${path}`, { headers: { cookie: `ov_token=${u.token}` } }).then((x) => x.text());
const page = (path, u) => fetch(`${h.base}${path}`, { headers: { cookie: `ov_token=${u.token}` } }).then((x) => x.text());