openvibers/openvibe.chat Commits

Support inboxes: Vibe sees the pictures a signed-in visitor sends (as images, with their words) (#161)

OP OpenVibers committed 19 days ago · 55906892cc · 3 files, +31 −4

Changes

ddiff --git a/README.md b/README.md
iindex 785e316..7b50368 100644
---- a/README.md
++++ b/README.md
@@@ -518,7 +518,8 @@ for people and for companies, with "direct live support with an agent", "fully m
inbox show inline (only openvibe.pics, at most four a message). **A visitor signed in to OpenVibe** sends one the
same way from the panel's picture button (`POST /api/chat/widget/inbox/:id/picture`, the body is the picture, their
token in `X-Visitor-Token`, a line in `X-Caption`): it goes to OpenVibe.Pics as them, and their messages carry
-- `account: true`, so only their Pics links show as pictures. Anonymous visitors are asked to sign in (403
++ `account: true`, so only their Pics links show as pictures. **Vibe sees them too:** the newest picture in a
++ signed-in visitor's latest messages (two at most) goes to the model as an image with their words. Anonymous visitors are asked to sign in (403
`inbox.sign_in_to_attach`) until Pics scans anonymous uploads, and a link one pastes stays a link. Wherever a
message's pictures show, their links are not repeated as text, and lists say "📷 Picture".
- **Translation** (`server/inbox/translate.js`): "Translate" under a visitor's message puts it in the reader's
ddiff --git a/server/inbox/inbox.js b/server/inbox/inbox.js
iindex aee906f..89e2ea1 100644
---- a/server/inbox/inbox.js
++++ b/server/inbox/inbox.js
@@@ -567,6 +567,24 @@ async function inboxTools(inbox, conv) {
return { defs, run };
}
++/**
++ * Vibe sees the pictures a signed-in visitor sent (their OpenVibe.Pics links; anyone else's stay links): in the visitor's
++ * latest messages, the newest that has any, at most two, given to the model with that message (rooms/pictures forModel).
++ */
++async function showPictures(said, transcript) {
++ const pictures = require('../rooms/pictures');
++ for (let i = said.length - 1; i >= 0 && said[i].author === 'visitor'; i--) {
++ if (said[i].author_user_id == null) continue;
++ const links = pictures.linksIn(said[i].body).slice(0, 2);
++ if (!links.length) continue;
++ const images = (await Promise.all(links.map((l) => pictures.forModel(l).catch(() => null)))).filter(Boolean);
++ if (images.length) {
++ transcript[i] = { role: 'user', content: [{ type: 'text', text: `${transcript[i].content}\n\n(${images.length === 1 ? 'The picture they sent is' : 'The pictures they sent are'} attached.)` }, ...images.map((url) => ({ type: 'image_url', image_url: { url } }))] };
++ }
++ return;
++ }
++}
++
async function aiAnswer(inbox, convId, deps = {}) {
if (!Number(inbox.ai_enabled) || inFlight.has(convId) || !config.oauth.clientSecret) return null;
const conv = await db.get('SELECT * FROM inbox_conversations WHERE id = ?', [convId]);
@@@ -585,8 +603,9 @@ async function aiAnswer(inbox, convId, deps = {}) {
try {
broadcaster(deps)(inbox.id, conv.id, { type: 'inbox_typing', inbox: inbox.id, conversation: conv.id, who: 'Vibe' });
const history = await db.all('SELECT * FROM inbox_messages WHERE conversation_id = ? ORDER BY id DESC LIMIT 20', [conv.id]);
-- const transcript = history.reverse().filter((m) => m.author !== 'system')
-- .map((m) => ({ role: m.author === 'visitor' ? 'user' : 'assistant', content: `${m.author === 'agent' ? `(${m.author_name || 'a person'} from ${inbox.name}) ` : ''}${String(m.body).slice(0, 2000)}` }));
++ const said = history.reverse().filter((m) => m.author !== 'system');
++ const transcript = said.map((m) => ({ role: m.author === 'visitor' ? 'user' : 'assistant', content: `${m.author === 'agent' ? `(${m.author_name || 'a person'} from ${inbox.name}) ` : ''}${String(m.body).slice(0, 2000)}` }));
++ await showPictures(said, transcript);
const tools = await inboxTools(inbox, conv);
const aiClient = deps.aiClient || require('../ai/client');
// What Vibe may know about where the visitor is and who they are (a verified name only: anyone can type a name).
ddiff --git a/test/inbox.test.js b/test/inbox.test.js
iindex 4bb8cb6..16de2b2 100644
---- a/test/inbox.test.js
++++ b/test/inbox.test.js
@@@ -32,6 +32,8 @@ function startPics() {
let n = 0;
picsServer = require('http').createServer((req, res) => {
req.resume();
++ // A picture itself (what Vibe is shown): a few PNG bytes.
++ if (req.method === 'GET') { res.writeHead(200, { 'content-type': 'image/png' }); return res.end(Buffer.from('89504e470d0a1a0a', 'hex')); }
req.on('end', () => {
const slug = `Pk${String(++n).padStart(6, '2')}`.replace(/[01]/g, '3');
res.writeHead(201, { 'content-type': 'application/json' });
@@@ -437,7 +439,7 @@ t('a signed-in visitor sends a picture from the panel (through OpenVibe.Pics as
.then(async (r) => ({ status: r.status, body: await r.json().catch(() => null) }));
const anon = (await widget('POST', '/visit', {})).body.token;
assert.deepStrictEqual([(await picture({ 'x-visitor-token': anon })).status, (await picture({ 'x-visitor-token': anon })).body.code], [403, 'inbox.sign_in_to_attach']);
-- const pasted = await widget('POST', '/messages', { token: anon, message: 'Look https://openvibe.pics/i/Pk222222.png' });
++ const pasted = await widget('POST', '/messages', { token: anon, message: 'Look https://openvibe.pics/i/PkZZZZZZ.png' });
assert.strictEqual(pasted.body.message.account, undefined, 'an anonymous visitor\'s link is only a link');
const eve = h.addUser('eve', { subject: ids.newId('user') });
await h.ctx.upsertUser(h.live.users.get(eve.id));
@@@ -449,6 +451,11 @@ t('a signed-in visitor sends a picture from the panel (through OpenVibe.Pics as
assert.ok(/^The crack, here\nhttps:\/\/openvibe\.pics\/i\/[2-9a-km-zA-HJ-NP-Z]{8}\.png$/.test(sent.body.message.body), sent.body.message.body);
assert.deepStrictEqual([sent.body.message.author, sent.body.message.account], ['visitor', true]);
const cid = sent.body.conversation.id;
++ const shown = await until(async () => turns.find((x) => x.input.messages.some((m) => Array.isArray(m.content) && m.content.some((c) => c.type === 'image_url'))), 'Vibe is shown the picture');
++ const withPic = shown.input.messages.find((m) => Array.isArray(m.content));
++ assert.ok(withPic.content[0].text.includes('The crack, here') && withPic.content[0].text.includes('The picture they sent is attached.'));
++ assert.ok(/^data:image\/png;base64,/.test(withPic.content[1].image_url.url), 'as an image, not a link');
++ assert.ok(!turns.some((x) => x.input.messages.some((m) => Array.isArray(m.content) && m.content[0].text.includes('PkZZZZZZ'))), 'an anonymous visitor\'s link stays text');
const typed = await widget('POST', '/messages', { token, message: 'And a second line' }, as);
assert.strictEqual(typed.body.message.account, true, 'everything they write signed in is marked');
const page = (path, u) => fetch(`${h.base}${path}`, { headers: { cookie: `ov_token=${u.token}` } }).then((x) => x.text());