openvibers/openvibe.chat Commits

Support calls: the visitor shares their screen with the team (#160)

OP OpenVibers committed 19 days ago · 05a3947acf · 9 files, +80 −15

* Support calls: the visitor shares their screen with the team (the browser's own picker), shown above the call bar; default greeting without a doubled period

* Docs: screen sharing in support calls

Changes

ddiff --git a/README.md b/README.md
iindex 790a07a..785e316 100644
---- a/README.md
++++ b/README.md
@@@ -501,7 +501,9 @@ for people and for companies, with "direct live support with an agent", "fully m
(`public/web/support-call.js`), at most four people. The visitor joins with their visitor token (a `support-visitor`
message); the inbox's people join as themselves; anyone else gets "not found". Either side ends it for everyone
(`DELETE …/call`, or the visitor's `POST /api/chat/widget/inbox/:id/call/end { token }`), and resolving the
-- conversation ends it too. A conversation carries `call: true` while one is open. The widget's frame may ask for the
++ conversation ends it too. In the call the visitor may **share their screen** (the browser's own picker; not on
++ phones): the team sees it above the call bar, with a full-screen button. A conversation carries `call: true` while one
++ is open. The widget's frame may ask for the
microphone (the browser still asks the visitor).
- **Records** (`server/inbox/records.js`): how long finished conversations stay, set in the settings ("Keep finished
conversations": forever, 30 or 90 days, 6 months, 1 or 2 years) or with `PATCH /api/chat/inboxes/:id { retention_days:
ddiff --git a/docs/platform.md b/docs/platform.md
iindex 04efe72..a13b94a 100644
---- a/docs/platform.md
++++ b/docs/platform.md
@@@ -58,7 +58,7 @@ center (MagicBrowse privacy), getstream.io (Stream vs Sendbird), rumbletalk.com,
3. **Community room** (built: `data-room`, bubble or inline, sign-in by ticket) **and site assistant** (built: an inbox
whose people do not answer) **modes** of the widget; pictures in the embed (built for signed-in visitors;
anonymous ones once Pics scans anonymous uploads), then video and files; call escalation from a support conversation
-- (built: a voice call the team starts, the visitor joins from the panel; next, sharing their screen in it).
++ (built: a voice call the team starts, the visitor joins from the panel and may share their screen in it).
4. **Developer platform**: `openvibe-sdk/chat` (built, SDK#78: inboxes, feeds, Vibe, `createInboxVisitor`,
`createFeedPublisher`), inbox webhooks (built: `chat.inbox.*` events signed like Events deliveries, Slack and
Discord lines). Next: capabilities in OpenVibe.Contracts (`chat.inbox.*`, `chat.feed.*`) so developer apps act for an
ddiff --git a/public/web/embed.js b/public/web/embed.js
iindex b6c4600..1d45317 100644
---- a/public/web/embed.js
++++ b/public/web/embed.js
@@@ -276,11 +276,16 @@
var row = el('div', 'em-call-row');
var mute = el('button', 'em-call-no', 'Mute'); mute.type = 'button'; mute.setAttribute('aria-pressed', 'false');
var hang = el('button', 'em-call-hang', 'Hang up'); hang.type = 'button';
-- row.appendChild(mute); row.appendChild(hang); callEl.appendChild(row);
++ var shareBtn = el('button', 'em-call-no', 'Share my screen'); shareBtn.type = 'button';
++ row.appendChild(mute);
++ if (navigator.mediaDevices && navigator.mediaDevices.getDisplayMedia) row.appendChild(shareBtn); // not on phones
++ row.appendChild(hang); callEl.appendChild(row);
++ shareBtn.addEventListener('click', function () { if (!call) return; if (call.sharing) call.stopSharing(); else call.share(); });
call = window.OVSupportCall.join({
channel: 'support-' + conv.id, visitorToken: token,
onState: function (s, t) { if (s === 'in') text.textContent = t || 'You are in the call.'; else if (s === 'ended' || s === 'error') { call = null; endCallCard(t); } else text.textContent = t; },
onPeople: function (names) { text.textContent = names.length ? 'In the call with ' + names.join(', ') + '.' : 'Waiting for the team to join…'; },
++ onSharing: function (on) { shareBtn.textContent = on ? 'Stop sharing' : 'Share my screen'; shareBtn.setAttribute('aria-pressed', on ? 'true' : 'false'); },
});
mute.addEventListener('click', function () { if (!call) return; call.mute(!call.muted); mute.textContent = call.muted ? 'Unmute' : 'Mute'; mute.setAttribute('aria-pressed', call.muted ? 'true' : 'false'); });
hang.addEventListener('click', function () { post('/call/end').catch(function () {}); if (call) call.leave(); });
ddiff --git a/public/web/inbox.js b/public/web/inbox.js
iindex fd6daaf..d1820a4 100644
---- a/public/web/inbox.js
++++ b/public/web/inbox.js
@@@ -252,11 +252,23 @@
channel: 'support-' + cfg.conversation, token: token(),
onState: function (s, t) { if (s === 'ended' || s === 'error') { call = null; callGone(t); } else if (t) callText.textContent = t; },
onPeople: function (names) { callText.textContent = names.length ? 'In the call with ' + names.join(', ') + '.' : 'Calling… the visitor sees an invitation in their chat.'; },
++ onScreen: showScreen,
});
}
++ // The visitor's shared screen, above the call bar.
++ var screenBox = document.getElementById('ib-screen'), screenVideo = document.getElementById('ib-screen-video'), screenWho = document.getElementById('ib-screen-who');
++ function showScreen(stream, name) {
++ if (!screenBox) return;
++ if (!stream) { screenBox.hidden = true; screenVideo.srcObject = null; return; }
++ screenWho.textContent = (name || 'The visitor') + ' is sharing their screen';
++ screenVideo.srcObject = stream; screenBox.hidden = false;
++ }
++ var fullBtn = document.getElementById('ib-screen-full');
++ if (fullBtn) fullBtn.addEventListener('click', function () { if (screenVideo.requestFullscreen) screenVideo.requestFullscreen().catch(function () {}); });
function callOffered(by) { if (call) return; showBar(by ? by + ' started a voice call.' : 'A voice call with this visitor is open.'); callJoin.hidden = false; callMute.hidden = true; }
function callGone(text) {
if (call) { var c = call; call = null; c.leave(); }
++ showScreen(null);
if (!callBar) return;
callBar.hidden = true;
if (callBtn) callBtn.hidden = false;
ddiff --git a/public/web/support-call.js b/public/web/support-call.js
iindex b21ee8b..1d9cba2 100644
---- a/public/web/support-call.js
++++ b/public/web/support-call.js
@@@ -4,16 +4,19 @@
var call = OVSupportCall.join({ channel: 'support-cnv_…', visitorToken: '…' | token: '…',
onState: function (state, text) {}, // joining | in | ended | error
-- onPeople: function (names) {} });
-- call.mute(true); call.leave();
++ onPeople: function (names) {},
++ onScreen: function (stream | null, name) {} }); // someone's shared screen
++ call.mute(true); call.share(); call.stopSharing(); call.leave();
-- Audio only; the microphone is asked for when joining (no microphone: you listen). */
++ Voice, and a shared screen: the visitor may share theirs (the browser's own picker decides what), shown to the team;
++ the microphone is asked for when joining (no microphone: you listen). The screen rides the same peer connections
++ (renegotiated); camera-off false/true tells the others it started or stopped. */
(function () {
'use strict';
if (!window.RTCPeerConnection || !window.WebSocket) return;
function join(opts) {
-- var ws = null, localStream = null, myPeerId = null, ended = false, muted = false;
++ var ws = null, localStream = null, myPeerId = null, ended = false, muted = false, screen = null;
var iceServers = [{ urls: 'stun:stun.l.google.com:19302' }];
var peers = new Map(), people = new Map();
var state = function (s, text) { if (opts.onState) opts.onState(s, text || ''); };
@@@ -37,18 +40,26 @@
peers.set(id, p);
if (localStream) localStream.getTracks().forEach(function (t) { pc.addTrack(t, localStream); });
else if (initiator) pc.addTransceiver('audio', { direction: 'recvonly' });
++ if (screen) p.screenSender = pc.addTrack(screen.getVideoTracks()[0], screen);
pc.onicecandidate = function (e) { if (e.candidate) send({ type: 'ice-candidate', targetPeerId: id, candidate: e.candidate.toJSON ? e.candidate.toJSON() : e.candidate }); };
pc.ontrack = function (e) {
++ if (e.track.kind === 'video') {
++ var who = people.get(id);
++ if (opts.onScreen) opts.onScreen(new MediaStream([e.track]), (who && (who.displayName || who.username)) || 'Someone');
++ e.track.addEventListener('ended', function () { if (opts.onScreen) opts.onScreen(null); });
++ return;
++ }
if (!p.audio) { p.audio = document.createElement('audio'); p.audio.autoplay = true; p.audio.hidden = true; document.body.appendChild(p.audio); }
p.audio.srcObject = (e.streams && e.streams[0]) || new MediaStream([e.track]);
};
-- if (initiator) {
-- pc.createOffer().then(function (o) { return pc.setLocalDescription(o); })
-- .then(function () { send({ type: 'offer', targetPeerId: id, sdp: { type: pc.localDescription.type, sdp: pc.localDescription.sdp } }); })
-- .catch(function () { state('error', 'Could not start the call.'); });
-- }
++ if (initiator) offer(id, p);
return p;
}
++ function offer(id, p) {
++ p.pc.createOffer().then(function (o) { return p.pc.setLocalDescription(o); })
++ .then(function () { send({ type: 'offer', targetPeerId: id, sdp: { type: p.pc.localDescription.type, sdp: p.pc.localDescription.sdp } }); })
++ .catch(function () { state('error', 'Could not start the call.'); });
++ }
function flush(p) { var c; while ((c = p.pending.shift())) p.pc.addIceCandidate(c).catch(function () {}); }
function onMessage(m) {
@@@ -59,7 +70,8 @@
state('in', localStream ? '' : 'No microphone: you can listen.'); names();
break;
case 'peer-joined': people.set(m.peerId, m); peer(m.peerId, false); names(); break;
-- case 'peer-left': people.delete(m.peerId); closePeer(m.peerId); names(); break;
++ case 'peer-left': people.delete(m.peerId); closePeer(m.peerId); names(); if (opts.onScreen) opts.onScreen(null); break;
++ case 'peer-camera': if (m.cameraOff && opts.onScreen) opts.onScreen(null); break;
case 'offer': {
var p = peers.get(m.fromPeerId) || peer(m.fromPeerId, false);
p.pc.setRemoteDescription(m.sdp).then(function () { flush(p); return p.pc.createAnswer(); })
@@@ -80,8 +92,31 @@
}
}
++ function stopSharing() {
++ if (!screen) return;
++ screen.getTracks().forEach(function (t) { t.stop(); });
++ screen = null;
++ peers.forEach(function (p, id) { if (p.screenSender) { try { p.pc.removeTrack(p.screenSender); } catch (e) { /* */ } p.screenSender = null; offer(id, p); } });
++ send({ type: 'camera-off', cameraOff: true });
++ if (opts.onSharing) opts.onSharing(false);
++ }
++ function share() {
++ if (screen || ended || !navigator.mediaDevices || !navigator.mediaDevices.getDisplayMedia) return Promise.resolve(false);
++ return navigator.mediaDevices.getDisplayMedia({ video: { frameRate: 10 }, audio: false }).then(function (s) {
++ if (ended) { s.getTracks().forEach(function (t) { t.stop(); }); return false; }
++ screen = s;
++ var track = s.getVideoTracks()[0];
++ track.addEventListener('ended', stopSharing); // the browser's own "Stop sharing"
++ peers.forEach(function (p, id) { p.screenSender = p.pc.addTrack(track, s); offer(id, p); });
++ send({ type: 'camera-off', cameraOff: false });
++ if (opts.onSharing) opts.onSharing(true);
++ return true;
++ }, function () { return false; });
++ }
++
function stop(s, text) {
if (ended) return;
++ if (screen) { screen.getTracks().forEach(function (t) { t.stop(); }); screen = null; }
ended = true;
peers.forEach(function (p, id) { closePeer(id); });
people.clear();
@@@ -110,6 +145,9 @@
return {
mute: function (on) { muted = !!on; if (localStream) localStream.getAudioTracks().forEach(function (t) { t.enabled = !muted; }); send({ type: 'mute', muted: muted }); },
leave: function () { stop('ended', 'You left the call.'); },
++ share: share,
++ stopSharing: stopSharing,
++ get sharing() { return !!screen; },
get muted() { return muted; },
};
}
ddiff --git a/public/web/vibe.css b/public/web/vibe.css
iindex f69198c..1b3770c 100644
---- a/public/web/vibe.css
++++ b/public/web/vibe.css
@@@ -246,6 +246,11 @@ body.oc-app-body { overflow: hidden; }
.ib-call-text { flex: 1 1 200px; font-weight: 600; }
.ib-callbar button { padding: 6px 12px; border-radius: 999px; border: 1px solid var(--oc-line); background: transparent; color: var(--oc-text); font: inherit; font-size: .85rem; font-weight: 700; cursor: pointer; }
.ib-callbar .ib-call-end { border-color: transparent; background: #dc2626; color: #fff; }
++.ib-screen { margin: 0 0 8px; padding: 8px; border-radius: 12px; border: 1px solid var(--oc-line); background: #000; display: grid; gap: 6px; }
++.ib-screen[hidden] { display: none; }
++.ib-screen figcaption { color: #e5e7eb; font-size: .85rem; font-weight: 600; }
++.ib-screen video { width: 100%; max-height: 50vh; border-radius: 8px; background: #000; object-fit: contain; }
++.ib-screen button { justify-self: end; padding: 4px 10px; border-radius: 999px; border: 1px solid #374151; background: transparent; color: #e5e7eb; font: inherit; font-size: .8rem; cursor: pointer; }
.ib-sys { text-align: center; color: var(--oc-muted); font-size: .82rem; }
.ib-who { margin: 0 0 2px; font-size: .74rem; font-weight: 700; color: var(--oc-muted); }
.ib-av { flex: none; width: 30px; height: 30px; border-radius: 50%; display: inline-grid; place-items: center; background: var(--oc-panel-2); color: var(--oc-text); font-weight: 700; }
ddiff --git a/server/inbox/inbox.js b/server/inbox/inbox.js
iindex a13d89c..aee906f 100644
---- a/server/inbox/inbox.js
++++ b/server/inbox/inbox.js
@@@ -123,7 +123,7 @@ async function create(user, input = {}) {
const id = newId('inb');
await db.tx(async () => {
await db.run(`INSERT INTO inboxes (id, owner_id, owner_subject, name, greeting, color, ai_enabled, handoff, ai_instructions, knowledge_urls, allowed_origins, hours)
-- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [id, user.id, user.subject_id || null, v.name, v.greeting || `Hi! Ask us anything about ${v.name}.`, v.color || '#3b82f6',
++ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [id, user.id, user.subject_id || null, v.name, v.greeting || `Hi! Ask us anything about ${v.name}${/[.!?]$/.test(v.name) ? '' : '.'}`, v.color || '#3b82f6',
v.ai_enabled == null ? 1 : v.ai_enabled, v.handoff == null ? 1 : v.handoff, v.ai_instructions || '', JSON.stringify(v.knowledge_urls || []), JSON.stringify(v.allowed_origins || []), v.hours ? JSON.stringify(v.hours) : null]);
await db.run("INSERT INTO inbox_members (inbox_id, user_id, role) VALUES (?, ?, 'owner')", [id, user.id]);
});
ddiff --git a/server/web/inbox-pages.js b/server/web/inbox-pages.js
iindex 3022722..822ed46 100644
---- a/server/web/inbox-pages.js
++++ b/server/web/inbox-pages.js
@@@ -163,7 +163,7 @@ ${Number(inbox.ai_enabled) && current.status !== 'bot' && current.status !== 're
${current ? `${Number(current.verified) ? `<p class="ib-page">Verified by your site${current.visitor_email ? ` · <a href="mailto:${esc(current.visitor_email)}">${esc(current.visitor_email)}</a>` : ''} · id ${esc(current.external_id || '')}</p>` : ''}${current.rating_comment ? `<p class="ib-page">Rated ${Number(current.rating) > 0 ? '👍' : '👎'}: “${esc(current.rating_comment)}”</p>` : ''}${current.page_url ? `<p class="ib-page">From <a href="${esc(current.page_url)}" rel="noopener nofollow" target="_blank">${esc(current.page_url)}</a></p>` : ''}<ol class="vb-thread" id="vb-thread">${msgs.map(turn).join('')}</ol>`
: `<div class="vb-hello"><h1>${list.length ? 'Pick a conversation' : 'No conversations yet'}</h1><p class="vb-sub">${list.length ? 'Waiting ones come first: those visitors asked for a person.' : 'Paste the snippet from Settings into your site. Visitors write; Vibe answers; you take over when it matters.'}</p></div>`}
</div>
--${current ? `<div class="vb-dock"><div class="ib-callbar" id="ib-callbar" role="group" aria-label="Voice call" hidden><span class="ib-call-text" id="ib-call-text"></span><button type="button" id="ib-call-join" hidden>Join the call</button><button type="button" id="ib-call-mute" aria-pressed="false" hidden>Mute</button><button type="button" class="ib-call-end" id="ib-call-end">End call</button></div><form class="vb-composer" id="vb-composer" method="post" action="/inbox/${esc(inbox.id)}/c/${esc(current.id)}">
++${current ? `<div class="vb-dock"><figure class="ib-screen" id="ib-screen" hidden><figcaption id="ib-screen-who"></figcaption><video id="ib-screen-video" autoplay playsinline muted></video><button type="button" id="ib-screen-full">Full screen</button></figure><div class="ib-callbar" id="ib-callbar" role="group" aria-label="Voice call" hidden><span class="ib-call-text" id="ib-call-text"></span><button type="button" id="ib-call-join" hidden>Join the call</button><button type="button" id="ib-call-mute" aria-pressed="false" hidden>Mute</button><button type="button" class="ib-call-end" id="ib-call-end">End call</button></div><form class="vb-composer" id="vb-composer" method="post" action="/inbox/${esc(inbox.id)}/c/${esc(current.id)}">
<label class="oc-sr" for="vb-input">Reply</label><textarea id="vb-input" name="message" rows="1" maxlength="4000" placeholder="Reply as ${esc(actor.user.display_name || actor.user.username)}${saved.length ? ' · / for saved replies' : ''}" required aria-autocomplete="list"></textarea>
${pictures.enabled() ? '<label class="ib-pic-btn" title="Send a picture"><input type="file" id="ib-pic" accept="image/jpeg,image/png,image/gif,image/webp,image/avif" hidden><span aria-hidden="true">&#128206;</span><span class="oc-sr">Send a picture</span></label>' : ''}<button class="vb-send" type="submit" aria-label="Send">&#8593;</button></form><p class="vb-fine">Replying takes the conversation from Vibe. <label class="ib-tr-pick" hidden>Translate my reply into <select id="ib-tr-to"><option value="">a language…</option>${TR_LANGS.map(([v, l]) => `<option value="${v}">${l}</option>`).join('')}</select></label></p></div>` : ''}
</section>
ddiff --git a/test/support-calls.test.js b/test/support-calls.test.js
iindex 8dc02c6..e9dd389 100644
---- a/test/support-calls.test.js
++++ b/test/support-calls.test.js
@@@ -86,6 +86,9 @@ t('resolving ends a call; a resolved conversation cannot be called; the pages ca
assert.strictEqual(visitor.first.type, 'welcome');
const page = await fetch(`${h.base}/inbox/${inbox.id}?c=${cid}`, { headers: { cookie: `ov_token=${bob.token}` } }).then((r) => r.text());
assert.ok(page.includes('id="ib-call"') && page.includes('id="ib-callbar"') && page.includes('support-call.js') && page.includes('"call":true'));
++ assert.ok(page.includes('id="ib-screen"') && page.includes('id="ib-screen-video"'), 'and the place where a visitor\'s shared screen shows');
++ const client = await fetch(`${h.base}/web/support-call.js`).then((r) => r.text());
++ assert.ok(client.includes('getDisplayMedia') && client.includes("cameraOff: false"), 'the client shares a screen and says so');
assert.strictEqual((await agentsApi('PATCH', `/${inbox.id}/conversations/${cid}`, bob, { status: 'resolved' })).status, 200);
await visitor.wait((m) => m.type === 'call-ended');
visitor.close();