diff --git a/config.json b/config.json index de6015a..57c577a 100644 --- a/config.json +++ b/config.json @@ -10,6 +10,10 @@ ], "allowCustomHomeservers": true, + "calling": { + "livekitServiceUrl": "https://b.ruv.wtf/matrix-rtc/livekit/jwt" + }, + "featuredCommunities": { "openAsDefault": false, "spaces": [ @@ -32,7 +36,7 @@ }, "hashRouter": { - "enabled": false, + "enabled": true, "basename": "/" } } diff --git a/package-lock.json b/package-lock.json index 158a611..44c7ea6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,7 @@ "jotai": "2.6.0", "linkify-react": "4.1.3", "linkifyjs": "4.1.3", + "livekit-client": "2.17.0", "matrix-js-sdk": "38.2.0", "millify": "6.1.0", "pdfjs-dist": "4.2.67", @@ -1649,6 +1650,12 @@ "node": ">=6.9.0" } }, + "node_modules/@bufbuild/protobuf": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.10.1.tgz", + "integrity": "sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, "node_modules/@emotion/hash": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz", @@ -2235,6 +2242,21 @@ "resolved": "https://registry.npmjs.org/@juggle/resize-observer/-/resize-observer-3.4.0.tgz", "integrity": "sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA==" }, + "node_modules/@livekit/mutex": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@livekit/mutex/-/mutex-1.1.1.tgz", + "integrity": "sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==", + "license": "Apache-2.0" + }, + "node_modules/@livekit/protocol": { + "version": "1.42.2", + "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.42.2.tgz", + "integrity": "sha512-0jeCwoMJKcwsZICg5S6RZM4xhJoF78qMvQELjACJQn6/VB+jmiySQKOSELTXvPBVafHfEbMlqxUw2UR1jTXs2g==", + "license": "Apache-2.0", + "dependencies": { + "@bufbuild/protobuf": "^1.10.0" + } + }, "node_modules/@mapbox/node-pre-gyp": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", @@ -8295,6 +8317,15 @@ "resolved": "https://registry.npmjs.org/javascript-stringify/-/javascript-stringify-2.1.0.tgz", "integrity": "sha512-JVAfqNPTvNq3sB/VHQJAFxN/sPgKnsKrCwyRt15zwNCdrMMJDdcEOdubuy+DuJYYdm0ox1J4uzEuYKkN+9yhVg==" }, + "node_modules/jose": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.1.3.tgz", + "integrity": "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/jotai": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/jotai/-/jotai-2.6.0.tgz", @@ -8504,6 +8535,27 @@ "resolved": "https://registry.npmjs.org/linkifyjs/-/linkifyjs-4.1.3.tgz", "integrity": "sha512-auMesunaJ8yfkHvK4gfg1K0SaKX/6Wn9g2Aac/NwX+l5VdmFZzo/hdPGxEOETj+ryRa4/fiOPjeeKURSAJx1sg==" }, + "node_modules/livekit-client": { + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.17.0.tgz", + "integrity": "sha512-BD1QUS44ancVTBdnAher0aO7DV5holFYH2lYradYT/HgXtn6R8xPyvtDAH3UH40jGcesDo9fEopCFwEdOgrIhg==", + "license": "Apache-2.0", + "dependencies": { + "@livekit/mutex": "1.1.1", + "@livekit/protocol": "1.42.2", + "events": "^3.3.0", + "jose": "^6.1.0", + "loglevel": "^1.9.2", + "sdp-transform": "^2.15.0", + "ts-debounce": "^4.0.0", + "tslib": "2.8.1", + "typed-emitter": "^2.1.0", + "webrtc-adapter": "^9.0.1" + }, + "peerDependencies": { + "@types/dom-mediacapture-record": "^1" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -9954,6 +10006,16 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/safe-array-concat": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.3.tgz", @@ -10056,6 +10118,12 @@ "compute-scroll-into-view": "^3.0.2" } }, + "node_modules/sdp": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/sdp/-/sdp-3.2.1.tgz", + "integrity": "sha512-lwsAIzOPlH8/7IIjjz3K0zYBk7aBVVcvjMwt3M4fLxpjMYyy7i3I97SLHebgn4YBjirkzfp3RvRDWSKsh/+WFw==", + "license": "MIT" + }, "node_modules/sdp-transform": { "version": "2.15.0", "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz", @@ -10751,6 +10819,12 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, + "node_modules/ts-debounce": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ts-debounce/-/ts-debounce-4.0.0.tgz", + "integrity": "sha512-+1iDGY6NmOGidq7i7xZGA4cm8DAa6fqdYcvO5Z6yBevH++Bdo9Qt/mN0TzHUgcCcKv1gmh9+W5dHqz8pMWbCbg==", + "license": "MIT" + }, "node_modules/tsconfig-paths": { "version": "3.15.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", @@ -10899,6 +10973,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz", + "integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==", + "license": "MIT", + "optionalDependencies": { + "rxjs": "*" + } + }, "node_modules/typescript": { "version": "4.9.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz", @@ -11653,6 +11736,19 @@ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, + "node_modules/webrtc-adapter": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.3.tgz", + "integrity": "sha512-5fALBcroIl31OeXAdd1YUntxiZl1eHlZZWzNg3U4Fn+J9/cGL3eT80YlrsWGvj2ojuz1rZr2OXkgCzIxAZ7vRQ==", + "license": "BSD-3-Clause", + "dependencies": { + "sdp": "^3.2.0" + }, + "engines": { + "node": ">=6.0.0", + "npm": ">=3.10.0" + } + }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", diff --git a/package.json b/package.json index 7316dcc..7d65af4 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "jotai": "2.6.0", "linkify-react": "4.1.3", "linkifyjs": "4.1.3", + "livekit-client": "2.17.0", "matrix-js-sdk": "38.2.0", "millify": "6.1.0", "pdfjs-dist": "4.2.67", diff --git a/src/app/features/call/CallOverlay.css.ts b/src/app/features/call/CallOverlay.css.ts new file mode 100644 index 0000000..f2d712b --- /dev/null +++ b/src/app/features/call/CallOverlay.css.ts @@ -0,0 +1,142 @@ +import { style } from '@vanilla-extract/css'; +import { config, DefaultReset, color } from 'folds'; + +export const CallOverlayContainer = style([ + DefaultReset, + { + position: 'fixed', + bottom: config.space.S400, + right: config.space.S400, + zIndex: 9999, + backgroundColor: color.Surface.Container, + borderRadius: config.radii.R400, + padding: config.space.S300, + boxShadow: `0 4px 12px ${color.Other.Shadow}`, + minWidth: '280px', + maxWidth: '400px', + transition: 'box-shadow 0.2s ease', + userSelect: 'none', + }, +]); + +export const CallOverlayDragging = style({ + boxShadow: `0 8px 24px ${color.Other.Shadow}`, + opacity: 0.95, +}); + +export const CallOverlayFullscreen = style({ + position: 'fixed', + top: 0, + left: 0, + right: 0, + bottom: 0, + width: '100vw', + height: '100vh', + maxWidth: 'none', + borderRadius: 0, + display: 'flex', + flexDirection: 'column', + transform: 'none !important', +}); + +export const CallOverlayHeader = style({ + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + marginBottom: config.space.S200, +}); + +export const CallOverlayControls = style({ + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + gap: config.space.S200, +}); + +export const CallDuration = style({ + fontVariantNumeric: 'tabular-nums', +}); + +export const VideoContainer = style({ + position: 'relative', + width: '100%', + aspectRatio: '16 / 9', + backgroundColor: color.Surface.ContainerActive, + borderRadius: config.radii.R300, + marginBottom: config.space.S200, + overflow: 'hidden', + selectors: { + [`${CallOverlayFullscreen} &`]: { + flex: 1, + aspectRatio: 'unset', + borderRadius: 0, + marginBottom: 0, + }, + }, +}); + +export const LocalVideo = style({ + position: 'absolute', + bottom: config.space.S200, + right: config.space.S200, + width: '80px', + aspectRatio: '4 / 3', + backgroundColor: color.Surface.ContainerLine, + borderRadius: config.radii.R300, + overflow: 'hidden', +}); + +export const VideoElement = style({ + width: '100%', + height: '100%', + objectFit: 'cover', +}); + +export const ScreenShareContainer = style({ + position: 'relative', + width: '100%', + aspectRatio: '16 / 9', + backgroundColor: color.Surface.ContainerActive, + borderRadius: config.radii.R300, + marginBottom: config.space.S200, + overflow: 'hidden', + border: `2px solid ${color.Primary.Main}`, + selectors: { + [`${CallOverlayFullscreen} &`]: { + flex: 1, + aspectRatio: 'unset', + borderRadius: 0, + marginBottom: 0, + }, + }, +}); + +export const ScreenShareLabel = style({ + position: 'absolute', + top: config.space.S100, + left: config.space.S100, + backgroundColor: 'rgba(0, 0, 0, 0.7)', + color: 'white', + padding: `${config.space.S100} ${config.space.S200}`, + borderRadius: config.radii.R300, + fontSize: '0.75rem', + display: 'flex', + alignItems: 'center', + gap: config.space.S100, + zIndex: 1, +}); + +export const ScreenShareVideoContainer = style({ + width: '100%', + height: '100%', + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + backgroundColor: '#000', +}); + +export const ScreenShareVideo = style({ + width: '100%', + height: '100%', + objectFit: 'contain', +}); diff --git a/src/app/features/call/CallOverlay.tsx b/src/app/features/call/CallOverlay.tsx new file mode 100644 index 0000000..70b68c4 --- /dev/null +++ b/src/app/features/call/CallOverlay.tsx @@ -0,0 +1,514 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { Box, Icon, IconButton, Icons, Text, Tooltip, TooltipProvider } from 'folds'; +import { useCall } from './useCall'; +import { CallState, CallType, ScreenShareTrackInfo } from './types'; +import * as css from './CallOverlay.css'; + +/** Drag position state */ +interface DragPosition { + x: number; + y: number; +} + +/** Custom hook for draggable functionality */ +function useDraggable(initialPosition: DragPosition = { x: 0, y: 0 }) { + const [position, setPosition] = useState(initialPosition); + const [isDragging, setIsDragging] = useState(false); + const dragRef = useRef(null); + const dragStartRef = useRef<{ x: number; y: number; startX: number; startY: number } | null>(null); + + const handleMouseDown = useCallback((e: React.MouseEvent) => { + if ((e.target as HTMLElement).closest('button')) return; + e.preventDefault(); + setIsDragging(true); + dragStartRef.current = { + x: position.x, + y: position.y, + startX: e.clientX, + startY: e.clientY, + }; + }, [position]); + + useEffect(() => { + if (!isDragging) return undefined; + + const handleMouseMove = (e: MouseEvent) => { + if (!dragStartRef.current) return; + const deltaX = e.clientX - dragStartRef.current.startX; + const deltaY = e.clientY - dragStartRef.current.startY; + setPosition({ + x: dragStartRef.current.x + deltaX, + y: dragStartRef.current.y + deltaY, + }); + }; + + const handleMouseUp = () => { + setIsDragging(false); + dragStartRef.current = null; + }; + + document.addEventListener('mousemove', handleMouseMove); + document.addEventListener('mouseup', handleMouseUp); + + return () => { + document.removeEventListener('mousemove', handleMouseMove); + document.removeEventListener('mouseup', handleMouseUp); + }; + }, [isDragging]); + + return { position, isDragging, handleMouseDown, dragRef }; +} + +/** + * Custom Screen share icon SVG (Phosphor-style) + */ +function ScreenShareIcon() { + return ( + + + + + ); +} + +/** + * Custom Screen share off icon SVG (Phosphor-style) + */ +function ScreenShareOffIcon() { + return ( + + + + + ); +} + +/** Fullscreen icon */ +function FullscreenIcon() { + return ( + + + + ); +} + +/** Exit fullscreen icon */ +function ExitFullscreenIcon() { + return ( + + + + ); +} + +/** Picture-in-Picture icon */ +function PipIcon() { + return ( + + + + ); +} + +/** + * Formats call duration in MM:SS format + */ +function formatDuration(seconds: number): string { + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`; +} + +/** + * Component that displays active call controls and video streams + */ +export function CallOverlay() { + const { activeCall, endCall, toggleMute, toggleVideo, toggleScreenShare } = useCall(); + const [duration, setDuration] = useState(0); + const [isTogglingScreenShare, setIsTogglingScreenShare] = useState(false); + const [screenShareTrack, setScreenShareTrack] = useState(null); + const [isFullscreen, setIsFullscreen] = useState(false); + const [isPip, setIsPip] = useState(false); + const localVideoRef = useRef(null); + const remoteVideoRef = useRef(null); + const containerRef = useRef(null); + const { position, isDragging, handleMouseDown } = useDraggable(); + + // Update screen share track state when activeCall changes + useEffect(() => { + if (activeCall?.remoteScreenTracks && activeCall.remoteScreenTracks.size > 0) { + const [firstTrackInfo] = activeCall.remoteScreenTracks.values(); + setScreenShareTrack(firstTrackInfo || null); + } else { + setScreenShareTrack(null); + } + }, [activeCall]); + + useEffect(() => { + if (!activeCall || activeCall.state !== CallState.Connected) { + setDuration(0); + return undefined; + } + + const { startTime } = activeCall; + const updateDuration = () => { + const elapsed = Math.floor((Date.now() - startTime) / 1000); + setDuration(elapsed); + }; + + updateDuration(); + const interval = setInterval(updateDuration, 1000); + + return () => clearInterval(interval); + }, [activeCall]); + + useEffect(() => { + if (localVideoRef.current && activeCall?.localStream) { + localVideoRef.current.srcObject = activeCall.localStream; + } + }, [activeCall?.localStream]); + + useEffect(() => { + if (remoteVideoRef.current && activeCall?.remoteStreams.size) { + const [firstStream] = activeCall.remoteStreams.values(); + if (firstStream) { + remoteVideoRef.current.srcObject = firstStream; + } + } + }, [activeCall?.remoteStreams]); + + // Attach screen share track to container using LiveKit's recommended approach + useEffect(() => { + const containerEl = document.getElementById('screen-share-container'); + + if (screenShareTrack?.track && containerEl) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const track = screenShareTrack.track as any; + + if (typeof track.attach === 'function') { + // Let LiveKit create and manage the video element + const videoEl = track.attach() as HTMLVideoElement; + videoEl.className = css.ScreenShareVideo; + + // Clear any existing content + containerEl.innerHTML = ''; + containerEl.appendChild(videoEl); + + // eslint-disable-next-line no-console + console.log('📺 Attached screen share - LiveKit created video element'); + + // Cleanup: detach when effect re-runs or unmounts + return () => { + if (typeof track.detach === 'function') { + track.detach(videoEl); + } + if (videoEl.parentNode) { + videoEl.parentNode.removeChild(videoEl); + } + }; + } + } + + // No track, clear container + if (containerEl && !screenShareTrack?.track) { + containerEl.innerHTML = ''; + } + + return undefined; + }, [screenShareTrack]); + + // Fullscreen toggle handler + const handleToggleFullscreen = useCallback(async () => { + if (!containerRef.current) return; + + try { + if (!isFullscreen) { + await containerRef.current.requestFullscreen(); + setIsFullscreen(true); + } else if (document.fullscreenElement) { + await document.exitFullscreen(); + setIsFullscreen(false); + } + } catch (err) { + // eslint-disable-next-line no-console + console.error('Fullscreen error:', err); + } + }, [isFullscreen]); + + // PiP toggle handler + const handleTogglePip = useCallback(async () => { + const videoEl = remoteVideoRef.current; + if (!videoEl) return; + + try { + if (!isPip && document.pictureInPictureEnabled) { + await videoEl.requestPictureInPicture(); + setIsPip(true); + } else if (document.pictureInPictureElement) { + await document.exitPictureInPicture(); + setIsPip(false); + } + } catch (err) { + // eslint-disable-next-line no-console + console.error('PiP error:', err); + } + }, [isPip]); + + // Listen for fullscreen change events + useEffect(() => { + const handleFullscreenChange = () => { + setIsFullscreen(!!document.fullscreenElement); + }; + document.addEventListener('fullscreenchange', handleFullscreenChange); + return () => document.removeEventListener('fullscreenchange', handleFullscreenChange); + }, []); + + // Listen for PiP change events + useEffect(() => { + const videoEl = remoteVideoRef.current; + if (!videoEl) return undefined; + + const handlePipEnter = () => setIsPip(true); + const handlePipLeave = () => setIsPip(false); + + videoEl.addEventListener('enterpictureinpicture', handlePipEnter); + videoEl.addEventListener('leavepictureinpicture', handlePipLeave); + + return () => { + videoEl.removeEventListener('enterpictureinpicture', handlePipEnter); + videoEl.removeEventListener('leavepictureinpicture', handlePipLeave); + }; + }, [activeCall]); + + if (!activeCall || activeCall.state === CallState.Idle || activeCall.state === CallState.Ended) { + return null; + } + + const isConnecting = activeCall.state === CallState.Connecting; + const isVideoCall = activeCall.callType === CallType.Video; + + // Check if anyone is sharing their screen + const hasRemoteScreenShare = screenShareTrack !== null; + + const handleEndCall = () => { + endCall(); + }; + + const handleToggleMute = () => { + toggleMute(); + }; + + const handleToggleVideo = () => { + toggleVideo(); + }; + + const handleToggleScreenShare = async () => { + if (isTogglingScreenShare) return; + setIsTogglingScreenShare(true); + try { + await toggleScreenShare(); + } finally { + setIsTogglingScreenShare(false); + } + }; + + return ( +
+ {/* eslint-disable-next-line jsx-a11y/no-static-element-interactions */} +
+ + + + {isConnecting ? 'Connecting...' : 'In Call'} + + + + + {formatDuration(duration)} + + {isVideoCall && ( + <> + {isPip ? 'Exit PiP' : 'Picture-in-Picture'}} + > + {(triggerRef) => ( + + + + + + )} + + {isFullscreen ? 'Exit Fullscreen' : 'Fullscreen'}} + > + {(triggerRef) => ( + + + {isFullscreen ? : } + + + )} + + + )} + +
+ + {/* Remote Screen Share Display */} + {hasRemoteScreenShare && screenShareTrack && ( +
+
+ + {screenShareTrack.participantId} +
+ {/* LiveKit will create and manage the video element inside this container */} +
+
+ )} + + {isVideoCall && ( +
+ {/* eslint-disable-next-line jsx-a11y/media-has-caption */} +
+ )} + +
+ + {activeCall.isMuted ? 'Unmute' : 'Mute'} + + } + > + {(triggerRef) => ( + + + + )} + + + {isVideoCall && ( + + {activeCall.isVideoEnabled ? 'Turn Off Camera' : 'Turn On Camera'} + + } + > + {(triggerRef) => ( + + + + )} + + )} + + + {activeCall.isScreenSharing ? 'Stop Screen Share' : 'Share Screen'} + + } + > + {(triggerRef) => ( + + + {activeCall.isScreenSharing ? : } + + + )} + + + + End Call + + } + > + {(triggerRef) => ( + + + + )} + +
+
+ ); +} diff --git a/src/app/features/call/CallProviderWrapper.tsx b/src/app/features/call/CallProviderWrapper.tsx new file mode 100644 index 0000000..da062b8 --- /dev/null +++ b/src/app/features/call/CallProviderWrapper.tsx @@ -0,0 +1,29 @@ +import React, { ReactNode } from 'react'; +import { CallProvider, useCallService } from './useCall'; +import { CallOverlay } from './CallOverlay'; +import { useClientConfig } from '../../hooks/useClientConfig'; + +interface CallProviderWrapperProps { + children: ReactNode; +} + +/** + * Wrapper component that initializes the call service and provides it to children + * Also renders the CallOverlay for active call controls + */ +export function CallProviderWrapper({ children }: CallProviderWrapperProps) { + const clientConfig = useClientConfig(); + const livekitServiceUrl = clientConfig.calling?.livekitServiceUrl; + + // eslint-disable-next-line no-console + console.log('CallProviderWrapper - config calling:', clientConfig.calling, 'livekitUrl:', livekitServiceUrl); + + const callContextValue = useCallService({ livekitServiceUrl }); + + return ( + + {children} + + + ); +} diff --git a/src/app/features/call/CallService.ts b/src/app/features/call/CallService.ts new file mode 100644 index 0000000..bbd5320 --- /dev/null +++ b/src/app/features/call/CallService.ts @@ -0,0 +1,737 @@ +/* eslint-disable no-console */ +/* eslint-disable import/no-extraneous-dependencies */ +import { + Room, + RoomEvent, + RemoteParticipant, + RemoteTrackPublication, + ConnectionState, + ParticipantEvent, + Participant, + LocalTrackPublication, + Track, +} from 'livekit-client'; +import { MatrixClient } from 'matrix-js-sdk'; +import { + ActiveCall, + CallEvent, + CallEventType, + CallServiceConfig, + CallState, + CallType, + LiveKitJWTResponse, + CallMemberEventContent, +} from './types'; +import { fetchLiveKitJWT, fetchWellKnownWithRTC, getLiveKitFocus, requestOpenIdToken } from './useRtcConfig'; +import { getAudioSettings, SCREEN_SHARE_RESOLUTIONS, SCREEN_SHARE_BITRATES } from '../settings/audio/Audio'; + +type CallEventListener = (event: CallEvent) => void; + +/** The Matrix state event type for call membership (MSC3401) */ +const CALL_MEMBER_EVENT_TYPE = 'org.matrix.msc3401.call.member'; + +/** + * Service for managing Matrix RTC calls via LiveKit + * Handles LiveKit connections, media streams, call state, and Matrix room signaling + */ +export class CallService { + private config: CallServiceConfig; + + private matrixClient: MatrixClient; + + private activeCall: ActiveCall | null = null; + + private listeners: Set = new Set(); + + private livekitRoom: Room | null = null; + + private livekitJwt: LiveKitJWTResponse | null = null; + + /** + * Creates a new CallService instance + * @param config - Service configuration including homeserver and LiveKit URLs + * @param matrixClient - The Matrix client for sending state events + */ + constructor(config: CallServiceConfig, matrixClient: MatrixClient) { + this.config = config; + this.matrixClient = matrixClient; + } + + /** + * Subscribes to call events + * @param listener - Callback function for call events + * @returns Unsubscribe function + */ + on(listener: CallEventListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + /** + * Emits a call event to all listeners + */ + private emit(event: CallEvent): void { + this.listeners.forEach((listener) => listener(event)); + } + + /** + * Updates call state and emits event + */ + private setState(state: CallState): void { + if (this.activeCall) { + this.activeCall.state = state; + this.emit({ + type: CallEventType.StateChanged, + roomId: this.activeCall.roomId, + data: { state }, + }); + } + } + + /** + * Gets the current active call + */ + getActiveCall(): ActiveCall | null { + return this.activeCall; + } + + /** + * Checks if calls are supported on this homeserver + */ + async isCallSupported(): Promise { + try { + const wellKnown = await fetchWellKnownWithRTC(this.config.homeserverBaseUrl); + return getLiveKitFocus(wellKnown) !== undefined; + } catch { + return false; + } + } + + /** + * Generates a unique call ID for this session + */ + private generateCallId(): string { + return `${this.config.deviceId}_${Date.now()}`; + } + + /** + * Sends a call member state event to announce participation in a call + * @param roomId - The Matrix room ID + * @param callId - The unique call ID + * @param active - Whether the user is actively in the call + */ + private async sendCallMemberEvent( + roomId: string, + callId: string, + active: boolean + ): Promise { + const stateKey = this.config.userId; + + if (active) { + const content: CallMemberEventContent = { + 'm.calls': [ + { + 'm.call_id': callId, + 'm.devices': [ + { + device_id: this.config.deviceId, + session_id: callId, + expires_ts: Date.now() + 1000 * 60 * 60, // 1 hour expiry + feeds: [ + { purpose: 'usermedia' }, + ], + }, + ], + }, + ], + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await (this.matrixClient as any).sendStateEvent( + roomId, + CALL_MEMBER_EVENT_TYPE, + content, + stateKey + ); + } else { + // Clear the call membership by sending empty calls array + const content: CallMemberEventContent = { + 'm.calls': [], + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await (this.matrixClient as any).sendStateEvent( + roomId, + CALL_MEMBER_EVENT_TYPE, + content, + stateKey + ); + } + } + + /** + * Starts a call in the specified room + * @param roomId - The Matrix room ID to start the call in + * @param callType - Voice or video call + */ + async startCall(roomId: string, callType: CallType): Promise { + if (this.activeCall) { + throw new Error('A call is already active'); + } + + const callId = this.generateCallId(); + + this.activeCall = { + roomId, + callType, + state: CallState.Connecting, + startTime: Date.now(), + remoteStreams: new Map(), + remoteScreenStreams: new Map(), + remoteScreenTracks: new Map(), + participants: [], + isMuted: false, + isVideoEnabled: callType === CallType.Video, + isScreenSharing: false, + }; + + this.emit({ + type: CallEventType.StateChanged, + roomId, + data: { state: CallState.Connecting }, + }); + + try { + // Step 1: Get an OpenID token from the Matrix homeserver + const openIdToken = await requestOpenIdToken( + this.config.homeserverBaseUrl, + this.config.userId, + this.config.accessToken + ); + + // Step 2: Exchange OpenID token for LiveKit JWT + console.log('Requesting LiveKit JWT for Matrix room:', roomId); + this.livekitJwt = await fetchLiveKitJWT( + this.config.livekitServiceUrl, + roomId, + this.config.userId, + this.config.deviceId, + openIdToken + ); + + console.log('Got LiveKit JWT, connecting to:', this.livekitJwt.url); + console.log('LiveKit JWT response:', JSON.stringify(this.livekitJwt)); + + // Step 3: Create and connect to LiveKit room + this.livekitRoom = new Room({ + adaptiveStream: true, + dynacast: true, + }); + + this.setupLiveKitEventHandlers(); + + await this.livekitRoom.connect(this.livekitJwt.url, this.livekitJwt.jwt); + + console.log('Connected to LiveKit room'); + console.log('LiveKit room name:', this.livekitRoom.name); + console.log('Local participant identity:', this.livekitRoom.localParticipant.identity); + console.log('Remote participants:', this.livekitRoom.remoteParticipants.size); + + // Get user's audio device preferences + const audioSettings = getAudioSettings(); + console.log('Using audio settings:', audioSettings); + + // Build audio capture options with processing settings + const audioCaptureOptions = { + deviceId: audioSettings.microphoneId || undefined, + noiseSuppression: audioSettings.noiseSuppression, + echoCancellation: audioSettings.echoCancellation, + autoGainControl: audioSettings.autoGainControl, + }; + + console.log('Audio capture options:', audioCaptureOptions); + + // Step 4: Publish local tracks based on call type with selected devices and processing + if (callType === CallType.Video) { + // For video calls, enable both camera and microphone + await this.livekitRoom.localParticipant.setCameraEnabled(true); + await this.livekitRoom.localParticipant.setMicrophoneEnabled(true, audioCaptureOptions); + } else { + // Voice call - only enable microphone with preferred device and processing + await this.livekitRoom.localParticipant.setMicrophoneEnabled(true, audioCaptureOptions); + } + + // Step 5: Send Matrix state event to announce participation + await this.sendCallMemberEvent(roomId, callId, true); + + console.log('Announced call participation to Matrix room'); + + this.setState(CallState.Connected); + } catch (error) { + console.error('Failed to start call:', error); + this.setState(CallState.Error); + this.emit({ + type: CallEventType.Error, + roomId, + data: { error }, + }); + throw error; + } + } + + /** + * Sets up LiveKit room event handlers + */ + private setupLiveKitEventHandlers(): void { + if (!this.livekitRoom) return; + + this.livekitRoom.on(RoomEvent.ConnectionStateChanged, (state: ConnectionState) => { + console.log('LiveKit connection state:', state); + + if (!this.activeCall) return; + + switch (state) { + case ConnectionState.Connected: + this.setState(CallState.Connected); + break; + case ConnectionState.Reconnecting: + this.setState(CallState.Reconnecting); + break; + case ConnectionState.Disconnected: + this.setState(CallState.Ended); + break; + default: + break; + } + }); + + this.livekitRoom.on( + RoomEvent.TrackSubscribed, + ( + track: RemoteTrackPublication['track'], + publication: RemoteTrackPublication, + participant: RemoteParticipant + ) => { + if (!track || !this.activeCall) return; + + const isScreenShare = publication.source === Track.Source.ScreenShare || + publication.source === Track.Source.ScreenShareAudio; + + console.log('Track subscribed:', track.kind, 'source:', publication.source, + 'isScreenShare:', isScreenShare, 'from', participant.identity); + console.log('Track mediaStreamTrack:', track.mediaStreamTrack); + console.log('Track mediaStreamTrack readyState:', track.mediaStreamTrack?.readyState); + console.log('Track mediaStreamTrack enabled:', track.mediaStreamTrack?.enabled); + + // For audio tracks, attach to an audio element to play + if (track.kind === 'audio') { + const audioElement = track.attach() as HTMLAudioElement; + audioElement.id = `audio-${participant.identity}-${publication.source}`; + + // Set the preferred output device if supported and selected + const audioSettings = getAudioSettings(); + if (audioSettings.speakerId && 'setSinkId' in audioElement) { + (audioElement as HTMLAudioElement & { setSinkId: (id: string) => Promise }) + .setSinkId(audioSettings.speakerId) + .then(() => console.log('Speaker set to:', audioSettings.speakerId)) + .catch((e) => console.warn('Could not set speaker device:', e)); + } + + document.body.appendChild(audioElement); + console.log('Audio element attached for', participant.identity); + } + + // Handle screen share video tracks separately + if (isScreenShare && track.kind === 'video') { + const mediaStream = new MediaStream([track.mediaStreamTrack]); + + console.log('📺 Screen share received from', participant.identity); + console.log('📺 Screen share track:', track); + console.log('📺 Screen share MediaStream:', mediaStream); + console.log('📺 Screen share MediaStream tracks:', mediaStream?.getTracks()); + + this.activeCall.remoteScreenStreams.set(participant.identity, mediaStream); + + // Store the actual track for proper attachment later + this.activeCall.remoteScreenTracks.set(participant.identity, { + participantId: participant.identity, + track, + stream: mediaStream, + }); + + this.emit({ + type: CallEventType.ScreenShareStarted, + roomId: this.activeCall.roomId, + data: { + participantId: participant.identity, + stream: mediaStream, + track, + }, + }); + } else if (track.kind === 'video') { + const mediaStream = new MediaStream([track.mediaStreamTrack]); + this.activeCall.remoteStreams.set(participant.identity, mediaStream); + + this.emit({ + type: CallEventType.RemoteStreamAdded, + roomId: this.activeCall.roomId, + data: { + participantId: participant.identity, + stream: mediaStream, + }, + }); + } + + // Update participants list + if (!this.activeCall.participants.includes(participant.identity)) { + this.activeCall.participants.push(participant.identity); + } + } + ); + + this.livekitRoom.on( + RoomEvent.TrackUnsubscribed, + ( + track: RemoteTrackPublication['track'], + publication: RemoteTrackPublication, + participant: RemoteParticipant + ) => { + if (!this.activeCall) return; + + const isScreenShare = publication.source === Track.Source.ScreenShare || + publication.source === Track.Source.ScreenShareAudio; + + console.log('Track unsubscribed:', track?.kind, 'source:', publication.source, + 'isScreenShare:', isScreenShare, 'from', participant.identity); + + // Remove audio element if it exists + if (track?.kind === 'audio') { + const audioElement = document.getElementById(`audio-${participant.identity}-${publication.source}`); + if (audioElement) { + audioElement.remove(); + console.log('Audio element removed for', participant.identity); + } + track.detach(); + } + + // Handle screen share removal + if (isScreenShare && track?.kind === 'video') { + // Detach the track from any elements + track.detach(); + this.activeCall.remoteScreenStreams.delete(participant.identity); + this.activeCall.remoteScreenTracks.delete(participant.identity); + console.log('📺 Screen share ended from', participant.identity); + + this.emit({ + type: CallEventType.ScreenShareStopped, + roomId: this.activeCall.roomId, + data: { participantId: participant.identity }, + }); + } else if (track?.kind === 'video') { + this.activeCall.remoteStreams.delete(participant.identity); + + this.emit({ + type: CallEventType.RemoteStreamRemoved, + roomId: this.activeCall.roomId, + data: { participantId: participant.identity }, + }); + } + } + ); + + this.livekitRoom.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => { + console.log('Participant connected:', participant.identity, 'in Matrix room:', this.activeCall?.roomId); + console.log('Current LiveKit room name:', this.livekitRoom?.name); + + // Listen for this participant's speaking state + participant.on(ParticipantEvent.IsSpeakingChanged, (speaking: boolean) => { + if (speaking) { + console.log(`🔊 ${participant.identity} is speaking`); + } else { + console.log(`🔈 ${participant.identity} stopped speaking`); + } + }); + + if (this.activeCall && !this.activeCall.participants.includes(participant.identity)) { + this.activeCall.participants.push(participant.identity); + this.emit({ + type: CallEventType.ParticipantJoined, + roomId: this.activeCall.roomId, + data: { participantId: participant.identity }, + }); + } + }); + + this.livekitRoom.on(RoomEvent.ParticipantDisconnected, (participant: RemoteParticipant) => { + console.log('Participant disconnected:', participant.identity); + + if (this.activeCall) { + this.activeCall.participants = this.activeCall.participants.filter( + (p) => p !== participant.identity + ); + this.activeCall.remoteStreams.delete(participant.identity); + + this.emit({ + type: CallEventType.ParticipantLeft, + roomId: this.activeCall.roomId, + data: { participantId: participant.identity }, + }); + } + }); + + this.livekitRoom.on(RoomEvent.LocalTrackPublished, (publication) => { + console.log('Local track published:', publication.kind); + + if (this.activeCall && publication.track) { + const mediaStream = new MediaStream([publication.track.mediaStreamTrack]); + this.activeCall.localStream = mediaStream; + } + }); + + // Voice activity detection - logs when speakers change + this.livekitRoom.on(RoomEvent.ActiveSpeakersChanged, (speakers: Participant[]) => { + if (speakers.length > 0) { + const speakerNames = speakers.map((s) => s.identity).join(', '); + console.log('🎤 Active speakers:', speakerNames); + } + }); + + // Track local participant speaking state + this.livekitRoom.localParticipant.on(ParticipantEvent.IsSpeakingChanged, (speaking: boolean) => { + if (speaking) { + console.log('🎙️ You are speaking'); + } else { + console.log('🔇 You stopped speaking'); + } + }); + + // Track audio level changes for local participant (more detailed) + this.livekitRoom.localParticipant.on(ParticipantEvent.AudioStreamAcquired, () => { + console.log('🎧 Audio stream acquired'); + }); + } + + /** + * Ends the current call + */ + async endCall(): Promise { + if (!this.activeCall) { + return; + } + + const { roomId } = this.activeCall; + this.setState(CallState.Disconnecting); + + try { + // Clear Matrix call membership + await this.sendCallMemberEvent(roomId, '', false); + } catch (error) { + console.error('Failed to clear call membership:', error); + } + + // Clean up all audio elements + document.querySelectorAll('[id^="audio-"]').forEach((el) => el.remove()); + + // Disconnect from LiveKit + if (this.livekitRoom) { + this.livekitRoom.disconnect(); + this.livekitRoom = null; + } + + this.activeCall = null; + this.livekitJwt = null; + + this.emit({ + type: CallEventType.StateChanged, + roomId, + data: { state: CallState.Ended }, + }); + } + + /** + * Toggles audio mute state + */ + toggleMute(): boolean { + if (!this.livekitRoom?.localParticipant) { + return false; + } + + const newMuteState = !this.activeCall?.isMuted; + this.livekitRoom.localParticipant.setMicrophoneEnabled(!newMuteState); + + if (this.activeCall) { + this.activeCall.isMuted = newMuteState; + } + + return newMuteState; + } + + /** + * Toggles video enabled state + */ + toggleVideo(): boolean { + if (!this.livekitRoom?.localParticipant) { + return false; + } + + const newVideoState = !this.activeCall?.isVideoEnabled; + this.livekitRoom.localParticipant.setCameraEnabled(newVideoState); + + if (this.activeCall) { + this.activeCall.isVideoEnabled = newVideoState; + } + + return newVideoState; + } + + /** + * Gets the LiveKit JWT for the current call + */ + getLiveKitCredentials(): LiveKitJWTResponse | null { + return this.livekitJwt; + } + + /** + * Gets the LiveKit room instance + */ + getLiveKitRoom(): Room | null { + return this.livekitRoom; + } + + /** + * Starts screen sharing with configurable resolution and bitrate + * @returns true if screen share started successfully + */ + async startScreenShare(): Promise { + if (!this.livekitRoom?.localParticipant || !this.activeCall) { + console.warn('Cannot start screen share: no active call or room'); + return false; + } + + if (this.activeCall.isScreenSharing) { + console.warn('Screen sharing is already active'); + return true; + } + + try { + // Preserve current mute state before starting screen share + const wasMuted = this.activeCall.isMuted; + + const audioSettings = getAudioSettings(); + const resolution = SCREEN_SHARE_RESOLUTIONS[audioSettings.screenShareResolution]; + + console.log('Starting screen share with settings:', { + resolution: audioSettings.screenShareResolution, + bitrate: audioSettings.screenShareBitrate, + frameRate: audioSettings.screenShareFrameRate, + }); + + // Build screen share options - LiveKit uses different format + // Resolution and framerate are set via constraints when creating track + // Note: audio here is for system/tab audio, not microphone + const screenShareOptions = { + audio: true, // Include system audio if available + resolution: audioSettings.screenShareResolution === 'source' + ? undefined + : { + width: resolution.width, + height: resolution.height, + frameRate: audioSettings.screenShareFrameRate, + }, + }; + + // Enable screen share with the configured options + await this.livekitRoom.localParticipant.setScreenShareEnabled(true, screenShareOptions); + + // Restore mute state - LiveKit might have changed mic state + if (wasMuted) { + this.livekitRoom.localParticipant.setMicrophoneEnabled(false); + } + + // Set bitrate on the screen share track publication + const screenSharePub = Array.from( + this.livekitRoom.localParticipant.trackPublications.values() + ).find( + (pub): pub is LocalTrackPublication => + pub.source === Track.Source.ScreenShare + ); + + if (screenSharePub?.track) { + console.log(`Screen share started with bitrate: ${SCREEN_SHARE_BITRATES[audioSettings.screenShareBitrate].bitrate}kbps`); + // LiveKit handles bitrate via simulcast and adaptive streaming + } + + this.activeCall.isScreenSharing = true; + + this.emit({ + type: CallEventType.StateChanged, + roomId: this.activeCall.roomId, + data: { state: this.activeCall.state, isScreenSharing: true }, + }); + + console.log('✅ Screen sharing started'); + return true; + } catch (error) { + console.error('Failed to start screen share:', error); + return false; + } + } + + /** + * Stops screen sharing + * @returns true if screen share stopped successfully + */ + async stopScreenShare(): Promise { + if (!this.livekitRoom?.localParticipant || !this.activeCall) { + return false; + } + + if (!this.activeCall.isScreenSharing) { + return true; + } + + try { + await this.livekitRoom.localParticipant.setScreenShareEnabled(false); + this.activeCall.isScreenSharing = false; + + this.emit({ + type: CallEventType.StateChanged, + roomId: this.activeCall.roomId, + data: { state: this.activeCall.state, isScreenSharing: false }, + }); + + console.log('🛑 Screen sharing stopped'); + return true; + } catch (error) { + console.error('Failed to stop screen share:', error); + return false; + } + } + + /** + * Toggles screen sharing on/off + * @returns The new screen sharing state + */ + async toggleScreenShare(): Promise { + if (this.activeCall?.isScreenSharing) { + await this.stopScreenShare(); + return false; + } + return this.startScreenShare(); + } + + /** + * Gets the current screen sharing state + */ + isScreenSharing(): boolean { + return this.activeCall?.isScreenSharing ?? false; + } + + /** + * Cleans up resources + */ + dispose(): void { + this.endCall(); + this.listeners.clear(); + } +} diff --git a/src/app/features/call/index.ts b/src/app/features/call/index.ts new file mode 100644 index 0000000..f3a6fce --- /dev/null +++ b/src/app/features/call/index.ts @@ -0,0 +1,7 @@ +export * from './types'; +export * from './useRtcConfig'; +export * from './CallService'; +export * from './useCall'; +export * from './CallProviderWrapper'; +export * from './CallOverlay'; +export * from './useRoomCallMembers'; diff --git a/src/app/features/call/types.ts b/src/app/features/call/types.ts new file mode 100644 index 0000000..b8a4d69 --- /dev/null +++ b/src/app/features/call/types.ts @@ -0,0 +1,185 @@ +/** + * Types for Matrix RTC calling functionality + * Uses LiveKit via the org.matrix.msc4143.rtc_foci configuration + */ + +/** + * LiveKit focus configuration from well-known + */ +export interface LiveKitFocus { + type: 'livekit'; + livekit_service_url: string; +} + +/** + * RTC Foci configuration from well-known response + */ +export type RtcFoci = LiveKitFocus; + +/** + * Extended well-known response with RTC configuration + */ +export interface WellKnownWithRTC { + 'm.homeserver': { + base_url: string; + }; + 'm.identity_server'?: { + base_url: string; + }; + 'io.element.e2ee'?: { + default?: boolean; + force_disable?: boolean; + }; + 'org.matrix.msc4143.rtc_foci'?: RtcFoci[]; +} + +/** + * OpenID token response from the Matrix homeserver + */ +export interface OpenIdToken { + access_token: string; + token_type: string; + matrix_server_name: string; + expires_in: number; +} + +/** + * Member information for LiveKit JWT request + */ +export interface LiveKitMember { + id: string; + claimed_user_id: string; + claimed_device_id: string; +} + +/** + * Request body for LiveKit JWT service + */ +export interface LiveKitJWTRequest { + openid_token: OpenIdToken; + room_id: string; + member: LiveKitMember; +} + +/** + * LiveKit JWT response from the service + */ +export interface LiveKitJWTResponse { + jwt: string; + url: string; +} + +/** + * Possible call states + */ +export enum CallState { + Idle = 'idle', + Connecting = 'connecting', + Connected = 'connected', + Reconnecting = 'reconnecting', + Disconnecting = 'disconnecting', + Ended = 'ended', + Error = 'error', +} + +/** + * Call type + */ +export enum CallType { + Voice = 'voice', + Video = 'video', +} + +/** + * Screen share track info with the actual track for proper attachment + */ +export interface ScreenShareTrackInfo { + participantId: string; + track: unknown; // RemoteTrack from livekit-client + stream: MediaStream; +} + +/** + * Active call information + */ +export interface ActiveCall { + roomId: string; + callType: CallType; + state: CallState; + startTime: number; + localStream?: MediaStream; + localScreenStream?: MediaStream; + remoteStreams: Map; + remoteScreenStreams: Map; + remoteScreenTracks: Map; + participants: string[]; + isMuted: boolean; + isVideoEnabled: boolean; + isScreenSharing: boolean; +} + +/** + * Call event types for the call service + */ +export enum CallEventType { + StateChanged = 'state_changed', + ParticipantJoined = 'participant_joined', + ParticipantLeft = 'participant_left', + RemoteStreamAdded = 'remote_stream_added', + RemoteStreamRemoved = 'remote_stream_removed', + ScreenShareStarted = 'screen_share_started', + ScreenShareStopped = 'screen_share_stopped', + Error = 'error', +} + +/** + * Call event payload + */ +export interface CallEvent { + type: CallEventType; + roomId: string; + data?: unknown; +} + +/** + * Call service configuration + */ +export interface CallServiceConfig { + livekitServiceUrl: string; + homeserverBaseUrl: string; + accessToken: string; + userId: string; + deviceId: string; +} + +/** + * Feed information for call member event + */ +export interface CallFeed { + purpose: 'usermedia' | 'screenshare'; +} + +/** + * Device information for call member event + */ +export interface CallDevice { + device_id: string; + session_id: string; + expires_ts: number; + feeds: CallFeed[]; +} + +/** + * Call information for call member event + */ +export interface CallInfo { + 'm.call_id': string; + 'm.devices': CallDevice[]; +} + +/** + * Content for org.matrix.msc3401.call.member state event + */ +export interface CallMemberEventContent { + 'm.calls': CallInfo[]; +} diff --git a/src/app/features/call/useCall.ts b/src/app/features/call/useCall.ts new file mode 100644 index 0000000..7270ce5 --- /dev/null +++ b/src/app/features/call/useCall.ts @@ -0,0 +1,238 @@ +import { createContext, useContext, useState, useCallback, useEffect, useMemo } from 'react'; +import { useMatrixClient } from '../../hooks/useMatrixClient'; +import { CallService } from './CallService'; +import { ActiveCall, CallEvent, CallEventType, CallState, CallType } from './types'; +import { fetchWellKnownWithRTC, getLiveKitFocus } from './useRtcConfig'; + +interface CallContextValue { + callService: CallService | null; + activeCall: ActiveCall | null; + callSupported: boolean; + callSupportLoading: boolean; + startCall: (roomId: string, callType: CallType) => Promise; + endCall: () => Promise; + toggleMute: () => boolean; + toggleVideo: () => boolean; + toggleScreenShare: () => Promise; +} + +const CallContext = createContext(null); + +/** + * Hook to access the call context + * @returns The call context value + */ +export function useCall(): CallContextValue { + const context = useContext(CallContext); + if (!context) { + throw new Error('useCall must be used within a CallProvider'); + } + return context; +} + +export const CallProvider = CallContext.Provider; + +interface UseCallServiceOptions { + livekitServiceUrl?: string; +} + +/** + * Hook to initialize and manage the call service + * Should be used at the app level to provide call context + * @param options - Optional configuration including livekitServiceUrl from client config + */ +export function useCallService(options?: UseCallServiceOptions): CallContextValue { + const mx = useMatrixClient(); + const [callService, setCallService] = useState(null); + const [activeCall, setActiveCall] = useState(null); + const [callSupported, setCallSupported] = useState(false); + const [callSupportLoading, setCallSupportLoading] = useState(true); + + const configuredLivekitUrl = options?.livekitServiceUrl; + + useEffect(() => { + let unsubscribe: (() => void) | undefined; + let service: CallService | undefined; + + const initCallService = async () => { + const baseUrl = mx.getHomeserverUrl(); + const accessToken = mx.getAccessToken(); + const userId = mx.getSafeUserId(); + const deviceId = mx.getDeviceId(); + + if (!accessToken || !deviceId) { + setCallSupportLoading(false); + return; + } + + try { + let livekitServiceUrl: string | undefined = configuredLivekitUrl; + + // If no configured URL, try to get from well-known + if (!livekitServiceUrl) { + const wellKnown = await fetchWellKnownWithRTC(baseUrl); + // eslint-disable-next-line no-console + console.log('Call service init - well-known:', wellKnown); + const livekitFocus = getLiveKitFocus(wellKnown); + // eslint-disable-next-line no-console + console.log('Call service init - LiveKit focus:', livekitFocus); + livekitServiceUrl = livekitFocus?.livekit_service_url; + } else { + // eslint-disable-next-line no-console + console.log('Call service init - Using configured LiveKit URL:', livekitServiceUrl); + } + + if (!livekitServiceUrl) { + // eslint-disable-next-line no-console + console.log('Call service init - No LiveKit service found, calls not supported'); + setCallSupported(false); + setCallSupportLoading(false); + return; + } + + service = new CallService( + { + livekitServiceUrl, + homeserverBaseUrl: baseUrl, + accessToken, + userId, + deviceId, + }, + mx + ); + + setCallService(service); + setCallSupported(true); + setCallSupportLoading(false); + // eslint-disable-next-line no-console + console.log('Call service init - Success! Calls are supported'); + + unsubscribe = service.on((event: CallEvent) => { + // Update state for any event that might change the call state + if (event.type === CallEventType.StateChanged || + event.type === CallEventType.ScreenShareStarted || + event.type === CallEventType.ScreenShareStopped || + event.type === CallEventType.RemoteStreamAdded || + event.type === CallEventType.RemoteStreamRemoved) { + const newCall = service?.getActiveCall(); + if (newCall) { + // Deep copy Maps to ensure React detects changes + setActiveCall({ + ...newCall, + remoteStreams: new Map(newCall.remoteStreams), + remoteScreenTracks: new Map(newCall.remoteScreenTracks), + remoteScreenStreams: new Map(newCall.remoteScreenStreams), + }); + } else { + setActiveCall(null); + } + } + }); + } catch (error) { + // eslint-disable-next-line no-console + console.error('Call service init - Error:', error); + setCallSupported(false); + setCallSupportLoading(false); + } + }; + + initCallService(); + + return () => { + unsubscribe?.(); + service?.dispose(); + }; + }, [mx, configuredLivekitUrl]); + + const startCall = useCallback( + async (roomId: string, callType: CallType) => { + if (!callService) { + throw new Error('Call service not initialized'); + } + await callService.startCall(roomId, callType); + }, + [callService] + ); + + const endCall = useCallback(async () => { + if (!callService) { + return; + } + await callService.endCall(); + }, [callService]); + + const toggleMute = useCallback(() => { + if (!callService) { + return false; + } + const newState = callService.toggleMute(); + const call = callService.getActiveCall(); + if (call) { + setActiveCall({ ...call }); + } + return newState; + }, [callService]); + + const toggleVideo = useCallback(() => { + if (!callService) { + return false; + } + const newState = callService.toggleVideo(); + const call = callService.getActiveCall(); + if (call) { + setActiveCall({ ...call }); + } + return newState; + }, [callService]); + + const toggleScreenShare = useCallback(async () => { + if (!callService) { + return false; + } + const newState = await callService.toggleScreenShare(); + const call = callService.getActiveCall(); + if (call) { + setActiveCall({ ...call }); + } + return newState; + }, [callService]); + + return useMemo( + () => ({ + callService, + activeCall, + callSupported, + callSupportLoading, + startCall, + endCall, + toggleMute, + toggleVideo, + toggleScreenShare, + }), + [callService, activeCall, callSupported, callSupportLoading, startCall, endCall, toggleMute, toggleVideo, toggleScreenShare] + ); +} + +/** + * Hook to check if a call is active in a specific room + */ +export function useRoomCall(roomId: string) { + const { activeCall, startCall, endCall, callSupported, callSupportLoading } = useCall(); + + const isInCall = activeCall?.roomId === roomId; + const callState = isInCall ? activeCall.state : CallState.Idle; + + const startVoiceCall = useCallback(() => startCall(roomId, CallType.Voice), [roomId, startCall]); + const startVideoCall = useCallback(() => startCall(roomId, CallType.Video), [roomId, startCall]); + + return { + isInCall, + callState, + activeCall: isInCall ? activeCall : null, + startVoiceCall, + startVideoCall, + endCall, + callSupported, + callSupportLoading, + }; +} diff --git a/src/app/features/call/useRoomCallMembers.ts b/src/app/features/call/useRoomCallMembers.ts new file mode 100644 index 0000000..e2063fe --- /dev/null +++ b/src/app/features/call/useRoomCallMembers.ts @@ -0,0 +1,114 @@ +import { useCallback, useEffect, useState } from 'react'; +import { MatrixEvent, Room, RoomStateEvent } from 'matrix-js-sdk'; +import { useMatrixClient } from '../../hooks/useMatrixClient'; + +/** The Matrix state event type for call membership (MSC3401) */ +const CALL_MEMBER_EVENT_TYPE = 'org.matrix.msc3401.call.member'; + +/** Represents an active call member in a room */ +export interface CallMember { + userId: string; + deviceId: string; + sessionId: string; + expiresTs: number; +} + +/** + * Extracts active call members from a room's state events + * @param room - The Matrix room to check + * @returns Array of active call members + */ +function getActiveCallMembers(room: Room): CallMember[] { + const members: CallMember[] = []; + const now = Date.now(); + + try { + const stateEvents = room.currentState.getStateEvents(CALL_MEMBER_EVENT_TYPE); + + for (const event of stateEvents) { + const content = event.getContent(); + const userId = event.getStateKey(); + + if (!userId || !content['m.calls']) continue; + + const calls = content['m.calls']; + if (!Array.isArray(calls)) continue; + + for (const call of calls) { + const devices = call['m.devices']; + if (!Array.isArray(devices)) continue; + + for (const device of devices) { + const expiresTs = device.expires_ts || 0; + + // Check if the call membership is still valid (not expired) + if (expiresTs > now) { + members.push({ + userId, + deviceId: device.device_id, + sessionId: device.session_id, + expiresTs, + }); + } + } + } + } + } catch (e) { + console.error('Error reading call member events:', e); + } + + return members; +} + +/** + * Hook to get the list of users currently in a call in a room + * @param roomId - The Matrix room ID to monitor + * @returns Object with call members and whether a call is active + */ +export function useRoomCallMembers(roomId: string) { + const mx = useMatrixClient(); + const [callMembers, setCallMembers] = useState([]); + + const updateCallMembers = useCallback(() => { + const room = mx.getRoom(roomId); + if (!room) { + setCallMembers([]); + return; + } + + const members = getActiveCallMembers(room); + setCallMembers(members); + }, [mx, roomId]); + + useEffect(() => { + updateCallMembers(); + + // Listen for state event changes + const handleStateEvent = (event: MatrixEvent) => { + if (event.getRoomId() === roomId && event.getType() === CALL_MEMBER_EVENT_TYPE) { + updateCallMembers(); + } + }; + + mx.on(RoomStateEvent.Events, handleStateEvent); + + // Also refresh periodically to handle expired memberships + const interval = setInterval(updateCallMembers, 30000); + + return () => { + mx.off(RoomStateEvent.Events, handleStateEvent); + clearInterval(interval); + }; + }, [mx, roomId, updateCallMembers]); + + const isCallActive = callMembers.length > 0; + const myUserId = mx.getUserId(); + const othersInCall = callMembers.filter((m) => m.userId !== myUserId); + + return { + callMembers, + isCallActive, + othersInCall, + callMemberCount: callMembers.length, + }; +} diff --git a/src/app/features/call/useRtcConfig.ts b/src/app/features/call/useRtcConfig.ts new file mode 100644 index 0000000..9456afd --- /dev/null +++ b/src/app/features/call/useRtcConfig.ts @@ -0,0 +1,170 @@ +import { useCallback } from 'react'; +import { useMatrixClient } from '../../hooks/useMatrixClient'; +import { LiveKitFocus, LiveKitJWTResponse, OpenIdToken, WellKnownWithRTC } from './types'; + +/** + * Fetches the LiveKit focus configuration from well-known data + * @param wellKnown - The well-known response that may contain RTC foci + * @returns The LiveKit focus configuration if available + */ +export function getLiveKitFocus(wellKnown: WellKnownWithRTC): LiveKitFocus | undefined { + const rtcFoci = wellKnown['org.matrix.msc4143.rtc_foci']; + if (!rtcFoci || !Array.isArray(rtcFoci)) { + return undefined; + } + + return rtcFoci.find((focus): focus is LiveKitFocus => focus.type === 'livekit'); +} + +/** + * Fetches the well-known client configuration from a homeserver + * @param baseUrl - The homeserver base URL + * @returns The well-known response with RTC configuration + */ +export async function fetchWellKnownWithRTC(baseUrl: string): Promise { + const wellKnownUrl = `${baseUrl}/.well-known/matrix/client`; + + const response = await fetch(wellKnownUrl); + if (!response.ok) { + throw new Error(`Failed to fetch well-known: ${response.status}`); + } + + return response.json(); +} + +/** + * Requests an OpenID token from the Matrix homeserver + * This token can be used to authenticate with third-party services + * @param homeserverBaseUrl - The Matrix homeserver base URL + * @param userId - The user's Matrix ID + * @param accessToken - The Matrix access token + * @returns The OpenID token response + */ +export async function requestOpenIdToken( + homeserverBaseUrl: string, + userId: string, + accessToken: string +): Promise { + const url = `${homeserverBaseUrl}/_matrix/client/v3/user/${encodeURIComponent(userId)}/openid/request_token`; + + const response = await fetch(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({}), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to request OpenID token: ${response.status} - ${errorText}`); + } + + return response.json(); +} + +/** + * Fetches a LiveKit JWT for joining a call using OpenID token authentication + * @param livekitServiceUrl - The LiveKit service URL from well-known or config + * @param roomId - The Matrix room ID to join + * @param userId - The Matrix user ID + * @param deviceId - The Matrix device ID + * @param openIdToken - The OpenID token from the Matrix homeserver + * @returns The LiveKit JWT and server URL + */ +export async function fetchLiveKitJWT( + livekitServiceUrl: string, + roomId: string, + userId: string, + deviceId: string, + openIdToken: OpenIdToken +): Promise { + // Generate a unique member ID for this participant (used for identity, not room assignment) + const memberId = `${deviceId}_${Date.now()}`; + + // slot_id must be constant so all participants in the same Matrix room + // get assigned to the same LiveKit room. The JWT service hashes room_id + slot_id + // to create the LiveKit room name. + const slotId = 'm.call'; + + const response = await fetch(livekitServiceUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + openid_token: openIdToken, + room_id: roomId, + slot_id: slotId, + member: { + id: memberId, + claimed_user_id: userId, + claimed_device_id: deviceId, + }, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to fetch LiveKit JWT: ${response.status} - ${errorText}`); + } + + return response.json(); +} + +/** + * Hook to get RTC configuration functions + * Provides utilities for fetching well-known data and LiveKit JWTs + */ +export function useRtcConfig() { + const mx = useMatrixClient(); + + const getWellKnown = useCallback(async (): Promise => { + const baseUrl = mx.getHomeserverUrl(); + return fetchWellKnownWithRTC(baseUrl); + }, [mx]); + + const getLiveKitJWT = useCallback( + async (roomId: string): Promise => { + const baseUrl = mx.getHomeserverUrl(); + const accessToken = mx.getAccessToken(); + const deviceId = mx.getDeviceId(); + const userId = mx.getSafeUserId(); + + if (!accessToken || !deviceId) { + throw new Error('Missing access token or device ID'); + } + + const wellKnown = await fetchWellKnownWithRTC(baseUrl); + const livekitFocus = getLiveKitFocus(wellKnown); + + if (!livekitFocus) { + return null; + } + + // Get OpenID token from homeserver + const openIdToken = await requestOpenIdToken(baseUrl, userId, accessToken); + + // Exchange for LiveKit JWT + return fetchLiveKitJWT(livekitFocus.livekit_service_url, roomId, userId, deviceId, openIdToken); + }, + [mx] + ); + + const isCallSupported = useCallback(async (): Promise => { + try { + const baseUrl = mx.getHomeserverUrl(); + const wellKnown = await fetchWellKnownWithRTC(baseUrl); + return getLiveKitFocus(wellKnown) !== undefined; + } catch { + return false; + } + }, [mx]); + + return { + getWellKnown, + getLiveKitJWT, + isCallSupported, + }; +} diff --git a/src/app/features/room/RoomViewHeader.tsx b/src/app/features/room/RoomViewHeader.tsx index 9b4bfd2..2fd460b 100644 --- a/src/app/features/room/RoomViewHeader.tsx +++ b/src/app/features/room/RoomViewHeader.tsx @@ -69,6 +69,8 @@ import { useRoomNavigate } from '../../hooks/useRoomNavigate'; import { useRoomCreators } from '../../hooks/useRoomCreators'; import { useRoomPermissions } from '../../hooks/useRoomPermissions'; import { InviteUserPrompt } from '../../components/invite-user-prompt'; +import { useRoomCall, useRoomCallMembers } from '../call'; +import { CallState, CallType } from '../call/types'; type RoomMenuProps = { room: Room; @@ -254,6 +256,170 @@ const RoomMenu = forwardRef(({ room, requestClose ); }); +type CallIndicatorProps = { + roomId: string; +}; + +/** + * Shows when others are in a call in this room + */ +function CallIndicator({ roomId }: CallIndicatorProps) { + const { othersInCall, isCallActive } = useRoomCallMembers(roomId); + + if (!isCallActive || othersInCall.length === 0) { + return null; + } + + const userCount = othersInCall.length; + const tooltipText = userCount === 1 + ? `${othersInCall[0].userId.split(':')[0].slice(1)} is in a call` + : `${userCount} people are in a call`; + + return ( + + {tooltipText} + + } + > + {(triggerRef) => ( + + + + {userCount} in call + + + )} + + ); +} + +type RoomCallButtonsProps = { + roomId: string; +}; + +/** + * Component for voice/video call buttons in the room header + */ +function RoomCallButtons({ roomId }: RoomCallButtonsProps) { + const { + isInCall, + callState, + activeCall, + startVoiceCall, + startVideoCall, + endCall, + callSupported, + callSupportLoading, + } = useRoomCall(roomId); + + // Debug logging + React.useEffect(() => { + // eslint-disable-next-line no-console + console.log('RoomCallButtons render:', { callSupportLoading, callSupported, roomId }); + }, [callSupportLoading, callSupported, roomId]); + + if (callSupportLoading || !callSupported) { + return null; + } + + const isConnecting = callState === CallState.Connecting; + const isConnected = callState === CallState.Connected; + const isActive = isConnecting || isConnected; + + const handleVoiceCall = async () => { + if (isInCall) { + await endCall(); + } else { + try { + await startVoiceCall(); + } catch (error) { + console.error('Failed to start voice call:', error); + } + } + }; + + const handleVideoCall = async () => { + if (isInCall) { + await endCall(); + } else { + try { + await startVideoCall(); + } catch (error) { + console.error('Failed to start video call:', error); + } + } + }; + + return ( + <> + + {isInCall ? 'End Call' : 'Voice Call'} + + } + > + {(triggerRef) => ( + + {isConnecting ? ( + + ) : ( + + )} + + )} + + + {isInCall ? 'End Call' : 'Video Call'} + + } + > + {(triggerRef) => ( + + {isConnecting ? ( + + ) : ( + + )} + + )} + + + ); +} + export function RoomViewHeader() { const navigate = useNavigate(); const mx = useMatrixClient(); @@ -369,7 +535,9 @@ export function RoomViewHeader() { )} - + + + {!ecryptedRoom && ( name: 'Notifications', icon: Icons.Bell, }, + { + page: SettingsPages.AudioPage, + name: 'Audio & Video', + icon: Icons.Phone, + }, { page: SettingsPages.DevicesPage, name: 'Devices', @@ -219,6 +226,9 @@ export function Settings({ initialPage, requestClose }: SettingsProps) { {activePage === SettingsPages.NotificationPage && ( )} + {activePage === SettingsPages.AudioPage && ( + + + + + + ); +} diff --git a/src/app/features/settings/audio/index.ts b/src/app/features/settings/audio/index.ts new file mode 100644 index 0000000..b7965c5 --- /dev/null +++ b/src/app/features/settings/audio/index.ts @@ -0,0 +1 @@ +export * from './Audio'; diff --git a/src/app/hooks/useClientConfig.ts b/src/app/hooks/useClientConfig.ts index e5fc6cc..d7592fe 100644 --- a/src/app/hooks/useClientConfig.ts +++ b/src/app/hooks/useClientConfig.ts @@ -5,6 +5,10 @@ export type HashRouterConfig = { basename?: string; }; +export type CallingConfig = { + livekitServiceUrl?: string; +}; + export type ClientConfig = { defaultHomeserver?: number; homeserverList?: string[]; @@ -17,6 +21,8 @@ export type ClientConfig = { servers?: string[]; }; + calling?: CallingConfig; + hashRouter?: HashRouterConfig; }; diff --git a/src/app/pages/client/ClientRoot.tsx b/src/app/pages/client/ClientRoot.tsx index e1a5dc0..96d97e3 100644 --- a/src/app/pages/client/ClientRoot.tsx +++ b/src/app/pages/client/ClientRoot.tsx @@ -35,6 +35,7 @@ import { stopPropagation } from '../../utils/keyboard'; import { SyncStatus } from './SyncStatus'; import { AuthMetadataProvider } from '../../hooks/useAuthMetadata'; import { getFallbackSession } from '../../state/sessions'; +import { CallProviderWrapper } from '../../features/call/CallProviderWrapper'; function ClientRootLoading() { return ( @@ -211,17 +212,19 @@ export function ClientRoot({ children }: ClientRootProps) { ) : ( - - {(serverConfigs) => ( - - - - {children} - - - - )} - + + + {(serverConfigs) => ( + + + + {children} + + + + )} + + )} diff --git a/vite.config.js b/vite.config.js index dfa02fc..57d9c55 100644 --- a/vite.config.js +++ b/vite.config.js @@ -73,7 +73,7 @@ export default defineConfig({ base: buildConfig.base, server: { port: 8080, - host: true, + host: '0.0.0.0', fs: { // Allow serving files from one level up to the project root allow: ['..'],