feat(call): add sound effects and incoming call notification UI

- Implemented sound management for call events including mute, unmute, deafen, undeafen, call joined, call depart, and dialing sounds.
- Created a new CallSounds class to handle sound playback and state management.
- Added IncomingCallNotification component to display incoming call alerts with caller information and action buttons.
- Styled the incoming call notification using vanilla-extract CSS for animations and layout.
- Integrated sound playback for incoming calls with a 15-second threshold for dialing sound.
This commit is contained in:
2026-02-05 03:24:18 +11:00
parent cd912025f1
commit 1a452f52ca
27 changed files with 1888 additions and 179 deletions

View File

@@ -1,7 +1,15 @@
import React, { useCallback, useEffect, useRef, useState } from 'react';
import { Box, Icon, IconButton, Icons, Text, Tooltip, TooltipProvider } from 'folds';
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Box, Icon, IconButton, Icons, Text, Tooltip, TooltipProvider, color } from 'folds';
import { useCall } from './useCall';
import { CallState, CallType, ScreenShareTrackInfo } from './types';
import { useRoomCallMembers } from './useRoomCallMembers';
import { CallState, CallType } from './types';
import { useMatrixClient } from '../../hooks/useMatrixClient';
import { getMemberDisplayName, getMemberAvatarMxc } from '../../utils/room';
import { getMxIdLocalPart, mxcUrlToHttp } from '../../utils/matrix';
import { useMediaAuthentication } from '../../hooks/useMediaAuthentication';
import { UserAvatar } from '../../components/user-avatar';
import { useSelectedRoom } from '../../hooks/router/useSelectedRoom';
import { useRoomNavigate } from '../../hooks/useRoomNavigate';
import * as css from './CallOverlay.css';
/** Drag position state */
@@ -10,9 +18,9 @@ interface DragPosition {
y: number;
}
/** Custom hook for draggable functionality */
function useDraggable(initialPosition: DragPosition = { x: 0, y: 0 }) {
const [position, setPosition] = useState<DragPosition>(initialPosition);
/** Custom hook for draggable functionality with viewport constraints */
function useDraggable() {
const [position, setPosition] = useState<DragPosition>({ x: 0, y: 0 });
const [isDragging, setIsDragging] = useState(false);
const dragRef = useRef<HTMLDivElement>(null);
const dragStartRef = useRef<{ x: number; y: number; startX: number; startY: number } | null>(null);
@@ -29,17 +37,92 @@ function useDraggable(initialPosition: DragPosition = { x: 0, y: 0 }) {
};
}, [position]);
// Clamp position to viewport bounds
const clampToViewport = useCallback(() => {
if (!dragRef.current) return;
const rect = dragRef.current.getBoundingClientRect();
const padding = 16;
// The element is positioned with right: S400 (16px) and top: 60px in CSS
// Transform moves it from that base position
// We need to ensure the element stays fully visible
let newX = position.x;
let newY = position.y;
let needsUpdate = false;
// Check if left edge is off screen (rect.left < padding)
if (rect.left < padding) {
newX = position.x + (padding - rect.left);
needsUpdate = true;
}
// Check if right edge is off screen
if (rect.right > window.innerWidth - padding) {
newX = position.x - (rect.right - window.innerWidth + padding);
needsUpdate = true;
}
// Check if top edge is off screen
if (rect.top < padding) {
newY = position.y + (padding - rect.top);
needsUpdate = true;
}
// Check if bottom edge is off screen
if (rect.bottom > window.innerHeight - padding) {
newY = position.y - (rect.bottom - window.innerHeight + padding);
needsUpdate = true;
}
if (needsUpdate) {
setPosition({ x: newX, y: newY });
}
}, [position]);
// Listen for window resize to keep element in bounds
useEffect(() => {
const handleResize = () => {
clampToViewport();
};
window.addEventListener('resize', handleResize);
return () => window.removeEventListener('resize', handleResize);
}, [clampToViewport]);
useEffect(() => {
if (!isDragging) return undefined;
const handleMouseMove = (e: MouseEvent) => {
if (!dragStartRef.current) return;
if (!dragStartRef.current || !dragRef.current) return;
const deltaX = e.clientX - dragStartRef.current.startX;
const deltaY = e.clientY - dragStartRef.current.startY;
setPosition({
x: dragStartRef.current.x + deltaX,
y: dragStartRef.current.y + deltaY,
});
// Get element dimensions for bounds checking
const rect = dragRef.current.getBoundingClientRect();
const padding = 16;
// Calculate new position
let newX = dragStartRef.current.x + deltaX;
let newY = dragStartRef.current.y + deltaY;
// Calculate where the element would be with the new transform
const newLeft = rect.left - position.x + newX;
const newRight = rect.right - position.x + newX;
const newTop = rect.top - position.y + newY;
const newBottom = rect.bottom - position.y + newY;
// Clamp to keep fully on screen
if (newLeft < padding) {
newX += padding - newLeft;
}
if (newRight > window.innerWidth - padding) {
newX -= newRight - window.innerWidth + padding;
}
if (newTop < padding) {
newY += padding - newTop;
}
if (newBottom > window.innerHeight - padding) {
newY -= newBottom - window.innerHeight + padding;
}
setPosition({ x: newX, y: newY });
};
const handleMouseUp = () => {
@@ -54,7 +137,7 @@ function useDraggable(initialPosition: DragPosition = { x: 0, y: 0 }) {
document.removeEventListener('mousemove', handleMouseMove);
document.removeEventListener('mouseup', handleMouseUp);
};
}, [isDragging]);
}, [isDragging, position]);
return { position, isDragging, handleMouseDown, dragRef };
}
@@ -119,30 +202,178 @@ function formatDuration(seconds: number): string {
return `${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
}
/** Screen share entry type */
interface ScreenShareEntry {
id: string;
isLocal: boolean;
participantId: string;
livekitId?: string;
track?: unknown;
stream?: MediaStream;
}
/** Props for ScreenShareItem component */
interface ScreenShareItemProps {
share: ScreenShareEntry;
}
/**
* Individual screen share display component
* Handles attaching the video stream/track properly
*/
function ScreenShareItem({ share }: ScreenShareItemProps) {
const containerRef = useRef<HTMLDivElement>(null);
useEffect(() => {
const container = containerRef.current;
if (!container) return undefined;
// Handle local screenshare (MediaStream)
if (share.isLocal && share.stream) {
const videoEl = document.createElement('video');
videoEl.className = css.ScreenShareVideo;
videoEl.autoplay = true;
videoEl.playsInline = true;
videoEl.muted = true;
videoEl.srcObject = share.stream;
container.innerHTML = '';
container.appendChild(videoEl);
return () => {
videoEl.srcObject = null;
if (videoEl.parentNode) {
videoEl.parentNode.removeChild(videoEl);
}
};
}
// Handle remote screenshare (LiveKit track)
if (!share.isLocal && share.track) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const track = share.track as any;
if (typeof track.attach === 'function') {
const videoEl = track.attach() as HTMLVideoElement;
videoEl.className = css.ScreenShareVideo;
container.innerHTML = '';
container.appendChild(videoEl);
return () => {
if (typeof track.detach === 'function') {
track.detach(videoEl);
}
if (videoEl.parentNode) {
videoEl.parentNode.removeChild(videoEl);
}
};
}
}
return undefined;
}, [share.isLocal, share.stream, share.track]);
/**
* Handles double-click to toggle fullscreen on the screenshare video element
*/
const handleDoubleClick = useCallback(async () => {
const container = containerRef.current;
if (!container) return;
// Find the video element inside the container
const videoEl = container.querySelector('video');
if (!videoEl) return;
try {
if (document.fullscreenElement) {
await document.exitFullscreen();
} else {
await videoEl.requestFullscreen();
}
} catch (err) {
// Fullscreen may not be available
}
}, []);
return (
<div className={css.ScreenShareContainer}>
{/* eslint-disable-next-line jsx-a11y/click-events-have-key-events, jsx-a11y/no-static-element-interactions */}
<div
ref={containerRef}
className={css.ScreenShareVideoContainer}
onDoubleClick={handleDoubleClick}
style={{ cursor: 'pointer' }}
/>
</div>
);
}
/**
* Component that displays active call controls and video streams
*/
export function CallOverlay() {
const { activeCall, endCall, toggleMute, toggleVideo, toggleScreenShare } = useCall();
const { activeCall, endCall, toggleMute, toggleDeafen, toggleVideo, toggleScreenShare } = useCall();
const mx = useMatrixClient();
const useAuthentication = useMediaAuthentication();
const selectedRoomId = useSelectedRoom();
const [duration, setDuration] = useState(0);
const [isTogglingScreenShare, setIsTogglingScreenShare] = useState(false);
const [screenShareTrack, setScreenShareTrack] = useState<ScreenShareTrackInfo | null>(null);
const [isFullscreen, setIsFullscreen] = useState(false);
const [isPip, setIsPip] = useState(false);
const [showParticipants, setShowParticipants] = useState(true);
const [collapsedScreenShares, setCollapsedScreenShares] = useState<Set<string>>(new Set());
const localVideoRef = useRef<HTMLVideoElement>(null);
const remoteVideoRef = useRef<HTMLVideoElement>(null);
const containerRef = useRef<HTMLDivElement>(null);
const { position, isDragging, handleMouseDown } = useDraggable();
const { position, isDragging, handleMouseDown, dragRef } = useDraggable();
const { navigateRoom } = useRoomNavigate();
// Get call members from Matrix state events (provides real Matrix user IDs)
const { callMembers } = useRoomCallMembers(activeCall?.roomId ?? '');
const myUserId = mx.getUserId() ?? '';
// Update screen share track state when activeCall changes
useEffect(() => {
if (activeCall?.remoteScreenTracks && activeCall.remoteScreenTracks.size > 0) {
const [firstTrackInfo] = activeCall.remoteScreenTracks.values();
setScreenShareTrack(firstTrackInfo || null);
} else {
setScreenShareTrack(null);
// Memoize the screenshare list to prevent re-renders causing flickering
const allScreenShares = useMemo((): ScreenShareEntry[] => {
const shares: ScreenShareEntry[] = [];
// Add local screenshare first if active
if (activeCall?.isScreenSharing && activeCall?.localScreenStream) {
shares.push({
id: 'local',
isLocal: true,
participantId: myUserId,
stream: activeCall.localScreenStream,
});
}
}, [activeCall]);
// Add remote screenshares
const remoteScreenTracks = activeCall?.remoteScreenTracks
? Array.from(activeCall.remoteScreenTracks.values())
: [];
remoteScreenTracks.forEach((trackInfo) => {
// Try to extract deviceId from LiveKit identity (format: "deviceId_timestamp")
const livekitId = trackInfo.participantId;
const underscoreIndex = livekitId.lastIndexOf('_');
const possibleDeviceId = underscoreIndex > 0 ? livekitId.substring(0, underscoreIndex) : livekitId;
// Look up Matrix user ID by matching device ID from call members
const matchingMember = callMembers.find((m) => m.deviceId === possibleDeviceId);
const matrixUserId = matchingMember?.userId ?? livekitId;
shares.push({
id: livekitId,
isLocal: false,
participantId: matrixUserId,
livekitId,
track: trackInfo.track,
stream: trackInfo.stream,
});
});
return shares;
}, [activeCall?.isScreenSharing, activeCall?.localScreenStream, activeCall?.remoteScreenTracks, callMembers, myUserId]);
useEffect(() => {
if (!activeCall || activeCall.state !== CallState.Connected) {
@@ -177,46 +408,6 @@ export function CallOverlay() {
}
}, [activeCall?.remoteStreams]);
// Attach screen share track to container using LiveKit's recommended approach
useEffect(() => {
const containerEl = document.getElementById('screen-share-container');
if (screenShareTrack?.track && containerEl) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const track = screenShareTrack.track as any;
if (typeof track.attach === 'function') {
// Let LiveKit create and manage the video element
const videoEl = track.attach() as HTMLVideoElement;
videoEl.className = css.ScreenShareVideo;
// Clear any existing content
containerEl.innerHTML = '';
containerEl.appendChild(videoEl);
// eslint-disable-next-line no-console
console.log('📺 Attached screen share - LiveKit created video element');
// Cleanup: detach when effect re-runs or unmounts
return () => {
if (typeof track.detach === 'function') {
track.detach(videoEl);
}
if (videoEl.parentNode) {
videoEl.parentNode.removeChild(videoEl);
}
};
}
}
// No track, clear container
if (containerEl && !screenShareTrack?.track) {
containerEl.innerHTML = '';
}
return undefined;
}, [screenShareTrack]);
// Fullscreen toggle handler
const handleToggleFullscreen = useCallback(async () => {
if (!containerRef.current) return;
@@ -254,10 +445,11 @@ export function CallOverlay() {
}
}, [isPip]);
// Listen for fullscreen change events
// Listen for fullscreen change events (only for the call overlay itself, not screenshares)
useEffect(() => {
const handleFullscreenChange = () => {
setIsFullscreen(!!document.fullscreenElement);
// Only set fullscreen state if the call overlay container is fullscreened
setIsFullscreen(document.fullscreenElement === containerRef.current);
};
document.addEventListener('fullscreenchange', handleFullscreenChange);
return () => document.removeEventListener('fullscreenchange', handleFullscreenChange);
@@ -280,15 +472,90 @@ export function CallOverlay() {
};
}, [activeCall]);
// Assign both refs to the container - must be before early return
const setContainerRef = useCallback((el: HTMLDivElement | null) => {
(containerRef as React.MutableRefObject<HTMLDivElement | null>).current = el;
(dragRef as React.MutableRefObject<HTMLDivElement | null>).current = el;
}, [dragRef]);
if (!activeCall || activeCall.state === CallState.Idle || activeCall.state === CallState.Ended) {
return null;
}
const isConnecting = activeCall.state === CallState.Connecting;
const isVideoCall = activeCall.callType === CallType.Video;
// Check if anyone is sharing their screen
const hasRemoteScreenShare = screenShareTrack !== null;
// Get room for participant info
const room = mx.getRoom(activeCall.roomId);
// Build participant list from Matrix state events (provides real user IDs)
// callMembers has the actual Matrix user IDs from org.matrix.msc3401.call.member events
const allParticipants = callMembers.map((m) => m.userId);
const participantCount = allParticipants.length;
/**
* Gets avatar URL for a participant
*/
const getParticipantAvatar = (participantId: string): string | undefined => {
if (!room) return undefined;
const mxcUrl = getMemberAvatarMxc(room, participantId);
if (!mxcUrl) return undefined;
return mxcUrlToHttp(mx, mxcUrl, useAuthentication, 48, 48, 'crop') ?? undefined;
};
/**
* Gets display name for a participant
*/
const getParticipantName = (participantId: string): string => {
if (!room) return getMxIdLocalPart(participantId) ?? participantId;
return getMemberDisplayName(room, participantId) ?? getMxIdLocalPart(participantId) ?? participantId;
};
/**
* Checks if a participant is currently speaking
* Handles mapping between Matrix user IDs and LiveKit identities
*/
const isParticipantSpeaking = (userId: string): boolean => {
if (!activeCall) return false;
// Direct check for Matrix user ID (works for local user)
if (activeCall.activeSpeakers.has(userId)) return true;
// For remote users, activeSpeakers contains LiveKit identities (deviceId_timestamp)
// We need to find the deviceId for this user and check if any matching identity is speaking
const member = callMembers.find((m) => m.userId === userId);
if (!member) return false;
// Check if any active speaker has this user's deviceId
return Array.from(activeCall.activeSpeakers).some((speakerId) => {
const underscoreIndex = speakerId.lastIndexOf('_');
const deviceId = underscoreIndex > 0 ? speakerId.substring(0, underscoreIndex) : speakerId;
return deviceId === member.deviceId;
});
};
/**
* Checks if a participant is currently muted
* Handles mapping between Matrix user IDs and LiveKit identities
*/
const isParticipantMuted = (userId: string): boolean => {
if (!activeCall) return false;
// For local user, check activeCall.isMuted
if (userId === myUserId) return activeCall.isMuted;
// For remote users, mutedParticipants contains LiveKit identities (deviceId_timestamp)
// We need to find the deviceId for this user and check if any matching identity is muted
const member = callMembers.find((m) => m.userId === userId);
if (!member) return false;
// Check if any muted participant has this user's deviceId
return Array.from(activeCall.mutedParticipants).some((mutedId) => {
const underscoreIndex = mutedId.lastIndexOf('_');
const deviceId = underscoreIndex > 0 ? mutedId.substring(0, underscoreIndex) : mutedId;
return deviceId === member.deviceId;
});
};
const handleEndCall = () => {
endCall();
@@ -298,6 +565,10 @@ export function CallOverlay() {
toggleMute();
};
const handleToggleDeafen = () => {
toggleDeafen();
};
const handleToggleVideo = () => {
toggleVideo();
};
@@ -314,7 +585,7 @@ export function CallOverlay() {
return (
<div
ref={containerRef}
ref={setContainerRef}
className={`${css.CallOverlayContainer} ${isFullscreen ? css.CallOverlayFullscreen : ''} ${isDragging ? css.CallOverlayDragging : ''}`}
style={!isFullscreen ? {
transform: `translate(${position.x}px, ${position.y}px)`,
@@ -326,14 +597,36 @@ export function CallOverlay() {
onMouseDown={handleMouseDown}
style={{ cursor: isDragging ? 'grabbing' : 'grab' }}
>
<Box alignItems="Center" gap="200">
<Box alignItems="Center" gap="200" style={{ minWidth: 0, flex: 1 }}>
<Icon
size="200"
src={isVideoCall ? Icons.VideoCamera : Icons.Phone}
/>
<Text size="T300">
{isConnecting ? 'Connecting...' : 'In Call'}
<Text size="T300" truncate>
{isConnecting ? 'Connecting...' : `In Call - ${room?.name ?? 'Unknown Room'}`}
{selectedRoomId === activeCall.roomId && (
<span style={{ opacity: 0.7, fontSize: '0.85em' }}> (Current Room)</span>
)}
</Text>
{selectedRoomId !== activeCall.roomId && (
<TooltipProvider
position="Top"
offset={4}
tooltip={<Tooltip><Text>Go to Room</Text></Tooltip>}
>
{(triggerRef) => (
<IconButton
ref={triggerRef}
onClick={() => navigateRoom(activeCall.roomId)}
variant="Secondary"
size="300"
style={{ padding: '2px', marginLeft: '4px' }}
>
<Icon size="100" src={Icons.Link} />
</IconButton>
)}
</TooltipProvider>
)}
</Box>
<Box alignItems="Center" gap="100">
<Text size="T200" className={css.CallDuration}>
@@ -372,17 +665,109 @@ export function CallOverlay() {
</Box>
</div>
{/* Remote Screen Share Display */}
{hasRemoteScreenShare && screenShareTrack && (
<div className={css.ScreenShareContainer}>
<div className={css.ScreenShareLabel}>
<ScreenShareIcon />
<span>{screenShareTrack.participantId}</span>
{/* Participants Section */}
<div className={css.ParticipantsSection}>
<button
type="button"
className={css.ParticipantsHeader}
onClick={() => setShowParticipants(!showParticipants)}
>
<Box alignItems="Center" gap="200">
<Icon size="100" src={Icons.User} />
<Text size="T200" className={css.ParticipantsHeaderText}>
{participantCount} {participantCount === 1 ? 'Participant' : 'Participants'}
</Text>
</Box>
<Icon
size="100"
src={showParticipants ? Icons.ChevronTop : Icons.ChevronBottom}
/>
</button>
{showParticipants && (
<div className={css.ParticipantsList}>
{allParticipants.map((participantId) => {
const isMe = participantId === myUserId;
const displayName = getParticipantName(participantId);
const avatarUrl = getParticipantAvatar(participantId);
const isSpeaking = isParticipantSpeaking(participantId);
const isMuted = isParticipantMuted(participantId);
// Get screenshares for this participant
const participantScreenShares = allScreenShares.filter(
(share) => share.participantId === participantId
);
return (
<div key={participantId} className={css.ParticipantItem}>
<div className={`${css.ParticipantAvatarWrapper} ${isSpeaking ? css.ParticipantSpeaking : ''}`}>
<UserAvatar
className={css.ParticipantAvatar}
userId={participantId}
src={avatarUrl}
alt={displayName}
renderFallback={() => (
<Text size="H6" style={{ color: color.Surface.Container }}>
{displayName.charAt(0).toUpperCase()}
</Text>
)}
/>
</div>
<Box direction="Column" grow="Yes" style={{ minWidth: 0 }}>
<Text size="T200" className={css.ParticipantName} truncate>
{displayName}
{isMe && <span className={css.YouBadge}> (You)</span>}
</Text>
<Text size="T200" className={css.ParticipantId} truncate>
{participantId}
</Text>
</Box>
{isMuted && (
<Icon size="100" src={Icons.MicMute} className={css.ParticipantMutedIcon} />
)}
{participantScreenShares.length > 0 && (
<TooltipProvider
position="Top"
offset={4}
tooltip={<Tooltip><Text>{collapsedScreenShares.has(participantScreenShares[0].id) ? 'Show Screen' : 'Hide Screen'}</Text></Tooltip>}
>
{(triggerRef) => (
<IconButton
ref={triggerRef}
onClick={() => {
setCollapsedScreenShares((prev) => {
const next = new Set(prev);
// Toggle all screenshares for this participant
participantScreenShares.forEach((share) => {
if (next.has(share.id)) {
next.delete(share.id);
} else {
next.add(share.id);
}
});
return next;
});
}}
variant="Secondary"
size="300"
style={{ padding: '2px' }}
>
<ScreenShareIcon />
</IconButton>
)}
</TooltipProvider>
)}
</div>
);
})}
</div>
{/* LiveKit will create and manage the video element inside this container */}
<div id="screen-share-container" className={css.ScreenShareVideoContainer} />
</div>
)}
)}
</div>
{/* Screen shares rendered outside participant list */}
{allScreenShares
.filter((share) => !collapsedScreenShares.has(share.id))
.map((share) => (
<ScreenShareItem key={share.id} share={share} />
))}
{isVideoCall && (
<div className={css.VideoContainer}>
@@ -432,6 +817,29 @@ export function CallOverlay() {
)}
</TooltipProvider>
<TooltipProvider
position="Top"
offset={4}
tooltip={
<Tooltip>
<Text>{activeCall.isDeafened ? 'Undeafen' : 'Deafen'}</Text>
</Tooltip>
}
>
{(triggerRef) => (
<IconButton
ref={triggerRef}
onClick={handleToggleDeafen}
variant={activeCall.isDeafened ? 'Critical' : 'Secondary'}
>
<Icon
size="300"
src={activeCall.isDeafened ? Icons.VolumeMute : Icons.VolumeHigh}
/>
</IconButton>
)}
</TooltipProvider>
{isVideoCall && (
<TooltipProvider
position="Top"