1078 lines
35 KiB
TypeScript
1078 lines
35 KiB
TypeScript
/* eslint-disable no-console */
|
|
/* eslint-disable import/no-extraneous-dependencies */
|
|
import {
|
|
Room,
|
|
RoomEvent,
|
|
RemoteParticipant,
|
|
RemoteTrackPublication,
|
|
ConnectionState,
|
|
ParticipantEvent,
|
|
Participant,
|
|
LocalTrackPublication,
|
|
Track,
|
|
} from 'livekit-client';
|
|
import { MatrixClient } from 'matrix-js-sdk';
|
|
import {
|
|
ActiveCall,
|
|
CallEvent,
|
|
CallEventType,
|
|
CallServiceConfig,
|
|
CallState,
|
|
CallType,
|
|
LiveKitJWTResponse,
|
|
CallMemberEventContent,
|
|
} from './types';
|
|
import { fetchLiveKitJWT, fetchWellKnownWithRTC, getLiveKitFocus, requestOpenIdToken } from './useRtcConfig';
|
|
import { getAudioSettings, SCREEN_SHARE_RESOLUTIONS, SCREEN_SHARE_BITRATES } from '../settings/audio/Audio';
|
|
import { getCallSounds, CallSoundType } from './CallSounds';
|
|
|
|
type CallEventListener = (event: CallEvent) => void;
|
|
|
|
/** The Matrix state event type for call membership (MSC3401) */
|
|
const CALL_MEMBER_EVENT_TYPE = 'org.matrix.msc3401.call.member';
|
|
|
|
/**
|
|
* Service for managing Matrix RTC calls via LiveKit
|
|
* Handles LiveKit connections, media streams, call state, and Matrix room signaling
|
|
*/
|
|
/** How often to refresh call membership to prevent expiry (45 minutes) */
|
|
const MEMBERSHIP_REFRESH_INTERVAL_MS = 45 * 60 * 1000;
|
|
|
|
/** Call membership expiry time (1 hour) */
|
|
const MEMBERSHIP_EXPIRY_MS = 60 * 60 * 1000;
|
|
|
|
/**
|
|
* Check WebRTC capabilities and log diagnostic information
|
|
*/
|
|
function checkWebRTCSupport(): { supported: boolean; details: Record<string, unknown> } {
|
|
const details: Record<string, unknown> = {
|
|
userAgent: navigator.userAgent,
|
|
platform: navigator.platform,
|
|
};
|
|
|
|
// Check for key WebRTC APIs
|
|
details.hasGetUserMedia = !!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia);
|
|
details.hasRTCPeerConnection = !!(window.RTCPeerConnection);
|
|
details.hasRTCDataChannel = !!(window.RTCPeerConnection && window.RTCPeerConnection.prototype.createDataChannel);
|
|
|
|
// Check WebKit-specific APIs
|
|
details.hasWebkitGetUserMedia = !!(navigator.getUserMedia || (navigator as any).webkitGetUserMedia);
|
|
details.hasWebkitRTCPeerConnection = !!((window as any).webkitRTCPeerConnection);
|
|
|
|
// Check if mediaDevices API exists
|
|
details.hasMediaDevices = !!navigator.mediaDevices;
|
|
details.hasEnumerateDevices = !!(navigator.mediaDevices && navigator.mediaDevices.enumerateDevices);
|
|
|
|
const supported = !!(
|
|
(navigator.mediaDevices && navigator.mediaDevices.getUserMedia) &&
|
|
(window.RTCPeerConnection || (window as any).webkitRTCPeerConnection)
|
|
);
|
|
|
|
return { supported, details };
|
|
}
|
|
|
|
/**
|
|
* Service for managing Matrix RTC calls via LiveKit
|
|
* Handles LiveKit connections, media streams, call state, and Matrix room signaling
|
|
*/
|
|
export class CallService {
|
|
private config: CallServiceConfig;
|
|
|
|
private matrixClient: MatrixClient;
|
|
|
|
private activeCall: ActiveCall | null = null;
|
|
|
|
private listeners: Set<CallEventListener> = new Set();
|
|
|
|
private livekitRoom: Room | null = null;
|
|
|
|
private livekitJwt: LiveKitJWTResponse | null = null;
|
|
|
|
private membershipRefreshInterval: ReturnType<typeof setInterval> | null = null;
|
|
|
|
private currentCallId: string | null = null;
|
|
|
|
private wasMutedBeforeDeafen = false;
|
|
|
|
/**
|
|
* Creates a new CallService instance
|
|
* @param config - Service configuration including homeserver and LiveKit URLs
|
|
* @param matrixClient - The Matrix client for sending state events
|
|
*/
|
|
constructor(config: CallServiceConfig, matrixClient: MatrixClient) {
|
|
this.config = config;
|
|
this.matrixClient = matrixClient;
|
|
}
|
|
|
|
/**
|
|
* Subscribes to call events
|
|
* @param listener - Callback function for call events
|
|
* @returns Unsubscribe function
|
|
*/
|
|
on(listener: CallEventListener): () => void {
|
|
this.listeners.add(listener);
|
|
return () => this.listeners.delete(listener);
|
|
}
|
|
|
|
/**
|
|
* Emits a call event to all listeners
|
|
*/
|
|
private emit(event: CallEvent): void {
|
|
this.listeners.forEach((listener) => listener(event));
|
|
}
|
|
|
|
/**
|
|
* Updates call state and emits event
|
|
*/
|
|
private setState(state: CallState): void {
|
|
if (this.activeCall) {
|
|
this.activeCall.state = state;
|
|
this.emit({
|
|
type: CallEventType.StateChanged,
|
|
roomId: this.activeCall.roomId,
|
|
data: { state },
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Gets the current active call
|
|
*/
|
|
getActiveCall(): ActiveCall | null {
|
|
return this.activeCall;
|
|
}
|
|
|
|
/**
|
|
* Checks if calls are supported on this homeserver
|
|
*/
|
|
async isCallSupported(): Promise<boolean> {
|
|
try {
|
|
const wellKnown = await fetchWellKnownWithRTC(this.config.homeserverBaseUrl);
|
|
return getLiveKitFocus(wellKnown) !== undefined;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generates a unique call ID for this session
|
|
*/
|
|
private generateCallId(): string {
|
|
return `${this.config.deviceId}_${Date.now()}`;
|
|
}
|
|
|
|
/**
|
|
* Sends a call member state event to announce participation in a call
|
|
* @param roomId - The Matrix room ID
|
|
* @param callId - The unique call ID
|
|
* @param active - Whether the user is actively in the call
|
|
*/
|
|
private async sendCallMemberEvent(
|
|
roomId: string,
|
|
callId: string,
|
|
active: boolean
|
|
): Promise<void> {
|
|
const stateKey = this.config.userId;
|
|
|
|
if (active) {
|
|
const content: CallMemberEventContent = {
|
|
'm.calls': [
|
|
{
|
|
'm.call_id': callId,
|
|
'm.devices': [
|
|
{
|
|
device_id: this.config.deviceId,
|
|
session_id: callId,
|
|
expires_ts: Date.now() + MEMBERSHIP_EXPIRY_MS,
|
|
feeds: [
|
|
{ purpose: 'usermedia' },
|
|
],
|
|
},
|
|
],
|
|
},
|
|
],
|
|
};
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
await (this.matrixClient as any).sendStateEvent(
|
|
roomId,
|
|
CALL_MEMBER_EVENT_TYPE,
|
|
content,
|
|
stateKey
|
|
);
|
|
} else {
|
|
// Clear the call membership by sending empty calls array
|
|
const content: CallMemberEventContent = {
|
|
'm.calls': [],
|
|
};
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
await (this.matrixClient as any).sendStateEvent(
|
|
roomId,
|
|
CALL_MEMBER_EVENT_TYPE,
|
|
content,
|
|
stateKey
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Starts periodic refresh of call membership to prevent expiry
|
|
* Refreshes every 45 minutes to ensure the 1 hour expiry is never reached
|
|
* @param roomId - The Matrix room ID
|
|
* @param callId - The unique call ID
|
|
*/
|
|
private startMembershipRefresh(roomId: string, callId: string): void {
|
|
this.stopMembershipRefresh();
|
|
|
|
this.membershipRefreshInterval = setInterval(async () => {
|
|
if (!this.activeCall || this.activeCall.roomId !== roomId) {
|
|
this.stopMembershipRefresh();
|
|
return;
|
|
}
|
|
|
|
try {
|
|
console.log('Refreshing call membership to prevent expiry');
|
|
await this.sendCallMemberEvent(roomId, callId, true);
|
|
console.log('Call membership refreshed successfully');
|
|
} catch (error) {
|
|
console.error('Failed to refresh call membership:', error);
|
|
}
|
|
}, MEMBERSHIP_REFRESH_INTERVAL_MS);
|
|
|
|
console.log('Started call membership refresh interval (every 45 minutes)');
|
|
}
|
|
|
|
/**
|
|
* Stops the membership refresh interval
|
|
*/
|
|
private stopMembershipRefresh(): void {
|
|
if (this.membershipRefreshInterval) {
|
|
clearInterval(this.membershipRefreshInterval);
|
|
this.membershipRefreshInterval = null;
|
|
console.log('Stopped call membership refresh interval');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Starts a call in the specified room
|
|
* @param roomId - The Matrix room ID to start the call in
|
|
* @param callType - Voice or video call
|
|
*/
|
|
async startCall(roomId: string, callType: CallType): Promise<void> {
|
|
if (this.activeCall) {
|
|
throw new Error('A call is already active');
|
|
}
|
|
|
|
const callId = this.generateCallId();
|
|
|
|
this.activeCall = {
|
|
roomId,
|
|
callType,
|
|
state: CallState.Connecting,
|
|
startTime: Date.now(),
|
|
remoteStreams: new Map(),
|
|
remoteScreenStreams: new Map(),
|
|
remoteScreenTracks: new Map(),
|
|
participants: [],
|
|
activeSpeakers: new Set(),
|
|
mutedParticipants: new Set(),
|
|
isMuted: false,
|
|
isDeafened: false,
|
|
isVideoEnabled: callType === CallType.Video,
|
|
isScreenSharing: false,
|
|
};
|
|
|
|
// Play connecting/dialing sound
|
|
getCallSounds().playSound(CallSoundType.CallDialing, true);
|
|
|
|
this.emit({
|
|
type: CallEventType.StateChanged,
|
|
roomId,
|
|
data: { state: CallState.Connecting },
|
|
});
|
|
|
|
try {
|
|
// Step 1: Get an OpenID token from the Matrix homeserver
|
|
const openIdToken = await requestOpenIdToken(
|
|
this.config.homeserverBaseUrl,
|
|
this.config.userId,
|
|
this.config.accessToken
|
|
);
|
|
|
|
// Step 2: Determine LiveKit service URL (check room-level config first)
|
|
let livekitServiceUrl = this.config.livekitServiceUrl;
|
|
|
|
// Check if room has a custom LiveKit config
|
|
const room = this.matrixClient.getRoom(roomId);
|
|
if (room) {
|
|
const livekitConfigEvent = room.currentState.getStateEvents('im.paarrot.room.livekit_config', '');
|
|
if (livekitConfigEvent) {
|
|
const roomLivekitUrl = livekitConfigEvent.getContent()?.service_url;
|
|
if (roomLivekitUrl) {
|
|
console.log('Using room-level LiveKit URL:', roomLivekitUrl);
|
|
livekitServiceUrl = roomLivekitUrl;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Step 3: Exchange OpenID token for LiveKit JWT
|
|
console.log('Requesting LiveKit JWT for Matrix room:', roomId);
|
|
this.livekitJwt = await fetchLiveKitJWT(
|
|
livekitServiceUrl,
|
|
roomId,
|
|
this.config.userId,
|
|
this.config.deviceId,
|
|
openIdToken
|
|
);
|
|
|
|
console.log('Got LiveKit JWT, connecting to:', this.livekitJwt.url);
|
|
console.log('LiveKit JWT response:', JSON.stringify(this.livekitJwt));
|
|
|
|
// Check WebRTC support before attempting connection
|
|
const webrtcCheck = checkWebRTCSupport();
|
|
console.log('WebRTC support check:', webrtcCheck);
|
|
|
|
if (!webrtcCheck.supported) {
|
|
console.warn('WebRTC may not be fully supported. Attempting connection anyway...');
|
|
console.warn('WebRTC details:', JSON.stringify(webrtcCheck.details, null, 2));
|
|
}
|
|
|
|
// Step 4: Create and connect to LiveKit room
|
|
this.livekitRoom = new Room({
|
|
adaptiveStream: true,
|
|
dynacast: true,
|
|
});
|
|
|
|
this.setupLiveKitEventHandlers();
|
|
|
|
await this.livekitRoom.connect(this.livekitJwt.url, this.livekitJwt.jwt);
|
|
|
|
console.log('Connected to LiveKit room');
|
|
console.log('LiveKit room name:', this.livekitRoom.name);
|
|
console.log('Local participant identity:', this.livekitRoom.localParticipant.identity);
|
|
console.log('Remote participants:', this.livekitRoom.remoteParticipants.size);
|
|
|
|
// Get user's audio device preferences
|
|
const audioSettings = getAudioSettings();
|
|
console.log('Using audio settings:', audioSettings);
|
|
|
|
// Build audio capture options with processing settings
|
|
const audioCaptureOptions = {
|
|
deviceId: audioSettings.microphoneId || undefined,
|
|
noiseSuppression: audioSettings.noiseSuppression,
|
|
echoCancellation: audioSettings.echoCancellation,
|
|
autoGainControl: audioSettings.autoGainControl,
|
|
};
|
|
|
|
console.log('Audio capture options:', audioCaptureOptions);
|
|
|
|
// Step 5: Publish local tracks based on call type with selected devices and processing
|
|
if (callType === CallType.Video) {
|
|
// For video calls, enable both camera and microphone
|
|
await this.livekitRoom.localParticipant.setCameraEnabled(true);
|
|
await this.livekitRoom.localParticipant.setMicrophoneEnabled(true, audioCaptureOptions);
|
|
} else {
|
|
// Voice call - only enable microphone with preferred device and processing
|
|
await this.livekitRoom.localParticipant.setMicrophoneEnabled(true, audioCaptureOptions);
|
|
}
|
|
|
|
// Step 6: Send Matrix state event to announce participation
|
|
// This state event will be rendered in the timeline as a "joined the call" notification
|
|
await this.sendCallMemberEvent(roomId, callId, true);
|
|
this.currentCallId = callId;
|
|
|
|
// Step 7: Start periodic membership refresh to prevent expiry after 1 hour
|
|
this.startMembershipRefresh(roomId, callId);
|
|
|
|
console.log('Announced call participation to Matrix room');
|
|
|
|
// Stop dialing sound and play join sound
|
|
getCallSounds().stopDialingLoop();
|
|
getCallSounds().playSound(CallSoundType.CallJoined);
|
|
|
|
this.setState(CallState.Connected);
|
|
} catch (error) {
|
|
console.error('Failed to start call:', error);
|
|
this.setState(CallState.Error);
|
|
this.emit({
|
|
type: CallEventType.Error,
|
|
roomId,
|
|
data: { error },
|
|
});
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sets up LiveKit room event handlers
|
|
*/
|
|
private setupLiveKitEventHandlers(): void {
|
|
if (!this.livekitRoom) return;
|
|
|
|
this.livekitRoom.on(RoomEvent.ConnectionStateChanged, (state: ConnectionState) => {
|
|
console.log('LiveKit connection state:', state);
|
|
|
|
if (!this.activeCall) return;
|
|
|
|
switch (state) {
|
|
case ConnectionState.Connected:
|
|
this.setState(CallState.Connected);
|
|
break;
|
|
case ConnectionState.Reconnecting:
|
|
this.setState(CallState.Reconnecting);
|
|
break;
|
|
case ConnectionState.Disconnected:
|
|
this.setState(CallState.Ended);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
});
|
|
|
|
this.livekitRoom.on(
|
|
RoomEvent.TrackSubscribed,
|
|
(
|
|
track: RemoteTrackPublication['track'],
|
|
publication: RemoteTrackPublication,
|
|
participant: RemoteParticipant
|
|
) => {
|
|
if (!track || !this.activeCall) return;
|
|
|
|
const isScreenShare = publication.source === Track.Source.ScreenShare ||
|
|
publication.source === Track.Source.ScreenShareAudio;
|
|
|
|
console.log('Track subscribed:', track.kind, 'source:', publication.source,
|
|
'isScreenShare:', isScreenShare, 'from', participant.identity);
|
|
console.log('Track mediaStreamTrack:', track.mediaStreamTrack);
|
|
console.log('Track mediaStreamTrack readyState:', track.mediaStreamTrack?.readyState);
|
|
console.log('Track mediaStreamTrack enabled:', track.mediaStreamTrack?.enabled);
|
|
|
|
// For audio tracks, attach to an audio element to play
|
|
if (track.kind === 'audio') {
|
|
const audioElement = track.attach() as HTMLAudioElement;
|
|
audioElement.id = `audio-${participant.identity}-${publication.source}`;
|
|
|
|
// Set the preferred output device if supported and selected
|
|
const audioSettings = getAudioSettings();
|
|
if (audioSettings.speakerId && 'setSinkId' in audioElement) {
|
|
(audioElement as HTMLAudioElement & { setSinkId: (id: string) => Promise<void> })
|
|
.setSinkId(audioSettings.speakerId)
|
|
.then(() => console.log('Speaker set to:', audioSettings.speakerId))
|
|
.catch((e) => console.warn('Could not set speaker device:', e));
|
|
}
|
|
|
|
document.body.appendChild(audioElement);
|
|
console.log('Audio element attached for', participant.identity);
|
|
|
|
// If we're currently deafened, mute this new track so we don't hear them
|
|
if (this.activeCall?.isDeafened) {
|
|
track.setMuted(true);
|
|
console.log('Auto-muted new audio track from', participant.identity, 'due to deafen state');
|
|
}
|
|
}
|
|
|
|
// Handle screen share video tracks separately
|
|
if (isScreenShare && track.kind === 'video') {
|
|
const mediaStream = new MediaStream([track.mediaStreamTrack]);
|
|
|
|
console.log('📺 Screen share received from', participant.identity);
|
|
console.log('📺 Screen share track:', track);
|
|
console.log('📺 Screen share MediaStream:', mediaStream);
|
|
console.log('📺 Screen share MediaStream tracks:', mediaStream?.getTracks());
|
|
|
|
this.activeCall.remoteScreenStreams.set(participant.identity, mediaStream);
|
|
|
|
// Store the actual track for proper attachment later
|
|
this.activeCall.remoteScreenTracks.set(participant.identity, {
|
|
participantId: participant.identity,
|
|
track,
|
|
stream: mediaStream,
|
|
});
|
|
|
|
this.emit({
|
|
type: CallEventType.ScreenShareStarted,
|
|
roomId: this.activeCall.roomId,
|
|
data: {
|
|
participantId: participant.identity,
|
|
stream: mediaStream,
|
|
track,
|
|
},
|
|
});
|
|
} else if (track.kind === 'video') {
|
|
const mediaStream = new MediaStream([track.mediaStreamTrack]);
|
|
this.activeCall.remoteStreams.set(participant.identity, mediaStream);
|
|
|
|
this.emit({
|
|
type: CallEventType.RemoteStreamAdded,
|
|
roomId: this.activeCall.roomId,
|
|
data: {
|
|
participantId: participant.identity,
|
|
stream: mediaStream,
|
|
},
|
|
});
|
|
}
|
|
|
|
// Update participants list
|
|
if (!this.activeCall.participants.includes(participant.identity)) {
|
|
this.activeCall.participants.push(participant.identity);
|
|
}
|
|
}
|
|
);
|
|
|
|
this.livekitRoom.on(
|
|
RoomEvent.TrackUnsubscribed,
|
|
(
|
|
track: RemoteTrackPublication['track'],
|
|
publication: RemoteTrackPublication,
|
|
participant: RemoteParticipant
|
|
) => {
|
|
if (!this.activeCall) return;
|
|
|
|
const isScreenShare = publication.source === Track.Source.ScreenShare ||
|
|
publication.source === Track.Source.ScreenShareAudio;
|
|
|
|
console.log('Track unsubscribed:', track?.kind, 'source:', publication.source,
|
|
'isScreenShare:', isScreenShare, 'from', participant.identity);
|
|
|
|
// Remove audio element if it exists
|
|
if (track?.kind === 'audio') {
|
|
const audioElement = document.getElementById(`audio-${participant.identity}-${publication.source}`);
|
|
if (audioElement) {
|
|
audioElement.remove();
|
|
console.log('Audio element removed for', participant.identity);
|
|
}
|
|
track.detach();
|
|
}
|
|
|
|
// Handle screen share removal
|
|
if (isScreenShare && track?.kind === 'video') {
|
|
// Detach the track from any elements
|
|
track.detach();
|
|
this.activeCall.remoteScreenStreams.delete(participant.identity);
|
|
this.activeCall.remoteScreenTracks.delete(participant.identity);
|
|
console.log('📺 Screen share ended from', participant.identity);
|
|
|
|
this.emit({
|
|
type: CallEventType.ScreenShareStopped,
|
|
roomId: this.activeCall.roomId,
|
|
data: { participantId: participant.identity },
|
|
});
|
|
} else if (track?.kind === 'video') {
|
|
this.activeCall.remoteStreams.delete(participant.identity);
|
|
|
|
this.emit({
|
|
type: CallEventType.RemoteStreamRemoved,
|
|
roomId: this.activeCall.roomId,
|
|
data: { participantId: participant.identity },
|
|
});
|
|
}
|
|
}
|
|
);
|
|
|
|
this.livekitRoom.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {
|
|
console.log('Participant connected:', participant.identity, 'in Matrix room:', this.activeCall?.roomId);
|
|
console.log('Current LiveKit room name:', this.livekitRoom?.name);
|
|
|
|
// Listen for this participant's speaking state
|
|
participant.on(ParticipantEvent.IsSpeakingChanged, (speaking: boolean) => {
|
|
if (speaking) {
|
|
console.log(`🔊 ${participant.identity} is speaking`);
|
|
} else {
|
|
console.log(`🔈 ${participant.identity} stopped speaking`);
|
|
}
|
|
});
|
|
|
|
if (this.activeCall && !this.activeCall.participants.includes(participant.identity)) {
|
|
this.activeCall.participants.push(participant.identity);
|
|
|
|
// Play join sound for remote participant
|
|
getCallSounds().playSound(CallSoundType.CallJoined);
|
|
|
|
this.emit({
|
|
type: CallEventType.ParticipantJoined,
|
|
roomId: this.activeCall.roomId,
|
|
data: { participantId: participant.identity },
|
|
});
|
|
}
|
|
});
|
|
|
|
this.livekitRoom.on(RoomEvent.ParticipantDisconnected, (participant: RemoteParticipant) => {
|
|
console.log('Participant disconnected:', participant.identity);
|
|
|
|
if (this.activeCall) {
|
|
this.activeCall.participants = this.activeCall.participants.filter(
|
|
(p) => p !== participant.identity
|
|
);
|
|
this.activeCall.remoteStreams.delete(participant.identity);
|
|
|
|
// Play leave sound for remote participant
|
|
getCallSounds().playSound(CallSoundType.CallDepart);
|
|
|
|
this.emit({
|
|
type: CallEventType.ParticipantLeft,
|
|
roomId: this.activeCall.roomId,
|
|
data: { participantId: participant.identity },
|
|
});
|
|
}
|
|
});
|
|
|
|
this.livekitRoom.on(RoomEvent.LocalTrackPublished, (publication) => {
|
|
console.log('Local track published:', publication.kind);
|
|
|
|
if (this.activeCall && publication.track) {
|
|
const mediaStream = new MediaStream([publication.track.mediaStreamTrack]);
|
|
this.activeCall.localStream = mediaStream;
|
|
}
|
|
});
|
|
|
|
// Track mute/unmute events for remote participants
|
|
this.livekitRoom.on(RoomEvent.TrackMuted, (publication, participant) => {
|
|
// Only emit for remote participants and audio tracks
|
|
if (!this.activeCall) return;
|
|
if (participant.identity === this.livekitRoom?.localParticipant.identity) return;
|
|
if (publication.kind !== 'audio') return;
|
|
|
|
console.log(`🔇 ${participant.identity} muted their microphone`);
|
|
|
|
// Track muted state
|
|
this.activeCall.mutedParticipants.add(participant.identity);
|
|
|
|
// Play mute sound for remote participant
|
|
getCallSounds().playSound(CallSoundType.MicMute);
|
|
|
|
this.emit({
|
|
type: CallEventType.RemoteMuteChanged,
|
|
roomId: this.activeCall.roomId,
|
|
data: { participantId: participant.identity, isMuted: true },
|
|
});
|
|
});
|
|
|
|
this.livekitRoom.on(RoomEvent.TrackUnmuted, (publication, participant) => {
|
|
// Only emit for remote participants and audio tracks
|
|
if (!this.activeCall) return;
|
|
if (participant.identity === this.livekitRoom?.localParticipant.identity) return;
|
|
if (publication.kind !== 'audio') return;
|
|
|
|
console.log(`🔊 ${participant.identity} unmuted their microphone`);
|
|
|
|
// Track unmuted state
|
|
this.activeCall.mutedParticipants.delete(participant.identity);
|
|
|
|
// Play unmute sound for remote participant
|
|
getCallSounds().playSound(CallSoundType.MicUnmute);
|
|
|
|
this.emit({
|
|
type: CallEventType.RemoteMuteChanged,
|
|
roomId: this.activeCall.roomId,
|
|
data: { participantId: participant.identity, isMuted: false },
|
|
});
|
|
});
|
|
|
|
// Voice activity detection - logs when speakers change
|
|
// Note: This event tracks remote speakers. Local speaker is tracked separately.
|
|
// We preserve the existing activeSpeakers that may include local user
|
|
this.livekitRoom.on(RoomEvent.ActiveSpeakersChanged, (speakers: Participant[]) => {
|
|
if (!this.activeCall) return;
|
|
|
|
// Get remote speaker identities
|
|
const remoteSpeakerIds = speakers
|
|
.filter((s) => s.identity !== this.livekitRoom?.localParticipant.identity)
|
|
.map((s) => s.identity);
|
|
|
|
// Check if local participant is speaking (from LiveKit's list)
|
|
const localSpeaking = speakers.some(
|
|
(s) => s.identity === this.livekitRoom?.localParticipant.identity
|
|
);
|
|
|
|
// Build new activeSpeakers set - use Matrix user ID for local
|
|
const newSpeakers = new Set<string>(remoteSpeakerIds);
|
|
if (localSpeaking) {
|
|
newSpeakers.add(this.config.userId);
|
|
}
|
|
|
|
this.activeCall.activeSpeakers = newSpeakers;
|
|
|
|
if (speakers.length > 0) {
|
|
const speakerNames = speakers.map((s) => s.identity).join(', ');
|
|
console.log('🎤 Active speakers:', speakerNames);
|
|
}
|
|
|
|
this.emit({
|
|
type: CallEventType.SpeakersChanged,
|
|
roomId: this.activeCall.roomId,
|
|
data: { speakers: newSpeakers },
|
|
});
|
|
});
|
|
|
|
// Listen for participant metadata changes (for deafen state)
|
|
this.livekitRoom.on(RoomEvent.ParticipantMetadataChanged, (prevMetadata: string | undefined, participant: Participant) => {
|
|
if (!this.activeCall) return;
|
|
if (participant.identity === this.livekitRoom?.localParticipant.identity) return;
|
|
|
|
try {
|
|
const prevData = prevMetadata ? JSON.parse(prevMetadata) : {};
|
|
const newData = participant.metadata ? JSON.parse(participant.metadata) : {};
|
|
|
|
const wasDeafened = prevData.isDeafened === true;
|
|
const isDeafened = newData.isDeafened === true;
|
|
|
|
if (wasDeafened !== isDeafened) {
|
|
console.log(`🎧 ${participant.identity} ${isDeafened ? 'deafened' : 'undeafened'}`);
|
|
getCallSounds().playSound(isDeafened ? CallSoundType.Deafen : CallSoundType.Undeafen);
|
|
}
|
|
} catch {
|
|
// Ignore metadata parse errors
|
|
}
|
|
});
|
|
|
|
// Track local participant speaking state - use Matrix user ID for consistency
|
|
this.livekitRoom.localParticipant.on(ParticipantEvent.IsSpeakingChanged, (speaking: boolean) => {
|
|
if (!this.activeCall) return;
|
|
|
|
const myUserId = this.config.userId;
|
|
|
|
if (speaking) {
|
|
console.log('🎙️ You are speaking');
|
|
this.activeCall.activeSpeakers.add(myUserId);
|
|
} else {
|
|
console.log('🔇 You stopped speaking');
|
|
this.activeCall.activeSpeakers.delete(myUserId);
|
|
}
|
|
|
|
this.emit({
|
|
type: CallEventType.SpeakersChanged,
|
|
roomId: this.activeCall.roomId,
|
|
data: { speakers: this.activeCall.activeSpeakers },
|
|
});
|
|
});
|
|
|
|
// Track audio level changes for local participant (more detailed)
|
|
this.livekitRoom.localParticipant.on(ParticipantEvent.AudioStreamAcquired, () => {
|
|
console.log('🎧 Audio stream acquired');
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Ends the current call
|
|
*/
|
|
async endCall(): Promise<void> {
|
|
if (!this.activeCall) {
|
|
return;
|
|
}
|
|
|
|
const { roomId } = this.activeCall;
|
|
this.setState(CallState.Disconnecting);
|
|
|
|
// Stop the membership refresh interval
|
|
this.stopMembershipRefresh();
|
|
|
|
// Stop any dialing loop and play departure sound
|
|
getCallSounds().stopDialingLoop();
|
|
getCallSounds().playSound(CallSoundType.CallDepart);
|
|
|
|
try {
|
|
// Clear Matrix call membership - this state event will be rendered
|
|
// in the timeline as a "left the call" notification
|
|
await this.sendCallMemberEvent(roomId, '', false);
|
|
} catch (error) {
|
|
console.error('Failed to clear call membership:', error);
|
|
}
|
|
|
|
// Clean up all audio elements
|
|
document.querySelectorAll('[id^="audio-"]').forEach((el) => el.remove());
|
|
|
|
// Disconnect from LiveKit
|
|
if (this.livekitRoom) {
|
|
this.livekitRoom.disconnect();
|
|
this.livekitRoom = null;
|
|
}
|
|
|
|
this.activeCall = null;
|
|
this.livekitJwt = null;
|
|
this.currentCallId = null;
|
|
|
|
this.emit({
|
|
type: CallEventType.StateChanged,
|
|
roomId,
|
|
data: { state: CallState.Ended },
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Toggles audio mute state
|
|
* Unmuting will also undeafen if currently deafened
|
|
*/
|
|
toggleMute(): boolean {
|
|
if (!this.livekitRoom?.localParticipant || !this.activeCall) {
|
|
return false;
|
|
}
|
|
|
|
const newMuteState = !this.activeCall.isMuted;
|
|
|
|
// If unmuting while deafened, undeafen first
|
|
if (!newMuteState && this.activeCall.isDeafened) {
|
|
this.activeCall.isDeafened = false;
|
|
getCallSounds().setDeafened(false);
|
|
|
|
// Unmute all remote audio tracks
|
|
this.livekitRoom.remoteParticipants.forEach((participant) => {
|
|
participant.audioTrackPublications.forEach((publication) => {
|
|
if (publication.track) {
|
|
publication.track.setMuted(false);
|
|
}
|
|
});
|
|
});
|
|
|
|
// Broadcast undeafen state
|
|
const metadata = JSON.stringify({ isDeafened: false });
|
|
this.livekitRoom.localParticipant.setMetadata(metadata).catch((err) => {
|
|
console.warn('Failed to set participant metadata:', err);
|
|
});
|
|
|
|
// Play undeafen sound
|
|
getCallSounds().playSound(CallSoundType.Undeafen);
|
|
}
|
|
|
|
this.livekitRoom.localParticipant.setMicrophoneEnabled(!newMuteState);
|
|
this.activeCall.isMuted = newMuteState;
|
|
|
|
// Play mute/unmute sound
|
|
getCallSounds().playSound(
|
|
newMuteState ? CallSoundType.MicMute : CallSoundType.MicUnmute
|
|
);
|
|
|
|
return newMuteState;
|
|
}
|
|
|
|
/**
|
|
* Toggles deafen state (mutes all incoming audio and your mic)
|
|
* Also broadcasts deafen state to other participants via metadata
|
|
*/
|
|
toggleDeafen(): boolean {
|
|
if (!this.livekitRoom || !this.activeCall) {
|
|
return false;
|
|
}
|
|
|
|
const newDeafenState = !this.activeCall.isDeafened;
|
|
this.activeCall.isDeafened = newDeafenState;
|
|
|
|
// Update CallSounds deafen state
|
|
getCallSounds().setDeafened(newDeafenState);
|
|
|
|
// Mute/unmute all remote audio tracks (so we don't hear them)
|
|
this.livekitRoom.remoteParticipants.forEach((participant) => {
|
|
participant.audioTrackPublications.forEach((publication) => {
|
|
if (publication.track) {
|
|
publication.track.setMuted(newDeafenState);
|
|
}
|
|
});
|
|
});
|
|
|
|
// Handle mic muting when deafening
|
|
if (newDeafenState) {
|
|
// Remember if user was muted before deafening
|
|
this.wasMutedBeforeDeafen = this.activeCall.isMuted;
|
|
|
|
// Deafening - always mute mic
|
|
if (!this.activeCall.isMuted) {
|
|
this.livekitRoom.localParticipant.setMicrophoneEnabled(false);
|
|
this.activeCall.isMuted = true;
|
|
}
|
|
} else {
|
|
// Undeafening - only unmute if user wasn't muted before deafening
|
|
if (!this.wasMutedBeforeDeafen && this.activeCall.isMuted) {
|
|
this.livekitRoom.localParticipant.setMicrophoneEnabled(true);
|
|
this.activeCall.isMuted = false;
|
|
}
|
|
this.wasMutedBeforeDeafen = false;
|
|
}
|
|
|
|
// Broadcast deafen state to other participants via metadata
|
|
const metadata = JSON.stringify({ isDeafened: newDeafenState });
|
|
this.livekitRoom.localParticipant.setMetadata(metadata).catch((err) => {
|
|
console.warn('Failed to set participant metadata:', err);
|
|
});
|
|
|
|
// Play deafen/undeafen sound (these always play for user feedback)
|
|
getCallSounds().playSound(
|
|
newDeafenState ? CallSoundType.Deafen : CallSoundType.Undeafen
|
|
);
|
|
|
|
return newDeafenState;
|
|
}
|
|
|
|
/**
|
|
* Toggles video enabled state
|
|
*/
|
|
toggleVideo(): boolean {
|
|
if (!this.livekitRoom?.localParticipant) {
|
|
return false;
|
|
}
|
|
|
|
const newVideoState = !this.activeCall?.isVideoEnabled;
|
|
this.livekitRoom.localParticipant.setCameraEnabled(newVideoState);
|
|
|
|
if (this.activeCall) {
|
|
this.activeCall.isVideoEnabled = newVideoState;
|
|
}
|
|
|
|
return newVideoState;
|
|
}
|
|
|
|
/**
|
|
* Gets the LiveKit JWT for the current call
|
|
*/
|
|
getLiveKitCredentials(): LiveKitJWTResponse | null {
|
|
return this.livekitJwt;
|
|
}
|
|
|
|
/**
|
|
* Gets the LiveKit room instance
|
|
*/
|
|
getLiveKitRoom(): Room | null {
|
|
return this.livekitRoom;
|
|
}
|
|
|
|
/**
|
|
* Starts screen sharing with configurable resolution and bitrate
|
|
* @returns true if screen share started successfully
|
|
*/
|
|
async startScreenShare(): Promise<boolean> {
|
|
if (!this.livekitRoom?.localParticipant || !this.activeCall) {
|
|
console.warn('Cannot start screen share: no active call or room');
|
|
return false;
|
|
}
|
|
|
|
if (this.activeCall.isScreenSharing) {
|
|
console.warn('Screen sharing is already active');
|
|
return true;
|
|
}
|
|
|
|
try {
|
|
// Preserve current mute state before starting screen share
|
|
const wasMuted = this.activeCall.isMuted;
|
|
|
|
const audioSettings = getAudioSettings();
|
|
const resolution = SCREEN_SHARE_RESOLUTIONS[audioSettings.screenShareResolution];
|
|
|
|
console.log('Starting screen share with settings:', {
|
|
resolution: audioSettings.screenShareResolution,
|
|
bitrate: audioSettings.screenShareBitrate,
|
|
frameRate: audioSettings.screenShareFrameRate,
|
|
});
|
|
|
|
// Build screen share options - LiveKit uses different format
|
|
// Resolution and framerate are set via constraints when creating track
|
|
// Note: audio here is for system/tab audio, not microphone
|
|
const screenShareOptions = {
|
|
audio: true, // Include system audio if available
|
|
resolution: audioSettings.screenShareResolution === 'source'
|
|
? undefined
|
|
: {
|
|
width: resolution.width,
|
|
height: resolution.height,
|
|
frameRate: audioSettings.screenShareFrameRate,
|
|
},
|
|
};
|
|
|
|
// Enable screen share with the configured options
|
|
await this.livekitRoom.localParticipant.setScreenShareEnabled(true, screenShareOptions);
|
|
|
|
// Restore mute state - LiveKit might have changed mic state
|
|
if (wasMuted) {
|
|
this.livekitRoom.localParticipant.setMicrophoneEnabled(false);
|
|
}
|
|
|
|
// Set bitrate on the screen share track publication
|
|
const screenSharePub = Array.from(
|
|
this.livekitRoom.localParticipant.trackPublications.values()
|
|
).find(
|
|
(pub): pub is LocalTrackPublication =>
|
|
pub.source === Track.Source.ScreenShare
|
|
);
|
|
|
|
if (screenSharePub?.track) {
|
|
console.log(`Screen share started with bitrate: ${SCREEN_SHARE_BITRATES[audioSettings.screenShareBitrate].bitrate}kbps`);
|
|
// Capture local screen stream for preview
|
|
const mediaStream = new MediaStream([screenSharePub.track.mediaStreamTrack]);
|
|
this.activeCall.localScreenStream = mediaStream;
|
|
}
|
|
|
|
this.activeCall.isScreenSharing = true;
|
|
|
|
this.emit({
|
|
type: CallEventType.StateChanged,
|
|
roomId: this.activeCall.roomId,
|
|
data: { state: this.activeCall.state, isScreenSharing: true },
|
|
});
|
|
|
|
console.log('✅ Screen sharing started');
|
|
return true;
|
|
} catch (error) {
|
|
console.error('Failed to start screen share:', error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stops screen sharing
|
|
* @returns true if screen share stopped successfully
|
|
*/
|
|
async stopScreenShare(): Promise<boolean> {
|
|
if (!this.livekitRoom?.localParticipant || !this.activeCall) {
|
|
return false;
|
|
}
|
|
|
|
if (!this.activeCall.isScreenSharing) {
|
|
return true;
|
|
}
|
|
|
|
try {
|
|
await this.livekitRoom.localParticipant.setScreenShareEnabled(false);
|
|
this.activeCall.isScreenSharing = false;
|
|
this.activeCall.localScreenStream = undefined;
|
|
|
|
this.emit({
|
|
type: CallEventType.StateChanged,
|
|
roomId: this.activeCall.roomId,
|
|
data: { state: this.activeCall.state, isScreenSharing: false },
|
|
});
|
|
|
|
console.log('🛑 Screen sharing stopped');
|
|
return true;
|
|
} catch (error) {
|
|
console.error('Failed to stop screen share:', error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Toggles screen sharing on/off
|
|
* @returns The new screen sharing state
|
|
*/
|
|
async toggleScreenShare(): Promise<boolean> {
|
|
if (this.activeCall?.isScreenSharing) {
|
|
await this.stopScreenShare();
|
|
return false;
|
|
}
|
|
return this.startScreenShare();
|
|
}
|
|
|
|
/**
|
|
* Gets the current screen sharing state
|
|
*/
|
|
isScreenSharing(): boolean {
|
|
return this.activeCall?.isScreenSharing ?? false;
|
|
}
|
|
|
|
/**
|
|
* Cleans up resources
|
|
*/
|
|
dispose(): void {
|
|
this.stopMembershipRefresh();
|
|
this.endCall();
|
|
this.listeners.clear();
|
|
}
|
|
}
|