litruv/cinny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(call): add sound effects and incoming call notification UI

Browse Source 
- Implemented sound management for call events including mute, unmute, deafen, undeafen, call joined, call depart, and dialing sounds.
- Created a new CallSounds class to handle sound playback and state management.
- Added IncomingCallNotification component to display incoming call alerts with caller information and action buttons.
- Styled the incoming call notification using vanilla-extract CSS for animations and layout.
- Integrated sound playback for incoming calls with a 15-second threshold for dialing sound.
This commit is contained in:
Litruv
2026-02-05 03:24:18 +11:00
parent cd912025f1
commit 1a452f52ca
27 changed files with 1888 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
src/app/features/call/CallService.ts
View File

@@ -24,6 +24,7 @@ import {
} from './types';
import { fetchLiveKitJWT, fetchWellKnownWithRTC, getLiveKitFocus, requestOpenIdToken } from './useRtcConfig';
import { getAudioSettings, SCREEN_SHARE_RESOLUTIONS, SCREEN_SHARE_BITRATES } from '../settings/audio/Audio';
import { getCallSounds, CallSoundType } from './CallSounds';
type CallEventListener = (event: CallEvent) => void;
@@ -190,11 +191,17 @@ export class CallService {
remoteScreenStreams: new Map(),
remoteScreenTracks: new Map(),
participants: [],
activeSpeakers: new Set(),
mutedParticipants: new Set(),
isMuted: false,
isDeafened: false,
isVideoEnabled: callType === CallType.Video,
isScreenSharing: false,
};
// Play connecting/dialing sound
getCallSounds().playSound(CallSoundType.CallDialing, true);
this.emit({
type: CallEventType.StateChanged,
roomId,
@@ -262,10 +269,15 @@ export class CallService {
}
// Step 5: Send Matrix state event to announce participation
// This state event will be rendered in the timeline as a "joined the call" notification
await this.sendCallMemberEvent(roomId, callId, true);
console.log('Announced call participation to Matrix room');
// Stop dialing sound and play join sound
getCallSounds().stopDialingLoop();
getCallSounds().playSound(CallSoundType.CallJoined);
this.setState(CallState.Connected);
} catch (error) {
console.error('Failed to start call:', error);
@@ -339,6 +351,12 @@ export class CallService {
document.body.appendChild(audioElement);
console.log('Audio element attached for', participant.identity);
// If we're currently deafened, mute this new track so we don't hear them
if (this.activeCall?.isDeafened) {
track.setMuted(true);
console.log('Auto-muted new audio track from', participant.identity, 'due to deafen state');
}
}
// Handle screen share video tracks separately
@@ -454,6 +472,10 @@ export class CallService {
if (this.activeCall && !this.activeCall.participants.includes(participant.identity)) {
this.activeCall.participants.push(participant.identity);
// Play join sound for remote participant
getCallSounds().playSound(CallSoundType.CallJoined);
this.emit({
type: CallEventType.ParticipantJoined,
roomId: this.activeCall.roomId,
@@ -471,6 +493,9 @@ export class CallService {
);
this.activeCall.remoteStreams.delete(participant.identity);
// Play leave sound for remote participant
getCallSounds().playSound(CallSoundType.CallDepart);
this.emit({
type: CallEventType.ParticipantLeft,
roomId: this.activeCall.roomId,
@@ -488,21 +513,125 @@ export class CallService {
}
});
// Track mute/unmute events for remote participants
this.livekitRoom.on(RoomEvent.TrackMuted, (publication, participant) => {
// Only emit for remote participants and audio tracks
if (!this.activeCall) return;
if (participant.identity === this.livekitRoom?.localParticipant.identity) return;
if (publication.kind !== 'audio') return;
console.log(`🔇 ${participant.identity} muted their microphone`);
// Track muted state
this.activeCall.mutedParticipants.add(participant.identity);
// Play mute sound for remote participant
getCallSounds().playSound(CallSoundType.MicMute);
this.emit({
type: CallEventType.RemoteMuteChanged,
roomId: this.activeCall.roomId,
data: { participantId: participant.identity, isMuted: true },
});
});
this.livekitRoom.on(RoomEvent.TrackUnmuted, (publication, participant) => {
// Only emit for remote participants and audio tracks
if (!this.activeCall) return;
if (participant.identity === this.livekitRoom?.localParticipant.identity) return;
if (publication.kind !== 'audio') return;
console.log(`🔊 ${participant.identity} unmuted their microphone`);
// Track unmuted state
this.activeCall.mutedParticipants.delete(participant.identity);
// Play unmute sound for remote participant
getCallSounds().playSound(CallSoundType.MicUnmute);
this.emit({
type: CallEventType.RemoteMuteChanged,
roomId: this.activeCall.roomId,
data: { participantId: participant.identity, isMuted: false },
});
});
// Voice activity detection - logs when speakers change
// Note: This event tracks remote speakers. Local speaker is tracked separately.
// We preserve the existing activeSpeakers that may include local user
this.livekitRoom.on(RoomEvent.ActiveSpeakersChanged, (speakers: Participant[]) => {
if (!this.activeCall) return;
// Get remote speaker identities
const remoteSpeakerIds = speakers
.filter((s) => s.identity !== this.livekitRoom?.localParticipant.identity)
.map((s) => s.identity);
// Check if local participant is speaking (from LiveKit's list)
const localSpeaking = speakers.some(
(s) => s.identity === this.livekitRoom?.localParticipant.identity
);
// Build new activeSpeakers set - use Matrix user ID for local
const newSpeakers = new Set<string>(remoteSpeakerIds);
if (localSpeaking) {
newSpeakers.add(this.config.userId);
}
this.activeCall.activeSpeakers = newSpeakers;
if (speakers.length > 0) {
const speakerNames = speakers.map((s) => s.identity).join(', ');
console.log('🎤 Active speakers:', speakerNames);
}
this.emit({
type: CallEventType.SpeakersChanged,
roomId: this.activeCall.roomId,
data: { speakers: newSpeakers },
});
});
// Track local participant speaking state
// Listen for participant metadata changes (for deafen state)
this.livekitRoom.on(RoomEvent.ParticipantMetadataChanged, (prevMetadata: string | undefined, participant: Participant) => {
if (!this.activeCall) return;
if (participant.identity === this.livekitRoom?.localParticipant.identity) return;
try {
const prevData = prevMetadata ? JSON.parse(prevMetadata) : {};
const newData = participant.metadata ? JSON.parse(participant.metadata) : {};
const wasDeafened = prevData.isDeafened === true;
const isDeafened = newData.isDeafened === true;
if (wasDeafened !== isDeafened) {
console.log(`🎧 ${participant.identity} ${isDeafened ? 'deafened' : 'undeafened'}`);
getCallSounds().playSound(isDeafened ? CallSoundType.Deafen : CallSoundType.Undeafen);
}
} catch {
// Ignore metadata parse errors
}
});
// Track local participant speaking state - use Matrix user ID for consistency
this.livekitRoom.localParticipant.on(ParticipantEvent.IsSpeakingChanged, (speaking: boolean) => {
if (!this.activeCall) return;
const myUserId = this.config.userId;
if (speaking) {
console.log('🎙️ You are speaking');
this.activeCall.activeSpeakers.add(myUserId);
} else {
console.log('🔇 You stopped speaking');
this.activeCall.activeSpeakers.delete(myUserId);
}
this.emit({
type: CallEventType.SpeakersChanged,
roomId: this.activeCall.roomId,
data: { speakers: this.activeCall.activeSpeakers },
});
});
// Track audio level changes for local participant (more detailed)
@@ -522,8 +651,13 @@ export class CallService {
const { roomId } = this.activeCall;
this.setState(CallState.Disconnecting);
// Stop any dialing loop and play departure sound
getCallSounds().stopDialingLoop();
getCallSounds().playSound(CallSoundType.CallDepart);
try {
// Clear Matrix call membership
// Clear Matrix call membership - this state event will be rendered
// in the timeline as a "left the call" notification
await this.sendCallMemberEvent(roomId, '', false);
} catch (error) {
console.error('Failed to clear call membership:', error);
@@ -550,22 +684,97 @@ export class CallService {
/**
* Toggles audio mute state
* Unmuting will also undeafen if currently deafened
*/
toggleMute(): boolean {
if (!this.livekitRoom?.localParticipant) {
if (!this.livekitRoom?.localParticipant || !this.activeCall) {
return false;
}
const newMuteState = !this.activeCall?.isMuted;
this.livekitRoom.localParticipant.setMicrophoneEnabled(!newMuteState);
if (this.activeCall) {
this.activeCall.isMuted = newMuteState;
const newMuteState = !this.activeCall.isMuted;
// If unmuting while deafened, undeafen first
if (!newMuteState && this.activeCall.isDeafened) {
this.activeCall.isDeafened = false;
getCallSounds().setDeafened(false);
// Unmute all remote audio tracks
this.livekitRoom.remoteParticipants.forEach((participant) => {
participant.audioTrackPublications.forEach((publication) => {
if (publication.track) {
publication.track.setMuted(false);
}
});
});
// Broadcast undeafen state
const metadata = JSON.stringify({ isDeafened: false });
this.livekitRoom.localParticipant.setMetadata(metadata).catch((err) => {
console.warn('Failed to set participant metadata:', err);
});
}
this.livekitRoom.localParticipant.setMicrophoneEnabled(!newMuteState);
this.activeCall.isMuted = newMuteState;
// Play mute/unmute sound
getCallSounds().playSound(
newMuteState ? CallSoundType.MicMute : CallSoundType.MicUnmute
);
return newMuteState;
}
/**
* Toggles deafen state (mutes all incoming audio and your mic)
* Also broadcasts deafen state to other participants via metadata
*/
toggleDeafen(): boolean {
if (!this.livekitRoom || !this.activeCall) {
return false;
}
const newDeafenState = !this.activeCall.isDeafened;
this.activeCall.isDeafened = newDeafenState;
// Update CallSounds deafen state
getCallSounds().setDeafened(newDeafenState);
// Mute/unmute all remote audio tracks (so we don't hear them)
this.livekitRoom.remoteParticipants.forEach((participant) => {
participant.audioTrackPublications.forEach((publication) => {
if (publication.track) {
publication.track.setMuted(newDeafenState);
}
});
});
// Also mute/unmute our mic when deafening (silently - no mute sound)
// Only change mic state if it doesn't match deafen state
if (newDeafenState && !this.activeCall.isMuted) {
// Deafening - also mute mic
this.livekitRoom.localParticipant.setMicrophoneEnabled(false);
this.activeCall.isMuted = true;
} else if (!newDeafenState && this.activeCall.isMuted) {
// Undeafening - also unmute mic
this.livekitRoom.localParticipant.setMicrophoneEnabled(true);
this.activeCall.isMuted = false;
}
// Broadcast deafen state to other participants via metadata
const metadata = JSON.stringify({ isDeafened: newDeafenState });
this.livekitRoom.localParticipant.setMetadata(metadata).catch((err) => {
console.warn('Failed to set participant metadata:', err);
});
// Play deafen/undeafen sound (these always play for user feedback)
getCallSounds().playSound(
newDeafenState ? CallSoundType.Deafen : CallSoundType.Undeafen
);
return newDeafenState;
}
/**
* Toggles video enabled state
*/
@@ -658,7 +867,9 @@ export class CallService {
if (screenSharePub?.track) {
console.log(`Screen share started with bitrate: ${SCREEN_SHARE_BITRATES[audioSettings.screenShareBitrate].bitrate}kbps`);
// LiveKit handles bitrate via simulcast and adaptive streaming
// Capture local screen stream for preview
const mediaStream = new MediaStream([screenSharePub.track.mediaStreamTrack]);
this.activeCall.localScreenStream = mediaStream;
}
this.activeCall.isScreenSharing = true;
@@ -693,6 +904,7 @@ export class CallService {
try {
await this.livekitRoom.localParticipant.setScreenShareEnabled(false);
this.activeCall.isScreenSharing = false;
this.activeCall.localScreenStream = undefined;
this.emit({
type: CallEventType.StateChanged,