diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx
index 60f3e0f1..ee3e893a 100644
--- a/src/room/InCallView.tsx
+++ b/src/room/InCallView.tsx
@@ -110,7 +110,9 @@ export function ActiveCall(props: ActiveCallProps) {
     return null;
   }
 
-  livekitRoom.setE2EEEnabled(true);
+  if (props.e2eeConfig && !livekitRoom.isE2EEEnabled) {
+    livekitRoom.setE2EEEnabled(!!props.e2eeConfig);
+  }
 
   return (
     <RoomContext.Provider value={livekitRoom}>