diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts
index 06970b93..aa7f32be 100644
--- a/src/state/CallViewModel/CallViewModel.ts
+++ b/src/state/CallViewModel/CallViewModel.ts
@@ -8,12 +8,10 @@ Please see LICENSE in the repository root for full details.
 import {
   type BaseKeyProvider,
   type ConnectionState,
-  type E2EEOptions,
   ExternalE2EEKeyProvider,
   type Room as LivekitRoom,
   type RoomOptions,
 } from "livekit-client";
-import E2EEWorker from "livekit-client/e2ee-worker?worker";
 import { type Room as MatrixRoom } from "matrix-js-sdk";
 import {
   combineLatest,
@@ -179,19 +177,11 @@ export class CallViewModel {
   private readonly userId = this.matrixRoom.client.getUserId()!;
   private readonly deviceId = this.matrixRoom.client.getDeviceId()!;
 
-  private readonly livekitE2EEKeyProvider = getE2eeKeyProvider(
+  private readonly livekitKeyProvider = getE2eeKeyProvider(
     this.options.encryptionSystem,
     this.matrixRTCSession,
   );
 
-  private readonly e2eeLivekitOptions: E2EEOptions | undefined = this
-    .livekitE2EEKeyProvider
-    ? {
-        keyProvider: this.livekitE2EEKeyProvider,
-        worker: new E2EEWorker(),
-      }
-    : undefined;
-
   private memberships$ = createMemberships$(this.scope, this.matrixRTCSession);
 
   private membershipsAndTransports = membershipsAndTransports$(
@@ -215,7 +205,7 @@ export class CallViewModel {
     this.matrixRoom.client,
     this.mediaDevices,
     this.trackProcessorState$,
-    this.e2eeLivekitOptions,
+    this.livekitKeyProvider,
     getUrlParams().controlledAudioDevices,
   );
 
@@ -251,7 +241,7 @@ export class CallViewModel {
   private connectOptions$ = this.scope.behavior(
     matrixRTCMode.value$.pipe(
       map((mode) => ({
-        encryptMedia: this.e2eeLivekitOptions !== undefined,
+        encryptMedia: this.livekitKeyProvider !== undefined,
         // TODO. This might need to get called again on each cahnge of matrixRTCMode...
         matrixRTCMode: mode,
       })),
@@ -266,7 +256,6 @@ export class CallViewModel {
     matrixRTCSession: this.matrixRTCSession,
     matrixRoom: this.matrixRoom,
     localTransport$: this.localTransport$,
-    e2eeLivekitOptions: this.e2eeLivekitOptions,
     trackProcessorState$: this.trackProcessorState$,
     widget,
     options: this.connectOptions$,
diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts
index d4408f27..1d517643 100644
--- a/src/state/CallViewModel/localMember/LocalMembership.ts
+++ b/src/state/CallViewModel/localMember/LocalMembership.ts
@@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details.
 
 import {
   type LocalTrack,
-  type E2EEOptions,
   type Participant,
   ParticipantEvent,
   type LocalParticipant,
@@ -105,7 +104,6 @@ interface Props {
   matrixRTCSession: MatrixRTCSession;
   matrixRoom: MatrixRoom;
   localTransport$: Behavior<LivekitTransport | null>;
-  e2eeLivekitOptions: E2EEOptions | undefined;
   trackProcessorState$: Behavior<ProcessorState>;
   widget: WidgetHelpers | null;
 }
@@ -132,7 +130,6 @@ export const createLocalMembership$ = ({
   matrixRTCSession,
   localTransport$,
   matrixRoom,
-  e2eeLivekitOptions,
   trackProcessorState$,
   widget,
 }: Props): {
@@ -252,7 +249,6 @@ export const createLocalMembership$ = ({
           connection,
           mediaDevices,
           muteStates,
-          e2eeLivekitOptions,
           trackProcessorState$,
         ),
       );
diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts
index 1c436397..ff4afbd6 100644
--- a/src/state/CallViewModel/localMember/Publisher.ts
+++ b/src/state/CallViewModel/localMember/Publisher.ts
@@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 import {
-  type E2EEOptions,
   LocalVideoTrack,
   type Room as LivekitRoom,
   Track,
@@ -55,7 +54,6 @@ export class Publisher {
     private connection: Connection,
     devices: MediaDevices,
     private readonly muteStates: MuteStates,
-    e2eeLivekitOptions: E2EEOptions | undefined,
     trackerProcessorState$: Behavior<ProcessorState>,
     private logger?: Logger,
   ) {
@@ -64,7 +62,7 @@ export class Publisher {
 
     const room = connection.livekitRoom;
 
-    room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e: Error) => {
+    room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
       this.logger?.error("Failed to set E2EE enabled on room", e);
     });
 
diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
index a9e2b8fb..9f448cd9 100644
--- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
+++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
@@ -10,8 +10,10 @@ import {
   type E2EEOptions,
   Room as LivekitRoom,
   type RoomOptions,
+  type BaseKeyProvider,
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
+import E2EEWorker from "livekit-client/e2ee-worker?worker";
 
 import { type ObservableScope } from "../../ObservableScope.ts";
 import { Connection } from "./Connection.ts";
@@ -46,7 +48,7 @@ export class ECConnectionFactory implements ConnectionFactory {
     private client: OpenIDClientParts,
     private devices: MediaDevices,
     private processorState$: Behavior<ProcessorState>,
-    private e2eeLivekitOptions: E2EEOptions | undefined,
+    livekitKeyProvider: BaseKeyProvider | undefined,
     private controlledAudioDevices: boolean,
     livekitRoomFactory?: () => LivekitRoom,
   ) {
@@ -55,7 +57,12 @@ export class ECConnectionFactory implements ConnectionFactory {
         generateRoomOption(
           this.devices,
           this.processorState$.value,
-          this.e2eeLivekitOptions,
+          livekitKeyProvider && {
+            keyProvider: livekitKeyProvider,
+            // It's important that every room use a separate E2EE worker.
+            // They get confused if given streams from multiple rooms.
+            worker: new E2EEWorker(),
+          },
           this.controlledAudioDevices,
         ),
       );