diff --git a/package.json b/package.json
index 6c74b84c..2611587a 100644
--- a/package.json
+++ b/package.json
@@ -109,7 +109,7 @@
     "livekit-client": "^2.13.0",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4",
     "matrix-widget-api": "^1.14.0",
     "normalize.css": "^8.0.1",
     "observable-hooks": "^4.2.3",
diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts
index 166fd82c..962c9bc6 100644
--- a/src/e2ee/matrixKeyProvider.ts
+++ b/src/e2ee/matrixKeyProvider.ts
@@ -10,7 +10,6 @@ import {
   type MatrixRTCSession,
   MatrixRTCSessionEvent,
 } from "matrix-js-sdk/lib/matrixrtc";
-import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 const logger = rootLogger.getChild("[MatrixKeyProvider]");
 
@@ -27,10 +26,6 @@ export class MatrixKeyProvider extends BaseKeyProvider {
         MatrixRTCSessionEvent.EncryptionKeyChanged,
         this.onEncryptionKeyChanged,
       );
-      this.rtcSession.off(
-        MatrixRTCSessionEvent.MembershipsChanged,
-        this.onMembershipsChanged,
-      );
     }
 
     this.rtcSession = rtcSession;
@@ -39,63 +34,17 @@ export class MatrixKeyProvider extends BaseKeyProvider {
       MatrixRTCSessionEvent.EncryptionKeyChanged,
       this.onEncryptionKeyChanged,
     );
-    this.rtcSession.on(
-      MatrixRTCSessionEvent.MembershipsChanged,
-      this.onMembershipsChanged,
-    );
 
     // The new session could be aware of keys of which the old session wasn't,
     // so emit key changed events
     this.rtcSession.reemitEncryptionKeys();
   }
 
-  private keyCache = new Array<{
-    membership: CallMembershipIdentityParts;
-    encryptionKey: Uint8Array;
-    encryptionKeyIndex: number;
-  }>();
-
-  private onMembershipsChanged = (): void => {
-    const duplicatedArray = this.keyCache;
-    // Reset key cache first. It will get repopulated when calling `onEncryptionKeyChanged`
-    this.keyCache = [];
-    let next = duplicatedArray.pop();
-    while (next !== undefined) {
-      logger.debug(
-        "[KeyCache] remove key event from the cache and try adding it again. For membership: ",
-        next.membership,
-      );
-      this.onEncryptionKeyChanged(
-        next.encryptionKey,
-        next.encryptionKeyIndex,
-        next.membership,
-      );
-      next = duplicatedArray.pop();
-    }
-  };
-
   private onEncryptionKeyChanged = (
     encryptionKey: Uint8Array,
     encryptionKeyIndex: number,
-    membership: CallMembershipIdentityParts,
+    rtcBackendIdentity: string,
   ): void => {
-    // This is the only way we can get the kind of the membership event we just received the key for.
-    // best case we want to recompute this once the memberships change (you can receive the key before the participant...)
-    const membershipFull = this.rtcSession?.memberships.find(
-      (m) =>
-        m.userId === membership.userId &&
-        m.deviceId === membership.deviceId &&
-        m.memberId === membership.memberId,
-    );
-    if (!membershipFull) {
-      logger.debug(
-        "[KeyCache] Added key event to the cache because we do not have a membership for it (yet): ",
-        membership,
-      );
-      this.keyCache.push({ membership, encryptionKey, encryptionKeyIndex });
-      return;
-    }
-
     crypto.subtle
       .importKey("raw", encryptionKey, "HKDF", false, [
         "deriveBits",
@@ -105,7 +54,7 @@ export class MatrixKeyProvider extends BaseKeyProvider {
         (keyMaterial) => {
           this.onSetEncryptionKey(
             keyMaterial,
-            membershipFull.rtcBackendIdentity,
+            rtcBackendIdentity,
             encryptionKeyIndex,
           );
 
diff --git a/yarn.lock b/yarn.lock
index c4b2e957..0ddaad61 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7533,7 +7533,7 @@ __metadata:
     livekit-client: "npm:^2.13.0"
     lodash-es: "npm:^4.17.21"
     loglevel: "npm:^1.9.1"
-    matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725"
+    matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4"
     matrix-widget-api: "npm:^1.14.0"
     normalize.css: "npm:^8.0.1"
     observable-hooks: "npm:^4.2.3"
@@ -10338,9 +10338,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725":
-  version: 39.3.0
-  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2bb3b03a248e689f7460f4e70d5ffbf10353c725"
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4":
+  version: 39.4.0
+  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4"
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
     "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0"
@@ -10356,7 +10356,7 @@ __metadata:
     sdp-transform: "npm:^3.0.0"
     unhomoglyph: "npm:^1.0.6"
     uuid: "npm:13"
-  checksum: 10c0/2e7061f6e648c91aaeb30b3e01626d855e24efcb330bbe432fcba199bd46b0b0d998cbc545748e1c72a7b643d25581f988fcad9bbaa42912a6ec96a27c41d0de
+  checksum: 10c0/2375dd3d9191f78fe589b0d3170f3da7792ed469a81d3ba3cd12f4915fd33a859f8af3491edb9cf0cdaa1f881a3ea7c1bf7539e850ad0360ec9981271f462c81
   languageName: node
   linkType: hard