diff --git a/src/livekit/auth/csApi.ts b/src/livekit/auth/csApi.ts
index 63b9d0cb9..d62111f02 100644
--- a/src/livekit/auth/csApi.ts
+++ b/src/livekit/auth/csApi.ts
@@ -5,12 +5,21 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type MatrixClient } from "matrix-js-sdk";
+import { ConnectionError, HTTPError, type MatrixClient } from "matrix-js-sdk";
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 
 import { extractFullConfigFromToken, type SFUConfig } from "./types.ts";
 import { doNetworkOperationWithRetry } from "../../utils/matrix.ts";
 
+/**
+ * In widget mode the host may refuse an action before any request is made
+ * which surfaces as a plain error. Asking again cannot change its answer,
+ * so only errors from the network are worth retrying.
+ */
+function isNetworkError(e: unknown): boolean {
+  return e instanceof HTTPError || e instanceof ConnectionError;
+}
+
 export type ClientGetTokenParts = Pick<
   MatrixClient,
   "_unstable_getLivekitToken"
@@ -55,14 +64,16 @@ export async function getSFUConfig({
   roomId,
   slotId,
 }: GetSFUConfigParams): Promise<SFUConfig> {
-  const res = await doNetworkOperationWithRetry(async () =>
-    client._unstable_getLivekitToken({
-      url,
-      server_name: serverName,
-      room_id: roomId,
-      slot_id: slotId,
-      member_id: membership.memberId,
-    }),
+  const res = await doNetworkOperationWithRetry(
+    async () =>
+      client._unstable_getLivekitToken({
+        url,
+        server_name: serverName,
+        room_id: roomId,
+        slot_id: slotId,
+        member_id: membership.memberId,
+      }),
+    isNetworkError,
   );
   return extractFullConfigFromToken({ url, jwt: res.jwt });
 }
@@ -112,13 +123,15 @@ export async function delegateDelayedLeave({
   slotId,
   delayId,
 }: DelegateDelayedLeaveParams): Promise<void> {
-  await doNetworkOperationWithRetry(async () =>
-    client._unstable_delegateDelayedLeave({
-      url,
-      room_id: roomId,
-      slot_id: slotId,
-      member_id: membership.memberId,
-      delay_id: delayId,
-    }),
+  await doNetworkOperationWithRetry(
+    async () =>
+      client._unstable_delegateDelayedLeave({
+        url,
+        room_id: roomId,
+        slot_id: slotId,
+        member_id: membership.memberId,
+        delay_id: delayId,
+      }),
+    isNetworkError,
   );
 }
diff --git a/src/livekit/auth/index.test.ts b/src/livekit/auth/index.test.ts
index 6112e4caf..f16987a34 100644
--- a/src/livekit/auth/index.test.ts
+++ b/src/livekit/auth/index.test.ts
@@ -158,6 +158,34 @@ describe("getSFUConfig", () => {
     expect(openID.getSFUConfig).toHaveBeenCalled();
   });
 
+  it("falls back to OpenID when the widget host refuses the CS API", async () => {
+    // In widget mode a host without MSC4533 rejects the request before it
+    // reaches the homeserver, with a plain error rather than a Matrix error
+    client._unstable_getLivekitToken.mockRejectedValueOnce(
+      new Error(
+        "The rtc_livekit_get_token action is not supported by the client.",
+      ),
+    );
+    expect(
+      await getSFUConfig({
+        ...defaultParams,
+        transport: urlAndServiceUrlTransport,
+      }),
+    ).toEqual(expect.objectContaining({ jwt: openIDJWTToken }));
+    expect(openID.getSFUConfig).toHaveBeenCalled();
+  });
+
+  it("surfaces the widget host's refusal with url transport", async () => {
+    const error = new Error(
+      "The rtc_livekit_get_token action is not supported by the client.",
+    );
+    client._unstable_getLivekitToken.mockRejectedValueOnce(error);
+    await expect(
+      getSFUConfig({ ...defaultParams, transport: urlTransport }),
+    ).rejects.toThrow(error);
+    expect(openID.getSFUConfig).not.toHaveBeenCalled();
+  });
+
   it("does not fall back to OpenID when publishing", async () => {
     client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError);
     await expect(
diff --git a/src/livekit/auth/index.ts b/src/livekit/auth/index.ts
index 20258c81d..018347cb3 100644
--- a/src/livekit/auth/index.ts
+++ b/src/livekit/auth/index.ts
@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 import { type Logger } from "matrix-js-sdk/lib/logger";
 
-import { MatrixError } from "matrix-js-sdk";
+import { ConnectionError, MatrixError } from "matrix-js-sdk";
 import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import * as openID from "./openID";
 import * as csApi from "./csApi";
@@ -38,6 +38,19 @@ function isEndpointUnsupported(e: unknown): boolean {
   );
 }
 
+/**
+ * Checks whether an error means the MSC4195 CS API cannot be used from here:
+ * the homeserver does not implement it, or the request never reached the
+ * homeserver because in widget mode the host does not expose the endpoint
+ * (MSC4533), which the widget API reports as a plain error rather than a
+ * Matrix error. A connection error is neither, as the endpoint may well exist.
+ * @param e The error to check.
+ */
+function isCsApiUnavailable(e: unknown): boolean {
+  if (isEndpointUnsupported(e)) return true;
+  return !(e instanceof MatrixError) && !(e instanceof ConnectionError);
+}
+
 export interface GetSFUConfigParams {
   /**
    * The Matrix client.
@@ -113,23 +126,25 @@ export async function getSFUConfig({
       slotId,
     });
   } catch (e) {
-    if (!isEndpointUnsupported(e)) throw e;
     // Publishers should never fall back. If a homeserver advertises a `url`
     // but doesn't support MSC4195 itself, that's a misconfiguration which
     // the admin should know about so they can prepare for the future
     // deprecation of `livekit_service_url`.
     const mayFallBack =
       role === "subscriber" && "livekit_service_url" in transport;
-    if (!mayFallBack) {
+    if (mayFallBack && isCsApiUnavailable(e)) {
+      logger.warn(
+        `Cannot use the MSC4195 get_token endpoint (${e}), falling back to the OpenID flow with service ${transport.livekit_service_url}`,
+      );
+      return await viaOpenID(transport.livekit_service_url);
+    }
+    if (isEndpointUnsupported(e)) {
       logger.error(
         `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${flow.url}`,
       );
       throw new NoMatrix2AuthorizationService(e as Error);
     }
-    logger.warn(
-      `Homeserver does not support the MSC4195 get_token endpoint, falling back to the OpenID flow with service ${transport.livekit_service_url}`,
-    );
-    return await viaOpenID(transport.livekit_service_url);
+    throw e;
   }
 }
 
diff --git a/src/utils/matrix.ts b/src/utils/matrix.ts
index 4e3ae3c32..823de6880 100644
--- a/src/utils/matrix.ts
+++ b/src/utils/matrix.ts
@@ -341,9 +341,13 @@ export function getRelativeRoomUrl(
  * Perform a network operation with retries on ConnectionError.
  * If the error is not retryable, or the max number of retries is reached, the error is rethrown.
  * Supports handling of matrix quotas.
+ * @param operation The operation to perform.
+ * @param shouldRetry Rules out errors that retrying cannot fix, on top of the
+ *   standard backoff rule. Retries everything by default.
  */
 export async function doNetworkOperationWithRetry<T>(
   operation: () => Promise<T>,
+  shouldRetry: (e: unknown) => boolean = () => true,
 ): Promise<T> {
   let currentRetryCount = 0;
 
@@ -353,7 +357,9 @@ export async function doNetworkOperationWithRetry<T>(
       return await operation();
     } catch (e) {
       currentRetryCount++;
-      const backoff = calculateRetryBackoff(e, currentRetryCount, true);
+      const backoff = shouldRetry(e)
+        ? calculateRetryBackoff(e, currentRetryCount, true)
+        : -1;
       if (backoff < 0) {
         // Max number of retries reached, or error is not retryable. rethrow the error
         throw e;