fix: lazily retry signing key fetch on JWT authentication (#1032)

If the gateway started before IAM was ready and exhausted its startup
retries, _signing_public_pem stayed None and all JWT authentication
permanently 401'd with no recovery path short of a restart. Now
_verify_jwt retries the fetch once on demand, so the gateway self-heals
as soon as IAM becomes available.
This commit is contained in:
cybermaggedon 2026-07-07 16:00:34 +01:00 committed by GitHub
parent f76f2abe08
commit 0374098ee9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -189,36 +189,44 @@ class IamAuth:
# Lifecycle # Lifecycle
# ------------------------------------------------------------------ # ------------------------------------------------------------------
async def _fetch_signing_key(self):
"""Single attempt to fetch the IAM signing public key.
Returns True on success."""
try:
async def _fetch(client):
return await client.get_signing_key_public()
pem = await self._with_client(_fetch)
if pem:
self._signing_public_pem = pem
logger.info(
"IamAuth: fetched IAM signing public key "
f"({len(pem)} bytes)"
)
return True
except Exception as e:
logger.debug(
f"IamAuth: signing key fetch failed: "
f"{type(e).__name__}: {e}"
)
return False
async def start(self, max_retries=30, retry_delay=2.0): async def start(self, max_retries=30, retry_delay=2.0):
"""Fetch the signing public key from IAM. Retries on """Fetch the signing public key from IAM. Retries on
failure the gateway may be starting before IAM is ready.""" failure the gateway may be starting before IAM is ready."""
async def _fetch(client):
return await client.get_signing_key_public()
for attempt in range(max_retries): for attempt in range(max_retries):
try: if await self._fetch_signing_key():
pem = await self._with_client(_fetch) return
if pem: logger.info(
self._signing_public_pem = pem f"IamAuth: waiting for IAM signing key; "
logger.info( f"retry {attempt + 1}/{max_retries}"
"IamAuth: fetched IAM signing public key " )
f"({len(pem)} bytes)"
)
return
except Exception as e:
logger.info(
f"IamAuth: waiting for IAM signing key "
f"({type(e).__name__}: {e}); "
f"retry {attempt + 1}/{max_retries}"
)
await asyncio.sleep(retry_delay) await asyncio.sleep(retry_delay)
# Don't prevent startup forever. A later authenticate() call
# will try again via the JWT path.
logger.warning( logger.warning(
"IamAuth: could not fetch IAM signing key at startup; " "IamAuth: could not fetch IAM signing key at startup; "
"JWT validation will fail until it's available" "JWT validation will retry on first request"
) )
# ------------------------------------------------------------------ # ------------------------------------------------------------------
@ -259,7 +267,7 @@ class IamAuth:
self._annotate_request(request, identity) self._annotate_request(request, identity)
return identity return identity
if token.count(".") == 2: if token.count(".") == 2:
identity = self._verify_jwt(token) identity = await self._verify_jwt(token)
self._annotate_request(request, identity) self._annotate_request(request, identity)
return identity return identity
raise _auth_failure() raise _auth_failure()
@ -281,7 +289,9 @@ class IamAuth:
except Exception: except Exception:
pass pass
def _verify_jwt(self, token): async def _verify_jwt(self, token):
if not self._signing_public_pem:
await self._fetch_signing_key()
if not self._signing_public_pem: if not self._signing_public_pem:
raise _auth_failure() raise _auth_failure()
try: try: