diff --git a/authlib/jose/rfc7515/jws.py b/authlib/jose/rfc7515/jws.py
index 1248c955..8a34e947 100644
--- a/authlib/jose/rfc7515/jws.py
+++ b/authlib/jose/rfc7515/jws.py
@@ -252,7 +252,7 @@ def _prepare_algorithm_key(self, header, payload, key):
         algorithm = self.ALGORITHMS_REGISTRY[alg]
         if callable(key):
             key = key(header, payload)
-        elif 'jwk' in header:
+        elif key is None and 'jwk' in header:
             key = header['jwk']
         key = algorithm.prepare_key(key)
         return algorithm, key
diff --git a/authlib/jose/rfc7516/jwe.py b/authlib/jose/rfc7516/jwe.py
index 0de8ea40..30228e7e 100644
--- a/authlib/jose/rfc7516/jwe.py
+++ b/authlib/jose/rfc7516/jwe.py
@@ -717,6 +717,6 @@ def _validate_private_headers(self, header, alg):
 def prepare_key(alg, header, key):
     if callable(key):
         key = key(header, None)
-    elif 'jwk' in header:
+    elif key is None and 'jwk' in header:
         key = header['jwk']
     return alg.prepare_key(key)