diff --git a/lib/src/main/java/com/auth0/jwt/JWTVerifier.java b/lib/src/main/java/com/auth0/jwt/JWTVerifier.java
index 94339dbc..5c901fc8 100644
--- a/lib/src/main/java/com/auth0/jwt/JWTVerifier.java
+++ b/lib/src/main/java/com/auth0/jwt/JWTVerifier.java
@@ -368,9 +368,11 @@ private boolean assertValidAudienceClaim(
                 List<String> expectedAudience,
                 boolean shouldContainAll
         ) {
-            // normalize to lists if null
-            actualAudience = actualAudience == null ? Collections.emptyList() : actualAudience;
-            expectedAudience = expectedAudience == null ? Collections.emptyList() : expectedAudience;
+            if (actualAudience == null && expectedAudience == null) {
+                return true;
+            } else if (actualAudience == null || expectedAudience == null) {
+                return false;
+            }
 
             if (shouldContainAll) {
                 // containsAll([]) always returns true