diff --git a/rustica-agent/src/lib.rs b/rustica-agent/src/lib.rs
index 8c84ba7..3306455 100644
--- a/rustica-agent/src/lib.rs
+++ b/rustica-agent/src/lib.rs
@@ -333,8 +333,15 @@ impl SshAgentHandler for Handler {
 
             return Ok(Response::SignResponse { signature });
         } else if let Signatory::Direct(privkey) = &self.signatory {
+            // Extract the pubkey fingerprint from either the SSH pubkey or the SSH cert
+            let fingerprint = match (Certificate::from_bytes(&pubkey), PublicKey::from_bytes(&pubkey)) {
+                (Ok(cert), _) => cert.key.fingerprint(),
+                (_, Ok(pubkey)) => pubkey.fingerprint(),
+                _ => return Err(AgentError::from("Invalid key blob")),
+            };
+
             // Don't sign requests if the requested key does not match the signatory
-            if privkey.pubkey.encode() != pubkey {
+            if privkey.pubkey.fingerprint() != fingerprint {
                 return Err(AgentError::from("No such key"));
             }