diff --git a/octavia/certificates/manager/barbican.py b/octavia/certificates/manager/barbican.py
index 049b23b06..533f7daba 100644
--- a/octavia/certificates/manager/barbican.py
+++ b/octavia/certificates/manager/barbican.py
@@ -19,6 +19,7 @@
 """
 from OpenSSL import crypto
 
+from barbicanclient import exceptions as barbican_exceptions
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import encodeutils
@@ -115,7 +116,12 @@ def get_cert(self, context, cert_ref, resource_ref=None, check_only=False,
             return pkcs12.PKCS12Cert(cert_secret.payload)
         except exceptions.UnreadablePKCS12:
             raise
-        except Exception as e:
+        except barbican_exceptions.HTTPClientError as e:
+            # we only want to try the legacy (container) based retrieval if the pkcs12 cert is not found,
+            # else, just raise the error so we retry the pkcs12 retrieval again
+            if e.status_code != 404:
+                raise
+
             LOG.warning('Failed to load PKCS12Cert for secret %s with %s',
                         cert_ref, str(e))
             LOG.warning('Falling back to the barbican_legacy implementation.')