diff --git a/rustica-agent-cli/src/config/mod.rs b/rustica-agent-cli/src/config/mod.rs
index f9ec62b..a8bdf33 100644
--- a/rustica-agent-cli/src/config/mod.rs
+++ b/rustica-agent-cli/src/config/mod.rs
@@ -53,7 +53,7 @@ pub enum RusticaAgentAction {
     ListPIVKeys(listpivkeys::ListPIVKeysConfig),
     ListFidoDevices,
     GitConfig(PublicKey),
-    RefreshAttestedX509(refresh_attested_x509_certificate::RefreshAttestedX509Config)
+    RefreshAttestedX509(refresh_attested_x509_certificate::RefreshAttestedX509Config),
 }
 
 impl From<std::io::Error> for ConfigurationError {
@@ -85,13 +85,13 @@ fn get_signatory(
     match (cmd_slot, config_slot, file, &config_key) {
         (Some(slot), _, _, _) => match slot_parser(slot) {
             Some(s) => Ok(Signatory::Yubikey(YubikeySigner {
-                yk: Yubikey::new().unwrap(),
+                yk: Yubikey::new().unwrap().into(),
                 slot: s,
             })),
             None => Err(ConfigurationError::BadSlot),
         },
         (_, _, Some(file), _) => match PrivateKey::from_path(file) {
-            Ok(p) => Ok(Signatory::Direct(p)),
+            Ok(p) => Ok(Signatory::Direct(p.into())),
             Err(e) => Err(ConfigurationError::CannotReadFile(format!(
                 "{}: {}",
                 e, file
@@ -99,12 +99,12 @@ fn get_signatory(
         },
         (_, Some(slot), _, _) => match slot_parser(slot) {
             Some(s) => Ok(Signatory::Yubikey(YubikeySigner {
-                yk: Yubikey::new().unwrap(),
+                yk: Yubikey::new().unwrap().into(),
                 slot: s,
             })),
             None => Err(ConfigurationError::BadSlot),
         },
-        (_, _, _, Some(key_string)) => Ok(Signatory::Direct(PrivateKey::from_string(key_string)?)),
+        (_, _, _, Some(key_string)) => Ok(Signatory::Direct(PrivateKey::from_string(key_string)?.into())),
         (None, None, None, None) => Err(ConfigurationError::MissingSSHKey),
     }
 }
@@ -166,7 +166,9 @@ pub fn add_daemon_options(cmd: Command) -> Command {
         )
 }
 
-fn parse_config_from_args(matches: &ArgMatches) -> Result<UpdatableConfiguration, ConfigurationError> {
+fn parse_config_from_args(
+    matches: &ArgMatches,
+) -> Result<UpdatableConfiguration, ConfigurationError> {
     UpdatableConfiguration::new(matches.value_of("config").unwrap())
         .map_err(|e| ConfigurationError::BadConfiguration(e))
 }
@@ -255,10 +257,11 @@ pub async fn configure() -> Result<RusticaAgentAction, ConfigurationError> {
             .about("Show the git configuration for code-signing with the provided key"),
     );
 
-    let refresh_x509 = refresh_attested_x509_certificate::add_configuration(new_run_agent_subcommand(
-        "refresh-attested-x509",
-        "Refresh an X509 certificate in a Yubikey slot",
-    ));
+    let refresh_x509 =
+        refresh_attested_x509_certificate::add_configuration(new_run_agent_subcommand(
+            "refresh-attested-x509",
+            "Refresh an X509 certificate in a Yubikey slot",
+        ));
 
     let command_configuration = command_configuration
         .subcommand(immediate_mode)
@@ -312,7 +315,8 @@ pub async fn configure() -> Result<RusticaAgentAction, ConfigurationError> {
     }
 
     if let Some(x509_config) = matches.subcommand_matches("refresh-x509") {
-        return refresh_attested_x509_certificate::configure_refresh_x509_certificate(x509_config).await;
+        return refresh_attested_x509_certificate::configure_refresh_x509_certificate(x509_config)
+            .await;
     }
 
     cc_help.print_help().unwrap();
diff --git a/rustica-agent-cli/src/config/multimode.rs b/rustica-agent-cli/src/config/multimode.rs
index 6164264..3e1d805 100644
--- a/rustica-agent-cli/src/config/multimode.rs
+++ b/rustica-agent-cli/src/config/multimode.rs
@@ -2,7 +2,7 @@ use std::{collections::HashMap, fs};
 
 use rustica_agent::{
     get_all_piv_keys, Handler, RusticaAgentLibraryError, Signatory, YubikeyPIVKeyDescriptor,
-    YubikeySigner
+    YubikeySigner,
 };
 
 use clap::{Arg, ArgMatches, Command};
@@ -91,7 +91,7 @@ fn get_signatory(
         let key = PublicKey::from_bytes(key).unwrap();
         if certificate_fingerprint == key.fingerprint().hash {
             let sig = Signatory::Yubikey(YubikeySigner {
-                yk: Yubikey::open(des.serial).unwrap(),
+                yk: Yubikey::open(des.serial).unwrap().into(),
                 slot: des.slot,
             });
             return Ok((des.public_key.clone(), sig));
@@ -102,7 +102,7 @@ fn get_signatory(
         if certificate_fingerprint == private_key.pubkey.fingerprint().hash {
             return Ok((
                 private_key.pubkey.clone(),
-                Signatory::Direct(private_key.clone()),
+                Signatory::Direct(private_key.clone().into()),
             ));
         }
     }
@@ -157,17 +157,16 @@ pub async fn configure_multimode(
     key_map.remove(&pubkey.encode().to_vec());
 
     let handler = Handler {
-        updatable_configuration,
-        cert: None,
+        updatable_configuration: updatable_configuration.into(),
+        cert: None.into(),
         pubkey: pubkey.clone(),
         signatory,
-        stale_at: 0,
+        stale_at: 0.into(),
         certificate_options,
-        identities: private_keys,
+        identities: private_keys.into(),
         piv_identities: key_map,
         notification_function: None,
         certificate_priority: matches.is_present("certificate-priority"),
-        
     };
 
     Ok(RusticaAgentAction::Run(RunConfig {
diff --git a/rustica-agent-cli/src/config/refresh_attested_x509_certificate.rs b/rustica-agent-cli/src/config/refresh_attested_x509_certificate.rs
index 6f67248..a3b1ae8 100644
--- a/rustica-agent-cli/src/config/refresh_attested_x509_certificate.rs
+++ b/rustica-agent-cli/src/config/refresh_attested_x509_certificate.rs
@@ -1,10 +1,12 @@
 use std::env;
 
-use clap::{Command, Arg, ArgMatches};
-use rustica_agent::{slot_validator, slot_parser, Signatory, YubikeySigner, config::UpdatableConfiguration};
+use clap::{Arg, ArgMatches, Command};
+use rustica_agent::{
+    config::UpdatableConfiguration, slot_parser, slot_validator, Signatory, YubikeySigner,
+};
 use sshcerts::yubikey::piv::Yubikey;
 
-use super::{RusticaAgentAction, ConfigurationError, parse_config_from_args};
+use super::{parse_config_from_args, ConfigurationError, RusticaAgentAction};
 
 pub struct RefreshAttestedX509Config {
     pub updatable_configuration: UpdatableConfiguration,
@@ -22,7 +24,7 @@ pub async fn configure_refresh_x509_certificate(
     let slot = slot_parser(&slot).unwrap();
 
     let signatory = Signatory::Yubikey(YubikeySigner {
-        yk: Yubikey::new().unwrap(),
+        yk: Yubikey::new().unwrap().into(),
         slot,
     });
 
@@ -37,42 +39,42 @@ pub async fn configure_refresh_x509_certificate(
         Err(_) => return Err(ConfigurationError::YubikeyManagementKeyInvalid),
     };
 
-
-    Ok(RusticaAgentAction::RefreshAttestedX509(RefreshAttestedX509Config {
-        updatable_configuration,
-        signatory,
-        pin,
-        management_key,
-    }))
+    Ok(RusticaAgentAction::RefreshAttestedX509(
+        RefreshAttestedX509Config {
+            updatable_configuration,
+            signatory,
+            pin,
+            management_key,
+        },
+    ))
 }
 
 pub fn add_configuration(cmd: Command) -> Command {
-    cmd
-        .arg(
-            Arg::new("slot")
-                .help("Numerical value for the slot on the yubikey to use for your private key")
-                .long("slot")
-                .short('s')
-                .required(true)
-                .validator(slot_validator)
-                .takes_value(true),
-        )
-        .arg(
-            Arg::new("pin-env")
-                .help("Specify a different pin environment variable")
-                .default_value("YK_PIN")
-                .long("pinenv")
-                .short('p')
-                .required(false)
-                .takes_value(true),
-        )
-        .arg(
-            Arg::new("management-key")
-                .help("Specify the management key")
-                .default_value("010203040506070801020304050607080102030405060708")
-                .long("mgmkey")
-                .short('m')
-                .required(false)
-                .takes_value(true),
-        )
+    cmd.arg(
+        Arg::new("slot")
+            .help("Numerical value for the slot on the yubikey to use for your private key")
+            .long("slot")
+            .short('s')
+            .required(true)
+            .validator(slot_validator)
+            .takes_value(true),
+    )
+    .arg(
+        Arg::new("pin-env")
+            .help("Specify a different pin environment variable")
+            .default_value("YK_PIN")
+            .long("pinenv")
+            .short('p')
+            .required(false)
+            .takes_value(true),
+    )
+    .arg(
+        Arg::new("management-key")
+            .help("Specify the management key")
+            .default_value("010203040506070801020304050607080102030405060708")
+            .long("mgmkey")
+            .short('m')
+            .required(false)
+            .takes_value(true),
+    )
 }
diff --git a/rustica-agent-cli/src/config/register.rs b/rustica-agent-cli/src/config/register.rs
index 45e5c89..4cba978 100644
--- a/rustica-agent-cli/src/config/register.rs
+++ b/rustica-agent-cli/src/config/register.rs
@@ -1,5 +1,5 @@
 use clap::{Arg, ArgMatches, Command};
-use rustica_agent::{slot_validator, PIVAttestation, Signatory, config::UpdatableConfiguration};
+use rustica_agent::{config::UpdatableConfiguration, slot_validator, PIVAttestation, Signatory};
 use yubikey::piv::SlotId;
 
 use super::{get_signatory, parse_config_from_args, ConfigurationError, RusticaAgentAction};
@@ -32,12 +32,10 @@ pub async fn configure_register(
             Signatory::Direct(_) => return Err(ConfigurationError::CannotAttestFileBasedKey),
         };
 
-        attestation.certificate = signer
-            .yk
-            .fetch_attestation(&signer.slot)
-            .unwrap_or_default();
-        attestation.intermediate = signer
-            .yk
+        let mut yk = signer.yk.lock().await;
+
+        attestation.certificate = yk.fetch_attestation(&signer.slot).unwrap_or_default();
+        attestation.intermediate = yk
             .fetch_certificate(&SlotId::Attestation)
             .unwrap_or_default();
 
diff --git a/rustica-agent-cli/src/config/singlemode.rs b/rustica-agent-cli/src/config/singlemode.rs
index 6132513..aea6dd0 100644
--- a/rustica-agent-cli/src/config/singlemode.rs
+++ b/rustica-agent-cli/src/config/singlemode.rs
@@ -22,11 +22,14 @@ pub async fn configure_singlemode(
 
     let mut signatory = get_signatory(&slot, &config.slot, &file, &config.key)?;
     let pubkey = match &mut signatory {
-        Signatory::Yubikey(signer) => match signer.yk.ssh_cert_fetch_pubkey(&signer.slot) {
-            Ok(cert) => cert,
-            Err(_) => return Err(ConfigurationError::YubikeyNoKeypairFound),
-        },
+        Signatory::Yubikey(signer) => {
+            match signer.yk.lock().await.ssh_cert_fetch_pubkey(&signer.slot) {
+                Ok(cert) => cert,
+                Err(_) => return Err(ConfigurationError::YubikeyNoKeypairFound),
+            }
+        }
         Signatory::Direct(privkey) => {
+            let mut privkey = privkey.lock().await;
             if let Some(path) = matches.value_of("fido-device-path") {
                 privkey.set_device_path(path);
             }
@@ -36,17 +39,16 @@ pub async fn configure_singlemode(
     };
 
     let handler = Handler {
-        updatable_configuration,
-        cert: None,
+        updatable_configuration: updatable_configuration.into(),
+        cert: None.into(),
         pubkey: pubkey.clone(),
         signatory,
-        stale_at: 0,
+        stale_at: 0.into(),
         certificate_options,
-        identities: HashMap::new(),
+        identities: HashMap::new().into(),
         piv_identities: HashMap::new(),
         notification_function: None,
         certificate_priority: matches.is_present("certificate-priority"),
-        
     };
 
     Ok(RusticaAgentAction::Run(RunConfig {
diff --git a/rustica-agent-cli/src/main.rs b/rustica-agent-cli/src/main.rs
index a4c0532..5953a51 100644
--- a/rustica-agent-cli/src/main.rs
+++ b/rustica-agent-cli/src/main.rs
@@ -52,7 +52,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 &config.management_key,
                 config.require_touch,
                 config.pin_policy,
-            ) {
+            )
+            .await
+            {
                 Some(_) => (),
                 None => {
                     println!("Provisioning Error");
@@ -65,7 +67,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         Ok(RusticaAgentAction::ProvisionAndRegisterFido(prf)) => {
             let new_fido_key = generate_new_ssh_key(&prf.app_name, &prf.comment, prf.pin, None)?;
 
-            let mut signatory = Signatory::Direct(new_fido_key.private_key.clone());
+            let mut signatory = Signatory::Direct(new_fido_key.private_key.clone().into());
             let u2f_attestation = U2FAttestation {
                 auth_data: new_fido_key.attestation.auth_data,
                 auth_data_sig: new_fido_key.attestation.auth_data_sig,
@@ -184,15 +186,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             .await;
         }
         Ok(RusticaAgentAction::RefreshAttestedX509(mut config)) => {
-            match rustica_agent::fetch_new_attested_x509_certificate(&config.updatable_configuration.get_configuration().servers, &mut config.signatory)
-                .await
+            match rustica_agent::fetch_new_attested_x509_certificate(
+                &config.updatable_configuration.get_configuration().servers,
+                &mut config.signatory,
+            )
+            .await
             {
                 Ok(cert) => match config.signatory {
-                    Signatory::Yubikey(mut yk) => {
-                        yk.yk
-                            .unlock(config.pin.as_bytes(), &config.management_key)
+                    Signatory::Yubikey(yk) => {
+                        let slot = yk.slot;
+                        let mut yk = yk.yk.lock().await;
+                        yk.unlock(config.pin.as_bytes(), &config.management_key)
                             .unwrap();
-                        yk.yk.write_certificate(&yk.slot, &cert).unwrap();
+                        yk.write_certificate(&slot, &cert).unwrap();
                     }
                     Signatory::Direct(_) => {
                         let parsed_cert = Certificate::from_bytes(cert.to_vec()).unwrap();
diff --git a/rustica-agent-gui/src/main.rs b/rustica-agent-gui/src/main.rs
index a604bda..a0397fd 100644
--- a/rustica-agent-gui/src/main.rs
+++ b/rustica-agent-gui/src/main.rs
@@ -137,7 +137,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let agent = load_environments()?; //.into_iter().map(|x| x.to_string_lossy().to_string()).collect();
 
     let options = eframe::NativeOptions::default();
-    eframe::run_native(
+    let _ = eframe::run_native(
         "Rustica Agent",
         options,
         Box::new(move |_cc| Box::new(agent)),
@@ -299,18 +299,18 @@ impl eframe::App for RusticaAgentGui {
                                         private_key.set_device_path(&self.fido_devices[*fido_device].path);
 
                                         let pubkey = private_key.pubkey.clone();
-                                        let signatory = Signatory::Direct(private_key);
+                                        let signatory = Signatory::Direct(private_key.into());
 
                                         let certificate_options = CertificateConfig::from(updatable_configuration.get_configuration().options.clone());
 
                                         let handler = rustica_agent::Handler {
-                                            updatable_configuration,
-                                            cert: None,
+                                            updatable_configuration: updatable_configuration.into(),
+                                            cert: None.into(),
                                             pubkey,
                                             signatory,
-                                            stale_at: 0,
+                                            stale_at: 0.into(),
                                             certificate_options,
-                                            identities: HashMap::new(),
+                                            identities:HashMap::new().into(),
                                             piv_identities: self.piv_keys.iter().filter_map(|x| if x.1.in_use {Some((x.0.clone(), x.1.descriptor.clone()))} else {None}).collect(),
                                             notification_function: None,
                                             certificate_priority: self.certificate_priority,
diff --git a/rustica-agent/src/ffi.rs b/rustica-agent/src/ffi.rs
index c9e1e50..fdc43cb 100644
--- a/rustica-agent/src/ffi.rs
+++ b/rustica-agent/src/ffi.rs
@@ -292,7 +292,7 @@ pub unsafe extern "C" fn generate_and_enroll_fido(
 
     let runtime_handle = runtime.handle().to_owned();
 
-    let mut signatory = Signatory::Direct(new_fido_key.private_key.clone());
+    let mut signatory = Signatory::Direct(new_fido_key.private_key.clone().into());
     let u2f_attestation = U2FAttestation {
         auth_data: new_fido_key.attestation.auth_data,
         auth_data_sig: new_fido_key.attestation.auth_data_sig,
@@ -425,7 +425,10 @@ pub unsafe extern "C" fn generate_and_enroll(
         Err(_) => return false,
     };
 
-    let mut signatory = Signatory::Yubikey(YubikeySigner { yk, slot });
+    let mut signatory = Signatory::Yubikey(YubikeySigner {
+        yk: yk.into(),
+        slot,
+    });
 
     let runtime = match Runtime::new() {
         Ok(rt) => rt,
@@ -643,13 +646,13 @@ pub unsafe extern "C" fn start_direct_rustica_agent_with_piv_idents(
     certificate_options.authority = authority;
 
     let handler = Handler {
-        updatable_configuration,
-        cert: None,
-        stale_at: 0,
+        updatable_configuration: updatable_configuration.into(),
+        cert: None.into(),
+        stale_at: 0.into(),
         pubkey: private_key.pubkey.clone(),
         certificate_options,
-        signatory: Signatory::Direct(private_key),
-        identities: HashMap::new(),
+        signatory: Signatory::Direct(private_key.into()),
+        identities: HashMap::new().into(),
         piv_identities,
         notification_function: Some(Box::new(notification_f)),
         certificate_priority,
@@ -747,16 +750,16 @@ pub unsafe extern "C" fn start_yubikey_rustica_agent(
     };
 
     let handler = Handler {
-        updatable_configuration,
-        cert: None,
-        stale_at: 0,
+        updatable_configuration: updatable_configuration.into(),
+        cert: None.into(),
+        stale_at: 0.into(),
         pubkey,
         certificate_options,
         signatory: Signatory::Yubikey(YubikeySigner {
-            yk: Yubikey::open(yubikey_serial).unwrap(),
+            yk: Yubikey::open(yubikey_serial).unwrap().into(),
             slot: SlotId::try_from(slot).unwrap(),
         }),
-        identities: HashMap::new(),
+        identities: HashMap::new().into(),
         piv_identities: HashMap::new(),
         notification_function: Some(Box::new(notification_f)),
         certificate_priority,
@@ -946,7 +949,10 @@ pub unsafe extern "C" fn ffi_refresh_x509_certificate(
         return false;
     }
 
-    let mut signatory = Signatory::Yubikey(YubikeySigner { yk, slot });
+    let mut signatory = Signatory::Yubikey(YubikeySigner {
+        yk: yk.into(),
+        slot,
+    });
 
     let runtime = match Runtime::new() {
         Ok(rt) => rt,
diff --git a/rustica-agent/src/lib.rs b/rustica-agent/src/lib.rs
index 37b26aa..f4f3305 100644
--- a/rustica-agent/src/lib.rs
+++ b/rustica-agent/src/lib.rs
@@ -28,6 +28,8 @@ use std::{convert::TryFrom, env};
 
 use std::time::SystemTime;
 
+use tokio::sync::Mutex;
+
 #[derive(Debug)]
 pub struct CertificateConfig {
     pub principals: Vec<String>,
@@ -48,14 +50,14 @@ pub struct RusticaServer {
 #[derive(Debug)]
 pub struct YubikeySigner {
     pub slot: SlotId,
-    pub yk: Yubikey,
+    pub yk: Mutex<Yubikey>,
 }
 
 #[derive(Debug)]
 #[allow(clippy::large_enum_variant)]
 pub enum Signatory {
     Yubikey(YubikeySigner),
-    Direct(PrivateKey),
+    Direct(Mutex<PrivateKey>),
 }
 
 #[derive(Debug, Clone)]
@@ -119,23 +121,24 @@ impl std::fmt::Display for RusticaAgentLibraryError {
 
 impl std::error::Error for RusticaAgentLibraryError {}
 
-
 pub struct Handler {
     /// Configuration path that can be updated if a server returns updated
     /// settings
-    pub updatable_configuration: UpdatableConfiguration,
+    pub updatable_configuration: Mutex<UpdatableConfiguration>,
     /// A previously issued certificate
-    pub cert: Option<Identity>,
+    pub cert: Mutex<Option<Identity>>,
     /// The public key we for the key we are providing a certificate for
     pub pubkey: PublicKey,
-    /// The signing method for the private part of our public key
+    /// The signing method for the private part of our public key. This needs to have
+    /// interior mutability because it's sometimes a Yubikey that requires exclusive
+    /// access to the USB interface
     pub signatory: Signatory,
     /// When our certificate expires and we must request a new one
-    pub stale_at: u64,
+    pub stale_at: Mutex<u64>,
     /// Any settings we wish to ask the server for in our certificate
     pub certificate_options: CertificateConfig,
     /// Any other identities added to our agent
-    pub identities: HashMap<Vec<u8>, PrivateKey>,
+    pub identities: Mutex<HashMap<Vec<u8>, PrivateKey>>,
     /// Other PIV identities
     pub piv_identities: HashMap<Vec<u8>, YubikeyPIVKeyDescriptor>,
     /// A function that we will call before calling the signatory
@@ -147,20 +150,12 @@ pub struct Handler {
 
 impl std::fmt::Debug for Handler {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Handler")
-            .field("server", &self.updatable_configuration.get_configuration().servers)
-            .field("cert", &self.cert)
-            .finish()
+        f.debug_struct("Handler").field("cert", &self.cert).finish()
     }
 }
 
 impl RusticaServer {
-    pub fn new(
-        address: String,
-        ca_pem: String,
-        mtls_cert: String,
-        mtls_key: String,
-    ) -> Self {
+    pub fn new(address: String, ca_pem: String, mtls_cert: String, mtls_key: String) -> Self {
         Self {
             address,
             ca_pem,
@@ -196,18 +191,20 @@ impl From<Option<Options>> for CertificateConfig {
 
 #[async_trait]
 impl SshAgentHandler for Handler {
-    fn add_identity(&mut self, private_key: PrivateKey) -> Result<Response, AgentError> {
+    async fn add_identity(&self, private_key: PrivateKey) -> Result<Response, AgentError> {
         trace!("Add Identity call");
         let public_key = private_key.pubkey.encode();
-        self.identities.insert(public_key, private_key);
+        self.identities.lock().await.insert(public_key, private_key);
         Ok(Response::Success)
     }
 
-    async fn identities(&mut self) -> Result<Response, AgentError> {
+    async fn identities(&self) -> Result<Response, AgentError> {
         trace!("Identities call");
         // We start building identies with the manually loaded keys
         let mut identities: Vec<Identity> = self
             .identities
+            .lock()
+            .await
             .iter()
             .map(|x| Identity {
                 key_blob: x.1.pubkey.encode().to_vec(),
@@ -226,8 +223,12 @@ impl SshAgentHandler for Handler {
             .unwrap()
             .as_secs();
 
+        let mut stale_at = self.stale_at.lock().await;
+        let mut existing_cert = self.cert.lock().await;
+        let mut configuration = self.updatable_configuration.lock().await;
+
         // Fetch a new certificate or use the cached one if it's still valid
-        let certificate = match (&self.cert, timestamp < self.stale_at) {
+        let certificate = match (&*existing_cert, timestamp < *stale_at) {
             // In the case we have a certificate and it's not expired.
             (Some(cert), true) => Ok(cert.clone()),
             // All other cases require us to fetch a certificate from one
@@ -242,8 +243,8 @@ impl SshAgentHandler for Handler {
 
                 // Fetch a new certificate from one of the servers
                 fetch_new_certificate(
-                    &mut self.updatable_configuration,
-                    &mut self.signatory,
+                    &mut configuration,
+                    &self.signatory,
                     &self.certificate_options,
                 )
                 .await
@@ -255,8 +256,8 @@ impl SshAgentHandler for Handler {
 
                     // This is ugly doing a mutation in a map
                     // Look for a better way to do this.
-                    self.cert = Some(ident.clone());
-                    self.stale_at = cert.valid_before;
+                    *existing_cert = Some(ident.clone());
+                    *stale_at = cert.valid_before;
 
                     ident
                 })
@@ -284,8 +285,8 @@ impl SshAgentHandler for Handler {
     }
 
     /// Sign a request coming in from an SSH command.
-    fn sign_request(
-        &mut self,
+    async fn sign_request(
+        &self,
         pubkey: Vec<u8>,
         data: Vec<u8>,
         _flags: u32,
@@ -303,8 +304,15 @@ impl SshAgentHandler for Handler {
         // key is the same process as keys added afterwards, we do this to prevent duplication
         // of the private key based signing code.
         // TODO: @obelisk make this better
-        let private_key: Option<&PrivateKey> = if self.identities.contains_key(&pubkey) {
-            Some(&self.identities[&pubkey])
+        if let Some(private_key) =
+            self.identities.lock().await.get(&pubkey).map(|x| x.clone())
+        {
+            let signature = match private_key.sign(&data) {
+                None => return Err(AgentError::from("Signing Error")),
+                Some(signature) => signature,
+            };
+
+            return Ok(Response::SignResponse { signature })
         } else if let Some(descriptor) = self.piv_identities.get(&pubkey) {
             let mut yk = Yubikey::open(descriptor.serial).map_err(|e| {
                 println!("Unable to open Yubikey: {e}");
@@ -341,16 +349,23 @@ impl SshAgentHandler for Handler {
 
             return Ok(Response::SignResponse { signature });
         } else if let Signatory::Direct(privkey) = &self.signatory {
+            let privkey = privkey.lock().await;
+
             // Don't sign requests if the requested key does not match the signatory
             if privkey.pubkey.fingerprint() != fingerprint {
                 return Err(AgentError::from("No such key"));
             }
 
-            Some(privkey)
-        } else if let Signatory::Yubikey(signer) = &mut self.signatory {
+            let signature = match privkey.sign(&data) {
+                None => return Err(AgentError::from("Signing Error")),
+                Some(signature) => signature,
+            };
+
+            return Ok(Response::SignResponse { signature })
+        } else if let Signatory::Yubikey(signer) = &self.signatory {
+            let mut yk = signer.yk.lock().await;
             // Don't sign requests if the requested key does not match the signatory
-            if signer
-                .yk
+            if yk
                 .ssh_cert_fetch_pubkey(&signer.slot)
                 .map_err(|e| {
                     println!("Yubikey Fetch Certificate Error: {e}");
@@ -370,30 +385,15 @@ impl SshAgentHandler for Handler {
                 f()
             }
 
-            let signature = signer
-                .yk
-                .ssh_cert_signer(&data, &signer.slot)
-                .map_err(|e| {
-                    println!("Signing Error: {e}");
-                    AgentError::from("Yubikey signing error")
-                })?;
+            let signature = yk.ssh_cert_signer(&data, &signer.slot).map_err(|e| {
+                println!("Signing Error: {e}");
+                AgentError::from("Yubikey signing error")
+            })?;
 
             return Ok(Response::SignResponse { signature });
         } else {
-            None
+            return Err(AgentError::from("Signing Error: No Valid Keys"));
         };
-
-        match private_key {
-            Some(key) => {
-                let signature = match key.sign(&data) {
-                    None => return Err(AgentError::from("Signing Error")),
-                    Some(signature) => signature,
-                };
-
-                Ok(Response::SignResponse { signature })
-            }
-            None => Err(AgentError::from("Signing Error: No Valid Keys")),
-        }
     }
 }
 
@@ -427,8 +427,8 @@ pub fn slot_validator(slot: &str) -> Result<(), String> {
 }
 
 /// Provisions a new keypair on the Yubikey with the given settings.
-pub fn provision_new_key(
-    mut yubikey: YubikeySigner,
+pub async fn provision_new_key(
+    yubikey: YubikeySigner,
     pin: &str,
     subj: &str,
     mgm_key: &[u8],
@@ -444,12 +444,14 @@ pub fn provision_new_key(
         TouchPolicy::Cached
     };
 
-    if yubikey.yk.unlock(pin.as_bytes(), mgm_key).is_err() {
+    let mut yk = yubikey.yk.lock().await;
+
+    if yk.unlock(pin.as_bytes(), mgm_key).is_err() {
         println!("Could not unlock key");
         return None;
     }
 
-    match yubikey.yk.provision(
+    match yk.provision(
         &yubikey.slot,
         subj,
         AlgorithmId::EccP384,
@@ -457,8 +459,8 @@ pub fn provision_new_key(
         pin_policy,
     ) {
         Ok(_) => {
-            let certificate = yubikey.yk.fetch_attestation(&yubikey.slot);
-            let intermediate = yubikey.yk.fetch_certificate(&SlotId::Attestation);
+            let certificate = yk.fetch_attestation(&yubikey.slot);
+            let intermediate = yk.fetch_certificate(&SlotId::Attestation);
 
             match (certificate, intermediate) {
                 (Ok(certificate), Ok(intermediate)) => Some(PIVAttestation {
@@ -564,7 +566,7 @@ pub fn git_config_from_public_key(public_key: &PublicKey) -> String {
 /// return a usable certificate
 pub async fn fetch_new_certificate(
     configuration: &mut UpdatableConfiguration,
-    signatory: &mut Signatory,
+    signatory: &Signatory,
     options: &CertificateConfig,
 ) -> Result<Certificate, RusticaAgentLibraryError> {
     for server in configuration.get_servers_mut() {
@@ -610,7 +612,10 @@ pub async fn fetch_new_attested_x509_certificate(
     signatory: &mut Signatory,
 ) -> Result<Vec<u8>, RusticaAgentLibraryError> {
     for server in servers.iter() {
-        match server.refresh_attested_x509_certificate_async(signatory).await {
+        match server
+            .refresh_attested_x509_certificate_async(signatory)
+            .await
+        {
             Ok(certificate) => return Ok(certificate),
             Err(e) => {
                 error!(
diff --git a/rustica-agent/src/rustica/cert.rs b/rustica-agent/src/rustica/cert.rs
index 97e5d67..d5d69e9 100644
--- a/rustica-agent/src/rustica/cert.rs
+++ b/rustica-agent/src/rustica/cert.rs
@@ -1,6 +1,6 @@
 use super::error::{RefreshError, ServerError};
 use super::{CertificateRequest, RusticaCert, Signatory};
-use crate::{CertificateConfig, RusticaServer, MtlsCredentials};
+use crate::{CertificateConfig, MtlsCredentials, RusticaServer};
 use sshcerts::Certificate;
 use tokio::runtime::Handle;
 
@@ -10,7 +10,7 @@ use std::time::SystemTime;
 impl RusticaServer {
     pub async fn refresh_certificate_async(
         &self,
-        signatory: &mut Signatory,
+        signatory: &Signatory,
         options: &CertificateConfig,
     ) -> Result<(RusticaCert, Option<MtlsCredentials>), RefreshError> {
         let (mut client, challenge) = super::complete_rustica_challenge(self, signatory).await?;
@@ -54,10 +54,13 @@ impl RusticaServer {
             None
         };
 
-        Ok((RusticaCert {
-            cert: response.certificate,
-            comment: "JITC".to_string(),
-        }, mtls_credentials))
+        Ok((
+            RusticaCert {
+                cert: response.certificate,
+                comment: "JITC".to_string(),
+            },
+            mtls_credentials,
+        ))
     }
 
     pub fn get_custom_certificate(
@@ -66,7 +69,6 @@ impl RusticaServer {
         options: &CertificateConfig,
         handle: &Handle,
     ) -> Result<(RusticaCert, Option<MtlsCredentials>), RefreshError> {
-        handle
-            .block_on(async { self.refresh_certificate_async(signatory, options).await })
+        handle.block_on(async { self.refresh_certificate_async(signatory, options).await })
     }
 }
diff --git a/rustica-agent/src/rustica/mod.rs b/rustica-agent/src/rustica/mod.rs
index 7c6378f..835616c 100644
--- a/rustica-agent/src/rustica/mod.rs
+++ b/rustica-agent/src/rustica/mod.rs
@@ -3,14 +3,15 @@ pub mod error;
 pub mod key;
 pub mod x509;
 
+use std::ops::Deref;
 use std::time::Duration;
 
 pub use error::RefreshError;
 
 pub use rustica_proto::rustica_client::RusticaClient;
 pub use rustica_proto::{
-    CertificateRequest, CertificateResponse, Challenge, ChallengeRequest, RegisterKeyRequest, AttestedX509CertificateRequest, AttestedX509CertificateResponse,
-    RegisterU2fKeyRequest,
+    AttestedX509CertificateRequest, AttestedX509CertificateResponse, CertificateRequest,
+    CertificateResponse, Challenge, ChallengeRequest, RegisterKeyRequest, RegisterU2fKeyRequest,
 };
 
 use sshcerts::ssh::Certificate as SSHCertificate;
@@ -28,7 +29,9 @@ pub struct RusticaCert {
     pub comment: String,
 }
 
-pub async fn get_rustica_client(server: &RusticaServer) -> Result<RusticaClient<tonic::transport::Channel>, RefreshError> {
+pub async fn get_rustica_client(
+    server: &RusticaServer,
+) -> Result<RusticaClient<tonic::transport::Channel>, RefreshError> {
     let client_identity = Identity::from_pem(&server.mtls_cert, &server.mtls_key);
 
     let channel = match Channel::from_shared(server.address.clone()) {
@@ -54,17 +57,18 @@ pub async fn get_rustica_client(server: &RusticaServer) -> Result<RusticaClient<
 
 pub async fn complete_rustica_challenge(
     server: &RusticaServer,
-    signatory: &mut Signatory,
+    signatory: &Signatory,
 ) -> Result<(RusticaClient<tonic::transport::Channel>, Challenge), RefreshError> {
     let ssh_pubkey = match signatory {
         Signatory::Yubikey(signer) => {
-            signer.yk.reconnect()?;
-            match signer.yk.ssh_cert_fetch_pubkey(&signer.slot) {
+            let mut yk = signer.yk.lock().await;
+            yk.reconnect()?;
+            match yk.ssh_cert_fetch_pubkey(&signer.slot) {
                 Ok(pkey) => pkey,
                 Err(_) => return Err(RefreshError::SigningError),
             }
         }
-        Signatory::Direct(ref privkey) => privkey.pubkey.clone(),
+        Signatory::Direct(privkey) => privkey.lock().await.pubkey.clone(),
     };
 
     let encoded_key = format!("{}", ssh_pubkey);
@@ -112,15 +116,20 @@ pub async fn complete_rustica_challenge(
         Signatory::Yubikey(signer) => {
             let signature = signer
                 .yk
+                .lock()
+                .await
                 .ssh_cert_signer(&challenge_certificate.tbs_certificate(), &signer.slot)
                 .map_err(|_| RefreshError::SigningError)?;
             challenge_certificate
                 .add_signature(&signature)
                 .map_err(|_| RefreshError::SigningError)?
         }
-        Signatory::Direct(privkey) => challenge_certificate
-            .sign(privkey)
-            .map_err(|_| RefreshError::SigningError)?,
+        Signatory::Direct(privkey) => {
+            let privkey = privkey.lock().await;
+            challenge_certificate
+            .sign(privkey.deref())
+            .map_err(|_| RefreshError::SigningError)?
+        }
     };
 
     Ok((
diff --git a/rustica-agent/src/rustica/x509.rs b/rustica-agent/src/rustica/x509.rs
index 29b9835..aabe967 100644
--- a/rustica-agent/src/rustica/x509.rs
+++ b/rustica-agent/src/rustica/x509.rs
@@ -3,40 +3,35 @@ use yubikey::piv::SlotId;
 
 use crate::{RusticaServer, Signatory};
 
-use super::{error::ServerError, get_rustica_client, RefreshError, AttestedX509CertificateRequest};
+use super::{error::ServerError, get_rustica_client, AttestedX509CertificateRequest, RefreshError};
 
 impl RusticaServer {
     pub async fn refresh_attested_x509_certificate_async(
         &self,
         signatory: &mut Signatory,
     ) -> Result<Vec<u8>, RefreshError> {
-        let yk_signatory = match signatory {
-            Signatory::Yubikey(yk) => yk,
+        let (mut yk, slot) = match signatory {
+            Signatory::Yubikey(yk) => (yk.yk.lock().await, yk.slot),
             _ => return Err(RefreshError::UnsupportedMode),
         };
 
         // The CN will be ignored by the backend
-        let csr = yk_signatory
-            .yk
-            .generate_csr(&yk_signatory.slot, "common_name")
-            .map_err(|_| {
-                RefreshError::ConfigurationError(format!(
-                    "Could not generate CSR for slot {}. Is it provisioned?",
-                    yk_signatory.slot
-                ))
-            })?;
+        let csr = yk.generate_csr(&slot, "common_name").map_err(|_| {
+            RefreshError::ConfigurationError(format!(
+                "Could not generate CSR for slot {}. Is it provisioned?",
+                slot
+            ))
+        })?;
 
-        yk_signatory.yk.reconnect().unwrap();
+        yk.reconnect().unwrap();
 
-        let attestation = yk_signatory.yk.fetch_attestation(&yk_signatory.slot).map_err(|e|
-            RefreshError::ConfigurationError(format!("Could not generate attestation for slot {}. Is it attestable (not imported)? Error {e}", yk_signatory.slot)))?;
+        let attestation = yk.fetch_attestation(&slot).map_err(|e|
+            RefreshError::ConfigurationError(format!("Could not generate attestation for slot {slot}. Is it attestable (not imported)? Error {e}")))?;
 
-        yk_signatory.yk.reconnect().unwrap();
+        yk.reconnect().unwrap();
 
-        let attestation_intermediate = yk_signatory
-            .yk
-            .fetch_certificate(&SlotId::Attestation)
-            .map_err(|_| {
+        let attestation_intermediate =
+            yk.fetch_certificate(&SlotId::Attestation).map_err(|_| {
                 RefreshError::ConfigurationError(format!(
                     "Could not fetch attestation intermediate. Have you manually removed it?"
                 ))
@@ -52,7 +47,10 @@ impl RusticaServer {
 
         let mut client = get_rustica_client(self).await?;
 
-        let response = client.attested_x509_certificate(request).await?.into_inner();
+        let response = client
+            .attested_x509_certificate(request)
+            .await?
+            .into_inner();
 
         match response.error_code {
             0 => Ok(response.certificate),
@@ -68,7 +66,9 @@ impl RusticaServer {
         signatory: &mut Signatory,
         handle: &Handle,
     ) -> Result<Vec<u8>, RefreshError> {
-        handle
-            .block_on(async { self.refresh_attested_x509_certificate_async(signatory).await })
+        handle.block_on(async {
+            self.refresh_attested_x509_certificate_async(signatory)
+                .await
+        })
     }
 }
diff --git a/rustica-agent/src/sshagent/agent.rs b/rustica-agent/src/sshagent/agent.rs
index 21c8c65..dda9858 100644
--- a/rustica-agent/src/sshagent/agent.rs
+++ b/rustica-agent/src/sshagent/agent.rs
@@ -1,3 +1,5 @@
+use std::sync::Arc;
+
 use tokio::net::UnixListener;
 use tokio::net::UnixStream;
 use tokio::select;
@@ -12,7 +14,7 @@ pub struct Agent;
 
 impl Agent {
     async fn handle_client<T: SshAgentHandler>(
-        handler: &mut T,
+        handler: Arc<T>,
         mut stream: UnixStream,
     ) -> HandleResult<()> {
         loop {
@@ -29,11 +31,12 @@ impl Agent {
     }
 
     pub async fn run_with_termination_channel<T: SshAgentHandler + 'static>(
-        mut handler: T,
+        handler: T,
         socket_path: String,
         term_channel: Option<Receiver<()>>,
     ) {
         let listener = UnixListener::bind(socket_path).unwrap();
+        let handler = Arc::new(handler);
 
         if let Some(mut term_channel) = term_channel {
             loop {
@@ -45,11 +48,14 @@ impl Agent {
                     v = listener.accept() => {
                         match v {
                             Ok(stream) => {
-                                debug!("Got connection from: {:?}", stream.1);
-                                match Agent::handle_client(&mut handler, stream.0).await {
-                                    Ok(_) => {}
-                                    Err(e) => debug!("handler: {:?}", e),
-                                }
+                                debug!("Got connection from: {:?}. Spawning thread to handle.", stream.1);
+                                let handler = handler.clone();
+                                tokio::spawn(async move {
+                                    match Agent::handle_client(handler, stream.0).await {
+                                        Ok(_) => {}
+                                        Err(e) => debug!("handler: {:?}", e),
+                                    }
+                                });
                             }
                             Err(e) => {
                                 // connection failed
@@ -66,10 +72,13 @@ impl Agent {
                     v = listener.accept() => {
                         match v {
                             Ok(stream) => {
-                                match Agent::handle_client(&mut handler, stream.0).await {
-                                    Ok(_) => {}
-                                    Err(e) => debug!("handler: {:?}", e),
-                                }
+                                let handler = handler.clone();
+                                tokio::spawn(async move {
+                                    match Agent::handle_client(handler, stream.0).await {
+                                        Ok(_) => {}
+                                        Err(e) => debug!("handler: {:?}", e),
+                                    }
+                                });
                             }
                             Err(e) => {
                                 // connection failed
diff --git a/rustica-agent/src/sshagent/handler.rs b/rustica-agent/src/sshagent/handler.rs
index 76abdb0..d36a3fe 100644
--- a/rustica-agent/src/sshagent/handler.rs
+++ b/rustica-agent/src/sshagent/handler.rs
@@ -8,24 +8,27 @@ use sshcerts::PrivateKey;
 
 #[async_trait]
 pub trait SshAgentHandler: Send + Sync {
-    fn add_identity(&mut self, key: PrivateKey) -> HandleResult<Response>;
-    async fn identities(&mut self) -> HandleResult<Response>;
-    fn sign_request(
-        &mut self,
+    async fn add_identity(&self, key: PrivateKey) -> HandleResult<Response>;
+    async fn identities(&self) -> HandleResult<Response>;
+    async fn sign_request(
+        &self,
         pubkey: Vec<u8>,
         data: Vec<u8>,
         flags: u32,
     ) -> HandleResult<Response>;
 
-    async fn handle_request(&mut self, request: Request) -> HandleResult<Response> {
+    async fn handle_request(&self, request: Request) -> HandleResult<Response> {
         match request {
             Request::Identities => self.identities().await,
             Request::Sign {
                 ref pubkey_blob,
                 ref data,
                 ref flags,
-            } => self.sign_request(pubkey_blob.clone(), data.clone(), *flags),
-            Request::AddIdentity { private_key } => self.add_identity(private_key),
+            } => {
+                self.sign_request(pubkey_blob.clone(), data.clone(), *flags)
+                    .await
+            }
+            Request::AddIdentity { private_key } => self.add_identity(private_key).await,
             Request::Unknown => Ok(Response::Failure),
         }
     }