merge: #4059

4059: feat: allow pulling pg cert from a remote url r=sprutton1 a=sprutton1

This allows `pg-data-nats` to grab root certs from a remote instead of as a file on disk or a base64 string. The file will be written to a temp file and then read as normal. This is specifically so the module-index can load a new root RDS cert from a url because the base64 encoded latest version of that cert is too big to fit in SecretsManager. I kind of like this approach better anyhow.

You should be able to pass `--pg-cert-url` or set `SI_MODULE_INDEX__PG__CERTIFICATE_URL` to the url. Note that these certs are public and therefore do not need to be encoded as secrets.

<img src="https://media4.giphy.com/media/nCRkXBaeCPyeX1fkrc/giphy.gif"/>

Co-authored-by: Scott Prutton <[email protected]>

bin/module-index/src/args.rs

@@ -87,6 +87,10 @@ pub(crate) struct Args {
     #[arg(long)]
     pub(crate) pg_cert_base64: Option<SensitiveString>,
 
+    /// PostgreSQL connection certificate url
+    #[arg(long)]
+    pub(crate) pg_cert_url: Option<String>,
+
     /// The address and port to bind the HTTP server to [example: 0.0.0.0:80]
     #[arg(long, env)]
     pub(crate) socket_addr: Option<String>,
@@ -144,10 +148,12 @@ impl TryFrom<Args> for Config {
             if let Some(cert) = args.pg_cert_base64 {
                 config_map.set("pg.certificate_base64", cert.to_string());
             }
+            if let Some(cert) = args.pg_cert_url {
+                config_map.set("pg.certificate_url", cert);
+            }
             if let Some(socket_addr) = args.socket_addr {
                 config_map.set("socket_addr", socket_addr);
             }
-
             if let Some(s3_access_key_id) = args.s3_access_key_id {
                 config_map.set("s3.access_key_id", s3_access_key_id.to_string());
             }

lib/si-data-pg/BUCK

@@ -13,10 +13,12 @@ rust_library(
         "//third-party/rust:num_cpus",
         "//third-party/rust:ouroboros",
         "//third-party/rust:refinery",
+        "//third-party/rust:reqwest",
         "//third-party/rust:rustls",
         "//third-party/rust:rustls-pemfile",
         "//third-party/rust:remain",
         "//third-party/rust:serde",
+        "//third-party/rust:tempfile",
         "//third-party/rust:thiserror",
         "//third-party/rust:tokio",
         "//third-party/rust:tokio-postgres",

lib/si-data-pg/Cargo.toml

@@ -18,11 +18,13 @@ num_cpus = { workspace = true }
 ouroboros = { workspace = true }
 refinery = { workspace = true }
 remain = { workspace = true }
+reqwest = { workspace = true }
 rustls = { workspace = true }
 rustls-pemfile = { workspace = true }
 serde = { workspace = true }
 si-std = { path = "../../lib/si-std" }
 telemetry = { path = "../../lib/telemetry-rs" }
+tempfile = { workspace = true  }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 tokio-postgres = { workspace = true }

lib/si-data-pg/src/lib.rs

@@ -11,12 +11,14 @@ use rustls::{
     pki_types::{CertificateDer, TrustAnchor},
     RootCertStore,
 };
+use tempfile::NamedTempFile;
 use tokio_postgres_rustls::MakeRustlsConnect;
 
 use base64::{engine::general_purpose, Engine};
 use std::{
     cmp,
     fmt::{self, Debug},
+    io::Write,
     net::ToSocketAddrs,
     path::Path,
     sync::Arc,
@@ -88,6 +90,8 @@ pub enum PgPoolError {
     ReadPem(std::io::Error),
     #[error("migration error: {0}")]
     Refinery(#[from] refinery::Error),
+    #[error("reqwest error: {0}")]
+    Reqwest(#[from] reqwest::Error),
     #[error("failed to resolve pg hostname")]
     ResolveHostname(std::io::Error),
     #[error("resolved hostname returned no entries")]
@@ -110,6 +114,7 @@ pub struct PgPoolConfig {
     pub password: SensitiveString,
     pub certificate_path: Option<CanonicalFile>,
     pub certificate_base64: Option<String>,
+    pub certificate_url: Option<String>,
     pub dbname: String,
     pub application_name: String,
     pub hostname: String,
@@ -129,6 +134,7 @@ impl Default for PgPoolConfig {
             password: SensitiveString::from("bugbear"),
             certificate_path: None,
             certificate_base64: None,
+            certificate_url: None,
             dbname: String::from("si"),
             application_name: String::from("si-unknown-app"),
             hostname: String::from("localhost"),
@@ -287,6 +293,11 @@ impl PgPool {
                 Self::get_certificate_from_base64(cert.to_string()).await?,
             );
         }
+        if let Some(cert) = &settings.certificate_url {
+            root_cert_store.add_parsable_certificates(
+                Self::get_certificate_from_remote(cert.to_string()).await?,
+            );
+        }
         let config = rustls::ClientConfig::builder()
             .with_root_certificates(root_cert_store)
             .with_no_client_auth();
@@ -325,6 +336,17 @@ impl PgPool {
         Self::get_certificate_from_bytes(&buf).await
     }
 
+    // Creates a Certificate object from a remote certificate
+    async fn get_certificate_from_remote(
+        url: String,
+    ) -> PgPoolResult<Vec<CertificateDer<'static>>> {
+        let contents = &reqwest::get(&url).await?.bytes().await?;
+
+        let mut temp_file = NamedTempFile::new()?;
+        temp_file.write_all(contents)?;
+        Self::get_certificate_from_path(&temp_file).await
+    }
+
     /// Attempts to establish a database connection and returns an error if not successful.
     #[instrument(
         name = "pool.test_connection",