Add support for certificate based authentication for HTTP sink (#595)

fennel-ai · Oct 31, 2024 · b3b0c2d · b3b0c2d
1 parent 074d4c9
commit b3b0c2d
Show file tree

Hide file tree

Showing 13 changed files with 187 additions and 70 deletions.
diff --git a/docs/api.yml b/docs/api.yml
@@ -74,6 +74,7 @@ sidebar:
   - slug: "api-reference/sink_connectors"
     title: "Sink Connectors"
     pages:
+      - "api-reference/sink_connectors/certificate"
       - "api-reference/sink_connectors/http"
       - "api-reference/sink_connectors/kafka"
       - "api-reference/sink_connectors/s3"

diff --git a/docs/examples/api-reference/sinks/http_sink.py b/docs/examples/api-reference/sinks/http_sink.py
@@ -39,13 +39,20 @@ class SomeDataset:
     from fennel.lib.params import inputs
 
     # docsnip basic
-    from fennel.connectors import sink, HTTP
+    from fennel.connectors import sink, HTTP, Certificate
+    from fennel.integrations.aws import Secret
 
     # docsnip-highlight start
+    aws_secret = Secret(
+        arn="arn:aws:secretsmanager:us-east-1:123456789012:secret:my-secret-name-I4hSKr",
+        role_arn="arn:aws:iam::123456789012:role/secret-access-role",
+    )
+
     http = HTTP(
         name="http",
         host="http://http-echo-server.harsha.svc.cluster.local:8081/",
         healthz="/health",
+        ca_cert=Certificate(aws_secret["ca_cert"]),
     )
     # docsnip-highlight end
 

diff --git a/docs/pages/api-reference/sink_connectors/certificate.md b/docs/pages/api-reference/sink_connectors/certificate.md
@@ -0,0 +1,18 @@
+---
+title: Certificate
+order: 0
+status: published
+---
+### Certificate
+Certificate to be used for HTTP-based authentication
+
+#### Parameters
+
+<Expandable title="cert" type='str | Secret'>
+CA Certificate required for client to authenticate the server
+</Expandable>
+
+<pre snippet="api-reference/sinks/http_sink#basic"
+    status="success" message="Using certificate with HTTP sink"
+    highlight="13-18, 20">
+</pre>
diff --git a/docs/pages/api-reference/sink_connectors/http.md b/docs/pages/api-reference/sink_connectors/http.md
@@ -19,6 +19,10 @@ The HTTP host URL. Example: https://127.0.0.1:8081
 The health check endpoint to verify the server's availability.
 </Expandable>
 
+<Expandable title="ca_cert" type="Certificate">
+Parameter for certificate-based authentication
+</Expandable>
+
 #### HTTP Path Parameters
 <Expandable title="endpoint" type="str">
 The specific endpoint where data will be sent

diff --git a/fennel/CHANGELOG.md b/fennel/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## [1.5.47] - 2024-10-31
+- Add support for certificate based authentication for HTTP sink
+
 ## [1.5.46] - 2024-10-30
 - Add support for AWS Secrets Manager
 

diff --git a/fennel/connectors/__init__.py b/fennel/connectors/__init__.py
@@ -33,5 +33,6 @@
     at_timestamp,
     Eval,
     eval,
+    Certificate,
 )
 import fennel.connectors.kinesis as kinesis
diff --git a/fennel/connectors/connectors.py b/fennel/connectors/connectors.py
@@ -760,9 +760,15 @@ def identifier(self) -> str:
         return f"[PubSub: {self.name}]"
 
 
+@dataclass
+class Certificate:
+    cert: Union[str, Secret]
+
+
 class HTTP(DataSource):
     host: Union[str, Secret]
     healthz: str
+    ca_cert: Certificate
 
     def required_fields(self) -> List[str]:
         return ["endpoint"]
@@ -777,7 +783,13 @@ def path(
 
     @staticmethod
     def get(name: str) -> HTTP:
-        return HTTP(name=name, _get=True, host="", healthz="")
+        return HTTP(
+            name=name,
+            _get=True,
+            host="",
+            healthz="",
+            auth=Certificate(""),
+        )
 
 
 # ------------------------------------------------------------------------------

diff --git a/fennel/connectors/test_connectors.py b/fennel/connectors/test_connectors.py
@@ -26,7 +26,7 @@
     eval,
     HTTP,
 )
-from fennel.connectors.connectors import CSV, Postgres, Sample
+from fennel.connectors.connectors import CSV, Postgres, Sample, Certificate
 from fennel.datasets import dataset, field, pipeline, Dataset
 from fennel.expr import col, lit
 from fennel.integrations.aws import Secret
@@ -545,14 +545,16 @@ class UserInfoDataset:
 
 http = HTTP(
     name="http_sink",
-    host="http://127.0.0.1:8081",
+    host="https://127.0.0.1:8081",
     healthz="/health",
+    ca_cert=Certificate(aws_secret["ca_cert"]),
 )
 
 http_with_secret = HTTP(
     name="http_sink_with_secret",
     host=aws_secret["http_host"],
     healthz="/health",
+    ca_cert=Certificate(aws_secret["ca_cert"]),
 )
 
 
@@ -902,8 +904,15 @@ def create_user_transactions(cls, dataset: Dataset):
                 "db": {
                     "name": "http_sink",
                     "http": {
-                        "host": "http://127.0.0.1:8081",
+                        "host": "https://127.0.0.1:8081",
                         "healthz": "/health",
+                        "ca_cert": {
+                            "secret_ref": {
+                                "secret_arn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1",
+                                "role_arn": "arn:aws:iam::123456789012:role/fennel-test-role",
+                                "path": ["ca_cert"],
+                            },
+                        },
                     },
                 },
                 "endpoint": "/sink",
@@ -935,12 +944,19 @@ def create_user_transactions(cls, dataset: Dataset):
                 "db": {
                     "name": "http_sink_with_secret",
                     "http": {
-                        "hostSecret": {
-                            "secretArn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1",
-                            "roleArn": "arn:aws:iam::123456789012:role/fennel-test-role",
+                        "host_secret": {
+                            "secret_arn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1",
+                            "role_arn": "arn:aws:iam::123456789012:role/fennel-test-role",
                             "path": ["http_host"],
                         },
                         "healthz": "/health",
+                        "ca_cert": {
+                            "secret_ref": {
+                                "secret_arn": "arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1",
+                                "role_arn": "arn:aws:iam::123456789012:role/fennel-test-role",
+                                "path": ["ca_cert"],
+                            },
+                        },
                     },
                 },
                 "endpoint": "/sink",

diff --git a/fennel/connectors/test_invalid_connectors.py b/fennel/connectors/test_invalid_connectors.py
@@ -19,9 +19,11 @@
     BigQuery,
     S3Connector,
     HTTP,
+    Certificate,
 )
 from fennel.datasets import dataset, field
 from fennel.expr import col
+from fennel.integrations import Secret
 from fennel.lib import meta
 
 # noinspection PyUnresolvedReferences
@@ -93,10 +95,16 @@
     password="password",
 )
 
+aws_secret = Secret(
+    arn="arn:aws:secretsmanager:us-west-2:123456789012:secret:fennel-test-secret-1",
+    role_arn="arn:aws:iam::123456789012:role/fennel-test-role",
+)
+
 http = HTTP(
     name="http_sink",
-    host="http://127.0.0.1:8081",
+    host="https://127.0.0.1:8081",
     healthz="/health",
+    ca_cert=Certificate(aws_secret["ca_cert"]),
 )