Deprecate Stream Enrich (close #788)

snowplow · Oct 16, 2023 · 09b9ee5 · 09b9ee5
1 parent b13099e
commit 09b9ee5
Show file tree

Hide file tree

Showing 111 changed files with 3,306 additions and 11,378 deletions.
diff --git a/.github/workflows/lacework.yml b/.github/workflows/lacework.yml
@@ -60,48 +60,6 @@ jobs:
           LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
         run: ./lw-scanner image evaluate snowplow/snowplow-enrich-kinesis ${{ steps.ver.outputs.tag }}-distroless --build-id ${{ github.run_id }} --no-pull
 
-      - name: Scan Stream Enrich Kinesis
-        env:
-          LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
-          LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
-          LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
-        run: ./lw-scanner image evaluate snowplow/stream-enrich-kinesis ${{ steps.ver.outputs.tag }} --build-id ${{ github.run_id }} --no-pull
-
-      - name: Scan Stream Enrich Kinesis distroless
-        env:
-          LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
-          LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
-          LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
-        run: ./lw-scanner image evaluate snowplow/stream-enrich-kinesis ${{ steps.ver.outputs.tag }}-distroless --build-id ${{ github.run_id }} --no-pull
-
-      - name: Scan Stream Enrich Kafka
-        env:
-          LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
-          LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
-          LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
-        run: ./lw-scanner image evaluate snowplow/stream-enrich-kafka ${{ steps.ver.outputs.tag }} --build-id ${{ github.run_id }} --no-pull
-
-      - name: Scan Stream Enrich Kafka distroless
-        env:
-          LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
-          LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
-          LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
-        run: ./lw-scanner image evaluate snowplow/stream-enrich-kafka ${{ steps.ver.outputs.tag }}-distroless --build-id ${{ github.run_id }} --no-pull
-
-      - name: Scan Stream Enrich NSQ
-        env:
-          LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
-          LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
-          LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
-        run: ./lw-scanner image evaluate snowplow/stream-enrich-nsq ${{ steps.ver.outputs.tag }} --build-id ${{ github.run_id }} --no-pull
-
-      - name: Scan Stream Enrich NSQ distroless
-        env:
-          LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
-          LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
-          LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
-        run: ./lw-scanner image evaluate snowplow/stream-enrich-nsq ${{ steps.ver.outputs.tag }}-distroless --build-id ${{ github.run_id }} --no-pull
-
       - name: Scan enrich-kafka
         env:
           LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -23,10 +23,7 @@ jobs:
       run: echo "::set-output name=tag::${GITHUB_REF#refs/tags/}"
     - name: Assemble fat jars
       run: |
-        sbt "project streamKinesis; set assembly / test := {}; assembly" \
-            "project streamKafka; set assembly / test := {}; assembly" \
-            "project streamNsq; set assembly / test := {}; assembly" \
-            "project pubsub; set assembly / test := {}; assembly" \
+        sbt "project pubsub; set assembly / test := {}; assembly" \
             "project kinesis; set assembly / test := {}; assembly" \
             "project rabbitmq; set assembly / test := {}; assembly" \
             "project kafka; set assembly / test := {}; assembly" \
@@ -39,9 +36,6 @@ jobs:
         name: ${{ steps.ver.outputs.tag }}
         tag_name: ${{ steps.ver.outputs.tag }}
         files: |
-          modules/stream/kinesis/target/scala-2.12/snowplow-stream-enrich-kinesis-${{ steps.ver.outputs.tag }}.jar
-          modules/stream/kafka/target/scala-2.12/snowplow-stream-enrich-kafka-${{ steps.ver.outputs.tag }}.jar
-          modules/stream/nsq/target/scala-2.12/snowplow-stream-enrich-nsq-${{ steps.ver.outputs.tag }}.jar
           modules/pubsub/target/scala-2.12/snowplow-enrich-pubsub-${{ steps.ver.outputs.tag }}.jar
           modules/kinesis/target/scala-2.12/snowplow-enrich-kinesis-${{ steps.ver.outputs.tag }}.jar
           modules/rabbitmq/target/scala-2.12/snowplow-enrich-rabbitmq-${{ steps.ver.outputs.tag }}.jar
@@ -56,9 +50,6 @@ jobs:
     strategy:
       matrix:
         app:
-          - streamKinesis
-          - streamKafka
-          - streamNsq
           - pubsub
           - kinesis
           - kafka

diff --git a/build.sbt b/build.sbt
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012-2022 Snowplow Analytics Ltd. All rights reserved.
+ * Copyright (c) 2012-2023 Snowplow Analytics Ltd. All rights reserved.
  *
  * This program is licensed to you under the Apache License Version 2.0, and
  * you may not use this file except in compliance with the Apache License
@@ -23,80 +23,14 @@ lazy val root = project.in(file("."))
   .settings(projectSettings)
   .settings(compilerSettings)
   .settings(resolverSettings)
-  .aggregate(common, commonFs2, pubsub, pubsubDistroless, kinesis, kinesisDistroless, streamCommon, streamKinesis, streamKinesisDistroless, streamKafka, streamKafkaDistroless, streamNsq, streamNsqDistroless, streamStdin, kafka, kafkaDistroless, rabbitmq, rabbitmqDistroless, nsq, nsqDistroless)
+  .aggregate(common, commonFs2, pubsub, pubsubDistroless, kinesis, kinesisDistroless, kafka, kafkaDistroless, rabbitmq, rabbitmqDistroless, nsq, nsqDistroless)
 
 lazy val common = project
   .in(file("modules/common"))
   .settings(commonBuildSettings)
   .settings(libraryDependencies ++= commonDependencies)
   .settings(excludeDependencies ++= exclusions)
 
-lazy val streamCommon = project
-  .in(file("modules/stream/common"))
-  .enablePlugins(BuildInfoPlugin)
-  .settings(streamCommonBuildSettings)
-  .settings(libraryDependencies ++= streamCommonDependencies)
-  .settings(excludeDependencies ++= exclusions)
-  .dependsOn(common % "test->test;compile->compile")
-
-lazy val streamKinesis = project
-  .in(file("modules/stream/kinesis"))
-  .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin)
-  .settings(streamKinesisBuildSettings)
-  .settings(libraryDependencies ++= streamKinesisDependencies)
-  .settings(excludeDependencies ++= exclusions)
-  .dependsOn(streamCommon)
-
-lazy val streamKinesisDistroless = project
-  .in(file("modules/distroless/stream/kinesis"))
-  .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin)
-  .settings(sourceDirectory := (streamKinesis / sourceDirectory).value)
-  .settings(streamKinesisDistrolessBuildSettings)
-  .settings(libraryDependencies ++= streamKinesisDependencies)
-  .settings(excludeDependencies ++= exclusions)
-  .dependsOn(streamCommon)
-
-lazy val streamKafka = project
-  .in(file("modules/stream/kafka"))
-  .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin)
-  .settings(streamKafkaBuildSettings)
-  .settings(libraryDependencies ++= streamKafkaDependencies)
-  .settings(excludeDependencies ++= exclusions)
-  .dependsOn(streamCommon)
-
-lazy val streamKafkaDistroless = project
-  .in(file("modules/distroless/stream/kafka"))
-  .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin)
-  .settings(sourceDirectory := (streamKafka / sourceDirectory).value)
-  .settings(streamKafkaDistrolessBuildSettings)
-  .settings(libraryDependencies ++= streamKafkaDependencies)
-  .settings(excludeDependencies ++= exclusions)
-  .dependsOn(streamCommon)
-
-lazy val streamNsq = project
-  .in(file("modules/stream/nsq"))
-  .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin)
-  .settings(streamNsqBuildSettings)
-  .settings(libraryDependencies ++= streamNsqDependencies)
-  .settings(excludeDependencies ++= exclusions)
-  .dependsOn(streamCommon)
-
-lazy val streamNsqDistroless = project
-  .in(file("modules/distroless/stream/nsq"))
-  .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin)
-  .settings(sourceDirectory := (streamNsq / sourceDirectory).value)
-  .settings(streamNsqDistrolessBuildSettings)
-  .settings(libraryDependencies ++= streamNsqDependencies)
-  .settings(excludeDependencies ++= exclusions)
-  .dependsOn(streamCommon)
-
-lazy val streamStdin = project
-  .in(file("modules/stream/stdin"))
-  .settings(streamStdinBuildSettings)
-  .settings(libraryDependencies ++= streamCommonDependencies)
-  .settings(excludeDependencies ++= exclusions)
-  .dependsOn(streamCommon)
-
 lazy val commonFs2 = project
   .in(file("modules/common-fs2"))
   .enablePlugins(BuildInfoPlugin)

diff --git a/config/stream-enrich-kafka.hocon b/config/stream-enrich-kafka.hocon
diff --git a/...s/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala b/...s/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2022 Snowplow Analytics Ltd. All rights reserved.
+ * Copyright (c) 2020-2023 Snowplow Analytics Ltd. All rights reserved.
  *
  * This program is licensed to you under the Apache License Version 2.0,
  * and you may not use this file except in compliance with the Apache License Version 2.0.
@@ -33,7 +33,7 @@ import fs2.io.file.{exists, move, readAll, tempFileResource, writeAll}
 import org.typelevel.log4cats.Logger
 import org.typelevel.log4cats.slf4j.Slf4jLogger
 
-import com.snowplowanalytics.snowplow.enrich.common.utils.{BlockerF, HttpClient, ShiftExecution}
+import com.snowplowanalytics.snowplow.enrich.common.utils.{HttpClient, ShiftExecution}
 
 import com.snowplowanalytics.snowplow.enrich.common.fs2.io.Clients
 
@@ -243,7 +243,7 @@ object Assets {
 
       _ <- Logger[F].info("Reinitializing enrichments")
       old <- enrichments.get
-      fresh <- old.reinitialize(BlockerF.ofBlocker(blocker), shifter)
+      fresh <- old.reinitialize(blocker, shifter)
       _ <- enrichments.set(fresh)
     } yield ()