chore(plugin-server): run node-rdkafka with cooperative rebalancing p…

…atched in (#17747)

* Remove node-rdkafka-acosom

* Add node-rdkafka

* Replace node-rdkafka-acosom imports with node-rdkafka

* Patch node-rdkafka with changes from https://github.com/PostHog/node-rdkafka/

* Add patch directions to README

plugin-server/README.md

@@ -181,6 +181,31 @@ New functions called here are:
 > Note:
 > An `organization_id` is tied to a _company_ and its _installed plugins_, a `team_id` is tied to a _project_ and its _plugin configs_ (enabled/disabled+extra config).
 
+### Patching node-rdkafka
+
+We carry a node-rdkafka patch that adds cooperative rebalancing. To generate this patch:
+
+    # setup a local node-rdkafka clone
+    git clone [email protected]:PostHog/node-rdkafka.git
+    cd node-rdkafka
+    git remote add blizzard [email protected]:Blizzard/node-rdkafka.git
+    git fetch blizzard
+
+    # generate the diff
+    git diff blizzard/master > ~/node-rdkafka.diff
+
+    # in the plugin-server directory, this will output a temporary working directory
+    pnpm patch [email protected]
+
+    # enter the temporary directory from the previous command
+    cd /private/var/folders/b7/bmmghlpx5qdd6gpyvmz1k1_m0000gn/T/6082767a6879b3b4e11182f944f5cca3
+
+    # if asked, skip any missing files
+    patch -p1 < ~/node-rdkafka.diff
+
+    # in the plugin-server directory, target the temporary directory from the previous command
+    pnpm patch-commit /private/var/folders/b7/bmmghlpx5qdd6gpyvmz1k1_m0000gn/T/6082767a6879b3b4e11182f944f5cca3
+
 ## Questions?
 
 ### [Join our Slack community. 🦔](https://posthog.com/slack)

plugin-server/functional_tests/kafka.ts

@@ -1,6 +1,6 @@
 import { CompressionCodecs, CompressionTypes } from 'kafkajs'
 import SnappyCodec from 'kafkajs-snappy'
-import { HighLevelProducer } from 'node-rdkafka-acosom'
+import { HighLevelProducer } from 'node-rdkafka'
 
 import { defaultConfig } from '../src/config/config'
 import { produce as defaultProduce } from '../src/kafka/producer'

plugin-server/package.json

@@ -73,7 +73,7 @@
         "lru-cache": "^6.0.0",
         "luxon": "^1.27.0",
         "node-fetch": "^2.6.1",
-        "node-rdkafka-acosom": "2.16.1",
+        "node-rdkafka": "^2.17.0",
         "node-schedule": "^2.1.0",
         "pg": "^8.6.0",
         "pino": "^8.6.0",
@@ -138,7 +138,8 @@
     },
     "pnpm": {
         "patchedDependencies": {
-            "[email protected]": "patches/[email protected]"
+            "[email protected]": "patches/[email protected]",
+            "[email protected]": "patches/[email protected]"
         }
     }
 }