chore(capture): allow to skip sha hashing of the kafka messages (#21850)

posthog/api/capture.py

@@ -625,7 +625,11 @@ def capture_internal(
     ):
         kafka_partition_key = None
     else:
-        kafka_partition_key = hashlib.sha256(candidate_partition_key.encode()).hexdigest()
+        if settings.CAPTURE_SKIP_KEY_HASHING:
+            kafka_partition_key = candidate_partition_key
+        else:
+            # TODO: remove after progressive rollout of the option
+            kafka_partition_key = hashlib.sha256(candidate_partition_key.encode()).hexdigest()
 
     return log_event(parsed_event, event["event"], partition_key=kafka_partition_key, historical=historical)
 

posthog/settings/ingestion.py

@@ -17,6 +17,9 @@
 # partitioning-related settings below.
 CAPTURE_ALLOW_RANDOM_PARTITIONING = get_from_env("CAPTURE_ALLOW_RANDOM_PARTITIONING", True, type_cast=str_to_bool)
 
+# TOOD: make default after rollout on both prods: remove the superfluous hashing of the Kafka message key
+CAPTURE_SKIP_KEY_HASHING = get_from_env("CAPTURE_SKIP_KEY_HASHING", type_cast=bool, default=False)
+
 # A list of <team_id:distinct_id> pairs (in the format 2:myLovelyId) that we should use
 # random partitioning for when producing events to the Kafka topic consumed by the plugin server.
 # This is a measure to handle hot partitions in ad-hoc cases.