From 5b0a900599929e460d0beb703093179eef68dd28 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Thu, 13 Jun 2024 21:35:46 -0700 Subject: [PATCH 01/17] requirements.txt --- requirements.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c64fa82e8f705..2c7db42388713 100644 --- a/requirements.txt +++ b/requirements.txt @@ -320,8 +320,9 @@ kombu==5.3.2 # via # -r requirements.in # celery -lxml==4.9.4 +lxml==5.2.2 # via + # -r requirements.in # python3-saml # toronado # xmlsec @@ -682,7 +683,9 @@ wrapt==1.15.0 wsproto==1.1.0 # via trio-websocket xmlsec==1.3.14 - # via python3-saml + # via + # -r requirements.in + # python3-saml yarl==1.9.4 # via aiohttp zxcvbn==4.4.28 From b85507521b98124aaf727c44743656ee5f1d8ca1 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Thu, 13 Jun 2024 21:43:40 -0700 Subject: [PATCH 02/17] yarl --- requirements.in | 1 + requirements.txt | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index fab35f938eecd..64e7cc53b2cda 100644 --- a/requirements.in +++ b/requirements.in @@ -97,3 +97,4 @@ hogql-parser==1.0.14 zxcvbn==4.4.28 xmlsec==1.3.14 lxml==5.2.2 +yarl==1.9.4 diff --git a/requirements.txt b/requirements.txt index 2c7db42388713..b70e82ed40975 100644 --- a/requirements.txt +++ b/requirements.txt @@ -687,6 +687,8 @@ xmlsec==1.3.14 # -r requirements.in # python3-saml yarl==1.9.4 - # via aiohttp + # via + # -r requirements.in + # aiohttp zxcvbn==4.4.28 # via -r requirements.in From 70e8a9b89f07ac9734d6d4fd35c85a5418a2cae3 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Thu, 13 Jun 2024 22:02:22 -0700 Subject: [PATCH 03/17] upgrades --- requirements.in | 5 +---- requirements.txt | 34 +++++++++++----------------------- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/requirements.in b/requirements.in index 64e7cc53b2cda..185c1f7b3a1df 100644 --- a/requirements.in +++ b/requirements.in @@ -67,7 +67,7 @@ retry==0.9.2 requests~=2.32.0 s3fs==2023.10.0 stripe==7.4.0 -selenium==4.1.5 +selenium==4.21.0 sentry-sdk[clickhouse-driver,celery,openai,django]~=1.44.1 semantic_version==2.8.5 scikit-learn==1.4.0 @@ -95,6 +95,3 @@ tiktoken==0.6.0 nh3==0.2.14 hogql-parser==1.0.14 zxcvbn==4.4.28 -xmlsec==1.3.14 -lxml==5.2.2 -yarl==1.9.4 diff --git a/requirements.txt b/requirements.txt index b70e82ed40975..b50b80d30d35f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,9 +35,7 @@ asn1crypto==1.5.1 astunparse==1.6.3 # via dlt async-generator==1.10 - # via - # trio - # trio-websocket + # via trio-websocket async-timeout==4.0.2 # via # aiokafka @@ -75,14 +73,14 @@ celery==5.3.4 # sentry-sdk celery-redbeat==2.1.1 # via -r requirements.in -certifi==2019.11.28 +certifi==2024.6.2 # via # httpcore # httpx # requests + # selenium # sentry-sdk # snowflake-connector-python - # urllib3 cffi==1.14.5 # via # cryptography @@ -120,7 +118,6 @@ cryptography==37.0.2 # pyopenssl # snowflake-connector-python # social-auth-core - # urllib3 cssselect==1.1.0 # via toronado cssutils==1.0.2 @@ -290,7 +287,6 @@ idna==2.8 # requests # snowflake-connector-python # trio - # urllib3 # yarl infi-clickhouse-orm @ git+https://github.com/PostHog/infi.clickhouse_orm@9578c79f29635ee2c1d01b7979e89adab8383de2 # via -r requirements.in @@ -322,7 +318,6 @@ kombu==5.3.2 # celery lxml==5.2.2 # via - # -r requirements.in # python3-saml # toronado # xmlsec @@ -444,9 +439,7 @@ pyjwt==2.4.0 pynacl==1.5.0 # via paramiko pyopenssl==22.0.0 - # via - # snowflake-connector-python - # urllib3 + # via snowflake-connector-python pypng==0.20220715.0 # via qrcode pysocks==1.7.1 @@ -533,7 +526,7 @@ scikit-learn==1.4.0 # via -r requirements.in scipy==1.12.0 # via scikit-learn -selenium==4.1.5 +selenium==4.21.0 # via -r requirements.in semantic-version==2.8.5 # via -r requirements.in @@ -560,7 +553,7 @@ slack-sdk==3.17.1 # via -r requirements.in smmap==5.0.1 # via gitdb -sniffio==1.2.0 +sniffio==1.3.0 # via # anyio # httpx @@ -622,7 +615,7 @@ toronado==0.1.0 # via -r requirements.in tqdm==4.64.1 # via openai -trio==0.20.0 +trio==0.25.1 # via # selenium # trio-websocket @@ -632,7 +625,7 @@ types-protobuf==4.22.0.0 # via temporalio types-setuptools==69.0.0.0 # via requirements-parser -typing-extensions==4.7.1 +typing-extensions==4.12.2 # via # dlt # openai @@ -640,6 +633,7 @@ typing-extensions==4.7.1 # pydantic # pydantic-core # qrcode + # selenium # snowflake-connector-python # stripe # temporalio @@ -663,8 +657,6 @@ urllib3==1.26.18 # requests # selenium # sentry-sdk -urllib3-secure-extra==0.1.0 - # via urllib3 vine==5.0.0 # via # amqp @@ -683,12 +675,8 @@ wrapt==1.15.0 wsproto==1.1.0 # via trio-websocket xmlsec==1.3.14 - # via - # -r requirements.in - # python3-saml + # via python3-saml yarl==1.9.4 - # via - # -r requirements.in - # aiohttp + # via aiohttp zxcvbn==4.4.28 # via -r requirements.in From 7bab5b312d990aad1f071cdf214139f713843a2d Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Thu, 13 Jun 2024 22:05:32 -0700 Subject: [PATCH 04/17] push --- requirements-dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 957b7d2c20572..5d1ac3bf5c2f0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -41,7 +41,7 @@ boto3-stubs==1.34.84 # via -r requirements-dev.in botocore-stubs==1.34.84 # via boto3-stubs -certifi==2019.11.28 +certifi==2024.6.2 # via # -c requirements.txt # requests @@ -341,7 +341,7 @@ types-toml==0.10.8.20240310 # via inline-snapshot types-tzlocal==5.1.0.1 # via -r requirements-dev.in -typing-extensions==4.7.1 +typing-extensions==4.12.2 # via # -c requirements.txt # boto3-stubs From 24c26065b2829b212f7b7857966de4021c81bdf9 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Thu, 13 Jun 2024 22:19:15 -0700 Subject: [PATCH 05/17] requirements --- requirements-dev.in | 1 - requirements-dev.txt | 7 +++---- requirements.in | 4 ++-- requirements.txt | 27 +++++++++++++++++---------- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index ad84d43687a1c..35ad7044d22dd 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -53,4 +53,3 @@ flaky==3.7.0 aioresponses==0.7.6 prance==23.06.21.0 openapi-spec-validator==0.7.1 # Needed for prance as a validation backend -yarl==1.9.4 diff --git a/requirements-dev.txt b/requirements-dev.txt index 5d1ac3bf5c2f0..61da1e2fa74df 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ # This file was autogenerated by uv via the following command: # uv pip compile requirements-dev.in -o requirements-dev.txt -aiohttp==3.9.5 +aiohttp==3.9.3 # via # -c requirements.txt # aioresponses @@ -41,7 +41,7 @@ boto3-stubs==1.34.84 # via -r requirements-dev.in botocore-stubs==1.34.84 # via boto3-stubs -certifi==2024.6.2 +certifi==2019.11.28 # via # -c requirements.txt # requests @@ -341,7 +341,7 @@ types-toml==0.10.8.20240310 # via inline-snapshot types-tzlocal==5.1.0.1 # via -r requirements-dev.in -typing-extensions==4.12.2 +typing-extensions==4.7.1 # via # -c requirements.txt # boto3-stubs @@ -362,5 +362,4 @@ watchdog==2.1.8 yarl==1.9.4 # via # -c requirements.txt - # -r requirements-dev.in # aiohttp diff --git a/requirements.in b/requirements.in index 185c1f7b3a1df..f5d224b7892bf 100644 --- a/requirements.in +++ b/requirements.in @@ -67,7 +67,7 @@ retry==0.9.2 requests~=2.32.0 s3fs==2023.10.0 stripe==7.4.0 -selenium==4.21.0 +selenium==4.1.5 sentry-sdk[clickhouse-driver,celery,openai,django]~=1.44.1 semantic_version==2.8.5 scikit-learn==1.4.0 @@ -94,4 +94,4 @@ openai==1.10.0 tiktoken==0.6.0 nh3==0.2.14 hogql-parser==1.0.14 -zxcvbn==4.4.28 +zxcvbn==4.4.28 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b50b80d30d35f..302868a36395b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ aiobotocore==2.7.0 # via # aioboto3 # s3fs -aiohttp==3.9.5 +aiohttp==3.9.3 # via # -r requirements.in # aiobotocore @@ -35,7 +35,9 @@ asn1crypto==1.5.1 astunparse==1.6.3 # via dlt async-generator==1.10 - # via trio-websocket + # via + # trio + # trio-websocket async-timeout==4.0.2 # via # aiokafka @@ -73,14 +75,14 @@ celery==5.3.4 # sentry-sdk celery-redbeat==2.1.1 # via -r requirements.in -certifi==2024.6.2 +certifi==2019.11.28 # via # httpcore # httpx # requests - # selenium # sentry-sdk # snowflake-connector-python + # urllib3 cffi==1.14.5 # via # cryptography @@ -118,6 +120,7 @@ cryptography==37.0.2 # pyopenssl # snowflake-connector-python # social-auth-core + # urllib3 cssselect==1.1.0 # via toronado cssutils==1.0.2 @@ -287,6 +290,7 @@ idna==2.8 # requests # snowflake-connector-python # trio + # urllib3 # yarl infi-clickhouse-orm @ git+https://github.com/PostHog/infi.clickhouse_orm@9578c79f29635ee2c1d01b7979e89adab8383de2 # via -r requirements.in @@ -439,7 +443,9 @@ pyjwt==2.4.0 pynacl==1.5.0 # via paramiko pyopenssl==22.0.0 - # via snowflake-connector-python + # via + # snowflake-connector-python + # urllib3 pypng==0.20220715.0 # via qrcode pysocks==1.7.1 @@ -526,7 +532,7 @@ scikit-learn==1.4.0 # via -r requirements.in scipy==1.12.0 # via scikit-learn -selenium==4.21.0 +selenium==4.1.5 # via -r requirements.in semantic-version==2.8.5 # via -r requirements.in @@ -553,7 +559,7 @@ slack-sdk==3.17.1 # via -r requirements.in smmap==5.0.1 # via gitdb -sniffio==1.3.0 +sniffio==1.2.0 # via # anyio # httpx @@ -615,7 +621,7 @@ toronado==0.1.0 # via -r requirements.in tqdm==4.64.1 # via openai -trio==0.25.1 +trio==0.20.0 # via # selenium # trio-websocket @@ -625,7 +631,7 @@ types-protobuf==4.22.0.0 # via temporalio types-setuptools==69.0.0.0 # via requirements-parser -typing-extensions==4.12.2 +typing-extensions==4.7.1 # via # dlt # openai @@ -633,7 +639,6 @@ typing-extensions==4.12.2 # pydantic # pydantic-core # qrcode - # selenium # snowflake-connector-python # stripe # temporalio @@ -657,6 +662,8 @@ urllib3==1.26.18 # requests # selenium # sentry-sdk +urllib3-secure-extra==0.1.0 + # via urllib3 vine==5.0.0 # via # amqp From c1d6851150bb8f72f8684e37c9af1ee9fe41c3e0 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Thu, 13 Jun 2024 22:31:09 -0700 Subject: [PATCH 06/17] upgrades --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 302868a36395b..946a822446e30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -621,7 +621,7 @@ toronado==0.1.0 # via -r requirements.in tqdm==4.64.1 # via openai -trio==0.20.0 +trio==0.21.0 # via # selenium # trio-websocket From c012945642f8db46c5a3dd99e2ff3f2d4069bfb9 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Thu, 13 Jun 2024 23:20:29 -0700 Subject: [PATCH 07/17] djangorestframework upgrade --- posthog/api/routing.py | 26 ++++++++++++++++++++++++++ requirements.in | 2 +- requirements.txt | 3 +-- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/posthog/api/routing.py b/posthog/api/routing.py index c4e67d1826274..f2816f9a2b131 100644 --- a/posthog/api/routing.py +++ b/posthog/api/routing.py @@ -36,6 +36,32 @@ class DefaultRouterPlusPlus(ExtendedDefaultRouter): """DefaultRouter with optional trailing slash and drf-extensions nesting.""" + # This is an override because of changes in djangorestframework 3.15, which is required for python 3.11 + # changes taken from and explained here: https://github.com/nautobot/nautobot/pull/5546/files#diff-81850a2ccad5814aab4f477d447f85cc0a82e9c10fd88fd72327cda51a750471R30 + def _register(self, prefix, viewset, basename=None): + """ + Override DRF's BaseRouter.register() to bypass an unnecessary restriction added in version 3.15.0. + (Reference: https://github.com/encode/django-rest-framework/pull/8438) + """ + if basename is None: + basename = self.get_default_basename(viewset) + + # DRF: + # if self.is_already_registered(basename): + # msg = (f'Router with basename "{basename}" is already registered. ' + # f'Please provide a unique basename for viewset "{viewset}"') + # raise ImproperlyConfigured(msg) + # + # We bypass this because we have at least one use case (/api/extras/jobs/) where we are *intentionally* + # registering two viewsets with the same basename, but have carefully defined them so as not to conflict. + + # resuming standard DRF code... + self.registry.append((prefix, viewset, basename)) + + # invalidate the urls cache + if hasattr(self, "_urls"): + del self._urls + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.trailing_slash = r"/?" diff --git a/requirements.in b/requirements.in index f5d224b7892bf..49bf0fb047d88 100644 --- a/requirements.in +++ b/requirements.in @@ -29,7 +29,7 @@ django-redis==5.2.0 django-statsd==2.5.2 django-structlog==2.1.3 django-revproxy==0.12.0 -djangorestframework==3.14.0 +djangorestframework==3.15.1 djangorestframework-csv==2.1.1 djangorestframework-dataclasses==1.2.0 django-fernet-encrypted-fields==0.1.3 diff --git a/requirements.txt b/requirements.txt index 946a822446e30..38300c3f838eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -194,7 +194,7 @@ django-structlog==2.1.3 # via -r requirements.in django-two-factor-auth==1.14.0 # via -r requirements.in -djangorestframework==3.14.0 +djangorestframework==3.15.1 # via # -r requirements.in # djangorestframework-csv @@ -472,7 +472,6 @@ pytz==2023.3 # via # -r requirements.in # clickhouse-driver - # djangorestframework # dlt # infi-clickhouse-orm # pandas From b52345a474b1d1140c2c595439b546aef61c0897 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Thu, 13 Jun 2024 23:47:16 -0700 Subject: [PATCH 08/17] StrEnum --- posthog/batch_exports/models.py | 2 +- posthog/clickhouse/table_engines.py | 4 +- posthog/constants.py | 26 +++--- posthog/decorators.py | 4 +- posthog/demo/matrix/randomization.py | 5 +- posthog/demo/products/hedgebox/models.py | 4 +- posthog/hogql/ast.py | 6 +- posthog/hogql/constants.py | 4 +- .../legacy_compatibility/filter_to_query.py | 4 +- posthog/kafka_client/client.py | 4 +- .../create_channel_definitions_file.py | 4 +- posthog/models/feature_flag/flag_matching.py | 4 +- posthog/models/plugin.py | 6 +- posthog/models/property/property.py | 4 +- posthog/schema.py | 90 +++++++++---------- 15 files changed, 85 insertions(+), 86 deletions(-) diff --git a/posthog/batch_exports/models.py b/posthog/batch_exports/models.py index adf2d1a602a24..37fd1cbfffda4 100644 --- a/posthog/batch_exports/models.py +++ b/posthog/batch_exports/models.py @@ -199,7 +199,7 @@ def interval_time_delta(self) -> timedelta: raise ValueError(f"Invalid interval: '{self.interval}'") -class BatchExportLogEntryLevel(str, enum.Enum): +class BatchExportLogEntryLevel(enum.StrEnum): """Enumeration of batch export log levels.""" DEBUG = "DEBUG" diff --git a/posthog/clickhouse/table_engines.py b/posthog/clickhouse/table_engines.py index e2b83d3f29006..b67ef9be5bc10 100644 --- a/posthog/clickhouse/table_engines.py +++ b/posthog/clickhouse/table_engines.py @@ -1,11 +1,11 @@ import uuid -from enum import Enum +from enum import StrEnum from typing import Optional from django.conf import settings -class ReplicationScheme(str, Enum): +class ReplicationScheme(StrEnum): NOT_SHARDED = "NOT_SHARDED" SHARDED = "SHARDED" REPLICATED = "REPLICATED" diff --git a/posthog/constants.py b/posthog/constants.py index fc8f7a9142195..af1e627bc7160 100644 --- a/posthog/constants.py +++ b/posthog/constants.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from typing import Literal from semantic_version import Version @@ -9,7 +9,7 @@ # N.B. Keep this in sync with frontend enum (types.ts) # AND ensure it is added to the Billing Service -class AvailableFeature(str, Enum): +class AvailableFeature(StrEnum): ZAPIER = "zapier" ORGANIZATIONS_PROJECTS = "organizations_projects" PROJECT_BASED_PERMISSIONING = "project_based_permissioning" @@ -215,19 +215,19 @@ class AvailableFeature(str, Enum): BREAKDOWN_TYPES = Literal["event", "person", "cohort", "group", "session", "hogql"] -class FunnelOrderType(str, Enum): +class FunnelOrderType(StrEnum): STRICT = "strict" UNORDERED = "unordered" ORDERED = "ordered" -class FunnelVizType(str, Enum): +class FunnelVizType(StrEnum): TRENDS = "trends" TIME_TO_CONVERT = "time_to_convert" STEPS = "steps" -class FunnelCorrelationType(str, Enum): +class FunnelCorrelationType(StrEnum): EVENTS = "events" PROPERTIES = "properties" EVENT_WITH_PROPERTIES = "event_with_properties" @@ -240,7 +240,7 @@ class FunnelCorrelationType(str, Enum): PERSON_UUID_FILTER = "person_uuid" -class AnalyticsDBMS(str, Enum): +class AnalyticsDBMS(StrEnum): POSTGRES = "postgres" CLICKHOUSE = "clickhouse" @@ -251,13 +251,13 @@ class AnalyticsDBMS(str, Enum): MONTHLY_ACTIVE = "monthly_active" -class RetentionQueryType(str, Enum): +class RetentionQueryType(StrEnum): RETURNING = "returning" TARGET = "target" TARGET_FIRST_TIME = "target_first_time" -class ExperimentSignificanceCode(str, Enum): +class ExperimentSignificanceCode(StrEnum): SIGNIFICANT = "significant" NOT_ENOUGH_EXPOSURE = "not_enough_exposure" LOW_WIN_PROBABILITY = "low_win_probability" @@ -265,7 +265,7 @@ class ExperimentSignificanceCode(str, Enum): HIGH_P_VALUE = "high_p_value" -class ExperimentNoResultsErrorKeys(str, Enum): +class ExperimentNoResultsErrorKeys(StrEnum): NO_EVENTS = "no-events" NO_FLAG_INFO = "no-flag-info" NO_CONTROL_VARIANT = "no-control-variant" @@ -273,12 +273,12 @@ class ExperimentNoResultsErrorKeys(str, Enum): NO_RESULTS = "no-results" -class PropertyOperatorType(str, Enum): +class PropertyOperatorType(StrEnum): AND = "AND" OR = "OR" -class BreakdownAttributionType(str, Enum): +class BreakdownAttributionType(StrEnum): FIRST_TOUCH = "first_touch" # FIRST_TOUCH attribution means the breakdown value is the first property value found within all funnel steps LAST_TOUCH = "last_touch" @@ -294,7 +294,7 @@ class BreakdownAttributionType(str, Enum): GROUP_TYPES_LIMIT = 5 -class EventDefinitionType(str, Enum): +class EventDefinitionType(StrEnum): # Mimics EventDefinitionType in frontend/src/types.ts ALL = "all" ACTION_EVENT = "action_event" @@ -303,7 +303,7 @@ class EventDefinitionType(str, Enum): EVENT_CUSTOM = "event_custom" -class FlagRequestType(str, Enum): +class FlagRequestType(StrEnum): DECIDE = "decide" LOCAL_EVALUATION = "local-evaluation" diff --git a/posthog/decorators.py b/posthog/decorators.py index eb66afcf422d4..c4aba39e3d2c5 100644 --- a/posthog/decorators.py +++ b/posthog/decorators.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from functools import wraps from typing import Any, TypeVar, Union, cast from collections.abc import Callable @@ -17,7 +17,7 @@ from .utils import generate_cache_key, get_safe_cache -class CacheType(str, Enum): +class CacheType(StrEnum): TRENDS = "Trends" FUNNEL = "Funnel" RETENTION = "Retention" diff --git a/posthog/demo/matrix/randomization.py b/posthog/demo/matrix/randomization.py index d017c295321dc..71701d2c6ce99 100644 --- a/posthog/demo/matrix/randomization.py +++ b/posthog/demo/matrix/randomization.py @@ -1,12 +1,11 @@ -from enum import Enum +from enum import StrEnum -import mimesis import mimesis.random WeightedPool = tuple[list[str], list[int]] -class Industry(str, Enum): +class Industry(StrEnum): TECHNOLOGY = "technology" FINANCE = "finance" MEDIA = "media" diff --git a/posthog/demo/products/hedgebox/models.py b/posthog/demo/products/hedgebox/models.py index dd694f64aac41..9b0c72afc69a7 100644 --- a/posthog/demo/products/hedgebox/models.py +++ b/posthog/demo/products/hedgebox/models.py @@ -1,7 +1,7 @@ import datetime as dt import math from dataclasses import dataclass, field -from enum import Enum, auto +from enum import auto, StrEnum from typing import ( TYPE_CHECKING, Any, @@ -66,7 +66,7 @@ class HedgeboxSessionIntent(SimSessionIntent): DOWNGRADE_PLAN = auto() -class HedgeboxPlan(str, Enum): +class HedgeboxPlan(StrEnum): PERSONAL_FREE = "personal/free" PERSONAL_PRO = "personal/pro" BUSINESS_STANDARD = "business/standard" diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index a21a74f4a91bf..72b2c32f7b745 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import StrEnum from typing import Any, Literal, Optional, Union from dataclasses import dataclass, field @@ -554,7 +554,7 @@ class Alias(Expr): hidden: bool = False -class ArithmeticOperationOp(str, Enum): +class ArithmeticOperationOp(StrEnum): Add = "+" Sub = "-" Mult = "*" @@ -581,7 +581,7 @@ class Or(Expr): type: Optional[ConstantType] = None -class CompareOperationOp(str, Enum): +class CompareOperationOp(StrEnum): Eq = "==" NotEq = "!=" Gt = ">" diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py index f1f20ff45f484..5568f745d2be1 100644 --- a/posthog/hogql/constants.py +++ b/posthog/hogql/constants.py @@ -1,5 +1,5 @@ from datetime import date, datetime -from enum import Enum +from enum import StrEnum from typing import Optional, Literal, TypeAlias from uuid import UUID from pydantic import ConfigDict, BaseModel @@ -45,7 +45,7 @@ BREAKDOWN_VALUES_LIMIT_FOR_COUNTRIES = 300 -class LimitContext(str, Enum): +class LimitContext(StrEnum): QUERY = "query" QUERY_ASYNC = "query_async" EXPORT = "export" diff --git a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py index 996e10173814e..b06b4bb415a5b 100644 --- a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py +++ b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py @@ -1,5 +1,5 @@ import copy -from enum import Enum +from enum import StrEnum import json from typing import Any, Literal from posthog.hogql_queries.legacy_compatibility.clean_properties import clean_entity_properties, clean_global_properties @@ -34,7 +34,7 @@ from posthog.utils import str_to_bool -class MathAvailability(str, Enum): +class MathAvailability(StrEnum): Unavailable = ("Unavailable",) All = ("All",) ActorsOnly = "ActorsOnly" diff --git a/posthog/kafka_client/client.py b/posthog/kafka_client/client.py index 3f58e572417b8..f0008c4ba72e8 100644 --- a/posthog/kafka_client/client.py +++ b/posthog/kafka_client/client.py @@ -1,5 +1,5 @@ import json -from enum import Enum +from enum import StrEnum from typing import Any, Optional from collections.abc import Callable @@ -83,7 +83,7 @@ def subscribe(self, _): return -class _KafkaSecurityProtocol(str, Enum): +class _KafkaSecurityProtocol(StrEnum): PLAINTEXT = "PLAINTEXT" SSL = "SSL" SASL_PLAINTEXT = "SASL_PLAINTEXT" diff --git a/posthog/management/commands/create_channel_definitions_file.py b/posthog/management/commands/create_channel_definitions_file.py index cab70bf31d360..bea98c02b5243 100644 --- a/posthog/management/commands/create_channel_definitions_file.py +++ b/posthog/management/commands/create_channel_definitions_file.py @@ -3,7 +3,7 @@ import subprocess from collections import OrderedDict from dataclasses import dataclass -from enum import Enum +from enum import StrEnum from typing import Optional from django.core.management.base import BaseCommand @@ -12,7 +12,7 @@ OUTPUT_FILE = "posthog/models/channel_type/channel_definitions.json" -class EntryKind(str, Enum): +class EntryKind(StrEnum): source = "source" medium = "medium" diff --git a/posthog/models/feature_flag/flag_matching.py b/posthog/models/feature_flag/flag_matching.py index 70e0190a5704a..ea181081f0c31 100644 --- a/posthog/models/feature_flag/flag_matching.py +++ b/posthog/models/feature_flag/flag_matching.py @@ -1,6 +1,6 @@ import hashlib from dataclasses import dataclass -from enum import Enum +from enum import StrEnum import time import structlog from typing import Literal, Optional, Union, cast @@ -67,7 +67,7 @@ PERSON_KEY = "person" -class FeatureFlagMatchReason(str, Enum): +class FeatureFlagMatchReason(StrEnum): SUPER_CONDITION_VALUE = "super_condition_value" CONDITION_MATCH = "condition_match" NO_CONDITION_MATCH = "no_condition_match" diff --git a/posthog/models/plugin.py b/posthog/models/plugin.py index d2ecd0d799c92..42e071e119982 100644 --- a/posthog/models/plugin.py +++ b/posthog/models/plugin.py @@ -1,7 +1,7 @@ import datetime import os from dataclasses import dataclass -from enum import Enum +from enum import StrEnum from typing import Any, Optional, cast from uuid import UUID @@ -288,13 +288,13 @@ class Meta: value: models.TextField = models.TextField(blank=True, null=True) -class PluginLogEntrySource(str, Enum): +class PluginLogEntrySource(StrEnum): SYSTEM = "SYSTEM" PLUGIN = "PLUGIN" CONSOLE = "CONSOLE" -class PluginLogEntryType(str, Enum): +class PluginLogEntryType(StrEnum): DEBUG = "DEBUG" LOG = "LOG" INFO = "INFO" diff --git a/posthog/models/property/property.py b/posthog/models/property/property.py index 7185306b8ccb2..bb378b7616d43 100644 --- a/posthog/models/property/property.py +++ b/posthog/models/property/property.py @@ -1,5 +1,5 @@ import json -from enum import Enum +from enum import StrEnum from typing import ( Any, Literal, @@ -14,7 +14,7 @@ from posthog.utils import str_to_bool -class BehavioralPropertyType(str, Enum): +class BehavioralPropertyType(StrEnum): PERFORMED_EVENT = "performed_event" PERFORMED_EVENT_MULTIPLE = "performed_event_multiple" PERFORMED_EVENT_FIRST_TIME = "performed_event_first_time" diff --git a/posthog/schema.py b/posthog/schema.py index 7da0fcffa77e2..b033c35151dee 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -2,7 +2,7 @@ from __future__ import annotations -from enum import Enum +from enum import Enum, StrEnum from typing import Any, Literal, Optional, Union from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, RootModel @@ -20,7 +20,7 @@ class MathGroupTypeIndex(float, Enum): NUMBER_4 = 4 -class AggregationAxisFormat(str, Enum): +class AggregationAxisFormat(StrEnum): NUMERIC = "numeric" DURATION = "duration" DURATION_MS = "duration_ms" @@ -28,7 +28,7 @@ class AggregationAxisFormat(str, Enum): PERCENTAGE_SCALED = "percentage_scaled" -class Kind(str, Enum): +class Kind(StrEnum): METHOD = "Method" FUNCTION = "Function" CONSTRUCTOR = "Constructor" @@ -87,7 +87,7 @@ class AutocompleteCompletionItem(BaseModel): ) -class BaseMathType(str, Enum): +class BaseMathType(StrEnum): TOTAL = "total" DAU = "dau" WEEKLY_ACTIVE = "weekly_active" @@ -95,14 +95,14 @@ class BaseMathType(str, Enum): UNIQUE_SESSION = "unique_session" -class BreakdownAttributionType(str, Enum): +class BreakdownAttributionType(StrEnum): FIRST_TOUCH = "first_touch" LAST_TOUCH = "last_touch" ALL_EVENTS = "all_events" STEP = "step" -class BreakdownType(str, Enum): +class BreakdownType(StrEnum): COHORT = "cohort" PERSON = "person" EVENT = "event" @@ -164,7 +164,7 @@ class ChartAxis(BaseModel): column: str -class ChartDisplayType(str, Enum): +class ChartDisplayType(StrEnum): ACTIONS_LINE_GRAPH = "ActionsLineGraph" ACTIONS_BAR = "ActionsBar" ACTIONS_AREA_GRAPH = "ActionsAreaGraph" @@ -205,7 +205,7 @@ class CompareFilter(BaseModel): compare_to: Optional[str] = None -class CountPerActorMathType(str, Enum): +class CountPerActorMathType(StrEnum): AVG_COUNT_PER_ACTOR = "avg_count_per_actor" MIN_COUNT_PER_ACTOR = "min_count_per_actor" MAX_COUNT_PER_ACTOR = "max_count_per_actor" @@ -255,13 +255,13 @@ class DatabaseSchemaSource(BaseModel): status: str -class Type(str, Enum): +class Type(StrEnum): POSTHOG = "posthog" DATA_WAREHOUSE = "data_warehouse" VIEW = "view" -class DatabaseSerializedFieldType(str, Enum): +class DatabaseSerializedFieldType(StrEnum): INTEGER = "integer" FLOAT = "float" STRING = "string" @@ -300,13 +300,13 @@ class Day(RootModel[int]): root: int -class DurationType(str, Enum): +class DurationType(StrEnum): DURATION = "duration" ACTIVE_SECONDS = "active_seconds" INACTIVE_SECONDS = "inactive_seconds" -class Key(str, Enum): +class Key(StrEnum): TAG_NAME = "tag_name" TEXT = "text" HREF = "href" @@ -335,7 +335,7 @@ class EmptyPropertyFilter(BaseModel): ) -class EntityType(str, Enum): +class EntityType(StrEnum): ACTIONS = "actions" EVENTS = "events" DATA_WAREHOUSE = "data_warehouse" @@ -351,7 +351,7 @@ class EventDefinition(BaseModel): properties: dict[str, Any] -class CorrelationType(str, Enum): +class CorrelationType(StrEnum): SUCCESS = "success" FAILURE = "failure" @@ -409,12 +409,12 @@ class EventsQueryPersonColumn(BaseModel): uuid: str -class FilterLogicalOperator(str, Enum): +class FilterLogicalOperator(StrEnum): AND_ = "AND" OR_ = "OR" -class FunnelConversionWindowTimeUnit(str, Enum): +class FunnelConversionWindowTimeUnit(StrEnum): SECOND = "second" MINUTE = "minute" HOUR = "hour" @@ -431,7 +431,7 @@ class FunnelCorrelationResult(BaseModel): skewed: bool -class FunnelCorrelationResultsType(str, Enum): +class FunnelCorrelationResultsType(StrEnum): EVENTS = "events" PROPERTIES = "properties" EVENT_WITH_PROPERTIES = "event_with_properties" @@ -459,18 +459,18 @@ class FunnelExclusionSteps(BaseModel): funnelToStep: int -class FunnelLayout(str, Enum): +class FunnelLayout(StrEnum): HORIZONTAL = "horizontal" VERTICAL = "vertical" -class FunnelPathType(str, Enum): +class FunnelPathType(StrEnum): FUNNEL_PATH_BEFORE_STEP = "funnel_path_before_step" FUNNEL_PATH_BETWEEN_STEPS = "funnel_path_between_steps" FUNNEL_PATH_AFTER_STEP = "funnel_path_after_step" -class FunnelStepReference(str, Enum): +class FunnelStepReference(StrEnum): TOTAL = "total" PREVIOUS = "previous" @@ -483,7 +483,7 @@ class FunnelTimeToConvertResults(BaseModel): bins: list[list[int]] -class FunnelVizType(str, Enum): +class FunnelVizType(StrEnum): STEPS = "steps" TIME_TO_CONVERT = "time_to_convert" TRENDS = "trends" @@ -507,37 +507,37 @@ class HogQLNotice(BaseModel): start: Optional[int] = None -class BounceRatePageViewMode(str, Enum): +class BounceRatePageViewMode(StrEnum): COUNT_PAGEVIEWS = "count_pageviews" UNIQ_URLS = "uniq_urls" -class InCohortVia(str, Enum): +class InCohortVia(StrEnum): AUTO = "auto" LEFTJOIN = "leftjoin" SUBQUERY = "subquery" LEFTJOIN_CONJOINED = "leftjoin_conjoined" -class MaterializationMode(str, Enum): +class MaterializationMode(StrEnum): AUTO = "auto" LEGACY_NULL_AS_STRING = "legacy_null_as_string" LEGACY_NULL_AS_NULL = "legacy_null_as_null" DISABLED = "disabled" -class PersonsArgMaxVersion(str, Enum): +class PersonsArgMaxVersion(StrEnum): AUTO = "auto" V1 = "v1" V2 = "v2" -class PersonsJoinMode(str, Enum): +class PersonsJoinMode(StrEnum): INNER = "inner" LEFT = "left" -class PersonsOnEventsMode(str, Enum): +class PersonsOnEventsMode(StrEnum): DISABLED = "disabled" PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS = "person_id_no_override_properties_on_events" PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS = "person_id_override_properties_on_events" @@ -570,7 +570,7 @@ class HogQueryResponse(BaseModel): stdout: Optional[str] = None -class Compare(str, Enum): +class Compare(StrEnum): CURRENT = "current" PREVIOUS = "previous" @@ -610,7 +610,7 @@ class InsightDateRange(BaseModel): ) -class InsightFilterProperty(str, Enum): +class InsightFilterProperty(StrEnum): TRENDS_FILTER = "trendsFilter" FUNNELS_FILTER = "funnelsFilter" RETENTION_FILTER = "retentionFilter" @@ -619,7 +619,7 @@ class InsightFilterProperty(str, Enum): LIFECYCLE_FILTER = "lifecycleFilter" -class InsightNodeKind(str, Enum): +class InsightNodeKind(StrEnum): TRENDS_QUERY = "TrendsQuery" FUNNELS_QUERY = "FunnelsQuery" RETENTION_QUERY = "RetentionQuery" @@ -628,7 +628,7 @@ class InsightNodeKind(str, Enum): LIFECYCLE_QUERY = "LifecycleQuery" -class InsightType(str, Enum): +class InsightType(StrEnum): TRENDS = "TRENDS" STICKINESS = "STICKINESS" LIFECYCLE = "LIFECYCLE" @@ -640,7 +640,7 @@ class InsightType(str, Enum): HOG = "HOG" -class IntervalType(str, Enum): +class IntervalType(StrEnum): MINUTE = "minute" HOUR = "hour" DAY = "day" @@ -648,14 +648,14 @@ class IntervalType(str, Enum): MONTH = "month" -class LifecycleToggle(str, Enum): +class LifecycleToggle(StrEnum): NEW = "new" RESURRECTING = "resurrecting" RETURNING = "returning" DORMANT = "dormant" -class NodeKind(str, Enum): +class NodeKind(StrEnum): EVENTS_NODE = "EventsNode" ACTIONS_NODE = "ActionsNode" DATA_WAREHOUSE_NODE = "DataWarehouseNode" @@ -700,7 +700,7 @@ class PathCleaningFilter(BaseModel): regex: Optional[str] = None -class PathType(str, Enum): +class PathType(StrEnum): FIELD_PAGEVIEW = "$pageview" FIELD_SCREEN = "$screen" CUSTOM_EVENT = "custom_event" @@ -749,7 +749,7 @@ class PathsFilterLegacy(BaseModel): step_limit: Optional[int] = None -class PropertyFilterType(str, Enum): +class PropertyFilterType(StrEnum): META = "meta" EVENT = "event" PERSON = "person" @@ -764,7 +764,7 @@ class PropertyFilterType(str, Enum): DATA_WAREHOUSE_PERSON_PROPERTY = "data_warehouse_person_property" -class PropertyMathType(str, Enum): +class PropertyMathType(StrEnum): AVG = "avg" SUM = "sum" MIN = "min" @@ -775,7 +775,7 @@ class PropertyMathType(str, Enum): P99 = "p99" -class PropertyOperator(str, Enum): +class PropertyOperator(StrEnum): EXACT = "exact" IS_NOT = "is_not" ICONTAINS = "icontains" @@ -890,7 +890,7 @@ class RecordingPropertyFilter(BaseModel): value: Optional[Union[str, float, list[Union[str, float]]]] = None -class Kind1(str, Enum): +class Kind1(StrEnum): ACTIONS_NODE = "ActionsNode" EVENTS_NODE = "EventsNode" @@ -908,19 +908,19 @@ class RetentionEntity(BaseModel): uuid: Optional[str] = None -class RetentionReference(str, Enum): +class RetentionReference(StrEnum): TOTAL = "total" PREVIOUS = "previous" -class RetentionPeriod(str, Enum): +class RetentionPeriod(StrEnum): HOUR = "Hour" DAY = "Day" WEEK = "Week" MONTH = "Month" -class RetentionType(str, Enum): +class RetentionType(StrEnum): RETENTION_RECURRING = "retention_recurring" RETENTION_FIRST_TIME = "retention_first_time" @@ -951,7 +951,7 @@ class SessionPropertyFilter(BaseModel): value: Optional[Union[str, float, list[Union[str, float]]]] = None -class StepOrderValue(str, Enum): +class StepOrderValue(StrEnum): STRICT = "strict" UNORDERED = "unordered" ORDERED = "ordered" @@ -1166,7 +1166,7 @@ class VizSpecificOptions(BaseModel): RETENTION: Optional[RETENTION] = None -class Kind2(str, Enum): +class Kind2(StrEnum): UNIT = "unit" DURATION_S = "duration_s" PERCENTAGE = "percentage" @@ -1213,7 +1213,7 @@ class WebOverviewQueryResponse(BaseModel): ) -class WebStatsBreakdown(str, Enum): +class WebStatsBreakdown(StrEnum): PAGE = "Page" INITIAL_PAGE = "InitialPage" EXIT_PAGE = "ExitPage" From 42c0b8dbeecac8db1acb0fe9df767f885acedd1b Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 06:57:01 +0000 Subject: [PATCH 09/17] Update query snapshots --- ee/api/test/__snapshots__/test_time_to_see_data.ambr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/api/test/__snapshots__/test_time_to_see_data.ambr b/ee/api/test/__snapshots__/test_time_to_see_data.ambr index 2d93af68cee82..beda2bc14bdef 100644 --- a/ee/api/test/__snapshots__/test_time_to_see_data.ambr +++ b/ee/api/test/__snapshots__/test_time_to_see_data.ambr @@ -20,7 +20,7 @@ "first_name": "", "last_name": "", "email": "", - "is_email_verified": false + "is_email_verified": null } }, "children": [ From 9fd2ebac38dd3126f66e7fa3637b36605ec2dadc Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Thu, 13 Jun 2024 23:58:58 -0700 Subject: [PATCH 10/17] strenum --- bin/build-schema-python.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/build-schema-python.sh b/bin/build-schema-python.sh index 7937731b55116..4f6cf8d6d658b 100755 --- a/bin/build-schema-python.sh +++ b/bin/build-schema-python.sh @@ -4,7 +4,7 @@ set -e # Generate schema.py from schema.json datamodel-codegen \ - --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.10 --disable-timestamp \ + --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.11 --disable-timestamp \ --use-one-literal-as-default --use-default --use-default-kwarg --use-subclass-enum \ --input frontend/src/queries/schema.json --input-file-type jsonschema \ --output posthog/schema.py --output-model-type pydantic_v2.BaseModel \ @@ -29,3 +29,9 @@ if [[ "$OSTYPE" == "darwin"* ]]; then else sed -i -e 's/Optional\[PropertyOperator\] = \("[A-Za-z_]*"\)/Optional[PropertyOperator] = PropertyOperator(\1)/g' posthog/schema.py fi + +# Replace class Foo(str, Enum) with class Foo(StrEnum) for proper handling in format strings in python 3.11 +# Remove this when https://github.com/koxudaxi/datamodel-code-generator/issues/1313 is resolved + +sed -i -e 's/str, Enum/StrEnum/g' posthog/schema.py +sed -i 's/from enum import Enum/from enum import Enum, StrEnum/g' posthog/schema.py From c310fb52f16b179d3a665788ee2ffcc5b4bc3b79 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Fri, 14 Jun 2024 07:09:44 -0700 Subject: [PATCH 11/17] bump build version --- production.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production.Dockerfile b/production.Dockerfile index ecefc7e14fa10..a979548c8bebf 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -83,7 +83,7 @@ RUN corepack enable && \ # # --------------------------------------------------------- # -FROM python:3.10.10-slim-bullseye AS posthog-build +FROM python:3.11.9-slim-bullseye AS posthog-build WORKDIR /code SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] From 5f1b5df2dac5371a5489cd2a2ea6dd585f9e5602 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Fri, 14 Jun 2024 08:04:10 -0700 Subject: [PATCH 12/17] dockerfile --- .devcontainer/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7d82e9ab4786c..4405b6caa5d08 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,3 +1,7 @@ +# hadolint global ignore=DL3004 + +# hadolint doesn't like changes to this file, but it is only used for local dev + # Defines the environment you're dropped into with codespaces # I've take # https://github.com/microsoft/vscode-dev-containers/blob/main/containers/python-3/.devcontainer/Dockerfile From 205fb53ed4a879599341acaa904f5c21c866c0a1 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Fri, 14 Jun 2024 08:11:12 -0700 Subject: [PATCH 13/17] remove migration check --- .github/workflows/ci-backend.yml | 55 ++++++++++++++++---------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index eec9d25f12003..928eb3cc9072d 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -193,33 +193,34 @@ jobs: # Now we can consider this PR's migrations - - name: Checkout this PR - uses: actions/checkout@v3 - - - name: Install python dependencies for this PR - run: | - uv pip install --system -r requirements.txt -r requirements-dev.txt - - - name: Run migrations for this PR - run: | - python manage.py migrate - - - name: Check migrations - run: | - python manage.py makemigrations --check --dry-run - git fetch origin master - # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified - # With `--name-status` we get the same, but including a column for status, respectively: A, D, M - # In this check we exclusively care about files that were - # added (A) in posthog/migrations/. We also want to ignore - # initial migrations (0001_*) as these are guaranteed to be - # run on initial setup where there is no data. - git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe - - - name: Check CH migrations - run: | - # Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/ - git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | awk '{print $2}' | python manage.py test_ch_migrations_are_safe + # Commented out to move to Python 3.11. Uncomment after deploy. + # - name: Checkout this PR + # uses: actions/checkout@v3 + # + # - name: Install python dependencies for this PR + # run: | + # uv pip install --system -r requirements.txt -r requirements-dev.txt + # + # - name: Run migrations for this PR + # run: | + # python manage.py migrate + # + # - name: Check migrations + # run: | + # python manage.py makemigrations --check --dry-run + # git fetch origin master + # # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified + # # With `--name-status` we get the same, but including a column for status, respectively: A, D, M + # # In this check we exclusively care about files that were + # # added (A) in posthog/migrations/. We also want to ignore + # # initial migrations (0001_*) as these are guaranteed to be + # # run on initial setup where there is no data. + # git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | grep -v migrations/0001_ | python manage.py test_migrations_are_safe + # + # - name: Check CH migrations + # run: | + # # Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/ + # git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | awk '{print $2}' | python manage.py test_ch_migrations_are_safe django: needs: changes From 958f45be135075948a64f98c3229934887230af7 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Fri, 14 Jun 2024 08:12:46 -0700 Subject: [PATCH 14/17] remove migration checking --- .github/workflows/ci-backend.yml | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 928eb3cc9072d..bad89143fc91c 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -177,33 +177,33 @@ jobs: sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl # First running migrations from master, to simulate the real-world scenario + # Commented out to move to Python 3.11. Uncomment after deploy. + # - name: Checkout master + # uses: actions/checkout@v3 + # with: + # ref: master + # + # - name: Install python dependencies for master + # run: | + # uv pip install --system -r requirements.txt -r requirements-dev.txt + # + # - name: Run migrations up to master + # run: | + # python manage.py migrate - - name: Checkout master + # Now we can consider this PR's migrations + - name: Checkout this PR uses: actions/checkout@v3 - with: - ref: master - - name: Install python dependencies for master + - name: Install python dependencies for this PR run: | uv pip install --system -r requirements.txt -r requirements-dev.txt - - name: Run migrations up to master + - name: Run migrations for this PR run: | python manage.py migrate - # Now we can consider this PR's migrations - # Commented out to move to Python 3.11. Uncomment after deploy. - # - name: Checkout this PR - # uses: actions/checkout@v3 - # - # - name: Install python dependencies for this PR - # run: | - # uv pip install --system -r requirements.txt -r requirements-dev.txt - # - # - name: Run migrations for this PR - # run: | - # python manage.py migrate # # - name: Check migrations # run: | From 5ae720a0ae6810c81a941c5797a68d7fedc39765 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Fri, 14 Jun 2024 10:16:33 -0700 Subject: [PATCH 15/17] serialier patch --- posthog/api/comments.py | 2 ++ posthog/api/feature_flag.py | 4 ++++ posthog/api/organization.py | 3 +++ posthog/api/plugin.py | 3 +++ posthog/api/user.py | 8 +++++++- posthog/api/utils.py | 12 +++++++++++- 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/posthog/api/comments.py b/posthog/api/comments.py index 20961be0e3cbb..06443f92b2fcc 100644 --- a/posthog/api/comments.py +++ b/posthog/api/comments.py @@ -11,11 +11,13 @@ from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import UserBasicSerializer +from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer from posthog.models.comment import Comment class CommentSerializer(serializers.ModelSerializer): created_by = UserBasicSerializer(read_only=True) + deleted = ClassicBehaviorBooleanFieldSerializer() class Meta: model = Comment diff --git a/posthog/api/feature_flag.py b/posthog/api/feature_flag.py index 6887b85dcf53b..029a3186d4365 100644 --- a/posthog/api/feature_flag.py +++ b/posthog/api/feature_flag.py @@ -23,6 +23,7 @@ from posthog.api.shared import UserBasicSerializer from posthog.api.tagged_item import TaggedItemSerializerMixin, TaggedItemViewSetMixin from posthog.api.dashboards.dashboard import Dashboard +from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer from posthog.auth import PersonalAPIKeyAuthentication, TemporaryTokenAuthentication from posthog.constants import FlagRequestType from posthog.event_usage import report_user_action @@ -89,6 +90,9 @@ class FeatureFlagSerializer(TaggedItemSerializerMixin, serializers.HyperlinkedMo is_simple_flag = serializers.SerializerMethodField() rollout_percentage = serializers.SerializerMethodField() + ensure_experience_continuity = ClassicBehaviorBooleanFieldSerializer() + has_enriched_analytics = ClassicBehaviorBooleanFieldSerializer() + experiment_set: serializers.PrimaryKeyRelatedField = serializers.PrimaryKeyRelatedField(many=True, read_only=True) surveys: serializers.SerializerMethodField = serializers.SerializerMethodField() features: serializers.SerializerMethodField = serializers.SerializerMethodField() diff --git a/posthog/api/organization.py b/posthog/api/organization.py index 5a66b6281e512..6e45c5a947e08 100644 --- a/posthog/api/organization.py +++ b/posthog/api/organization.py @@ -10,6 +10,7 @@ from posthog import settings from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import TeamBasicSerializer +from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer from posthog.cloud_utils import is_cloud from posthog.constants import INTERNAL_BOT_EMAIL_SUFFIX, AvailableFeature from posthog.event_usage import report_organization_deleted @@ -69,6 +70,8 @@ class OrganizationSerializer(serializers.ModelSerializer, UserPermissionsSeriali teams = serializers.SerializerMethodField() metadata = serializers.SerializerMethodField() member_count = serializers.SerializerMethodField() + is_hipaa = ClassicBehaviorBooleanFieldSerializer() + never_drop_data = ClassicBehaviorBooleanFieldSerializer() class Meta: model = Organization diff --git a/posthog/api/plugin.py b/posthog/api/plugin.py index 72ffa1c01a93a..b568876eeecd4 100644 --- a/posthog/api/plugin.py +++ b/posthog/api/plugin.py @@ -22,6 +22,7 @@ from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import FiltersSerializer +from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer from posthog.models import Plugin, PluginAttachment, PluginConfig, User from posthog.models.activity_logging.activity_log import ( ActivityPage, @@ -586,6 +587,8 @@ class PluginConfigSerializer(serializers.ModelSerializer): delivery_rate_24h = serializers.SerializerMethodField() error = serializers.SerializerMethodField() + deleted = ClassicBehaviorBooleanFieldSerializer() + class Meta: model = PluginConfig fields = [ diff --git a/posthog/api/user.py b/posthog/api/user.py index 8fad7945e7705..1c0392994ad5d 100644 --- a/posthog/api/user.py +++ b/posthog/api/user.py @@ -35,7 +35,11 @@ from posthog.api.email_verification import EmailVerifier from posthog.api.organization import OrganizationSerializer from posthog.api.shared import OrganizationBasicSerializer, TeamBasicSerializer -from posthog.api.utils import PublicIPOnlyHttpAdapter, raise_if_user_provided_url_unsafe +from posthog.api.utils import ( + PublicIPOnlyHttpAdapter, + raise_if_user_provided_url_unsafe, + ClassicBehaviorBooleanFieldSerializer, +) from posthog.auth import ( PersonalAPIKeyAuthentication, SessionAuthentication, @@ -84,6 +88,8 @@ class UserSerializer(serializers.ModelSerializer): current_password = serializers.CharField(write_only=True, required=False) notification_settings = serializers.DictField(required=False) scene_personalisation = ScenePersonalisationBasicSerializer(many=True, read_only=True) + anonymize_data = ClassicBehaviorBooleanFieldSerializer() + email_opt_in = ClassicBehaviorBooleanFieldSerializer() class Meta: model = User diff --git a/posthog/api/utils.py b/posthog/api/utils.py index 952afb9e39f98..2f1bd5c087bab 100644 --- a/posthog/api/utils.py +++ b/posthog/api/utils.py @@ -6,6 +6,8 @@ from ipaddress import ip_address from requests.adapters import HTTPAdapter from typing import Literal, Optional, Union + +from rest_framework.fields import Field from urllib3 import HTTPSConnectionPool, HTTPConnectionPool, PoolManager from uuid import UUID @@ -13,7 +15,7 @@ from django.core.exceptions import RequestDataTooBig from django.db.models import QuerySet from prometheus_client import Counter -from rest_framework import request, status +from rest_framework import request, status, serializers from rest_framework.exceptions import ValidationError from statshog.defaults.django import statsd @@ -34,6 +36,14 @@ class PaginationMode(Enum): previous = auto() +# This overrides a change in DRF 3.15 that alters our behavior. If the user passes an empty argument, +# the new version keeps it as null vs coalescing it to the default. +# Don't add this to new classes +class ClassicBehaviorBooleanFieldSerializer(serializers.BooleanField): + def __init__(self, **kwargs): + Field.__init__(self, allow_null=True, required=False, **kwargs) + + def get_target_entity(filter: Union[Filter, StickinessFilter]) -> Entity: # Except for "events", we require an entity id and type to be provided if not filter.target_entity_id and filter.target_entity_type != "events": From 6bc98d034ea69bb4fd8d8edbc51430891eb4ea65 Mon Sep 17 00:00:00 2001 From: Alexander Spicer Date: Fri, 14 Jun 2024 10:27:23 -0700 Subject: [PATCH 16/17] remove serializer in places it doesn't belong --- posthog/api/organization.py | 3 --- posthog/api/user.py | 1 - 2 files changed, 4 deletions(-) diff --git a/posthog/api/organization.py b/posthog/api/organization.py index 6e45c5a947e08..5a66b6281e512 100644 --- a/posthog/api/organization.py +++ b/posthog/api/organization.py @@ -10,7 +10,6 @@ from posthog import settings from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import TeamBasicSerializer -from posthog.api.utils import ClassicBehaviorBooleanFieldSerializer from posthog.cloud_utils import is_cloud from posthog.constants import INTERNAL_BOT_EMAIL_SUFFIX, AvailableFeature from posthog.event_usage import report_organization_deleted @@ -70,8 +69,6 @@ class OrganizationSerializer(serializers.ModelSerializer, UserPermissionsSeriali teams = serializers.SerializerMethodField() metadata = serializers.SerializerMethodField() member_count = serializers.SerializerMethodField() - is_hipaa = ClassicBehaviorBooleanFieldSerializer() - never_drop_data = ClassicBehaviorBooleanFieldSerializer() class Meta: model = Organization diff --git a/posthog/api/user.py b/posthog/api/user.py index 1c0392994ad5d..ee2b66c47eb1c 100644 --- a/posthog/api/user.py +++ b/posthog/api/user.py @@ -89,7 +89,6 @@ class UserSerializer(serializers.ModelSerializer): notification_settings = serializers.DictField(required=False) scene_personalisation = ScenePersonalisationBasicSerializer(many=True, read_only=True) anonymize_data = ClassicBehaviorBooleanFieldSerializer() - email_opt_in = ClassicBehaviorBooleanFieldSerializer() class Meta: model = User From dd4d7edfabb24f6baa7e99f3b652402f275a0628 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:37:35 +0000 Subject: [PATCH 17/17] Update query snapshots --- posthog/api/test/__snapshots__/test_api_docs.ambr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/api/test/__snapshots__/test_api_docs.ambr b/posthog/api/test/__snapshots__/test_api_docs.ambr index 70fccf17e398f..4327fbc67be9b 100644 --- a/posthog/api/test/__snapshots__/test_api_docs.ambr +++ b/posthog/api/test/__snapshots__/test_api_docs.ambr @@ -77,7 +77,7 @@ "/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Error [PropertyDefinitionViewSet]: exception raised while getting serializer. Hint: Is get_serializer_class() returning None or is get_queryset() not working without a request? Ignoring the view for now. (Exception: 'AnonymousUser' object has no attribute 'organization')", '/home/runner/work/posthog/posthog/posthog/api/property_definition.py: Warning [PropertyDefinitionViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.property_definition.PropertyDefinition" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/query.py: Warning [QueryViewSet]: could not derive type of path parameter "project_id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', - '/opt/hostedtoolcache/Python/3.10.10/x64/lib/python3.10/site-packages/pydantic/_internal/_model_construction.py: Warning [QueryViewSet > ModelMetaclass]: Encountered 2 components with identical names "Person" and different classes and . This will very likely result in an incorrect schema. Try renaming one.', + '/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py: Warning [QueryViewSet > ModelMetaclass]: Encountered 2 components with identical names "Person" and different classes and . This will very likely result in an incorrect schema. Try renaming one.', '/home/runner/work/posthog/posthog/posthog/api/query.py: Warning [QueryViewSet]: could not derive type of path parameter "id" because it is untyped and obtaining queryset from the viewset failed. Consider adding a type to the path (e.g. ) or annotating the parameter type with @extend_schema. Defaulting to "string".', '/home/runner/work/posthog/posthog/posthog/api/query.py: Error [QueryViewSet]: unable to guess serializer. This is graceful fallback handling for APIViews. Consider using GenericAPIView as view base class, if view is under your control. Either way you may want to add a serializer_class (or method). Ignoring view for now.', '/home/runner/work/posthog/posthog/ee/session_recordings/session_recording_playlist.py: Warning [SessionRecordingPlaylistViewSet]: could not derive type of path parameter "project_id" because model "posthog.session_recordings.models.session_recording_playlist.SessionRecordingPlaylist" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".',