From 233d3a9098764096ccde6a698d02590b18abaf73 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 22 Apr 2024 22:52:47 +0200 Subject: [PATCH] chore(deps): Remove unused Python requirements (#21580) * chore(deps): Remove unused Python requirements * Also remove `dnspython` * Recompile txts * Revert "Also remove `dnspython`" This reverts commit 7eaf89979b774808cc0920c34a3a1cf1d75d2f14. * Re-recompile txts * Remove more deps based on `deptry` * Restore `phonenumberslite` * Restore `geoip2` * Restore `python3-saml` * Move `prance` and `openapi-spec-validator` to dev reqs * Reorder `geoip2` * Force HTTP 1.1 in downloading MMDB Should fix CI failures caused by `curl: (92) HTTP/2 stream 0 was not closed cleanly: INTERNAL_ERROR (err 2)` (e.g. https://github.com/PostHog/posthog/actions/runs/8741861825/job/23988870688?pr=21580). --- .github/actions/run-backend-tests/action.yml | 2 +- bin/start | 2 +- production.Dockerfile | 2 +- requirements-dev.in | 3 + requirements-dev.txt | 61 +++++++++++++++ requirements.in | 14 ---- requirements.txt | 82 +------------------- 7 files changed, 71 insertions(+), 95 deletions(-) diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml index 3dc9ef416a095..3092caba27f82 100644 --- a/.github/actions/run-backend-tests/action.yml +++ b/.github/actions/run-backend-tests/action.yml @@ -125,7 +125,7 @@ runs: touch frontend/dist/index.html touch frontend/dist/layout.html touch frontend/dist/exporter.html - [ ! -f ./share/GeoLite2-City.mmdb ] && ( curl -L "https://mmdbcdn.posthog.net/" | brotli --decompress --output=./share/GeoLite2-City.mmdb ) + [ ! -f ./share/GeoLite2-City.mmdb ] && ( curl -L "https://mmdbcdn.posthog.net/" --http1.1 | brotli --decompress --output=./share/GeoLite2-City.mmdb ) - name: Wait for Clickhouse & Kafka shell: bash diff --git a/bin/start b/bin/start index 4cb46f4ee7a22..38f83accb3968 100755 --- a/bin/start +++ b/bin/start @@ -15,7 +15,7 @@ service_warning() { nc -z localhost 9092 || ( service_warning 'Kafka'; bin/check_kafka_clickhouse_up ) curl -s 'http://localhost:8123/ping' || ( service_warning 'ClickHouse'; bin/check_kafka_clickhouse_up ) -[ ! -f ./share/GeoLite2-City.mmdb ] && ( curl -L "https://mmdbcdn.posthog.net/" | brotli --decompress --output=./share/GeoLite2-City.mmdb ) +[ ! -f ./share/GeoLite2-City.mmdb ] && ( curl -L "https://mmdbcdn.posthog.net/" --http1.1 | brotli --decompress --output=./share/GeoLite2-City.mmdb ) ./bin/start-worker & ./bin/start-backend & diff --git a/production.Dockerfile b/production.Dockerfile index 0d4c8880a4556..4802bb7e51da7 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -131,7 +131,7 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* && \ mkdir share && \ - ( curl -s -L "https://mmdbcdn.posthog.net/" | brotli --decompress --output=./share/GeoLite2-City.mmdb ) && \ + ( curl -s -L "https://mmdbcdn.posthog.net/" --http1.1 | brotli --decompress --output=./share/GeoLite2-City.mmdb ) && \ chmod -R 755 ./share/GeoLite2-City.mmdb diff --git a/requirements-dev.in b/requirements-dev.in index 44bd424b1bcc4..034e348dc296e 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -24,6 +24,7 @@ fakeredis[lua]==2.11.0 freezegun==1.2.2 packaging==23.1 black~=22.8.0 +boto3-stubs[s3] types-markdown==3.3.9 types-PyYAML==6.0.1 types-freezegun==1.1.10 @@ -48,3 +49,5 @@ responses==0.23.1 syrupy~=4.6.0 flaky==3.7.0 aioresponses==0.7.6 +prance==23.06.21.0 +openapi-spec-validator==0.7.1 # Needed for prance as a validation backend diff --git a/requirements-dev.txt b/requirements-dev.txt index 854d543541b27..112447f2430ad 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -33,16 +33,24 @@ attrs==23.2.0 # via # -c requirements.txt # aiohttp + # jsonschema + # referencing black==22.8.0 # via # -r requirements-dev.in # datamodel-code-generator +boto3-stubs[s3]==1.34.84 + # via -r requirements-dev.in +botocore-stubs==1.34.84 + # via boto3-stubs build==0.10.0 # via pip-tools certifi==2019.11.28 # via # -c requirements.txt # requests +chardet==5.2.0 + # via prance charset-normalizer==2.1.0 # via # -c requirements.txt @@ -122,6 +130,20 @@ jinja2==3.1.3 # via # coreschema # datamodel-code-generator +jsonschema==4.20.0 + # via + # -c requirements.txt + # openapi-schema-validator + # openapi-spec-validator +jsonschema-path==0.3.2 + # via openapi-spec-validator +jsonschema-specifications==2023.12.1 + # via + # -c requirements.txt + # jsonschema + # openapi-schema-validator +lazy-object-proxy==1.10.0 + # via openapi-spec-validator lupa==1.14.1 # via fakeredis markupsafe==2.1.5 @@ -138,20 +160,29 @@ mypy==1.8.0 # djangorestframework-stubs mypy-baseline==0.6.1 # via -r requirements-dev.in +mypy-boto3-s3==1.34.65 + # via boto3-stubs mypy-extensions==1.0.0 # via # -r requirements-dev.in # black # mypy +openapi-schema-validator==0.6.2 + # via openapi-spec-validator +openapi-spec-validator==0.7.1 + # via -r requirements-dev.in packaging==23.1 # via # -c requirements.txt # -r requirements-dev.in # build # datamodel-code-generator + # prance # pytest parameterized==0.9.0 # via -r requirements-dev.in +pathable==0.4.3 + # via jsonschema-path pathspec==0.9.0 # via black pip-tools==7.3.0 @@ -164,6 +195,8 @@ pluggy==0.13.1 # via pytest pprintpp==0.4.0 # via pytest-icdiff +prance==23.6.21.0 + # via -r requirements-dev.in pydantic[email]==2.5.3 # via # -c requirements.txt @@ -212,25 +245,47 @@ pyyaml==6.0.1 # via # -c requirements.txt # datamodel-code-generator + # jsonschema-path # responses redis==4.5.4 # via # -c requirements.txt # fakeredis +referencing==0.31.1 + # via + # -c requirements.txt + # jsonschema + # jsonschema-path + # jsonschema-specifications requests==2.31.0 # via # -c requirements.txt # coreapi # djangorestframework-stubs + # jsonschema-path + # prance # responses responses==0.23.1 # via -r requirements-dev.in +rfc3339-validator==0.1.4 + # via openapi-schema-validator +rpds-py==0.16.2 + # via + # -c requirements.txt + # jsonschema + # referencing +ruamel-yaml==0.18.6 + # via prance +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml ruff==0.3.7 # via -r requirements-dev.in six==1.16.0 # via # -c requirements.txt + # prance # python-dateutil + # rfc3339-validator sortedcontainers==2.4.0 # via # -c requirements.txt @@ -253,6 +308,8 @@ tomli==1.2.3 # pip-tools # pyproject-hooks # pytest +types-awscrt==0.20.9 + # via botocore-stubs types-freezegun==1.1.10 # via -r requirements-dev.in types-markdown==3.3.9 @@ -273,16 +330,20 @@ types-requests==2.26.1 # via -r requirements-dev.in types-retry==0.9.9.4 # via -r requirements-dev.in +types-s3transfer==0.10.1 + # via boto3-stubs types-tzlocal==5.1.0.1 # via -r requirements-dev.in typing-extensions==4.7.1 # via # -c requirements.txt # asgiref + # boto3-stubs # django-stubs # django-stubs-ext # djangorestframework-stubs # mypy + # mypy-boto3-s3 # pydantic # pydantic-core uritemplate==4.1.1 diff --git a/requirements.in b/requirements.in index 7432ab40e6413..e286072a0f610 100644 --- a/requirements.in +++ b/requirements.in @@ -8,16 +8,13 @@ aiohttp>=3.9.0 aioboto3==12.0.0 aiokafka>=0.8 antlr4-python3-runtime==4.13.1 -amqp==5.1.1 boto3==1.28.16 -boto3-stubs[s3] brotli==1.1.0 celery==5.3.4 celery-redbeat==2.1.1 clickhouse-driver==0.2.6 clickhouse-pool==0.5.3 cryptography==37.0.2 -defusedxml==0.6.0 dj-database-url==0.5.0 Django~=4.2.11 django-axes==5.9.0 @@ -41,29 +38,20 @@ dnspython==2.2.1 drf-exceptions-hog==0.4.0 drf-extensions==0.7.0 drf-spectacular==0.27.1 -gevent==23.9.1 geoip2==4.6.0 google-cloud-bigquery==3.11.4 gunicorn==20.1.0 -idna==2.8 -importlib-metadata==6.8.0 infi-clickhouse-orm@ git+https://github.com/PostHog/infi.clickhouse_orm@9578c79f29635ee2c1d01b7979e89adab8383de2 kafka-python==2.0.2 -kafka-helper==0.2 kombu==5.3.2 lzstring==1.0.4 natsort==8.4.0 numpy==1.23.3 -openapi-spec-validator==0.7.1 openpyxl==3.1.2 orjson==3.9.10 pandas==2.2.0 -parso==0.8.1 -pexpect==4.7.0 -pickleshare==0.7.5 Pillow==10.2.0 posthoganalytics==3.5.0 -prance==23.06.21.0 psycopg2-binary==2.9.7 psycopg[binary]==3.1.13 pyarrow==15.0.0 @@ -75,7 +63,6 @@ pytz==2023.3 redis==4.5.4 retry==0.9.2 requests~=2.31.0 -requests-oauthlib==1.3.0 s3fs==2023.10.0 stripe==7.4.0 selenium==4.1.5 @@ -103,4 +90,3 @@ openai==1.10.0 tiktoken==0.6.0 nh3==0.2.14 hogql-parser==1.0.7 -urllib3[secure,socks]==1.26.18 diff --git a/requirements.txt b/requirements.txt index 50e0b66928940..e83a7064eff0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,9 +23,7 @@ aiokafka==0.8.1 aiosignal==1.2.0 # via aiohttp amqp==5.1.1 - # via - # -r requirements.in - # kombu + # via kombu annotated-types==0.5.0 # via pydantic antlr4-python3-runtime==4.13.1 @@ -64,15 +62,11 @@ boto3==1.28.16 # via # -r requirements.in # aiobotocore -boto3-stubs[s3]==1.26.138 - # via -r requirements.in botocore==1.31.64 # via # aiobotocore # boto3 # s3transfer -botocore-stubs==1.29.130 - # via boto3-stubs brotli==1.1.0 # via -r requirements.in cachetools==5.3.1 @@ -96,8 +90,6 @@ cffi==1.14.5 # via # cryptography # snowflake-connector-python -chardet==5.1.0 - # via prance charset-normalizer==2.1.0 # via # requests @@ -126,7 +118,6 @@ cryptography==37.0.2 # via # -r requirements.in # django-fernet-encrypted-fields - # kafka-helper # pyopenssl # snowflake-connector-python # social-auth-core @@ -139,7 +130,6 @@ decorator==5.1.1 # via retry defusedxml==0.6.0 # via - # -r requirements.in # python3-openid # social-auth-core distro==1.9.0 @@ -245,8 +235,6 @@ future==0.18.3 # via lzstring geoip2==4.6.0 # via -r requirements.in -gevent==23.9.1 - # via -r requirements.in gitdb==4.0.11 # via gitpython gitpython==3.1.40 @@ -273,8 +261,6 @@ googleapis-common-protos==1.60.0 # via # google-api-core # grpcio-status -greenlet==3.0.3 - # via gevent grpcio==1.57.0 # via # google-api-core @@ -300,7 +286,6 @@ humanize==4.9.0 # via dlt idna==2.8 # via - # -r requirements.in # anyio # httpx # requests @@ -308,8 +293,6 @@ idna==2.8 # trio # urllib3 # yarl -importlib-metadata==6.8.0 - # via -r requirements.in infi-clickhouse-orm @ git+https://github.com/PostHog/infi.clickhouse_orm@9578c79f29635ee2c1d01b7979e89adab8383de2 # via -r requirements.in inflection==0.5.1 @@ -327,18 +310,9 @@ joblib==1.3.2 jsonpath-ng==1.6.0 # via dlt jsonschema==4.20.0 - # via - # drf-spectacular - # openapi-schema-validator - # openapi-spec-validator -jsonschema-path==0.3.2 - # via openapi-spec-validator + # via drf-spectacular jsonschema-specifications==2023.12.1 - # via - # jsonschema - # openapi-schema-validator -kafka-helper==0.2 - # via -r requirements.in + # via jsonschema kafka-python==2.0.2 # via # -r requirements.in @@ -347,8 +321,6 @@ kombu==5.3.2 # via # -r requirements.in # celery -lazy-object-proxy==1.10.0 - # via openapi-spec-validator lxml==4.9.4 # via # python3-saml @@ -370,8 +342,6 @@ multidict==6.0.2 # via # aiohttp # yarl -mypy-boto3-s3==1.26.127 - # via boto3-stubs natsort==8.4.0 # via -r requirements.in nh3==0.2.14 @@ -391,10 +361,6 @@ openai==1.10.0 # via # -r requirements.in # sentry-sdk -openapi-schema-validator==0.6.2 - # via openapi-spec-validator -openapi-spec-validator==0.7.1 - # via -r requirements.in openpyxl==3.1.2 # via -r requirements.in orjson==3.9.10 @@ -408,25 +374,16 @@ packaging==23.1 # aiokafka # dlt # google-cloud-bigquery - # prance # snowflake-connector-python # webdriver-manager pandas==2.2.0 # via -r requirements.in -parso==0.8.1 - # via -r requirements.in -pathable==0.4.3 - # via jsonschema-path pathvalidate==3.2.0 # via dlt pendulum==2.1.2 # via dlt -pexpect==4.7.0 - # via -r requirements.in phonenumberslite==8.13.6 # via -r requirements.in -pickleshare==0.7.5 - # via -r requirements.in pillow==10.2.0 # via -r requirements.in platformdirs==3.11.0 @@ -435,8 +392,6 @@ ply==3.11 # via jsonpath-ng posthoganalytics==3.5.0 # via -r requirements.in -prance==23.6.21.0 - # via -r requirements.in prometheus-client==0.14.1 # via django-prometheus prompt-toolkit==3.0.39 @@ -457,8 +412,6 @@ psycopg-binary==3.1.13 # via psycopg psycopg2-binary==2.9.7 # via -r requirements.in -ptyprocess==0.6.0 - # via pexpect py==1.11.0 # via retry pyarrow==15.0.0 @@ -525,7 +478,6 @@ pyyaml==6.0.1 # via # dlt # drf-spectacular - # jsonschema-path qrcode==7.4.2 # via django-two-factor-auth redis==4.5.4 @@ -536,7 +488,6 @@ redis==4.5.4 referencing==0.31.1 # via # jsonschema - # jsonschema-path # jsonschema-specifications regex==2023.12.25 # via tiktoken @@ -548,9 +499,7 @@ requests==2.31.0 # google-api-core # google-cloud-bigquery # infi-clickhouse-orm - # jsonschema-path # posthoganalytics - # prance # requests-oauthlib # snowflake-connector-python # social-auth-core @@ -558,25 +507,17 @@ requests==2.31.0 # tiktoken # webdriver-manager requests-oauthlib==1.3.0 - # via - # -r requirements.in - # social-auth-core + # via social-auth-core requirements-parser==0.5.0 # via dlt retry==0.9.2 # via -r requirements.in -rfc3339-validator==0.1.4 - # via openapi-schema-validator rpds-py==0.16.2 # via # jsonschema # referencing rsa==4.9 # via google-auth -ruamel-yaml==0.17.21 - # via prance -ruamel-yaml-clib==0.2.7 - # via ruamel-yaml s3fs==2023.10.0 # via -r requirements.in s3transfer==0.6.0 @@ -602,9 +543,7 @@ six==1.16.0 # google-auth # isodate # posthoganalytics - # prance # python-dateutil - # rfc3339-validator slack-sdk==3.17.1 # via -r requirements.in smmap==5.0.1 @@ -669,14 +608,8 @@ trio==0.20.0 # trio-websocket trio-websocket==0.9.2 # via selenium -types-awscrt==0.16.17 - # via - # botocore-stubs - # types-s3transfer types-protobuf==4.22.0.0 # via temporalio -types-s3transfer==0.6.1 - # via boto3-stubs types-setuptools==69.0.0.0 # via requirements-parser typing-extensions==4.7.1 @@ -706,7 +639,6 @@ uritemplate==4.1.1 # via drf-spectacular urllib3[secure,socks]==1.26.18 # via - # -r requirements.in # botocore # django-revproxy # geoip2 @@ -737,12 +669,6 @@ xmlsec==1.3.13 # via python3-saml yarl==1.7.2 # via aiohttp -zipp==3.17.0 - # via importlib-metadata -zope-event==5.0 - # via gevent -zope-interface==6.1 - # via gevent # The following packages are considered to be unsafe in a requirements file: # setuptools