From d87f3fa449a1ffd3e2c6683aa6cf3b2d2b484ae5 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 21 Feb 2024 10:43:16 -0800 Subject: [PATCH 01/21] Upgrade all dependencies I'm not really sure this is a good idea but I'm curious to see how badly the tests fail. If it's not too hard to fix them I might chip away at this branch so we can close some of the open PRs from dependabot --- requirements-dev.txt | 95 +++++++++--------- requirements.txt | 224 +++++++++++++++++++++++-------------------- 2 files changed, 169 insertions(+), 150 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index dbe193a3be..8e0c60e816 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,53 +1,52 @@ -alabaster==0.7.10 -appdirs==1.4.4 -astroid==2.4.2 -asynctest==0.12.0 -Babel==2.9.1 -cfgv==2.0.1 -coverage==6.5.0 -debugpy==1.8.0 -distlib==0.3.4 -docutils==0.12 -exceptiongroup==1.1.2 -filelock==3.0.12 -flake8==3.5.0 -freezegun==0.3.7 -hypothesis==6.79.4 -identify==2.4.12 -imagesize==0.7.1 +alabaster==0.7.13 +astroid==2.13.3 +asynctest==0.13.0 +Babel==2.14.0 +cfgv==3.4.0 +coverage==7.4.2 +debugpy==1.8.1 +dill==0.3.7 +distlib==0.3.8 +docutils==0.19 +filelock==3.13.1 +flake8==7.0.0 +freezegun==1.4.0 +hypothesis==6.98.9 +identify==2.5.35 +imagesize==1.4.1 +importlib-metadata==7.0.1 iniconfig==2.0.0 -isort==4.2.5 -lazy-object-proxy==1.4.3 -mccabe==0.6.1 -mock==2.0.0 -nodeenv==1.1.4 -packaging==20.9 -path.py==8.1 -pbr==3.1.1 -pep8==1.5.7 -pluggy==1.2.0 -pre-commit==2.17.0 -pycodestyle==2.3.1 -pyfakefs==4.1.0 -pyflakes==1.6.0 -Pygments==2.7.4 -pylint==2.6.0 -pyparsing==2.4.7 -pytest==7.4.0 -pytest-asyncio==0.21.1 -requirements-tools==1.1.2 -snowballstemmer==1.2.0 +isort==5.13.2 +lazy-object-proxy==1.10.0 +mccabe==0.7.0 +mock==5.1.0 +nodeenv==1.8.0 +path==16.4.0 +path.py==12.5.0 +pep8==1.7.1 +pluggy==1.4.0 +pre-commit==3.5.0 +pycodestyle==2.11.1 +pyfakefs==5.3.5 +pyflakes==3.2.0 +pygments==2.17.2 +pylint==2.15.10 +pytest==8.0.1 +pytest-asyncio==0.23.5 +requirements-tools==2.1.0 +snowballstemmer==2.2.0 sortedcontainers==2.4.0 -Sphinx==2.2.0 -sphinx-rtd-theme==1.0.0 -sphinxcontrib-applehelp==1.0.1 -sphinxcontrib-devhelp==1.0.1 -sphinxcontrib-htmlhelp==1.0.2 +sphinx==7.1.1 +sphinx-rtd-theme==2.0.0 +sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-jquery==4.1 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.2 -sphinxcontrib-serializinghtml==1.1.3 -toml==0.9.4 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 tomli==2.0.1 -virtualenv==20.4.3 -wrapt==1.12.1 +tomlkit==0.12.3 +virtualenv==20.25.0 +wrapt==1.16.0 zake==0.2.2 diff --git a/requirements.txt b/requirements.txt index aa6201a9bb..67d17e942d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,124 +1,144 @@ a-sync==0.5.0 -addict==2.1.0 -aiohttp==3.7.4.post0 -argcomplete==0.8.1 +addict==2.4.0 +aiohttp==3.9.3 +aiosignal==1.3.1 +argcomplete==3.2.2 argparse==1.4.0 -arrow==1.1.0 -async-timeout==3.0.0 -attrs==19.2.0 +arrow==1.3.0 +async-timeout==4.0.3 +attrs==23.2.0 +backports.zoneinfo==0.2.1 binaryornot==0.4.4 -boto==2.48.0 -boto3==1.34.22 +boto==2.49.0.post1 +boto3==1.34.46 boto3-type-annotations==0.3.1 -botocore==1.34.22 -bravado==10.4.1 -bravado-core==5.12.1 -cachetools==2.0.1 -certifi==2017.11.5 -chardet==3.0.4 +botocore==1.34.46 +bravado==11.0.3 +bravado-core==6.1.1 +cachetools==5.3.2 +cattrs==22.2.0 +certifi==2024.2.2 +chardet==5.1.0 +charset-normalizer==3.3.2 choice==0.1 -click==6.6 -cookiecutter==1.4.0 -croniter==1.3.4 -decorator==4.1.2 -docker-py==1.2.3 -dulwich==0.17.3 -ephemeral-port-reserve==1.1.0 -future==0.16.0 -google-auth==1.2.0 -graphviz==0.8.2 -gunicorn==19.8.1 +click==8.1.7 +cookiecutter==2.1.1 +croniter==2.0.1 +decorator==5.1.1 +docker-py==1.10.6 +docker-pycreds==0.4.0 +dulwich==0.21.7 +ephemeral-port-reserve==1.1.4 +exceptiongroup==1.2.0 +fqdn==1.5.1 +frozenlist==1.4.1 +google-auth==2.28.0 +graphviz==0.20.1 +gunicorn==21.2.0 http-parser==0.9.0 -humanfriendly==4.18 -humanize==0.5.1 -hupper==1.0 -idna==2.6 +humanfriendly==10.0 +humanize==4.9.0 +hupper==1.12.1 +idna==3.6 +importlib-resources==5.12.0 inotify==0.2.8 -ipaddress==1.0.22 -isodate==0.5.1 -itsdangerous==2.0.1 -Jinja2==2.11.3 -jinja2-time==0.1.0 -jmespath==0.9.3 -jsonref==0.1 -jsonschema==2.5.1 -kazoo==2.8.0 +ipaddress==1.0.23 +isodate==0.6.1 +isoduration==20.11.0 +Jinja2==3.1.3 +jinja2-time==0.2.0 +jmespath==1.0.1 +jsonpointer==2.4 +jsonref==1.1.0 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +kazoo==2.9.0 kubernetes==21.7.0 -ldap3==2.6 -manhole==1.5.0 +ldap3==2.9.1 +manhole==1.8.0 marathon==0.12.0 -MarkupSafe==1.1.1 -monotonic==1.4 -msgpack-python==0.5.6 -multidict==4.7.6 -mypy-extensions==0.4.1 +MarkupSafe==2.1.5 +monotonic==1.6 +msgpack==1.0.7 +multidict==6.0.5 +mypy-extensions==1.0.0 nulltype==2.3.1 -oauthlib==3.1.0 -objgraph==3.4.0 -PasteDeploy==1.5.2 -ply==3.4 -poyo==0.4.0 -progressbar2==3.10.1 -prometheus-client==0.7.1 -protobuf==3.15.0 -py==1.5.2 -pyasn1==0.4.2 -pyasn1-modules==0.2.1 +oauthlib==3.2.2 +objgraph==3.5.0 +packaging==23.2 +PasteDeploy==3.1.0 +pkgutil-resolve-name==1.3.10 +plaster==1.1.2 +plaster-pastedeploy==1.0.1 +platformdirs==3.10.0 +ply==3.11 +progressbar2==4.3.2 +prometheus-client==0.20.0 +protobuf==4.25.3 +py==1.11.0 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 pyformance==0.4 pyinotify==0.9.6 -pymesos==0.3.9 -pyramid==1.8.4 -pyramid-swagger==2.4.1 -pyrsistent==0.13.0 -pysensu-yelp==0.4.1 -PyStaticConfiguration==0.10.3 -python-crontab==2.1.1 -python-dateutil==2.8.1 +pymesos==0.3.15 +pyramid==2.0.2 +pyramid-swagger==2.7.0 +pyrsistent==0.20.0 +pysensu-yelp==1.0.3 +PyStaticConfiguration==0.11.1 +python-crontab==2.4.1 +python-dateutil==2.8.2 python-iptables==1.0.1 -python-utils==2.0.1 -pytimeparse==1.1.5 -pytz==2016.10 -pyyaml==6.0.1 -repoze.lru==0.6 -requests==2.25.0 -requests-cache==0.6.3 -requests-oauthlib==1.2.0 -requests-toolbelt==0.9.1 +python-slugify==4.0.1 +python-utils==3.8.2 +pytimeparse==1.1.8 +pytz==2024.1 +PyYAML==6.0.1 +referencing==0.33.0 +requests==2.31.0 +requests-cache==1.2.0 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 retry==0.9.2 -rfc3987==1.3.7 -rsa==4.7.2 -ruamel.yaml==0.15.96 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rfc3987==1.3.8 +rpds-py==0.18.0 +rsa==4.9 +ruamel.yaml==0.18.6 +ruamel.yaml.clib==0.2.8 s3transfer==0.10.0 -sensu-plugin==0.3.1 -service-configuration-lib==2.18.11 -setuptools==39.0.1 -signalfx==1.0.17 -simplejson==3.10.0 -six==1.15.0 -slackclient==1.2.1 -splunk-sdk==1.7.0 +sensu-plugin==0.8.0 +service-configuration-lib==2.18.12 +setuptools==69.1.0 +signalfx==1.1.3 +simplejson==3.19.2 +six==1.16.0 +slackclient==2.9.4 +splunk-sdk==1.7.4 sseclient-py==1.7 -# make sure you bump sticht in extra_requirements_yelp.txt as well sticht==1.1.18 -strict-rfc3339==0.7 -swagger-spec-validator==2.1.0 -syslogmp==0.2.2 -task-processing==0.1.3 -thriftpy2==0.4.11 -transitions==0.6.9 +swagger-spec-validator==3.0.3 +syslogmp==0.4 +task-processing==0.13.0 +text-unidecode==1.3 +thriftpy2==0.4.17 +transitions==0.9.0 translationstring==1.3 -typing-extensions==4.3.0 -tzlocal==1.2 -url-normalize==1.4.2 +types-python-dateutil==2.8.19.20240106 +typing-extensions==4.9.0 +tzlocal==5.2 +uri-template==1.2.0 +url-normalize==1.4.3 urllib3==1.26.18 utaw==0.2.0 -venusian==1.1.0 -webcolors==1.7 -WebOb==1.7.4 -websocket-client==0.44.0 -whichcraft==0.4.0 +venusian==3.0.0 +webcolors==1.13 +WebOb==1.8.7 +websocket-client==1.7.0 ws4py==0.5.1 wsgicors==0.7.0 -yarl==1.1.1 -zope.deprecation==4.2.0 -zope.interface==4.1.2 +yarl==1.9.4 +zipp==3.17.0 +zope.deprecation==5.0 +zope.interface==6.2 From ab63cacb6a7f1684aa08fe37116f075577f33eaa Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 21 Feb 2024 10:52:08 -0800 Subject: [PATCH 02/21] Sync other requirements files --- requirements-bootstrap.txt | 2 +- yelp_package/extra_requirements_yelp.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt index b24f00fb9b..25e9079c3c 100644 --- a/requirements-bootstrap.txt +++ b/requirements-bootstrap.txt @@ -1,4 +1,4 @@ pip==18.1 -setuptools==39.0.1 +setuptools==69.1.0 venv-update==3.2.4 wheel==0.32.3 diff --git a/yelp_package/extra_requirements_yelp.txt b/yelp_package/extra_requirements_yelp.txt index 959d7b80f9..d627791a57 100644 --- a/yelp_package/extra_requirements_yelp.txt +++ b/yelp_package/extra_requirements_yelp.txt @@ -27,7 +27,7 @@ scribereader==0.15.0 signalform-tools==0.0.16 slo-transcoder==3.3.0 smmap2==2.0.3 -splunk-sdk==1.7.0 +splunk-sdk==1.7.4 srv-configs==1.1.0 sticht[yelp_internal]==1.1.18 vault-tools==0.9.2 From b31a5ba187bbd6dd4576e2f453aded0a1da8b2b7 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 21 Feb 2024 11:06:22 -0800 Subject: [PATCH 03/21] Bump pip and remove venv-update New pip needed since a dependency doesn't have setup.py. venv-update ties us to an old pip. Try not using venv-update which is maybe a shame but the only way forward. Maybe we should move to poetry but not sure how easy that would be? --- Makefile | 3 ++- requirements-bootstrap.txt | 3 +-- tox.ini | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index cb9ded6341..d89aa6438f 100644 --- a/Makefile +++ b/Makefile @@ -63,9 +63,10 @@ dev-api: .tox/py38-linux .paasta/bin/activate: requirements.txt requirements-dev.txt test -d .paasta/bin/activate || virtualenv -p python3.8 .paasta .paasta/bin/pip install -U \ - pip==18.1 \ + pip==22.0.4 \ virtualenv==16.2.0 \ tox==3.7.0 \ + tox-requirements-bootstrap==1.1.0 \ tox-pip-extensions==1.4.2 touch .paasta/bin/activate diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt index 25e9079c3c..594fd53892 100644 --- a/requirements-bootstrap.txt +++ b/requirements-bootstrap.txt @@ -1,4 +1,3 @@ -pip==18.1 +pip==22.0.4 setuptools==69.1.0 -venv-update==3.2.4 wheel==0.32.3 diff --git a/tox.ini b/tox.ini index 472ba7f09e..c824ba48a3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,9 @@ [tox] skipsdist=True envlist=py38-linux -tox_pip_extensions_ext_venv_update = true docker_compose_version = 1.26.2 +requires = + tox-requirements-bootstrap==1.1.0 [testenv] basepython = python3.8 From c564632d4a2e7e29f8884fae6f8425fca91a216c Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 21 Feb 2024 11:37:59 -0800 Subject: [PATCH 04/21] Sync another extra_requirement --- yelp_package/extra_requirements_yelp.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yelp_package/extra_requirements_yelp.txt b/yelp_package/extra_requirements_yelp.txt index d627791a57..6e6c884959 100644 --- a/yelp_package/extra_requirements_yelp.txt +++ b/yelp_package/extra_requirements_yelp.txt @@ -21,7 +21,7 @@ pycparser==2.20 pygpgme==0.3 pyopenssl==19.0.0 PySubnetTree==0.34 -python-jsonschema-objects==0.3.1 +python-jsonschema-objects==0.5.2 retrying==1.3.3 scribereader==0.15.0 signalform-tools==0.0.16 From 16c80e2685c05f047e4c665c354159631077bf2e Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 21 Feb 2024 11:38:34 -0800 Subject: [PATCH 05/21] Downgrade slackclient Can't just upgrade this, lots of stuff has changed and prob would need to bump in sticht. Also websockets is linked and tied to requirements in kubernetes module too. So avoid this for now... --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 67d17e942d..a287bc2494 100644 --- a/requirements.txt +++ b/requirements.txt @@ -114,7 +114,7 @@ setuptools==69.1.0 signalfx==1.1.3 simplejson==3.19.2 six==1.16.0 -slackclient==2.9.4 +slackclient==1.2.1 splunk-sdk==1.7.4 sseclient-py==1.7 sticht==1.1.18 @@ -135,7 +135,7 @@ utaw==0.2.0 venusian==3.0.0 webcolors==1.13 WebOb==1.8.7 -websocket-client==1.7.0 +websocket-client==0.44.0 ws4py==0.5.1 wsgicors==0.7.0 yarl==1.9.4 From ff07675725f6821abf064cbdf02670f130dfa900 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 21 Feb 2024 11:45:00 -0800 Subject: [PATCH 06/21] Sync CI with Make --- .github/workflows/ci.yml | 4 ++-- .github/workflows/pypi.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3732c0107b..cc87be64c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: with: python-version: 3.7 - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve + - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 tox-requirements-bootstrap==1.1.0 ephemeral-port-reserve - run: tox -e ${{ matrix.toxenv }} k8s_itests: runs-on: ubuntu-20.04 @@ -44,7 +44,7 @@ jobs: - run: python -m pip install --upgrade pip virtualenv - run: curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve + - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 tox-requirements-bootstrap==1.1.0 ephemeral-port-reserve - run: make k8s_itests build_debs: runs-on: ubuntu-20.04 diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 92df627f2c..c0d1b669ae 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -23,7 +23,7 @@ jobs: with: python-version: 3.7 - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve + - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 tox-requirements-bootstrap==1.1.0 ephemeral-port-reserve - run: tox -e ${{ matrix.toxenv }} pypi: # lets run tests before we push anything to pypi, much like we do internally From 0337c4f76dd86a4023eeb4e619512fcf1f40c705 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Thu, 22 Feb 2024 03:57:19 -0800 Subject: [PATCH 07/21] tox-requirements-bootstrap is internal only --- .github/workflows/ci.yml | 4 ++-- tox.ini | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc87be64c0..b3e30c2f43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: with: python-version: 3.7 - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 tox-requirements-bootstrap==1.1.0 ephemeral-port-reserve + - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 ephemeral-port-reserve - run: tox -e ${{ matrix.toxenv }} k8s_itests: runs-on: ubuntu-20.04 @@ -44,7 +44,7 @@ jobs: - run: python -m pip install --upgrade pip virtualenv - run: curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 tox-requirements-bootstrap==1.1.0 ephemeral-port-reserve + - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 ephemeral-port-reserve - run: make k8s_itests build_debs: runs-on: ubuntu-20.04 diff --git a/tox.ini b/tox.ini index c824ba48a3..9b3e385804 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,6 @@ skipsdist=True envlist=py38-linux docker_compose_version = 1.26.2 -requires = - tox-requirements-bootstrap==1.1.0 [testenv] basepython = python3.8 From b5f6c509058a456a48e5476444a49f99db69faf6 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Thu, 22 Feb 2024 03:59:45 -0800 Subject: [PATCH 08/21] We need a boto available everywhere --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a287bc2494..8eb03aff28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ async-timeout==4.0.3 attrs==23.2.0 backports.zoneinfo==0.2.1 binaryornot==0.4.4 -boto==2.49.0.post1 +boto==2.49.0 boto3==1.34.46 boto3-type-annotations==0.3.1 botocore==1.34.46 From f568b05573d29863533bc08f1efcc34ed9c194be Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Thu, 22 Feb 2024 10:00:51 -0800 Subject: [PATCH 09/21] Try using tox for bootstrap requirements No idea if this is okay tbh but let's see if it at least works. I stripped out the paasta venv because it seems unneccesary but I could be wrong... --- .github/workflows/ci.yml | 5 ++-- .github/workflows/pypi.yml | 1 - Makefile | 54 ++++++++++++++++---------------------- k8s_itests/Makefile | 2 +- requirements-bootstrap.txt | 3 --- tox.ini | 7 +++++ 6 files changed, 32 insertions(+), 40 deletions(-) delete mode 100644 requirements-bootstrap.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3e30c2f43..9d23e665cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,8 +29,7 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.7 - - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 ephemeral-port-reserve + - run: pip install tox==3.24.4 - run: tox -e ${{ matrix.toxenv }} k8s_itests: runs-on: ubuntu-20.04 @@ -44,7 +43,7 @@ jobs: - run: python -m pip install --upgrade pip virtualenv - run: curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 ephemeral-port-reserve + - run: pip install ephemeral-port-reserve - run: make k8s_itests build_debs: runs-on: ubuntu-20.04 diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index c0d1b669ae..60920d07e4 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -23,7 +23,6 @@ jobs: with: python-version: 3.7 - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.7.0 tox-pip-extensions==1.4.2 tox-requirements-bootstrap==1.1.0 ephemeral-port-reserve - run: tox -e ${{ matrix.toxenv }} pypi: # lets run tests before we push anything to pypi, much like we do internally diff --git a/Makefile b/Makefile index d89aa6438f..10c61994ce 100644 --- a/Makefile +++ b/Makefile @@ -32,46 +32,36 @@ endif .PHONY: all docs test itest k8s_itests quick-test -dev: .paasta/bin/activate - .paasta/bin/tox +dev: + tox -docs: .paasta/bin/activate - .paasta/bin/tox -e docs +docs: + tox -e docs -test: .paasta/bin/activate +test: if [ "$(PAASTA_ENV)" != "YELP" ]; then \ - .paasta/bin/tox -e tests; \ + tox -e tests; \ else \ - .paasta/bin/tox -e tests-yelpy; \ + tox -e tests-yelpy; \ fi -test-yelpy: .paasta/bin/activate - .paasta/bin/tox -e tests-yelpy +test-yelpy: + tox -e tests-yelpy -test-not-yelpy: .paasta/bin/activate - .paasta/bin/tox -e tests +test-not-yelpy: + tox -e tests quick-test: .tox/py38-linux TZ=UTC .tox/py38-linux/bin/py.test --failed-first -x --disable-warnings -- tests -.tox/py38-linux: .paasta/bin/activate - .paasta/bin/tox +.tox/py38-linux: + tox dev-api: .tox/py38-linux - .paasta/bin/tox -e dev-api + tox -e dev-api -.paasta/bin/activate: requirements.txt requirements-dev.txt - test -d .paasta/bin/activate || virtualenv -p python3.8 .paasta - .paasta/bin/pip install -U \ - pip==22.0.4 \ - virtualenv==16.2.0 \ - tox==3.7.0 \ - tox-requirements-bootstrap==1.1.0 \ - tox-pip-extensions==1.4.2 - touch .paasta/bin/activate - -itest: test .paasta/bin/activate - .paasta/bin/tox -e general_itests +itest: test + tox -e general_itests itest_%: # See the makefile in yelp_package/Makefile for packaging stuff @@ -101,7 +91,7 @@ help: install-hooks: tox -e install-hooks -k8s_itests: .paasta/bin/activate +k8s_itests: make -C k8s_itests all .PHONY: k8s_fake_cluster @@ -109,7 +99,7 @@ k8s_fake_cluster: .tox/py38-linux make -C k8s_itests .fake_cluster .PHONY: k8s_clean -k8s_clean: .paasta/bin/activate +k8s_clean: make -C k8s_itests clean # image source: openapitools/openapi-generator-cli:latest @@ -136,10 +126,10 @@ swagger-validate: -i paasta_tools/api/api_docs/swagger.json .PHONY: vscode_settings -vscode_settings: .paasta/bin/activate .tox/py38-linux - .paasta/bin/python paasta_tools/contrib/ide_helper.py +vscode_settings: .tox/py38-linux + .tox/py38-linux/bin/python paasta_tools/contrib/ide_helper.py -etc_paasta_playground soa_config_playground: .paasta/bin/activate .tox/py38-linux +etc_paasta_playground soa_config_playground: .tox/py38-linux .tox/py38-linux/bin/python paasta_tools/contrib/create_paasta_playground.py .PHONY: generate_deployments_for_service @@ -152,7 +142,7 @@ generate_deployments_for_service: | soa_config_playground .tox/py38-linux .PHONY: playground-api playground-api: .tox/py38-linux | soa_config_playground - .paasta/bin/tox -e playground-api + tox -e playground-api .PHONY: setup-kubernetes-job setup-kubernetes-job: k8s_fake_cluster generate_deployments_for_service diff --git a/k8s_itests/Makefile b/k8s_itests/Makefile index 35272470a6..ca1b339811 100644 --- a/k8s_itests/Makefile +++ b/k8s_itests/Makefile @@ -37,7 +37,7 @@ ifeq (,$(wildcard ./kind)) endif itest: .create_cluster - ../.paasta/bin/tox -e k8s_itests + tox -e k8s_itests clean: ./scripts/clean.sh $(PAASTA_CONFIG_DIR) $(CLUSTER) 2>/dev/null || true diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt deleted file mode 100644 index 594fd53892..0000000000 --- a/requirements-bootstrap.txt +++ /dev/null @@ -1,3 +0,0 @@ -pip==22.0.4 -setuptools==69.1.0 -wheel==0.32.3 diff --git a/tox.ini b/tox.ini index 9b3e385804..b2070a739d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,13 @@ skipsdist=True envlist=py38-linux docker_compose_version = 1.26.2 +requires= + tox==3.7.0 + tox-pip-extensions==1.4.2 + setuptools==69.1.0 + pip==22.0.4 + virtualenv==16.2.0 + ephemeral-port-reserve==1.1.4 [testenv] basepython = python3.8 From b5bef46033d18a9eea4fd141280f8eb13bf54593 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Thu, 22 Feb 2024 10:15:29 -0800 Subject: [PATCH 10/21] Newer python on CI? --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d23e665cf..c117055aca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - run: pip install tox==3.24.4 - run: tox -e ${{ matrix.toxenv }} k8s_itests: @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - run: python -m pip install --upgrade pip virtualenv - run: curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - run: python -m pip install --upgrade pip @@ -55,7 +55,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - run: sudo apt-get update - run: sudo apt-get install -yq devscripts - run: make itest_${{ matrix.dist }} From fa4090207a5d4c64519f14ea771766e9df6af9e4 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Thu, 22 Feb 2024 10:24:39 -0800 Subject: [PATCH 11/21] Pin back mypy-extensions to match pin in tox --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8eb03aff28..6f4f5b9a4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -61,7 +61,7 @@ MarkupSafe==2.1.5 monotonic==1.6 msgpack==1.0.7 multidict==6.0.5 -mypy-extensions==1.0.0 +mypy-extensions==0.4.1 nulltype==2.3.1 oauthlib==3.2.2 objgraph==3.5.0 From 14076adddad42f000d44e5a036f264484732d26b Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Thu, 22 Feb 2024 10:31:24 -0800 Subject: [PATCH 12/21] Pin back progressbar New version needs more modern python I think... --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f4f5b9a4c..79a0df9bc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -72,7 +72,7 @@ plaster==1.1.2 plaster-pastedeploy==1.0.1 platformdirs==3.10.0 ply==3.11 -progressbar2==4.3.2 +progressbar2==3.10.1 prometheus-client==0.20.0 protobuf==4.25.3 py==1.11.0 From c6bd05e91339151ea4204bbf25da3d1993144d33 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Thu, 22 Feb 2024 10:35:25 -0800 Subject: [PATCH 13/21] Pin back hypothesis Also needs a newer python I think --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8e0c60e816..e138d13e89 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ docutils==0.19 filelock==3.13.1 flake8==7.0.0 freezegun==1.4.0 -hypothesis==6.98.9 +hypothesis==6.79.4 identify==2.5.35 imagesize==1.4.1 importlib-metadata==7.0.1 From 9369a10c36690a0ba2b66a7349b553da65c5184e Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Fri, 23 Feb 2024 08:11:23 -0800 Subject: [PATCH 14/21] Revert "Pin back progressbar" This reverts commit 7927a1e6b93ede4ab1dd0a65078376314fdee42f. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 79a0df9bc5..6f4f5b9a4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -72,7 +72,7 @@ plaster==1.1.2 plaster-pastedeploy==1.0.1 platformdirs==3.10.0 ply==3.11 -progressbar2==3.10.1 +progressbar2==4.3.2 prometheus-client==0.20.0 protobuf==4.25.3 py==1.11.0 From 71033376215b372730126394df781e81fee11575 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Fri, 23 Feb 2024 08:11:34 -0800 Subject: [PATCH 15/21] Revert "Pin back hypothesis" This reverts commit 170a00acb48d028b39e389e3947a526b57eb50e4. --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e138d13e89..8e0c60e816 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ docutils==0.19 filelock==3.13.1 flake8==7.0.0 freezegun==1.4.0 -hypothesis==6.79.4 +hypothesis==6.98.9 identify==2.5.35 imagesize==1.4.1 importlib-metadata==7.0.1 From cadb9cba322f9a7876d4b06519365835b8874f11 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Fri, 23 Feb 2024 08:15:51 -0800 Subject: [PATCH 16/21] Bump mypy Needed to support walrus operator in packages we use --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b2070a739d..72b11c3cf6 100644 --- a/tox.ini +++ b/tox.ini @@ -158,7 +158,7 @@ commands = basepython = python3.8 deps = {[testenv]deps} - mypy==0.720 + mypy==0.730 setenv = MYPYPATH = {toxinidir} ignore_errors = True From 51dbe998df9595874e99892c4f62b76f5e0d01cb Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 13 Mar 2024 10:27:18 -0700 Subject: [PATCH 17/21] Add back the paasta venv I think in retrospect I still want this and can pin the tox versions outside of tox itself --- .github/workflows/ci.yml | 6 ++--- .github/workflows/pypi.yml | 2 +- Makefile | 49 +++++++++++++++++++++----------------- requirements-bootstrap.txt | 6 +++++ tox.ini | 7 ------ 5 files changed, 36 insertions(+), 34 deletions(-) create mode 100644 requirements-bootstrap.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c117055aca..c3a51ca3a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: - uses: actions/setup-python@v2 with: python-version: '3.10' - - run: pip install tox==3.24.4 + - run: pip install -U -r requirements-bootstrap.txt - run: tox -e ${{ matrix.toxenv }} k8s_itests: runs-on: ubuntu-20.04 @@ -40,10 +40,8 @@ jobs: - uses: actions/setup-python@v2 with: python-version: '3.10' - - run: python -m pip install --upgrade pip virtualenv + - run: pip install -U -r requirements-bootstrap.txt - run: curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - - run: python -m pip install --upgrade pip - - run: pip install ephemeral-port-reserve - run: make k8s_itests build_debs: runs-on: ubuntu-20.04 diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 60920d07e4..583c2f5079 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.7 - - run: python -m pip install --upgrade pip + - run: pip install -U -r requirements-bootstrap.txt - run: tox -e ${{ matrix.toxenv }} pypi: # lets run tests before we push anything to pypi, much like we do internally diff --git a/Makefile b/Makefile index 10c61994ce..558dfd7676 100644 --- a/Makefile +++ b/Makefile @@ -32,36 +32,41 @@ endif .PHONY: all docs test itest k8s_itests quick-test -dev: - tox +dev: .paasta/bin/activate + .paasta/bin/tox -docs: - tox -e docs +docs: .paasta/bin/activate + .paasta/bin/tox -e docs -test: +test: .paasta/bin/activate if [ "$(PAASTA_ENV)" != "YELP" ]; then \ - tox -e tests; \ + .paasta/bin/tox -e tests; \ else \ - tox -e tests-yelpy; \ + .paasta/bin/tox -e tests-yelpy; \ fi -test-yelpy: - tox -e tests-yelpy +test-yelpy: .paasta/bin/activate + .paasta/bin/tox -e tests-yelpy -test-not-yelpy: - tox -e tests +test-not-yelpy: .paasta/bin/activate + .paasta/bin/tox -e tests quick-test: .tox/py38-linux TZ=UTC .tox/py38-linux/bin/py.test --failed-first -x --disable-warnings -- tests -.tox/py38-linux: - tox +.tox/py38-linux: .paasta/bin/activate + .paasta/bin/tox dev-api: .tox/py38-linux - tox -e dev-api + .paasta/bin/tox -e dev-api -itest: test - tox -e general_itests +.paasta/bin/activate: requirements.txt requirements-dev.txt + test -d .paasta/bin/activate || virtualenv -p python3.8 .paasta + .paasta/bin/pip install -U -r requirements-bootstrap.txt + touch .paasta/bin/activate + +itest: test .paasta/bin/activate + .paasta/bin/tox -e general_itests itest_%: # See the makefile in yelp_package/Makefile for packaging stuff @@ -91,7 +96,7 @@ help: install-hooks: tox -e install-hooks -k8s_itests: +k8s_itests: .paasta/bin/activate make -C k8s_itests all .PHONY: k8s_fake_cluster @@ -99,7 +104,7 @@ k8s_fake_cluster: .tox/py38-linux make -C k8s_itests .fake_cluster .PHONY: k8s_clean -k8s_clean: +k8s_clean: .paasta/bin/activate make -C k8s_itests clean # image source: openapitools/openapi-generator-cli:latest @@ -126,10 +131,10 @@ swagger-validate: -i paasta_tools/api/api_docs/swagger.json .PHONY: vscode_settings -vscode_settings: .tox/py38-linux - .tox/py38-linux/bin/python paasta_tools/contrib/ide_helper.py +vscode_settings: .paasta/bin/activate .tox/py38-linux + .paasta/bin/python paasta_tools/contrib/ide_helper.py -etc_paasta_playground soa_config_playground: .tox/py38-linux +etc_paasta_playground soa_config_playground: .paasta/bin/activate .tox/py38-linux .tox/py38-linux/bin/python paasta_tools/contrib/create_paasta_playground.py .PHONY: generate_deployments_for_service @@ -142,7 +147,7 @@ generate_deployments_for_service: | soa_config_playground .tox/py38-linux .PHONY: playground-api playground-api: .tox/py38-linux | soa_config_playground - tox -e playground-api + .paasta/bin/tox -e playground-api .PHONY: setup-kubernetes-job setup-kubernetes-job: k8s_fake_cluster generate_deployments_for_service diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt new file mode 100644 index 0000000000..674c481464 --- /dev/null +++ b/requirements-bootstrap.txt @@ -0,0 +1,6 @@ +ephemeral-port-reserve==1.1.4 +pip==22.0.4 +setuptools==69.1.0 +tox==3.7.0 +tox-pip-extensions==1.4.2 +virtualenv==16.2.0 diff --git a/tox.ini b/tox.ini index 72b11c3cf6..465c8c5326 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,6 @@ skipsdist=True envlist=py38-linux docker_compose_version = 1.26.2 -requires= - tox==3.7.0 - tox-pip-extensions==1.4.2 - setuptools==69.1.0 - pip==22.0.4 - virtualenv==16.2.0 - ephemeral-port-reserve==1.1.4 [testenv] basepython = python3.8 From 2f5aae05a6090dca8b8074e0bb8979d678904a84 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 13 Mar 2024 10:35:27 -0700 Subject: [PATCH 18/21] Bump behave version Old version uses 2to3 via setuptools which is now deprecated, bump to latest dev version including a Yelp patch --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 465c8c5326..c27b9a35bf 100644 --- a/tox.ini +++ b/tox.ini @@ -143,7 +143,7 @@ deps = behave==1.2.5 commands = # TODO: upgrade behave if they ever take this reasonable PR - pip install git+https://github.com/Yelp/behave@1.2.5-issue_533-fork + pip install git+https://github.com/Yelp/behave@v1.2.7.dev5-issue_533-fork pylint -E paasta_tools/mesos/ --ignore master.py,task.py behave {posargs} From 0e1b0e20f604998decee7317eed016b7b4e92b7f Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 13 Mar 2024 10:49:54 -0700 Subject: [PATCH 19/21] Try pinning pip version in k8s itest --- yelp_package/dockerfiles/itest/k8s/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile index 84957612cb..9c1392bd8f 100644 --- a/yelp_package/dockerfiles/itest/k8s/Dockerfile +++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile @@ -45,6 +45,7 @@ WORKDIR /work ADD requirements.txt /work/ RUN virtualenv /venv -ppython3.8 --no-download ENV PATH=/venv/bin:$PATH +RUN pip install -r requirements-bootstrap.txt RUN pip install -r requirements.txt ADD yelp_package/dockerfiles/itest/k8s/wait_paasta_api.sh /venv/bin From f766499ceeb85084ce3c49b2469887472387fbb4 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 13 Mar 2024 11:50:36 -0700 Subject: [PATCH 20/21] Disable pylint on mesos files Not sure how this worked before but I don't think we care anyway --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index c27b9a35bf..f56d641d3f 100644 --- a/tox.ini +++ b/tox.ini @@ -144,7 +144,6 @@ deps = commands = # TODO: upgrade behave if they ever take this reasonable PR pip install git+https://github.com/Yelp/behave@v1.2.7.dev5-issue_533-fork - pylint -E paasta_tools/mesos/ --ignore master.py,task.py behave {posargs} [testenv:mypy] From b002f849ee21f8a9c22f7f8d84b18801bd689aa8 Mon Sep 17 00:00:00 2001 From: Matthew Mead-Briggs Date: Wed, 13 Mar 2024 11:53:55 -0700 Subject: [PATCH 21/21] fixup! Try pinning pip version in k8s itest --- yelp_package/dockerfiles/itest/api/Dockerfile | 2 ++ yelp_package/dockerfiles/itest/k8s/Dockerfile | 1 + 2 files changed, 3 insertions(+) diff --git a/yelp_package/dockerfiles/itest/api/Dockerfile b/yelp_package/dockerfiles/itest/api/Dockerfile index eb0bc51dee..989df8babd 100644 --- a/yelp_package/dockerfiles/itest/api/Dockerfile +++ b/yelp_package/dockerfiles/itest/api/Dockerfile @@ -34,8 +34,10 @@ RUN apt-get update > /dev/null && \ WORKDIR /work ADD requirements.txt /work/ +ADD requirements-bootstrap.txt /work/ RUN virtualenv /venv -ppython3.8 --no-download ENV PATH=/venv/bin:$PATH +RUN pip install -r requirements-bootstrap.txt RUN pip install -r requirements.txt COPY yelp_package/dockerfiles/xenial/mesos-slave-secret /etc/ diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile index 9c1392bd8f..17ed25f895 100644 --- a/yelp_package/dockerfiles/itest/k8s/Dockerfile +++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile @@ -43,6 +43,7 @@ RUN mv ./kubectl /usr/local/bin/kubectl WORKDIR /work ADD requirements.txt /work/ +ADD requirements-bootstrap.txt /work/ RUN virtualenv /venv -ppython3.8 --no-download ENV PATH=/venv/bin:$PATH RUN pip install -r requirements-bootstrap.txt