diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3732c0107b..c3a51ca3a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,9 +28,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.7 - - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve + python-version: '3.10' + - run: pip install -U -r requirements-bootstrap.txt - run: tox -e ${{ matrix.toxenv }} k8s_itests: runs-on: ubuntu-20.04 @@ -40,11 +39,9 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.7 - - run: python -m pip install --upgrade pip virtualenv + python-version: '3.10' + - run: pip install -U -r requirements-bootstrap.txt - run: curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve - run: make k8s_itests build_debs: runs-on: ubuntu-20.04 @@ -56,7 +53,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.10' - run: sudo apt-get update - run: sudo apt-get install -yq devscripts - run: make itest_${{ matrix.dist }} diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 92df627f2c..583c2f5079 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -22,8 +22,7 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.7 - - run: python -m pip install --upgrade pip - - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve + - run: pip install -U -r requirements-bootstrap.txt - run: tox -e ${{ matrix.toxenv }} pypi: # lets run tests before we push anything to pypi, much like we do internally diff --git a/Makefile b/Makefile index cb9ded6341..558dfd7676 100644 --- a/Makefile +++ b/Makefile @@ -62,11 +62,7 @@ dev-api: .tox/py38-linux .paasta/bin/activate: requirements.txt requirements-dev.txt test -d .paasta/bin/activate || virtualenv -p python3.8 .paasta - .paasta/bin/pip install -U \ - pip==18.1 \ - virtualenv==16.2.0 \ - tox==3.7.0 \ - tox-pip-extensions==1.4.2 + .paasta/bin/pip install -U -r requirements-bootstrap.txt touch .paasta/bin/activate itest: test .paasta/bin/activate diff --git a/k8s_itests/Makefile b/k8s_itests/Makefile index 35272470a6..ca1b339811 100644 --- a/k8s_itests/Makefile +++ b/k8s_itests/Makefile @@ -37,7 +37,7 @@ ifeq (,$(wildcard ./kind)) endif itest: .create_cluster - ../.paasta/bin/tox -e k8s_itests + tox -e k8s_itests clean: ./scripts/clean.sh $(PAASTA_CONFIG_DIR) $(CLUSTER) 2>/dev/null || true diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt index b24f00fb9b..674c481464 100644 --- a/requirements-bootstrap.txt +++ b/requirements-bootstrap.txt @@ -1,4 +1,6 @@ -pip==18.1 -setuptools==39.0.1 -venv-update==3.2.4 -wheel==0.32.3 +ephemeral-port-reserve==1.1.4 +pip==22.0.4 +setuptools==69.1.0 +tox==3.7.0 +tox-pip-extensions==1.4.2 +virtualenv==16.2.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index dbe193a3be..8e0c60e816 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,53 +1,52 @@ -alabaster==0.7.10 -appdirs==1.4.4 -astroid==2.4.2 -asynctest==0.12.0 -Babel==2.9.1 -cfgv==2.0.1 -coverage==6.5.0 -debugpy==1.8.0 -distlib==0.3.4 -docutils==0.12 -exceptiongroup==1.1.2 -filelock==3.0.12 -flake8==3.5.0 -freezegun==0.3.7 -hypothesis==6.79.4 -identify==2.4.12 -imagesize==0.7.1 +alabaster==0.7.13 +astroid==2.13.3 +asynctest==0.13.0 +Babel==2.14.0 +cfgv==3.4.0 +coverage==7.4.2 +debugpy==1.8.1 +dill==0.3.7 +distlib==0.3.8 +docutils==0.19 +filelock==3.13.1 +flake8==7.0.0 +freezegun==1.4.0 +hypothesis==6.98.9 +identify==2.5.35 +imagesize==1.4.1 +importlib-metadata==7.0.1 iniconfig==2.0.0 -isort==4.2.5 -lazy-object-proxy==1.4.3 -mccabe==0.6.1 -mock==2.0.0 -nodeenv==1.1.4 -packaging==20.9 -path.py==8.1 -pbr==3.1.1 -pep8==1.5.7 -pluggy==1.2.0 -pre-commit==2.17.0 -pycodestyle==2.3.1 -pyfakefs==4.1.0 -pyflakes==1.6.0 -Pygments==2.7.4 -pylint==2.6.0 -pyparsing==2.4.7 -pytest==7.4.0 -pytest-asyncio==0.21.1 -requirements-tools==1.1.2 -snowballstemmer==1.2.0 +isort==5.13.2 +lazy-object-proxy==1.10.0 +mccabe==0.7.0 +mock==5.1.0 +nodeenv==1.8.0 +path==16.4.0 +path.py==12.5.0 +pep8==1.7.1 +pluggy==1.4.0 +pre-commit==3.5.0 +pycodestyle==2.11.1 +pyfakefs==5.3.5 +pyflakes==3.2.0 +pygments==2.17.2 +pylint==2.15.10 +pytest==8.0.1 +pytest-asyncio==0.23.5 +requirements-tools==2.1.0 +snowballstemmer==2.2.0 sortedcontainers==2.4.0 -Sphinx==2.2.0 -sphinx-rtd-theme==1.0.0 -sphinxcontrib-applehelp==1.0.1 -sphinxcontrib-devhelp==1.0.1 -sphinxcontrib-htmlhelp==1.0.2 +sphinx==7.1.1 +sphinx-rtd-theme==2.0.0 +sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-jquery==4.1 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.2 -sphinxcontrib-serializinghtml==1.1.3 -toml==0.9.4 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 tomli==2.0.1 -virtualenv==20.4.3 -wrapt==1.12.1 +tomlkit==0.12.3 +virtualenv==20.25.0 +wrapt==1.16.0 zake==0.2.2 diff --git a/requirements.txt b/requirements.txt index aa6201a9bb..6f4f5b9a4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,124 +1,144 @@ a-sync==0.5.0 -addict==2.1.0 -aiohttp==3.7.4.post0 -argcomplete==0.8.1 +addict==2.4.0 +aiohttp==3.9.3 +aiosignal==1.3.1 +argcomplete==3.2.2 argparse==1.4.0 -arrow==1.1.0 -async-timeout==3.0.0 -attrs==19.2.0 +arrow==1.3.0 +async-timeout==4.0.3 +attrs==23.2.0 +backports.zoneinfo==0.2.1 binaryornot==0.4.4 -boto==2.48.0 -boto3==1.34.22 +boto==2.49.0 +boto3==1.34.46 boto3-type-annotations==0.3.1 -botocore==1.34.22 -bravado==10.4.1 -bravado-core==5.12.1 -cachetools==2.0.1 -certifi==2017.11.5 -chardet==3.0.4 +botocore==1.34.46 +bravado==11.0.3 +bravado-core==6.1.1 +cachetools==5.3.2 +cattrs==22.2.0 +certifi==2024.2.2 +chardet==5.1.0 +charset-normalizer==3.3.2 choice==0.1 -click==6.6 -cookiecutter==1.4.0 -croniter==1.3.4 -decorator==4.1.2 -docker-py==1.2.3 -dulwich==0.17.3 -ephemeral-port-reserve==1.1.0 -future==0.16.0 -google-auth==1.2.0 -graphviz==0.8.2 -gunicorn==19.8.1 +click==8.1.7 +cookiecutter==2.1.1 +croniter==2.0.1 +decorator==5.1.1 +docker-py==1.10.6 +docker-pycreds==0.4.0 +dulwich==0.21.7 +ephemeral-port-reserve==1.1.4 +exceptiongroup==1.2.0 +fqdn==1.5.1 +frozenlist==1.4.1 +google-auth==2.28.0 +graphviz==0.20.1 +gunicorn==21.2.0 http-parser==0.9.0 -humanfriendly==4.18 -humanize==0.5.1 -hupper==1.0 -idna==2.6 +humanfriendly==10.0 +humanize==4.9.0 +hupper==1.12.1 +idna==3.6 +importlib-resources==5.12.0 inotify==0.2.8 -ipaddress==1.0.22 -isodate==0.5.1 -itsdangerous==2.0.1 -Jinja2==2.11.3 -jinja2-time==0.1.0 -jmespath==0.9.3 -jsonref==0.1 -jsonschema==2.5.1 -kazoo==2.8.0 +ipaddress==1.0.23 +isodate==0.6.1 +isoduration==20.11.0 +Jinja2==3.1.3 +jinja2-time==0.2.0 +jmespath==1.0.1 +jsonpointer==2.4 +jsonref==1.1.0 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +kazoo==2.9.0 kubernetes==21.7.0 -ldap3==2.6 -manhole==1.5.0 +ldap3==2.9.1 +manhole==1.8.0 marathon==0.12.0 -MarkupSafe==1.1.1 -monotonic==1.4 -msgpack-python==0.5.6 -multidict==4.7.6 +MarkupSafe==2.1.5 +monotonic==1.6 +msgpack==1.0.7 +multidict==6.0.5 mypy-extensions==0.4.1 nulltype==2.3.1 -oauthlib==3.1.0 -objgraph==3.4.0 -PasteDeploy==1.5.2 -ply==3.4 -poyo==0.4.0 -progressbar2==3.10.1 -prometheus-client==0.7.1 -protobuf==3.15.0 -py==1.5.2 -pyasn1==0.4.2 -pyasn1-modules==0.2.1 +oauthlib==3.2.2 +objgraph==3.5.0 +packaging==23.2 +PasteDeploy==3.1.0 +pkgutil-resolve-name==1.3.10 +plaster==1.1.2 +plaster-pastedeploy==1.0.1 +platformdirs==3.10.0 +ply==3.11 +progressbar2==4.3.2 +prometheus-client==0.20.0 +protobuf==4.25.3 +py==1.11.0 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 pyformance==0.4 pyinotify==0.9.6 -pymesos==0.3.9 -pyramid==1.8.4 -pyramid-swagger==2.4.1 -pyrsistent==0.13.0 -pysensu-yelp==0.4.1 -PyStaticConfiguration==0.10.3 -python-crontab==2.1.1 -python-dateutil==2.8.1 +pymesos==0.3.15 +pyramid==2.0.2 +pyramid-swagger==2.7.0 +pyrsistent==0.20.0 +pysensu-yelp==1.0.3 +PyStaticConfiguration==0.11.1 +python-crontab==2.4.1 +python-dateutil==2.8.2 python-iptables==1.0.1 -python-utils==2.0.1 -pytimeparse==1.1.5 -pytz==2016.10 -pyyaml==6.0.1 -repoze.lru==0.6 -requests==2.25.0 -requests-cache==0.6.3 -requests-oauthlib==1.2.0 -requests-toolbelt==0.9.1 +python-slugify==4.0.1 +python-utils==3.8.2 +pytimeparse==1.1.8 +pytz==2024.1 +PyYAML==6.0.1 +referencing==0.33.0 +requests==2.31.0 +requests-cache==1.2.0 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 retry==0.9.2 -rfc3987==1.3.7 -rsa==4.7.2 -ruamel.yaml==0.15.96 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rfc3987==1.3.8 +rpds-py==0.18.0 +rsa==4.9 +ruamel.yaml==0.18.6 +ruamel.yaml.clib==0.2.8 s3transfer==0.10.0 -sensu-plugin==0.3.1 -service-configuration-lib==2.18.11 -setuptools==39.0.1 -signalfx==1.0.17 -simplejson==3.10.0 -six==1.15.0 +sensu-plugin==0.8.0 +service-configuration-lib==2.18.12 +setuptools==69.1.0 +signalfx==1.1.3 +simplejson==3.19.2 +six==1.16.0 slackclient==1.2.1 -splunk-sdk==1.7.0 +splunk-sdk==1.7.4 sseclient-py==1.7 -# make sure you bump sticht in extra_requirements_yelp.txt as well sticht==1.1.18 -strict-rfc3339==0.7 -swagger-spec-validator==2.1.0 -syslogmp==0.2.2 -task-processing==0.1.3 -thriftpy2==0.4.11 -transitions==0.6.9 +swagger-spec-validator==3.0.3 +syslogmp==0.4 +task-processing==0.13.0 +text-unidecode==1.3 +thriftpy2==0.4.17 +transitions==0.9.0 translationstring==1.3 -typing-extensions==4.3.0 -tzlocal==1.2 -url-normalize==1.4.2 +types-python-dateutil==2.8.19.20240106 +typing-extensions==4.9.0 +tzlocal==5.2 +uri-template==1.2.0 +url-normalize==1.4.3 urllib3==1.26.18 utaw==0.2.0 -venusian==1.1.0 -webcolors==1.7 -WebOb==1.7.4 +venusian==3.0.0 +webcolors==1.13 +WebOb==1.8.7 websocket-client==0.44.0 -whichcraft==0.4.0 ws4py==0.5.1 wsgicors==0.7.0 -yarl==1.1.1 -zope.deprecation==4.2.0 -zope.interface==4.1.2 +yarl==1.9.4 +zipp==3.17.0 +zope.deprecation==5.0 +zope.interface==6.2 diff --git a/tox.ini b/tox.ini index 472ba7f09e..f56d641d3f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] skipsdist=True envlist=py38-linux -tox_pip_extensions_ext_venv_update = true docker_compose_version = 1.26.2 [testenv] @@ -144,15 +143,14 @@ deps = behave==1.2.5 commands = # TODO: upgrade behave if they ever take this reasonable PR - pip install git+https://github.com/Yelp/behave@1.2.5-issue_533-fork - pylint -E paasta_tools/mesos/ --ignore master.py,task.py + pip install git+https://github.com/Yelp/behave@v1.2.7.dev5-issue_533-fork behave {posargs} [testenv:mypy] basepython = python3.8 deps = {[testenv]deps} - mypy==0.720 + mypy==0.730 setenv = MYPYPATH = {toxinidir} ignore_errors = True diff --git a/yelp_package/dockerfiles/itest/api/Dockerfile b/yelp_package/dockerfiles/itest/api/Dockerfile index eb0bc51dee..989df8babd 100644 --- a/yelp_package/dockerfiles/itest/api/Dockerfile +++ b/yelp_package/dockerfiles/itest/api/Dockerfile @@ -34,8 +34,10 @@ RUN apt-get update > /dev/null && \ WORKDIR /work ADD requirements.txt /work/ +ADD requirements-bootstrap.txt /work/ RUN virtualenv /venv -ppython3.8 --no-download ENV PATH=/venv/bin:$PATH +RUN pip install -r requirements-bootstrap.txt RUN pip install -r requirements.txt COPY yelp_package/dockerfiles/xenial/mesos-slave-secret /etc/ diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile index 84957612cb..17ed25f895 100644 --- a/yelp_package/dockerfiles/itest/k8s/Dockerfile +++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile @@ -43,8 +43,10 @@ RUN mv ./kubectl /usr/local/bin/kubectl WORKDIR /work ADD requirements.txt /work/ +ADD requirements-bootstrap.txt /work/ RUN virtualenv /venv -ppython3.8 --no-download ENV PATH=/venv/bin:$PATH +RUN pip install -r requirements-bootstrap.txt RUN pip install -r requirements.txt ADD yelp_package/dockerfiles/itest/k8s/wait_paasta_api.sh /venv/bin diff --git a/yelp_package/extra_requirements_yelp.txt b/yelp_package/extra_requirements_yelp.txt index 959d7b80f9..6e6c884959 100644 --- a/yelp_package/extra_requirements_yelp.txt +++ b/yelp_package/extra_requirements_yelp.txt @@ -21,13 +21,13 @@ pycparser==2.20 pygpgme==0.3 pyopenssl==19.0.0 PySubnetTree==0.34 -python-jsonschema-objects==0.3.1 +python-jsonschema-objects==0.5.2 retrying==1.3.3 scribereader==0.15.0 signalform-tools==0.0.16 slo-transcoder==3.3.0 smmap2==2.0.3 -splunk-sdk==1.7.0 +splunk-sdk==1.7.4 srv-configs==1.1.0 sticht[yelp_internal]==1.1.18 vault-tools==0.9.2