From 0b86bbb1bd8c8aa95534c4315ce8d79b6c95dc87 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 25 Sep 2023 15:18:09 -0400 Subject: [PATCH 1/3] chore: Unpin tubular requirements, make upgrade --- requirements/base.in | 33 ++++++++-------- requirements/base.txt | 73 ++++++++++++++++++++---------------- requirements/constraints.txt | 4 +- requirements/testing.in | 2 +- requirements/testing.txt | 21 +++++------ 5 files changed, 70 insertions(+), 63 deletions(-) diff --git a/requirements/base.in b/requirements/base.in index 43f8409a..70360814 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,26 +1,27 @@ -c constraints.txt -backoff==1.5.0 +backoff boto3 click-log -click>=7.0.0 +click cloudflare -edx-opaque-keys==0.4.0 -edx-rest-api-client==5.5.0 -freezegun==0.3.8 -GitPython==3.1.18 -google-api-python-client==1.7.3 -jenkinsapi==0.3.3 -kubernetes==12.0.1 +edx-opaque-keys +edx-rest-api-client +freezegun +GitPython +google-api-python-client +jenkinsapi +jinja2 +kubernetes PyGithub -pymongo==3.5.1 -pytz==2016.10 -pyyaml==5.4 -requests>=2.9.1,<3.0 +pymongo +pytz +pyyaml +requests sailthru-client simple-salesforce six -unicodecsv==0.14.1 +unicodecsv validators -yagocd==0.4.4 -wrapt==1.11.* +yagocd +wrapt diff --git a/requirements/base.txt b/requirements/base.txt index 414565a4..f71a548c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -10,17 +10,14 @@ attrs==23.1.0 # via # jsonlines # zeep -backoff==1.5.0 +backoff==2.2.1 # via -r requirements/base.in beautifulsoup4==4.12.2 # via cloudflare -boto3==1.20.54 - # via - # -c requirements/constraints.txt - # -r requirements/base.in -botocore==1.23.54 +boto3==1.28.53 + # via -r requirements/base.in +botocore==1.31.53 # via - # -c requirements/constraints.txt # boto3 # s3transfer cachetools==5.3.1 @@ -64,25 +61,30 @@ easydict==1.10 # via yagocd edx-django-utils==5.7.0 # via edx-rest-api-client -edx-opaque-keys==0.4 +edx-opaque-keys==2.5.1 # via -r requirements/base.in -edx-rest-api-client==5.5.0 +edx-rest-api-client==5.6.0 # via -r requirements/base.in -freezegun==0.3.8 +freezegun==1.2.2 # via -r requirements/base.in gitdb==4.0.10 # via gitpython -gitpython==3.1.18 +gitpython==3.1.37 # via -r requirements/base.in -google-api-python-client==1.7.3 +google-api-core==2.12.0 + # via google-api-python-client +google-api-python-client==2.101.0 # via -r requirements/base.in google-auth==2.23.0 # via + # google-api-core # google-api-python-client # google-auth-httplib2 # kubernetes google-auth-httplib2==0.1.1 # via google-api-python-client +googleapis-common-protos==1.60.0 + # via google-api-core httplib2==0.22.0 # via # google-api-python-client @@ -91,30 +93,40 @@ idna==3.4 # via requests isodate==0.6.1 # via zeep -jenkinsapi==0.3.3 +jenkinsapi==0.3.13 + # via -r requirements/base.in +jinja2==3.1.2 # via -r requirements/base.in -jmespath==0.10.0 +jmespath==1.0.1 # via # boto3 # botocore jsonlines==4.0.0 # via cloudflare -kubernetes==12.0.1 +kubernetes==28.1.0 # via -r requirements/base.in lxml==4.9.3 # via zeep +markupsafe==2.1.3 + # via jinja2 more-itertools==10.1.0 # via simple-salesforce newrelic==9.0.0 # via edx-django-utils oauthlib==3.2.2 - # via requests-oauthlib + # via + # kubernetes + # requests-oauthlib pbr==5.11.1 # via stevedore pendulum==2.1.2 # via simple-salesforce platformdirs==3.10.0 # via zeep +protobuf==4.24.3 + # via + # google-api-core + # googleapis-common-protos psutil==5.9.5 # via edx-django-utils pyasn1==0.5.0 @@ -132,7 +144,7 @@ pyjwt[crypto]==2.8.0 # edx-rest-api-client # pygithub # simple-salesforce -pymongo==3.5.1 +pymongo==3.13.0 # via # -r requirements/base.in # edx-opaque-keys @@ -148,7 +160,7 @@ python-dateutil==2.8.2 # freezegun # kubernetes # pendulum -pytz==2016.10 +pytz==2023.3.post1 # via # -r requirements/base.in # django @@ -156,7 +168,7 @@ pytz==2016.10 # zeep pytzdata==2020.1 # via pendulum -pyyaml==5.4 +pyyaml==6.0.1 # via # -r requirements/base.in # cloudflare @@ -166,6 +178,7 @@ requests==2.31.0 # -r requirements/base.in # cloudflare # edx-rest-api-client + # google-api-core # jenkinsapi # kubernetes # pygithub @@ -185,7 +198,7 @@ requests-toolbelt==1.0.0 # via zeep rsa==4.9 # via google-auth -s3transfer==0.5.2 +s3transfer==0.6.2 # via boto3 sailthru-client==2.3.5 # via -r requirements/base.in @@ -196,14 +209,11 @@ simplejson==3.19.1 six==1.16.0 # via # -r requirements/base.in - # edx-opaque-keys - # freezegun - # google-api-python-client # isodate + # jenkinsapi # kubernetes # python-dateutil # requests-file - # stevedore # yagocd slumber==0.7.1 # via edx-rest-api-client @@ -213,15 +223,17 @@ soupsieve==2.5 # via beautifulsoup4 sqlparse==0.4.4 # via django -stevedore==1.32.0 +stevedore==5.1.0 # via # edx-django-utils # edx-opaque-keys typing-extensions==4.8.0 - # via asgiref + # via + # asgiref + # edx-opaque-keys unicodecsv==0.14.1 # via -r requirements/base.in -uritemplate==3.0.1 +uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.16 # via @@ -233,14 +245,11 @@ validators==0.22.0 # via -r requirements/base.in websocket-client==1.6.3 # via kubernetes -wrapt==1.11.2 +wrapt==1.15.0 # via # -r requirements/base.in # deprecated -yagocd==0.4.4 +yagocd==1.0.1 # via -r requirements/base.in zeep==4.2.1 # via simple-salesforce - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 592ca185..350284d5 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -10,8 +10,8 @@ # TODO: Many pinned dependencies should be unpinned and/or moved to this constraints file. -boto3==1.20.54 -botocore>=1.23.54,<1.24.0 #https://github.com/boto/boto3/commit/d226d9db05b4fb9d24b4e090efcc752ace5d4985#diff-67fbd34f176dc812829309d74467869a35bf5af2764b626fc1cfb4656c94c633R21 +#boto3==1.20.54 +#botocore>=1.23.54,<1.24.0 #https://github.com/boto/boto3/commit/d226d9db05b4fb9d24b4e090efcc752ace5d4985#diff-67fbd34f176dc812829309d74467869a35bf5af2764b626fc1cfb4656c94c633R21 responses<0.21.0 django<4.0 setuptools<60 diff --git a/requirements/testing.in b/requirements/testing.in index 20c80cb1..99e1e5c7 100644 --- a/requirements/testing.in +++ b/requirements/testing.in @@ -15,4 +15,4 @@ pytest-xdist pytest-pylint responses requests_mock -wrapt==1.11.* +wrapt diff --git a/requirements/testing.txt b/requirements/testing.txt index a6df3613..e4afe98b 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -4,18 +4,15 @@ # # make upgrade # -astroid==2.15.6 +astroid==2.15.7 # via # -r requirements/testing.in # pylint # pylint-celery -boto3==1.20.54 - # via - # -c requirements/constraints.txt - # moto -botocore==1.23.54 +boto3==1.28.53 + # via moto +botocore==1.31.53 # via - # -c requirements/constraints.txt # boto3 # moto # s3transfer @@ -60,7 +57,7 @@ jinja2==3.1.2 # via # code-annotations # moto -jmespath==0.10.0 +jmespath==1.0.1 # via # boto3 # botocore @@ -74,7 +71,7 @@ mccabe==0.7.0 # via pylint mock==5.1.0 # via -r requirements/testing.in -moto==4.2.3 +moto==4.2.4 # via -r requirements/testing.in packaging==23.1 # via pytest @@ -92,7 +89,7 @@ pycodestyle==2.11.0 # pytest-pycodestyle pycparser==2.21 # via cffi -pylint==2.17.5 +pylint==2.17.6 # via # -r requirements/testing.in # edx-lint @@ -143,7 +140,7 @@ responses==0.20.0 # -c requirements/constraints.txt # -r requirements/testing.in # moto -s3transfer==0.5.2 +s3transfer==0.6.2 # via boto3 six==1.16.0 # via @@ -174,7 +171,7 @@ urllib3==1.26.16 # responses werkzeug==2.3.7 # via moto -wrapt==1.11.2 +wrapt==1.15.0 # via # -r requirements/testing.in # astroid From 14d652a248db9668594606771dfd8e165ab905b7 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Wed, 20 Sep 2023 16:03:17 -0400 Subject: [PATCH 2/3] fix: Update mock data for google library > 1.7.5 Content-ID changed in https://github.com/googleapis/google-api-python-client/pull/545/files see https://github.com/googleapis/google-api-python-client/issues/164 --- tubular/tests/test_google_api.py | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tubular/tests/test_google_api.py b/tubular/tests/test_google_api.py index 7d1bb53a..04f7be5e 100644 --- a/tubular/tests/test_google_api.py +++ b/tubular/tests/test_google_api.py @@ -115,7 +115,7 @@ def test_delete_file_success(self, mock_from_service_account_file): # pylint: d batch_response = b'''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 204 OK ETag: "etag/pony"\r\n\r\n @@ -123,7 +123,7 @@ def test_delete_file_success(self, mock_from_service_account_file): # pylint: d --batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 204 OK ETag: "etag/sheep"\r\n\r\n @@ -158,7 +158,7 @@ def test_delete_file_with_nonexistent_file(self, mock_from_service_account_file) batch_response = b'''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 404 NOT FOUND Content-Type: application/json @@ -182,7 +182,7 @@ def test_delete_file_with_nonexistent_file(self, mock_from_service_account_file) --batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 204 OK ETag: "etag/sheep"\r\n\r\n @@ -482,7 +482,7 @@ def test_comment_files_success(self, mock_from_service_account_file): # pylint: batch_response = b'''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 204 OK ETag: "etag/pony"\r\n\r\n{"id": "fake-comment-id0"} @@ -490,7 +490,7 @@ def test_comment_files_success(self, mock_from_service_account_file): # pylint: --batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 204 OK ETag: "etag/sheep"\r\n\r\n{"id": "fake-comment-id1"} @@ -524,7 +524,7 @@ def test_comment_files_batching_retries(self, mock_from_service_account_file): '''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 204 OK ETag: "etag/pony{idx}"\r\n\r\n{{"id": "fake-comment-id{idx}"}} @@ -536,7 +536,7 @@ def test_comment_files_batching_retries(self, mock_from_service_account_file): '''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 204 OK ETag: "etag/pony{idx}"\r\n\r\n{{"id": "fake-comment-id{idx}"}} @@ -548,7 +548,7 @@ def test_comment_files_batching_retries(self, mock_from_service_account_file): '''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 500 Internal Server Error ETag: "etag/pony{idx}"\r\n\r\n @@ -560,7 +560,7 @@ def test_comment_files_batching_retries(self, mock_from_service_account_file): '''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 204 OK ETag: "etag/pony{idx}"\r\n\r\n{{"id": "fake-comment-id{idx}"}} @@ -600,7 +600,7 @@ def test_comment_files_with_nonexistent_file(self, mock_from_service_account_fil batch_response = b'''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 404 NOT FOUND Content-Type: application/json @@ -624,7 +624,7 @@ def test_comment_files_with_nonexistent_file(self, mock_from_service_account_fil --batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 204 OK ETag: "etag/sheep"\r\n\r\n{"id": "fake-comment-id1"} @@ -671,7 +671,7 @@ def test_list_permissions_success(self, mock_from_service_account_file): # pyli batch_response = b'''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 200 OK Content-Type: application/json @@ -680,7 +680,7 @@ def test_list_permissions_success(self, mock_from_service_account_file): # pyli --batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 200 OK Content-Type: application/json @@ -712,7 +712,7 @@ def test_list_permissions_one_failure(self, mock_from_service_account_file): # batch_response = b'''--batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 200 OK Content-Type: application/json @@ -721,7 +721,7 @@ def test_list_permissions_one_failure(self, mock_from_service_account_file): # --batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 200 OK Content-Type: application/json @@ -730,7 +730,7 @@ def test_list_permissions_one_failure(self, mock_from_service_account_file): # --batch_foobarbaz Content-Type: application/http Content-Transfer-Encoding: binary -Content-ID: +Content-ID: HTTP/1.1 404 NOT FOUND Content-Type: application/json From f5546cfe52a320b15037eea35db4f04a838fa079 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Wed, 20 Sep 2023 16:14:30 -0400 Subject: [PATCH 3/3] fix: Pin google-api-python-client under 2.0 google-api-python-client >2.0 fails tests --- requirements/base.txt | 9 ++++++--- requirements/constraints.txt | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index f71a548c..0347f5e7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -73,8 +73,10 @@ gitpython==3.1.37 # via -r requirements/base.in google-api-core==2.12.0 # via google-api-python-client -google-api-python-client==2.101.0 - # via -r requirements/base.in +google-api-python-client==1.12.11 + # via + # -c requirements/constraints.txt + # -r requirements/base.in google-auth==2.23.0 # via # google-api-core @@ -209,6 +211,7 @@ simplejson==3.19.1 six==1.16.0 # via # -r requirements/base.in + # google-api-python-client # isodate # jenkinsapi # kubernetes @@ -233,7 +236,7 @@ typing-extensions==4.8.0 # edx-opaque-keys unicodecsv==0.14.1 # via -r requirements/base.in -uritemplate==4.1.1 +uritemplate==3.0.1 # via google-api-python-client urllib3==1.26.16 # via diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 350284d5..39a7b42f 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -15,3 +15,4 @@ responses<0.21.0 django<4.0 setuptools<60 +google-api-python-client<2