From 5a9e2041a1032648560bcb46b421ba3f42df8b65 Mon Sep 17 00:00:00 2001 From: Jerry Morrison <1fish2@users.noreply.github.com> Date: Fri, 7 Apr 2023 21:09:45 -0700 Subject: [PATCH] v0.12.0: newer Python and pips --- borealis/docker_task.py | 3 +- borealis/setup/how-to-install-gce-server.txt | 10 +- borealis/setup/requirements.txt | 104 +++++++++---------- docs/changes.md | 3 + docs/developer-setup.md | 2 +- docs/docker-build.md | 4 +- docs/install-tools.md | 10 +- requirements.txt | 24 +++-- setup.py | 7 +- 9 files changed, 88 insertions(+), 79 deletions(-) diff --git a/borealis/docker_task.py b/borealis/docker_task.py index c7b0337..7e56b84 100644 --- a/borealis/docker_task.py +++ b/borealis/docker_task.py @@ -282,7 +282,8 @@ def _terminate(self, container, logger, reason, terminated): and the Docker client. NOTE: "The KeyboardInterrupt exception will be received by an arbitrary - thread." -- https://docs.python.org/3.8/library/_thread.html + thread. (When the signal module is available, interrupts always go to + the main thread.)" -- https://docs.python.org/3.8/library/_thread.html """ name = self['name'] logger.debug('Terminating task {} for {}...'.format(name, reason)) diff --git a/borealis/setup/how-to-install-gce-server.txt b/borealis/setup/how-to-install-gce-server.txt index 68e7a7c..a5d1a47 100644 --- a/borealis/setup/how-to-install-gce-server.txt +++ b/borealis/setup/how-to-install-gce-server.txt @@ -107,7 +107,7 @@ echo y | gcloud auth configure-docker ## level Image that's common to the team (containing runtime components like ## Linux apts, Python, and pips), then preinstalling the runtime Image will ## save Docker Image build time as well as task startup time. -### Example: docker pull python:3.8.9 +### Example: docker pull python:3.11.3 ### Example: docker pull gcr.io/$(gcloud config get-value core/project)/my-app-runtime @@ -138,10 +138,10 @@ sudo su -l fireworker mkdir ~/borealis && cd ~/borealis -pyenv install 3.8.9 +pyenv install 3.11.3 -pyenv global 3.8.9 -pyenv local 3.8.9 +pyenv global 3.11.3 +pyenv local 3.11.3 pip install --upgrade pip setuptools virtualenv virtualenvwrapper virtualenv-clone wheel pyenv virtualenv fireworker @@ -262,7 +262,7 @@ sudo systemctl stop borealis-fireworker sudo journalctl --vacuum-time=2d ## Pull any Docker images that you want to preload for speed. -docker pull python:3.8.9 # or similar image(s) including your project's runtime Image +docker pull python:3.11.3 # or similar image(s) including your project's runtime Image ## Prune Docker images: docker image prune diff --git a/borealis/setup/requirements.txt b/borealis/setup/requirements.txt index 80acbe0..2e52d4e 100644 --- a/borealis/setup/requirements.txt +++ b/borealis/setup/requirements.txt @@ -1,57 +1,57 @@ -# Direct requirements -# dnspython is needed for pymongo to access server clusters. -dnspython==2.1.0 -docker==5.0.0 -FireWorks==1.9.7 -google-cloud-logging==2.3.1 -google-cloud-storage==1.37.1 -requests==2.25.1 -ruamel.yaml==0.17.4 -ruamel.yaml.clib==0.2.2 +# Direct requirements with frozen versions for reproducible testing. +# FireWorks requires pymongo; dnspython enables pymongo to access server clusters. +dnspython==2.3.0 +docker==6.0.1 +FireWorks==2.0.3 +google-cloud-logging==3.5.0 +google-cloud-storage==2.8.0 +requests==2.28.2 +ruamel.yaml==0.17.21 +ruamel.yaml.clib==0.2.7 -# Indirect requirements -appdirs==1.4.4 -cachetools==4.2.1 -certifi==2020.12.5 -cffi==1.14.5 -chardet==4.0.0 -click==7.1.2 -distlib==0.3.1 -filelock==3.0.12 -Flask==1.1.2 -flask-paginate==0.8.1 -google-api-core==1.26.3 -google-auth==1.29.0 -google-cloud-core==1.6.0 -google-crc32c==1.1.2 -google-resumable-media==1.2.0 -googleapis-common-protos==1.53.0 -grpcio==1.37.0 +# Indirect requirements added by pip freeze. +cachetools==5.3.0 +certifi==2022.12.7 +charset-normalizer==3.1.0 +click==8.1.3 +distlib==0.3.6 +filelock==3.11.0 +Flask==2.2.3 +flask-paginate==2022.1.8 +google-api-core==2.11.0 +google-auth==2.17.2 +google-cloud-appengine-logging==1.3.0 +google-cloud-audit-log==0.2.5 +google-cloud-core==2.3.2 +google-crc32c==1.5.0 +google-resumable-media==2.4.1 +googleapis-common-protos==1.59.0 +grpc-google-iam-v1==0.12.6 +grpcio==1.53.0 +grpcio-status==1.53.0 gunicorn==20.1.0 -idna==2.10 -itsdangerous==1.1.0 -Jinja2==2.11.3 -MarkupSafe==1.1.1 -monty==2021.3.3 -packaging==20.9 -pbr==5.5.1 -proto-plus==1.18.1 -protobuf==3.15.8 +idna==3.4 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.2 +monty==2022.9.9 +packaging==23.0 +pbr==5.11.1 +platformdirs==3.2.0 +proto-plus==1.22.2 +protobuf==4.22.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 -pycparser==2.20 -pymongo==3.11.3 -pyparsing==2.4.7 -python-dateutil==2.8.1 -pytz==2021.1 -rsa==4.7.2 -six==1.15.0 -stevedore==3.3.0 -tabulate==0.8.9 -tqdm==4.60.0 -urllib3==1.26.5 -virtualenv==20.4.3 -virtualenv-clone==0.5.4 +pymongo==4.3.3 +python-dateutil==2.8.2 +rsa==4.9 +six==1.16.0 +stevedore==5.0.0 +tabulate==0.9.0 +tqdm==4.65.0 +urllib3==1.26.15 +virtualenv==20.21.0 +virtualenv-clone==0.5.7 virtualenvwrapper==4.8.4 -websocket-client==0.58.0 -Werkzeug==1.0.1 +websocket-client==1.5.1 +Werkzeug==2.2.3 diff --git a/docs/changes.md b/docs/changes.md index 6a6d71e..f8b630b 100644 --- a/docs/changes.md +++ b/docs/changes.md @@ -1,5 +1,8 @@ # Change Log +## v0.12.0 +* Update the installation instructions to use Python 3.11.3 and newer pip libraries. + ## v0.11.0 * In the `gce create` CLI command, implement LaunchPad config `uri_mode` by setting the `host`, `uri_mode`, and `db` metadata fields. diff --git a/docs/developer-setup.md b/docs/developer-setup.md index 68f34a0..1265e4c 100644 --- a/docs/developer-setup.md +++ b/docs/developer-setup.md @@ -126,7 +126,7 @@ statement to your shell `.profile` or `.bash_profile` file: Then run that `export` command or create a new shell. -1. In a fresh development directory, install Python 3.8, create a Python +1. In a fresh development directory, install Python 3.11, create a Python virtual environment, and install the borealis-fireworks pip. 1. Create or augment your `my_launchpad.yaml` file: diff --git a/docs/docker-build.md b/docs/docker-build.md index bf1e753..091458b 100644 --- a/docs/docker-build.md +++ b/docs/docker-build.md @@ -13,7 +13,7 @@ First write a `Dockerfile` in your project directory. Example: # The ARG lets you pick another base Image but DON'T USE AN ALPINE base since # its floating point library produces different results; also # see https://pythonspeed.com/articles/alpine-docker-python/ -ARG from=python:3.8.6 +ARG from=python:3.8.16 FROM ${from} # Install Linux packages. @@ -59,7 +59,7 @@ docker build --tag application101 . You can base it on a different "FROM" Image like this: ```shell script -docker build --build-arg from=python:3.8.1 --tag application101 . +docker build --build-arg from=python:3.11.3 --tag application101 . ``` but **don't use an Alpine base image** because its floating point library produces diff --git a/docs/install-tools.md b/docs/install-tools.md index 03dce09..176bbf2 100644 --- a/docs/install-tools.md +++ b/docs/install-tools.md @@ -7,12 +7,12 @@ Also see [Handy Links](handy-links.md). 1. Install Python and Fireworks: - * Install Python: `pyenv install 3.8.6` + * Install Python: `pyenv install 3.11.3` * Install a Python virtual environment manager like `virtualenv`, `pyenv-virtualenv`, and `pyenv-virtualenvwrapper`; or `venv` * Create a Python virtual environment in your project directory, e.g.: ```shell script - pyenv local 3.8.6 + pyenv local 3.11.3 pyenv virtualenv myproject pyenv local myproject ``` @@ -27,21 +27,21 @@ Also see [Handy Links](handy-links.md). 1. Setup Python for the `gcloud` and `gsutil` tools. - **Note:** `gsutil -m` is broken in Python 3.8 so use Python 3.6 or 3.7. + **Note:** `gsutil -m` is still broken in Python 3.8+ so use Python 3.6 or 3.7. ([gsutil issue #961](https://github.com/GoogleCloudPlatform/gsutil/issues/961).) Install Python 3.6 or 3.7 if you don't already have one of them, e.g. using `pyenv`: ```shell script - pyenv install 3.7.9 + pyenv install 3.7.16 ``` Set `$CLOUDSDK_PYTHON` in your shell profile: ```shell script # Set the Python version for Cloud SDK. - export CLOUDSDK_PYTHON=$(pyenv shell 3.7.9; pyenv which python) + export CLOUDSDK_PYTHON=$(pyenv shell 3.7.16; pyenv which python) ``` Then open a new shell (or run this `export` command). diff --git a/requirements.txt b/requirements.txt index 9fc87cd..22c177a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,21 @@ ## Create the "fireworker" python virtual environment for development, and select it in this directory: -# pyenv install 3.8.9 +# pyenv install 3.11.3 # -# pyenv virtualenv 3.8.9 fireworker && pyenv local fireworker +# pyenv virtualenv 3.11.3 fireworker && pyenv local fireworker # pip install --upgrade pip setuptools virtualenv virtualenvwrapper virtualenv-clone wheel # pip install -r requirements.txt && pyenv rehash -# Direct dependents -# dnspython is needed for pymongo to access server clusters. -dnspython==2.1.0 -google-cloud-logging>=2.3.1 -google-cloud-storage>=1.37.1 -docker>=5.0.0 -FireWorks>=1.9.6 -requests>=2.25.1 -ruamel.yaml>=0.17.4 +# Direct requirements. +# FireWorks requires pymongo; dnspython enables pymongo to access server clusters. +# The binary extension ruamel.yaml.clib helps ruamel.yaml run faster, right? +dnspython +google-cloud-logging +google-cloud-storage +docker +FireWorks +requests +ruamel.yaml +ruamel.yaml.clib # Dev tools twine diff --git a/setup.py b/setup.py index 56a7a92..9aa5330 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ setup( name='borealis-fireworks', - version='0.11.0', + version='0.12.0', packages=['borealis', 'borealis.util'], url='https://github.com/CovertLab/borealis', project_urls={ @@ -34,7 +34,7 @@ 'google-cloud-logging>=2.0.0', 'google-cloud-storage>=1.28.0', 'docker>=4.1.0', - 'FireWorks>=1.9.5', + 'FireWorks>=1.9.7', 'requests>=2.22.0', 'ruamel.yaml>=0.16.9', ], @@ -58,6 +58,9 @@ 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ], keywords='fireworks workflow', )