Python 3.9 update

.github/workflows/build_worker_images.yml

@@ -23,7 +23,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip

.github/workflows/postcommit.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip

.github/workflows/stress.yml

@@ -10,7 +10,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - run: pip install -e .
       - name: Run tests
         run: python tests/stress/stress_test.py --instance https://worksheets-dev.codalab.org --heavy

.github/workflows/test.yml

@@ -13,10 +13,10 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 14.x
-      - name: Set up Python 3.7
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - run: npm ci
         working-directory: ./frontend
       - run: npm run check-ci
@@ -36,7 +36,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.9]
         os: [ubuntu-20.04, macos-latest]
     steps:
       - uses: actions/checkout@v3
@@ -78,7 +78,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -128,7 +128,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -198,7 +198,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -247,7 +247,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -303,7 +303,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -352,7 +352,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -401,7 +401,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -455,7 +455,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -521,7 +521,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -571,7 +571,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip

docker_config/dockerfiles/Dockerfile.server

@@ -7,27 +7,27 @@ RUN apt-get update && \
   apt-get install -y --no-install-recommends software-properties-common && \
   apt-get update && \
   apt-get install -y --no-install-recommends \
-    build-essential \
-    curl \
-    git \
-    libfuse-dev \
-    libjpeg-dev \
-    libmysqlclient-dev \
-    libssl-dev \
-    mysql-client \
-    zip \
-    unzip \
-    rsync && \
-    rm -rf /var/lib/apt/lists/*;
+  build-essential \
+  curl \
+  git \
+  libfuse-dev \
+  libjpeg-dev \
+  libmysqlclient-dev \
+  libssl-dev \
+  mysql-client \
+  zip \
+  unzip \
+  rsync && \
+  rm -rf /var/lib/apt/lists/*;
 
 ### Without this Python thinks we're ASCII and unicode chars fail
 ENV LANG C.UTF-8
 
-# Miniconda 4.5.11 installs Python 3.7 by default.
-RUN curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
-     chmod +x ~/miniconda.sh && \
-     ~/miniconda.sh -b -p /opt/conda && \
-     rm ~/miniconda.sh
+# Python 3.9 Version of miniconda 23.5.2
+RUN curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-py39_23.5.2-0-Linux-x86_64.sh && \
+  chmod +x ~/miniconda.sh && \
+  ~/miniconda.sh -b -p /opt/conda && \
+  rm ~/miniconda.sh
 ENV PATH /opt/conda/bin:$PATH
 RUN conda --version
 

docker_config/dockerfiles/Dockerfile.worker

@@ -6,25 +6,25 @@ RUN apt-get update && \
   apt-get install -y --no-install-recommends software-properties-common curl && \
   apt-get update && \
   apt-get install -y --no-install-recommends \
-    amazon-ecr-credential-helper \
-    build-essential \
-    libssl-dev \
-    uuid-dev \
-    libgpgme11-dev \
-    squashfs-tools \
-    libseccomp-dev \
-    pkg-config \
-    zip \ 
-    git \
-    wget \
-    unzip && \
-    rm -rf /var/lib/apt/lists/*;
+  amazon-ecr-credential-helper \
+  build-essential \
+  libssl-dev \
+  uuid-dev \
+  libgpgme11-dev \
+  squashfs-tools \
+  libseccomp-dev \
+  pkg-config \
+  zip \ 
+  git \
+  wget \
+  unzip && \
+  rm -rf /var/lib/apt/lists/*;
 
-# Miniconda 4.5.11 installs Python 3.7 by default.
-RUN curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
-     chmod +x ~/miniconda.sh && \
-     ~/miniconda.sh -b -p /opt/conda && \
-     rm ~/miniconda.sh
+# Python 3.9 Version of miniconda 23.5.2
+RUN curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-py39_23.5.2-0-Linux-x86_64.sh && \
+  chmod +x ~/miniconda.sh && \
+  ~/miniconda.sh -b -p /opt/conda && \
+  rm ~/miniconda.sh
 ENV PATH /opt/conda/bin:$PATH
 RUN conda --version
 
@@ -38,8 +38,8 @@ RUN mkdir ${WORKDIR}/scripts
 # Install dependencies
 COPY requirements.txt requirements.txt
 RUN python3 -m pip install --user --upgrade pip==20.3.4; \
-    python3 -m pip install setuptools --upgrade; \
-    python3 -m pip install --no-cache-dir -r requirements.txt;
+  python3 -m pip install setuptools --upgrade; \
+  python3 -m pip install --no-cache-dir -r requirements.txt;
 
 # Install code
 COPY setup.py setup.py

requirements.txt

@@ -1,24 +1,23 @@
-apache-beam[azure]==2.38.0
+apache-beam[azure]==2.48.0
 
 # GCP requirements for apache-beam. Note: when upgrading
 # apache-beam, make sure these requirements are also upgraded
 # accordingly according to Beam's GCP requirements
 # (https://github.com/apache/beam/blob/master/sdks/python/setup.py).
-cachetools>=3.1.0,<5
+cachetools>=3.1.0,<6
 google-apitools>=0.5.31,<0.5.32
 google-auth>=1.18.0,<3
 
 # Remaining requirements
 bottle==0.12.20
-dataclasses==0.7;python_version<'3.7'
 docker==4.3.0
 marshmallow-jsonapi==0.15.1
 marshmallow==2.15.1
 setuptools>=40.0.0
 argcomplete==1.12.3
 indexed_gzip==1.7.0
 ratarmountcore==0.1.3
-PyYAML==5.4
+PyYAML==6.0.1
 psutil==5.7.2
 six==1.15.0
 SQLAlchemy==1.3.19
@@ -31,7 +30,7 @@ diffimg==0.2.3
 selenium==3.141.0
 requests==2.25.0
 azure-storage-blob==12.4.0
-azure-core==1.8.0
+azure-core==1.29.5
 sentry-sdk==1.14.0
 requests-oauthlib==1.1.0
 oauthlib==2.1.0

setup.py

@@ -82,11 +82,11 @@ def get_requirements(*requirements_file_paths):
     packages=find_packages(exclude=["tests*"]),
     classifiers=[
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.9",
         "License :: OSI Approved :: Apache Software License",
     ],
     py_modules=['codalab_service'],
-    python_requires='~=3.6',
+    python_requires='~=3.9',
     cmdclass={'install': Install},
     include_package_data=True,
     install_requires=get_requirements('requirements.txt'),