From 60da8e37c111803c2b89a3f7ce4b8b38e62b505f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 16 May 2024 11:50:59 -0600 Subject: [PATCH 1/4] Upgrade WGCW image to Ubuntu 22.04 (#773) * Update WGCW image to Ubuntu 22.04 base * Update bake file * Fix WGCW tests --- Justfile | 3 ++- docker-bake.hcl | 4 +-- workbench-for-google-cloud-workstations/.env | 12 ++++----- ...rfile.ubuntu2004 => Dockerfile.ubuntu2204} | 26 +++++++++---------- .../conf/launcher-env | 2 +- .../conf/repos.conf | 4 +-- .../test/goss.yaml | 7 ++--- 7 files changed, 28 insertions(+), 30 deletions(-) rename workbench-for-google-cloud-workstations/{Dockerfile.ubuntu2004 => Dockerfile.ubuntu2204} (92%) diff --git a/Justfile b/Justfile index 656013a4..4386afdb 100644 --- a/Justfile +++ b/Justfile @@ -87,7 +87,8 @@ preview-plan branch="$(git branch --show-current)": # just test workbench test target="default" file="docker-bake.hcl": - python3 {{justfile_directory()}}/tools/test_bake_artifacts.py --target "{{target}}" --file "{{file}}" + GIT_SHA=$(git rev-parse --short HEAD) \ + python3 {{justfile_directory()}}/tools/test_bake_artifacts.py --target "{{target}}" --file "{{file}}" # just preview-test connect dev preview-test target="default" branch="$(git branch --show-current)": diff --git a/docker-bake.hcl b/docker-bake.hcl index 11f07f1e..ea7865cb 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -183,7 +183,7 @@ variable WORKBENCH_BUILD_MATRIX { variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { default = { builds = [ - {os = "ubuntu2004", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.7", py_alternate = "3.10.13"}, + {os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, ] } } @@ -479,7 +479,7 @@ target "workbench-for-google-cloud-workstations" { DRIVERS_VERSION = get_drivers_version(builds.os) RSW_VERSION = WORKBENCH_VERSION RSW_NAME = "rstudio-workbench" - RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/focal/amd64" + RSW_DOWNLOAD_URL = "https://download2.rstudio.org/server/jammy/amd64" } } diff --git a/workbench-for-google-cloud-workstations/.env b/workbench-for-google-cloud-workstations/.env index c9622919..c5c7cb4f 100644 --- a/workbench-for-google-cloud-workstations/.env +++ b/workbench-for-google-cloud-workstations/.env @@ -1,12 +1,12 @@ RSW_VERSION=2024.04.1+748.pro2 RSW_TAG_VERSION=2024.04.1-748.pro2 -RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/focal/amd64 +RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 RSW_NAME=rstudio-workbench -PYTHON_VERSION=3.10.13 -PYTHON_VERSION_ALT=3.9.18 -PYTHON_VERSION_JUPYTER=3.10.13 -R_VERSION=4.2.3 -R_VERSION_ALT=4.1.3 +PYTHON_VERSION=3.11.9 +PYTHON_VERSION_ALT=3.10.14 +PYTHON_VERSION_JUPYTER=3.10.14 +R_VERSION=4.4.0 +R_VERSION_ALT=4.3.3 DRIVERS_VERSION=2023.05.0 QUARTO_VERSION=1.3.340 IMAGE_REGISTRY_NAME=us-central1-docker.pkg.dev/posit-images/cloud-workstations/workbench diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 similarity index 92% rename from workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 rename to workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 index 65250eca..4398738b 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2004 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 @@ -1,17 +1,17 @@ -FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:last-ubuntu2004 as build +FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest as build ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive -ARG R_VERSION=4.2.3 -ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.11.7 -ARG PYTHON_VERSION_ALT=3.10.13 -ARG PYTHON_VERSION_JUPYTER=3.10.13 +ARG R_VERSION=4.4.0 +ARG R_VERSION_ALT=4.3.3 +ARG PYTHON_VERSION=3.11.9 +ARG PYTHON_VERSION_ALT=3.10.14 +ARG PYTHON_VERSION_JUPYTER=3.10.14 ARG JUPYTERLAB_VERSION=3.6.7 ARG DRIVERS_VERSION=2023.05.0 ARG RSW_VERSION=2024.04.1+748.pro2 ARG RSW_NAME=rstudio-workbench -ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/focal/amd64 +ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 ARG SCRIPTS_DIR=/opt/positscripts ENV STARTUP_DEBUG_MODE 0 @@ -40,8 +40,8 @@ RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyri && rm -rf /var/lib/apt/lists/* ### Install R versions ### -RUN curl -O https://cdn.rstudio.com/r/ubuntu-2004/pkgs/r-${R_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/r/ubuntu-2004/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \ +RUN curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION}_1_amd64.deb \ + && curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \ && apt-get update \ && apt-get install -yq --no-install-recommends ./r-${R_VERSION}_1_amd64.deb \ && apt-get install -yq --no-install-recommends ./r-${R_VERSION_ALT}_1_amd64.deb \ @@ -52,8 +52,8 @@ RUN curl -O https://cdn.rstudio.com/r/ubuntu-2004/pkgs/r-${R_VERSION}_1_amd64.de && rm -rf /var/lib/apt/lists/* ### Install Python versions ### -RUN curl -O https://cdn.rstudio.com/python/ubuntu-2004/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/python/ubuntu-2004/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \ +RUN curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \ + && curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \ && apt-get update \ && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION}_1_amd64.deb \ && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION_ALT}_1_amd64.deb \ @@ -95,7 +95,7 @@ RUN apt-get update \ && rm -f ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ && rm -rf /var/lib/apt/lists/* \ && cp /opt/rstudio-drivers/odbcinst.ini.sample /etc/odbcinst.ini \ - && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/focal/latest")' + && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/jammy/latest")' ### Install Workbench ### SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -117,7 +117,7 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION # TODO(ianpittwood): Replace monitor download with $RSW_VERSION after upgrading to 2023.06.0 RUN mkdir -p /opt/rstudio-license/ \ && mkdir -p /var/lib/rstudio-workbench \ - && curl -sL "https://s3.amazonaws.com/rstudio-ide-build/monitor/focal/rsp-monitor-workbench-gcpw-amd64-2023.06.0-419.pro1.tar.gz" | \ + && curl -sL "https://s3.amazonaws.com/rstudio-ide-build/monitor/jammy/rsp-monitor-workbench-gcpw-amd64-2023.06.0-419.pro1.tar.gz" | \ tar xzvf - --strip 2 -C /opt/rstudio-license/ \ && chmod 0755 /opt/rstudio-license/license-manager \ && mv /opt/rstudio-license/license-manager /opt/rstudio-license/license-manager-orig \ diff --git a/workbench-for-google-cloud-workstations/conf/launcher-env b/workbench-for-google-cloud-workstations/conf/launcher-env index 4a53e727..40f05ea8 100644 --- a/workbench-for-google-cloud-workstations/conf/launcher-env +++ b/workbench-for-google-cloud-workstations/conf/launcher-env @@ -3,4 +3,4 @@ Environment: LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 JobType: any -Environment: PATH=/opt/python/3.9.17/bin:$PATH +Environment: PATH=/opt/python/3.10.14/bin:$PATH diff --git a/workbench-for-google-cloud-workstations/conf/repos.conf b/workbench-for-google-cloud-workstations/conf/repos.conf index de4785ea..b8d02900 100644 --- a/workbench-for-google-cloud-workstations/conf/repos.conf +++ b/workbench-for-google-cloud-workstations/conf/repos.conf @@ -1,2 +1,2 @@ -CRAN=https://packagemanager.posit.co/cran/__linux__/focal/latest -RSPM=https://packagemanager.posit.co/cran/__linux__/focal/latest +CRAN=https://packagemanager.posit.co/cran/__linux__/jammy/latest +RSPM=https://packagemanager.posit.co/cran/__linux__/jammy/latest diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index fcb383d7..c55ff6d8 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -1,13 +1,10 @@ user: rstudio-server: exists: true - uid: 999 - gid: 998 group: rstudio-server: exists: true - gid: 998 package: rstudio-server: @@ -155,11 +152,11 @@ file: command: # Check OS release version (early heads up in case the base image is modified) - Check OS version is "focal": + Check OS version is "jammy": exec: "lsb_release -cs" exit-status: 0 stdout: [ - "focal" + "jammy" ] Test Workbench Monitor rstudio-server.log creation: exec: su rstudio-server -c 'touch /var/lib/rstudio-server/monitor/log/rstudio-server.log' From e5a3d812aea3e27f7ac752ab7d2273e5e0da2cf0 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 16 May 2024 13:56:09 -0600 Subject: [PATCH 2/4] Remove unused base images (#774) Bump WAML to R 4.4.0 --- docker-bake.hcl | 4 +--- docker-bake.preview.hcl | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index ea7865cb..34209304 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -96,8 +96,6 @@ function get_tags { variable BASE_BUILD_MATRIX { default = { builds = [ - {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, - {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17"}, {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, {os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, @@ -191,7 +189,7 @@ variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { variable WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX { default = { builds = [ - {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.14", py_alternate = "3.8.15"}, + {os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, ] } } diff --git a/docker-bake.preview.hcl b/docker-bake.preview.hcl index f61f07b5..40286806 100644 --- a/docker-bake.preview.hcl +++ b/docker-bake.preview.hcl @@ -114,8 +114,6 @@ function get_tags { variable BASE_BUILD_MATRIX { default = { builds = [ - {os = "centos7", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, - {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, {os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, ] } From 7a1c227972acab0b594f5a87fcebea61676539ba Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 28 May 2024 13:43:01 -0600 Subject: [PATCH 3/4] Enable user group creation for `rstudio-user` in Workbench image (#780) * Enable user group creation for `rstudio-user` in Workbench image * Fix default UID in script --- workbench/startup.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workbench/startup.sh b/workbench/startup.sh index 8e1b6743..a6b5f592 100644 --- a/workbench/startup.sh +++ b/workbench/startup.sh @@ -66,7 +66,10 @@ else if [ -z "$RSW_TESTUSER" ]; then echo "Empty 'RSW_TESTUSER' variables, not creating test user"; else - useradd -m -s /bin/bash -N -u $RSW_TESTUSER_UID $RSW_TESTUSER + if [ -z "$RSW_TESTUSER_UID" ]; then + RSW_TESTUSER_UID=10000 + fi + useradd -m -s /bin/bash -u $RSW_TESTUSER_UID -U $RSW_TESTUSER echo "$RSW_TESTUSER:$RSW_TESTUSER_PASSWD" | sudo chpasswd fi fi From c69133c4bb3487edf05c9f2a77861e89607b9f0e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 29 May 2024 07:58:43 -0600 Subject: [PATCH 4/4] Fix service image Python 3.12+ issues (#782) * Use `ensurepip` to make sure pip bundled with 3.12 is compatible * Add `ensurepip --upgrade` to install_python.sh * Downgrade WAML due to Azure ML Python 3.12 incompatibility --- docker-bake.hcl | 2 +- product/base/scripts/ubuntu/install_python.sh | 1 + workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 34209304..f9169170 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -189,7 +189,7 @@ variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { variable WORKBENCH_MICROSOFT_AZURE_ML_BUILD_MATRIX { default = { builds = [ - {os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, + {os = "ubuntu2204", r_primary = "4.2.3", r_alternate = "4.1.3", py_primary = "3.9.17", py_alternate = "3.8.17"}, ] } } diff --git a/product/base/scripts/ubuntu/install_python.sh b/product/base/scripts/ubuntu/install_python.sh index f96d783a..13515ef7 100755 --- a/product/base/scripts/ubuntu/install_python.sh +++ b/product/base/scripts/ubuntu/install_python.sh @@ -94,6 +94,7 @@ install_python() { apt-get install $APT_ARGS "/tmp/python-${PYTHON_VERSION}.deb" rm "/tmp/python-${PYTHON_VERSION}.deb" # Upgrade pip and setuptools to latest version + $PYTHON_BIN -m ensurepip --upgrade $PYTHON_BIN -m pip install -U setuptools $PYTHON_BIN -m pip install -U pip } diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 index 4398738b..9675ac76 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 @@ -59,10 +59,12 @@ RUN curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERS && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION_ALT}_1_amd64.deb \ && rm -rf python-${PYTHON_VERSION}_1_amd64.deb \ && rm -rf python-${PYTHON_VERSION_ALT}_1_amd64.deb \ + && /opt/python/${PYTHON_VERSION}/bin/python3 -m ensurepip --upgrade \ && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \ && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \ && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade pip \ && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ + && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m ensurepip --upgrade \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \ && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade pip \