From fbb08885b389372eddf8d8c2686e7e74b498eabf Mon Sep 17 00:00:00 2001 From: Steve Worley Date: Mon, 7 Aug 2023 10:36:34 +1000 Subject: [PATCH] [DDS-1576]: Ansible builder 3 (#195) --- .circleci/config.yml | 12 +- .gitignore | 1 + images/awx-ee/.gitignore | 1 - images/awx-ee/bindep.txt | 2 +- images/awx-ee/context/Dockerfile | 99 ----- images/awx-ee/context/_build/bindep.txt | 24 -- .../awx-ee/context/_build/bindep_combined.txt | 25 -- images/awx-ee/context/_build/requirements.txt | 9 - images/awx-ee/context/_build/requirements.yml | 15 - .../context/_build/requirements_combined.txt | 11 - images/awx-ee/context/_build/scripts/assemble | 171 -------- .../context/_build/scripts/check_ansible | 110 ----- .../context/_build/scripts/check_galaxy | 46 -- .../awx-ee/context/_build/scripts/entrypoint | 152 ------- .../_build/scripts/install-from-bindep | 107 ----- .../context/_build/scripts/introspect.py | 400 ------------------ images/awx-ee/context/azure-cli.repo | 6 - images/awx-ee/docker-bake.hcl | 25 ++ images/awx-ee/execution-environment.yml | 55 ++- images/awx-ee/requirements.txt | 10 +- 20 files changed, 85 insertions(+), 1196 deletions(-) create mode 100644 .gitignore delete mode 100644 images/awx-ee/.gitignore delete mode 100644 images/awx-ee/context/Dockerfile delete mode 100644 images/awx-ee/context/_build/bindep.txt delete mode 100644 images/awx-ee/context/_build/bindep_combined.txt delete mode 100644 images/awx-ee/context/_build/requirements.txt delete mode 100644 images/awx-ee/context/_build/requirements.yml delete mode 100644 images/awx-ee/context/_build/requirements_combined.txt delete mode 100755 images/awx-ee/context/_build/scripts/assemble delete mode 100755 images/awx-ee/context/_build/scripts/check_ansible delete mode 100755 images/awx-ee/context/_build/scripts/check_galaxy delete mode 100755 images/awx-ee/context/_build/scripts/entrypoint delete mode 100755 images/awx-ee/context/_build/scripts/install-from-bindep delete mode 100644 images/awx-ee/context/_build/scripts/introspect.py delete mode 100644 images/awx-ee/context/azure-cli.repo create mode 100644 images/awx-ee/docker-bake.hcl diff --git a/.circleci/config.yml b/.circleci/config.yml index a7656cbd1..5baa7ed08 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,14 +120,10 @@ jobs: IMAGE_TAG=$(.circleci/scripts/tag.sh) docker buildx bake -f bake.hcl --push --no-cache echo "==> Push the AWX Executor Environment image" pip install --upgrade ansible-builder - ansible-builder build \ - --context=images/awx-ee/context \ - --tag ghcr.io/dpc-sdp/bay/awx-ee:$(.circleci/scripts/tag.sh) \ - --tag singledigital/awx-ee:$(.circleci/scripts/tag.sh) \ - --container-runtime docker \ - -f images/awx-ee/execution-environment.yml - docker push singledigital/awx-ee:$(.circleci/scripts/tag.sh) - docker push ghcr.io/dpc-sdp/bay/awx-ee:$(.circleci/scripts/tag.sh) + cd images/awx-ee + ansible-builder create -f execution-environment.yml + docker buildx create --name sdp-amd-arm --platform linux/amd64,linux/arm64 --use + docker buildx bake --progress=plain --push --provenance false fi workflows: diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..50a93e55c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*/awx-ee/context \ No newline at end of file diff --git a/images/awx-ee/.gitignore b/images/awx-ee/.gitignore deleted file mode 100644 index 722e59f54..000000000 --- a/images/awx-ee/.gitignore +++ /dev/null @@ -1 +0,0 @@ -builder diff --git a/images/awx-ee/bindep.txt b/images/awx-ee/bindep.txt index f952ee947..166af1c31 100644 --- a/images/awx-ee/bindep.txt +++ b/images/awx-ee/bindep.txt @@ -1,4 +1,4 @@ -php ==7.4 +php php-cli php-json php-pdo diff --git a/images/awx-ee/context/Dockerfile b/images/awx-ee/context/Dockerfile deleted file mode 100644 index 6a00e24c3..000000000 --- a/images/awx-ee/context/Dockerfile +++ /dev/null @@ -1,99 +0,0 @@ -ARG EE_BASE_IMAGE="quay.io/ansible/ansible-runner:latest" -ARG EE_BUILDER_IMAGE="quay.io/ansible/ansible-builder:latest" -ARG PYCMD="/usr/bin/python3" -ARG PKGMGR_PRESERVE_CACHE="" -ARG ANSIBLE_GALAXY_CLI_COLLECTION_OPTS="" -ARG ANSIBLE_GALAXY_CLI_ROLE_OPTS="" - -# Base build stage -FROM $EE_BASE_IMAGE as base -USER root -ARG EE_BASE_IMAGE -ARG EE_BUILDER_IMAGE -ARG PYCMD -ARG PKGMGR_PRESERVE_CACHE -ARG ANSIBLE_GALAXY_CLI_COLLECTION_OPTS -ARG ANSIBLE_GALAXY_CLI_ROLE_OPTS - -RUN $PYCMD -m ensurepip -COPY _build/scripts/ /output/scripts/ -COPY _build/scripts/entrypoint /opt/builder/bin/entrypoint - -# Galaxy build stage -FROM base as galaxy -ARG EE_BASE_IMAGE -ARG EE_BUILDER_IMAGE -ARG PYCMD -ARG PKGMGR_PRESERVE_CACHE -ARG ANSIBLE_GALAXY_CLI_COLLECTION_OPTS -ARG ANSIBLE_GALAXY_CLI_ROLE_OPTS - -RUN /output/scripts/check_galaxy -COPY _build /build -WORKDIR /build - -RUN ansible-galaxy role install $ANSIBLE_GALAXY_CLI_ROLE_OPTS -r requirements.yml --roles-path "/usr/share/ansible/roles" -RUN ANSIBLE_GALAXY_DISABLE_GPG_VERIFY=1 ansible-galaxy collection install $ANSIBLE_GALAXY_CLI_COLLECTION_OPTS -r requirements.yml --collections-path "/usr/share/ansible/collections" - -# Builder build stage -FROM base as builder -WORKDIR /build -ARG EE_BASE_IMAGE -ARG EE_BUILDER_IMAGE -ARG PYCMD -ARG PKGMGR_PRESERVE_CACHE -ARG ANSIBLE_GALAXY_CLI_COLLECTION_OPTS -ARG ANSIBLE_GALAXY_CLI_ROLE_OPTS - -RUN $PYCMD -m pip install --no-cache-dir bindep pyyaml requirements-parser - -COPY --from=galaxy /usr/share/ansible /usr/share/ansible - -COPY _build/requirements.txt requirements.txt -COPY _build/bindep.txt bindep.txt -RUN $PYCMD /output/scripts/introspect.py introspect --sanitize --user-pip=requirements.txt --user-bindep=bindep.txt --write-bindep=/tmp/src/bindep.txt --write-pip=/tmp/src/requirements.txt -RUN /output/scripts/assemble - -# Final build stage -FROM base as final -ARG EE_BASE_IMAGE -ARG EE_BUILDER_IMAGE -ARG PYCMD -ARG PKGMGR_PRESERVE_CACHE -ARG ANSIBLE_GALAXY_CLI_COLLECTION_OPTS -ARG ANSIBLE_GALAXY_CLI_ROLE_OPTS - -LABEL org.opencontainers.image.authors="Digital Victoria" -LABEL org.opencontainers.image.description="Provides an AWX execution environment image optimised for use with SDP." -LABEL org.opencontainers.image.source="https://github.com/dpc-sdp/bay/blob/5.x/images/awx-ee/context/Dockerfile" -ARG LAGOON_CLI_VERSION=v0.15.4 -ARG NVM_INSTALL_VERSION=v0.39.1 -ARG NVM_DIR=$HOME/.nvm -ARG NODE_VERSION=v14.15.1 -RUN dnf module enable php:7.4 -y - -COPY --from=galaxy /usr/share/ansible /usr/share/ansible - -COPY --from=builder /output/ /output/ -RUN /output/scripts/install-from-bindep && rm -rf /output/wheels -RUN curl -L "https://github.com/uselagoon/lagoon-cli/releases/download/$LAGOON_CLI_VERSION/lagoon-cli-$LAGOON_CLI_VERSION-linux-amd64" -o /usr/local/bin/lagoon -RUN chmod +x /usr/local/bin/lagoon -RUN curl -L "https://github.com/github/hub/releases/download/v2.14.2/hub-linux-amd64-2.14.2.tgz" -o /tmp/hub && tar -xvf /tmp/hub -C /tmp && mv /tmp/hub-linux-amd64-2.14.2/bin/hub /usr/local/bin -RUN chmod +x /usr/local/bin/hub -RUN lagoon config feature --enable-local-dir-check false --force -RUN curl -sS https://getcomposer.org/download/2.3.7/composer.phar --output composer.phar -RUN chmod +x composer.phar -RUN mv composer.phar /usr/local/bin/composer -RUN composer global require szeidler/composer-patches-cli:^1.0 -RUN curl -L https://github.com/itchyny/gojq/releases/download/v0.12.4/gojq_v0.12.4_linux_amd64.tar.gz --output /tmp/gojq_v0.12.4_linux_amd64.tar.gz -RUN tar -C /tmp -xvf /tmp/gojq_v0.12.4_linux_amd64.tar.gz -RUN chmod +x /tmp/gojq_v0.12.4_linux_amd64/gojq -RUN mv /tmp/gojq_v0.12.4_linux_amd64/gojq /usr/local/bin -RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc -ADD azure-cli.repo /etc/yum.repos.d/azure-cli.repo -RUN dnf --assumeyes install azure-cli -RUN az aks install-cli -RUN touch $HOME/.bashrc && chmod +x $HOME/.bashrc -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/$NVM_INSTALL_VERSION/install.sh | bash -RUN rm -rf /output -LABEL ansible-execution-environment=true diff --git a/images/awx-ee/context/_build/bindep.txt b/images/awx-ee/context/_build/bindep.txt deleted file mode 100644 index f952ee947..000000000 --- a/images/awx-ee/context/_build/bindep.txt +++ /dev/null @@ -1,24 +0,0 @@ -php ==7.4 -php-cli -php-json -php-pdo -php-dom -php-gd -php-zip -php-mbstring -php-phar -php-soap - -which -git -git-lfs -patch - -unzip -zip - -ca-certificates -curl -openssl -jq -rsync diff --git a/images/awx-ee/context/_build/bindep_combined.txt b/images/awx-ee/context/_build/bindep_combined.txt deleted file mode 100644 index 237dd1cb5..000000000 --- a/images/awx-ee/context/_build/bindep_combined.txt +++ /dev/null @@ -1,25 +0,0 @@ -gcc-c++ [doc test platform:rpm] # from collection ansible.utils -python3-devel [test platform:rpm] # from collection ansible.utils -python3 [test platform:rpm] # from collection ansible.utils -kubernetes-client [platform:fedora] # from collection kubernetes.core -php ==7.4 # from collection user -php-cli # from collection user -php-json # from collection user -php-pdo # from collection user -php-dom # from collection user -php-gd # from collection user -php-zip # from collection user -php-mbstring # from collection user -php-phar # from collection user -php-soap # from collection user -which # from collection user -git # from collection user -git-lfs # from collection user -patch # from collection user -unzip # from collection user -zip # from collection user -ca-certificates # from collection user -curl # from collection user -openssl # from collection user -jq # from collection user -rsync # from collection user diff --git a/images/awx-ee/context/_build/requirements.txt b/images/awx-ee/context/_build/requirements.txt deleted file mode 100644 index c6ee55285..000000000 --- a/images/awx-ee/context/_build/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -boto>=2.49.0 -botocore>=1.12.249 -boto3>=1.9.249 -requests-oauthlib -ansible-lint -flake8 -yamllint -ruamel.yaml -pygithub \ No newline at end of file diff --git a/images/awx-ee/context/_build/requirements.yml b/images/awx-ee/context/_build/requirements.yml deleted file mode 100644 index 1ce652961..000000000 --- a/images/awx-ee/context/_build/requirements.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -collections: - - ansible.posix - - ansible.utils - - awx.awx - - community.general - - kubernetes.core - - name: lagoon.api - source: https://github.com/salsadigitalauorg/lagoon_ansible_collection.git - version: master - type: git - - name: section.api - source: https://github.com/salsadigitalauorg/section_ansible_collection.git - version: main - type: git diff --git a/images/awx-ee/context/_build/requirements_combined.txt b/images/awx-ee/context/_build/requirements_combined.txt deleted file mode 100644 index fd87f9ad5..000000000 --- a/images/awx-ee/context/_build/requirements_combined.txt +++ /dev/null @@ -1,11 +0,0 @@ -jsonschema==3.2.0 # from collection ansible.utils -textfsm # from collection ansible.utils -ttp # from collection ansible.utils -xmltodict # from collection ansible.utils -kubernetes>=12.0.0 # from collection kubernetes.core -requests-oauthlib # from collection kubernetes.core,user -jsonpatch # from collection kubernetes.core -ansible-builder # from collection user -boto>=2.49.0 # from collection user -botocore>=1.12.249 # from collection user -boto3>=1.9.249 # from collection user \ No newline at end of file diff --git a/images/awx-ee/context/_build/scripts/assemble b/images/awx-ee/context/_build/scripts/assemble deleted file mode 100755 index 65adabe12..000000000 --- a/images/awx-ee/context/_build/scripts/assemble +++ /dev/null @@ -1,171 +0,0 @@ -#!/bin/bash -# Copyright (c) 2019 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Make a list of bindep dependencies and a collection of built binary -# wheels for the repo in question as well as its python dependencies. -# Install javascript tools as well to support python that needs javascript -# at build time. -set -ex - -RELEASE=$(source /etc/os-release; echo $ID) - -# NOTE(pabelanger): Allow users to force either microdnf or dnf as a package -# manager. -PKGMGR="${PKGMGR:-}" -PKGMGR_OPTS="${PKGMGR_OPTS:-}" -PKGMGR_PRESERVE_CACHE="${PKGMGR_PRESERVE_CACHE:-}" - -PYCMD="${PYCMD:=/usr/bin/python3}" -PIPCMD="${PIPCMD:=$PYCMD -m pip}" - -$PYCMD -m ensurepip - -if [ -z $PKGMGR ]; then - # Expect dnf to be installed, however if we find microdnf default to it. - PKGMGR=/usr/bin/dnf - if [ -f "/usr/bin/microdnf" ]; then - PKGMGR=/usr/bin/microdnf - fi -fi - -if [ "$PKGMGR" = "/usr/bin/microdnf" ] -then - if [ -z $PKGMGR_OPTS ]; then - # NOTE(pabelanger): skip install docs and weak dependencies to - # make smaller images. Sadly, setting these in dnf.conf don't - # appear to work. - PKGMGR_OPTS="--nodocs --setopt install_weak_deps=0" - fi -fi - -# NOTE(pabelanger): Ensure all the directory we use exists regardless -# of the user first creating them or not. -mkdir -p /output/bindep -mkdir -p /output/wheels -mkdir -p /tmp/src - -cd /tmp/src - -function install_bindep { - # Protect from the bindep builder image use of the assemble script - # to produce a wheel. Note we append because we want all - # sibling packages in here too - if [ -f bindep.txt ] ; then - bindep -l newline | sort >> /output/bindep/run.txt || true - if [ "$RELEASE" == "centos" ] ; then - bindep -l newline -b epel | sort >> /output/bindep/stage.txt || true - grep -Fxvf /output/bindep/run.txt /output/bindep/stage.txt >> /output/bindep/epel.txt || true - rm -rf /output/bindep/stage.txt - fi - compile_packages=$(bindep -b compile || true) - if [ ! -z "$compile_packages" ] ; then - $PKGMGR install -y $PKGMGR_OPTS ${compile_packages} - fi - fi -} - -function install_wheels { - # NOTE(pabelanger): If there are build requirements to install, do so. - # However do not cache them as we do not want them in the final image. - if [ -f /tmp/src/build-requirements.txt ] && [ ! -f /tmp/src/.build-requirements.txt ] ; then - $PIPCMD install $CONSTRAINTS $PIP_OPTS --no-cache -r /tmp/src/build-requirements.txt - touch /tmp/src/.build-requirements.txt - fi - # Build a wheel so that we have an install target. - # pip install . in the container context with the mounted - # source dir gets ... exciting, if setup.py exists. - # We run sdist first to trigger code generation steps such - # as are found in zuul, since the sequencing otherwise - # happens in a way that makes wheel content copying unhappy. - # pip wheel isn't used here because it puts all of the output - # in the output dir and not the wheel cache, so it's not - # possible to tell what is the wheel for the project and - # what is the wheel cache. - if [ -f setup.py ] ; then - $PYCMD setup.py sdist bdist_wheel -d /output/wheels - fi - - # Install everything so that the wheel cache is populated with - # transitive depends. If a requirements.txt file exists, install - # it directly so that people can use git url syntax to do things - # like pick up patched but unreleased versions of dependencies. - # Only do this for the main package (i.e. only write requirements - # once). - if [ -f /tmp/src/requirements.txt ] && [ ! -f /output/requirements.txt ] ; then - $PIPCMD install $CONSTRAINTS $PIP_OPTS --cache-dir=/output/wheels -r /tmp/src/requirements.txt - cp /tmp/src/requirements.txt /output/requirements.txt - fi - # If we didn't build wheels, we can skip trying to install it. - if [ $(ls -1 /output/wheels/*whl 2>/dev/null | wc -l) -gt 0 ]; then - $PIPCMD uninstall -y /output/wheels/*.whl - $PIPCMD install $CONSTRAINTS $PIP_OPTS --cache-dir=/output/wheels /output/wheels/*whl - fi -} - -PACKAGES=$* -PIP_OPTS="${PIP_OPTS-}" - -# bindep the main package -install_bindep - -# go through ZUUL_SIBLINGS, if any, and build those wheels too -for sibling in ${ZUUL_SIBLINGS:-}; do - pushd .zuul-siblings/${sibling} - install_bindep - popd -done - -# Use a clean virtualenv for install steps to prevent things from the -# current environment making us not build a wheel. -# NOTE(pabelanger): We allow users to install distro python packages of -# libraries. This is important for projects that eventually want to produce -# an RPM or offline install. -$PYCMD -m venv /tmp/venv --system-site-packages --without-pip -source /tmp/venv/bin/activate - -# If there is an upper-constraints.txt file in the source tree, -# use it in the pip commands. -if [ -f /tmp/src/upper-constraints.txt ] ; then - cp /tmp/src/upper-constraints.txt /output/upper-constraints.txt - CONSTRAINTS="-c /tmp/src/upper-constraints.txt" -fi - -# If we got a list of packages, install them, otherwise install the -# main package. -if [[ $PACKAGES ]] ; then - $PIPCMD install $CONSTRAINTS $PIP_OPTS --cache-dir=/output/wheels $PACKAGES - for package in $PACKAGES ; do - echo "$package" >> /output/packages.txt - done -else - install_wheels -fi - -# go through ZUUL_SIBLINGS, if any, and build those wheels too -for sibling in ${ZUUL_SIBLINGS:-}; do - pushd .zuul-siblings/${sibling} - install_wheels - popd -done - -if [ -z $PKGMGR_PRESERVE_CACHE ]; then - $PKGMGR clean all - rm -rf /var/cache/{dnf,yum} -fi - -rm -rf /var/lib/dnf/history.* -rm -rf /var/log/{dnf.*,hawkey.log} -rm -rf /tmp/venv diff --git a/images/awx-ee/context/_build/scripts/check_ansible b/images/awx-ee/context/_build/scripts/check_ansible deleted file mode 100755 index 029be1f3a..000000000 --- a/images/awx-ee/context/_build/scripts/check_ansible +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -##################################################################### -# Script to validate that Ansible and Ansible Runner are installed. -# -# Usage: check_ansible -# -# Options: -# PYCMD - The path to the python executable to use. -##################################################################### - -set -x - -PYCMD=$1 - -if [ -z "$PYCMD" ] -then - echo "Usage: check_ansible " - exit 1 -fi - -if [ ! -x "$PYCMD" ] -then - echo "$PYCMD is not an executable" - exit 1 -fi - -ansible --version - -if [ $? -ne 0 ] -then - cat< /dev/null || true) # whoami-free way to get current username, falls back to current uid - -DEFAULT_HOME="/runner" -DEFAULT_SHELL="/bin/bash" - -if (( "$EP_DEBUG_TRACE" == 1 )); then - function log_debug() { echo "EP_DEBUG: $1" 1>&2; } -else - function log_debug() { :; } -fi - -log_debug "entrypoint.sh started" - -case "$EP_ON_ERROR" in - "fail") - function maybe_fail() { echo "EP_FAIL: $1" 1>&2; exit 1; } - ;; - "warn") - function maybe_fail() { echo "EP_WARN: $1" 1>&2; } - ;; - *) - function maybe_fail() { log_debug "EP_FAIL (ignored): $1"; } - ;; -esac - -function is_dir_writable() { - [ -d "$1" ] && [ -w "$1" ] && [ -x "$1" ] -} - -function ensure_current_uid_in_passwd() { - log_debug "is current uid ${CUR_UID} in /etc/passwd?" - - if ! getent passwd "${CUR_USERNAME}" &> /dev/null ; then - if [ -w "/etc/passwd" ]; then - log_debug "appending missing uid ${CUR_UID} into /etc/passwd" - # use the default homedir; we may have to rewrite it to another value later if it's inaccessible - echo "${CUR_UID}:x:${CUR_UID}:0:container user ${CUR_UID}:${DEFAULT_HOME}:${DEFAULT_SHELL}" >> /etc/passwd - else - maybe_fail "uid ${CUR_UID} is missing from /etc/passwd, which is not writable; this error is likely fatal" - fi - else - log_debug "current uid is already in /etc/passwd" - fi -} - -function ensure_writeable_homedir() { - if (is_dir_writable "${CANDIDATE_HOME}") ; then - log_debug "candidate homedir ${CANDIDATE_HOME} is valid and writeable" - else - if [ "${CANDIDATE_HOME}" == "/" ]; then - log_debug "skipping attempt to fix permissions on / as homedir" - return 1 - fi - - log_debug "candidate homedir ${CANDIDATE_HOME} is missing or not writeable; attempt to fix" - if ! (mkdir -p "${CANDIDATE_HOME}" >& /dev/null && chmod -R ug+rwx "${CANDIDATE_HOME}" >& /dev/null) ; then - log_debug "candidate homedir ${CANDIDATE_HOME} cannot be made writeable" - return 1 - else - log_debug "candidate homedir ${CANDIDATE_HOME} was successfully made writeable" - fi - fi - - # this might work; export it even if we end up not being able to update /etc/passwd - # this ensures the envvar matches current reality for this session; future sessions should set automatically if /etc/passwd is accurate - export HOME=${CANDIDATE_HOME} - - if [ "${CANDIDATE_HOME}" == "${PASSWD_HOME}" ] ; then - log_debug "candidate homedir ${CANDIDATE_HOME} matches /etc/passwd" - return 0 - fi - - if ! [ -w /etc/passwd ]; then - log_debug "candidate homedir ${CANDIDATE_HOME} is valid for ${CUR_USERNAME}, but /etc/passwd is not writable to update it" - return 1 - fi - - log_debug "resetting homedir for user ${CUR_USERNAME} to ${CANDIDATE_HOME} in /etc/passwd" - - # sed -i wants to create a tempfile next to the original, which won't work with /etc permissions in many cases, - # so just do it in memory and overwrite the existing file if we succeeded - NEWPW=$(sed -r "s;(^${CUR_USERNAME}:(.*:){4})(.*:);\1${CANDIDATE_HOME}:;g" /etc/passwd) - echo "${NEWPW}" > /etc/passwd -} - -ensure_current_uid_in_passwd - -log_debug "current value of HOME is ${HOME}" - -PASSWD_HOME=$(getent passwd "${CUR_USERNAME}" | cut -d: -f6) -log_debug "user ${CUR_USERNAME} homedir from /etc/passwd is ${PASSWD_HOME}" - -CANDIDATE_HOMES=("${PASSWD_HOME}" "${HOME}" "${DEFAULT_HOME}" "/tmp") - -# we'll set this in the loop as soon as we find a writeable dir -unset HOME - -for CANDIDATE_HOME in "${CANDIDATE_HOMES[@]}"; do - if ensure_writeable_homedir ; then - break - fi -done - -if ! [ -v HOME ] ; then - maybe_fail "a valid homedir could not be set for ${CUR_USERNAME}; this is likely fatal" -fi - -# chain exec whatever we were asked to run (ideally an init system) to keep any envvar state we've set -log_debug "chain exec-ing requested command $*" -exec "${@}" diff --git a/images/awx-ee/context/_build/scripts/install-from-bindep b/images/awx-ee/context/_build/scripts/install-from-bindep deleted file mode 100755 index 228ac3e8c..000000000 --- a/images/awx-ee/context/_build/scripts/install-from-bindep +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -# Copyright (c) 2019 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex -# NOTE(pabelanger): Allow users to force either microdnf or dnf as a package -# manager. -PKGMGR="${PKGMGR:-}" -PKGMGR_OPTS="${PKGMGR_OPTS:-}" -PKGMGR_PRESERVE_CACHE="${PKGMGR_PRESERVE_CACHE:-}" - -PYCMD="${PYCMD:=/usr/bin/python3}" -PIPCMD="${PIPCMD:=$PYCMD -m pip}" -PIP_OPTS="${PIP_OPTS-}" - -$PYCMD -m ensurepip - -if [ -z $PKGMGR ]; then - # Expect dnf to be installed, however if we find microdnf default to it. - PKGMGR=/usr/bin/dnf - if [ -f "/usr/bin/microdnf" ]; then - PKGMGR=/usr/bin/microdnf - fi -fi - -if [ "$PKGMGR" = "/usr/bin/microdnf" ] -then - if [ -z $PKGMGR_OPTS ]; then - # NOTE(pabelanger): skip install docs and weak dependencies to - # make smaller images. Sadly, setting these in dnf.conf don't - # appear to work. - PKGMGR_OPTS="--nodocs --setopt install_weak_deps=0" - fi -fi - -if [ -f /output/bindep/run.txt ] ; then - PACKAGES=$(cat /output/bindep/run.txt) - if [ ! -z "$PACKAGES" ]; then - $PKGMGR install -y $PKGMGR_OPTS $PACKAGES - fi -fi - -if [ -f /output/bindep/epel.txt ] ; then - EPEL_PACKAGES=$(cat /output/bindep/epel.txt) - if [ ! -z "$EPEL_PACKAGES" ]; then - $PKGMGR install -y $PKGMGR_OPTS --enablerepo epel $EPEL_PACKAGES - fi -fi - -# If there's a constraints file, use it. -if [ -f /output/upper-constraints.txt ] ; then - CONSTRAINTS="-c /output/upper-constraints.txt" -fi - -# If a requirements.txt file exists, -# install it directly so that people can use git url syntax -# to do things like pick up patched but unreleased versions -# of dependencies. -if [ -f /output/requirements.txt ] ; then - $PIPCMD install $CONSTRAINTS $PIP_OPTS --cache-dir=/output/wheels -r /output/requirements.txt -fi - -# Add any requested extras to the list of things to install -EXTRAS="" -for extra in $* ; do - EXTRAS="${EXTRAS} -r /output/$extra/requirements.txt" -done - -if [ -f /output/packages.txt ] ; then - # If a package list was passed to assemble, install that in the final - # image. - $PIPCMD install $CONSTRAINTS $PIP_OPTS --cache-dir=/output/wheels -r /output/packages.txt $EXTRAS -else - # Install the wheels. Uninstall any existing version as siblings maybe - # be built with the same version number as the latest release, but we - # really want the speculatively built wheels installed over any - # automatic dependencies. - # NOTE(pabelanger): It is possible a project may not have a wheel, but does have requirements.txt - if [ $(ls -1 /output/wheels/*whl 2>/dev/null | wc -l) -gt 0 ]; then - $PIPCMD uninstall -y /output/wheels/*.whl - $PIPCMD install $CONSTRAINTS $PIP_OPTS --cache-dir=/output/wheels /output/wheels/*.whl $EXTRAS - elif [ ! -z "$EXTRAS" ] ; then - $PIPCMD uninstall -y $EXTRAS - $PIPCMD install $CONSTRAINTS $PIP_OPTS --cache-dir=/output/wheels $EXTRAS - fi -fi - -# clean up after ourselves, unless requested to keep the cache -if [[ "$PKGMGR_PRESERVE_CACHE" != always ]]; then - $PKGMGR clean all - rm -rf /var/cache/{dnf,yum} -fi - -rm -rf /var/lib/dnf/history.* -rm -rf /var/log/{dnf.*,hawkey.log} diff --git a/images/awx-ee/context/_build/scripts/introspect.py b/images/awx-ee/context/_build/scripts/introspect.py deleted file mode 100644 index 0d84e84bd..000000000 --- a/images/awx-ee/context/_build/scripts/introspect.py +++ /dev/null @@ -1,400 +0,0 @@ -import argparse -import logging -import os -import sys -import yaml - -import requirements -import importlib.metadata - -base_collections_path = '/usr/share/ansible/collections' -default_file = 'execution-environment.yml' -logger = logging.getLogger(__name__) - - -def line_is_empty(line): - return bool((not line.strip()) or line.startswith('#')) - - -def read_req_file(path): - """Provide some minimal error and display handling for file reading""" - if not os.path.exists(path): - print('Expected requirements file not present at: {0}'.format(os.path.abspath(path))) - with open(path, 'r') as f: - return f.read() - - -def pip_file_data(path): - pip_content = read_req_file(path) - - pip_lines = [] - for line in pip_content.split('\n'): - if line_is_empty(line): - continue - if line.startswith('-r') or line.startswith('--requirement'): - _, new_filename = line.split(None, 1) - new_path = os.path.join(os.path.dirname(path or '.'), new_filename) - pip_lines.extend(pip_file_data(new_path)) - else: - pip_lines.append(line) - - return pip_lines - - -def bindep_file_data(path): - sys_content = read_req_file(path) - - sys_lines = [] - for line in sys_content.split('\n'): - if line_is_empty(line): - continue - sys_lines.append(line) - - return sys_lines - - -def process_collection(path): - """Return a tuple of (python_dependencies, system_dependencies) for the - collection install path given. - Both items returned are a list of dependencies. - - :param str path: root directory of collection (this would contain galaxy.yml file) - """ - CD = CollectionDefinition(path) - - py_file = CD.get_dependency('python') - pip_lines = [] - if py_file: - pip_lines = pip_file_data(os.path.join(path, py_file)) - - sys_file = CD.get_dependency('system') - bindep_lines = [] - if sys_file: - bindep_lines = bindep_file_data(os.path.join(path, sys_file)) - - return (pip_lines, bindep_lines) - - -def process(data_dir=base_collections_path, user_pip=None, user_bindep=None): - paths = [] - path_root = os.path.join(data_dir, 'ansible_collections') - - # build a list of all the valid collection paths - if os.path.exists(path_root): - for namespace in sorted(os.listdir(path_root)): - if not os.path.isdir(os.path.join(path_root, namespace)): - continue - for name in sorted(os.listdir(os.path.join(path_root, namespace))): - collection_dir = os.path.join(path_root, namespace, name) - if not os.path.isdir(collection_dir): - continue - files_list = os.listdir(collection_dir) - if 'galaxy.yml' in files_list or 'MANIFEST.json' in files_list: - paths.append(collection_dir) - - # populate the requirements content - py_req = {} - sys_req = {} - for path in paths: - col_pip_lines, col_sys_lines = process_collection(path) - CD = CollectionDefinition(path) - namespace, name = CD.namespace_name() - key = '{}.{}'.format(namespace, name) - - if col_pip_lines: - py_req[key] = col_pip_lines - - if col_sys_lines: - sys_req[key] = col_sys_lines - - # add on entries from user files, if they are given - if user_pip: - col_pip_lines = pip_file_data(user_pip) - if col_pip_lines: - py_req['user'] = col_pip_lines - if user_bindep: - col_sys_lines = bindep_file_data(user_bindep) - if col_sys_lines: - sys_req['user'] = col_sys_lines - - return { - 'python': py_req, - 'system': sys_req - } - - -def has_content(candidate_file): - """Beyond checking that the candidate exists, this also assures - that the file has something other than whitespace, - which can cause errors when given to pip. - """ - if not os.path.exists(candidate_file): - return False - with open(candidate_file, 'r') as f: - content = f.read() - return bool(content.strip().strip('\n')) - - -class CollectionDefinition: - """This class represents the dependency metadata for a collection - should be replaced by logic to hit the Galaxy API if made available - """ - - def __init__(self, collection_path): - self.reference_path = collection_path - meta_file = os.path.join(collection_path, 'meta', default_file) - if os.path.exists(meta_file): - with open(meta_file, 'r') as f: - self.raw = yaml.safe_load(f) - else: - self.raw = {'version': 1, 'dependencies': {}} - # Automatically infer requirements for collection - for entry, filename in [('python', 'requirements.txt'), ('system', 'bindep.txt')]: - candidate_file = os.path.join(collection_path, filename) - if has_content(candidate_file): - self.raw['dependencies'][entry] = filename - - def target_dir(self): - namespace, name = self.namespace_name() - return os.path.join( - base_collections_path, 'ansible_collections', - namespace, name - ) - - def namespace_name(self): - "Returns 2-tuple of namespace and name" - path_parts = [p for p in self.reference_path.split(os.path.sep) if p] - return tuple(path_parts[-2:]) - - def get_dependency(self, entry): - """A collection is only allowed to reference a file by a relative path - which is relative to the collection root - """ - req_file = self.raw.get('dependencies', {}).get(entry) - if req_file is None: - return None - elif os.path.isabs(req_file): - raise RuntimeError( - 'Collections must specify relative paths for requirements files. ' - 'The file {0} specified by {1} violates this.'.format( - req_file, self.reference_path - ) - ) - - return req_file - - -def simple_combine(reqs): - """Given a dictionary of requirement lines keyed off collections, - return a list with the most basic of de-duplication logic, - and comments indicating the sources based off the collection keys - """ - consolidated = [] - fancy_lines = [] - for collection, lines in reqs.items(): - for line in lines: - if line_is_empty(line): - continue - - base_line = line.split('#')[0].strip() - if base_line in consolidated: - i = consolidated.index(base_line) - fancy_lines[i] += ', {}'.format(collection) - else: - fancy_line = base_line + ' # from collection {}'.format(collection) - consolidated.append(base_line) - fancy_lines.append(fancy_line) - - return fancy_lines - - -def parse_args(args=sys.argv[1:]): - - parser = argparse.ArgumentParser( - prog='introspect', - description=( - 'ansible-builder introspection; injected and used during execution environment build' - ) - ) - - subparsers = parser.add_subparsers(help='The command to invoke.', dest='action') - subparsers.required = True - - create_introspect_parser(subparsers) - - args = parser.parse_args(args) - - return args - - -def run_introspect(args, logger): - data = process(args.folder, user_pip=args.user_pip, user_bindep=args.user_bindep) - if args.sanitize: - logger.info('# Sanitized dependencies for %s', args.folder) - data_for_write = data - data['python'] = sanitize_requirements(data['python']) - data['system'] = simple_combine(data['system']) - else: - logger.info('# Dependency data for %s', args.folder) - data_for_write = data.copy() - data_for_write['python'] = simple_combine(data['python']) - data_for_write['system'] = simple_combine(data['system']) - - print('---') - print(yaml.dump(data, default_flow_style=False)) - - if args.write_pip and data.get('python'): - write_file(args.write_pip, data_for_write.get('python') + ['']) - if args.write_bindep and data.get('system'): - write_file(args.write_bindep, data_for_write.get('system') + ['']) - - sys.exit(0) - - -def create_introspect_parser(parser): - introspect_parser = parser.add_parser( - 'introspect', - help='Introspects collections in folder.', - description=( - 'Loops over collections in folder and returns data about dependencies. ' - 'This is used internally and exposed here for verification. ' - 'This is targeted toward collection authors and maintainers.' - ) - ) - introspect_parser.add_argument('--sanitize', action='store_true', - help=('Sanitize and de-duplicate requirements. ' - 'This is normally done separately from the introspect script, but this ' - 'option is given to more accurately test collection content.')) - - introspect_parser.add_argument( - 'folder', default=base_collections_path, nargs='?', - help=( - 'Ansible collections path(s) to introspect. ' - 'This should have a folder named ansible_collections inside of it.' - ) - ) - # Combine user requirements and collection requirements into single file - # in the future, could look into passing multilple files to - # python-builder scripts to be fed multiple files as opposed to this - introspect_parser.add_argument( - '--user-pip', dest='user_pip', - help='An additional file to combine with collection pip requirements.' - ) - introspect_parser.add_argument( - '--user-bindep', dest='user_bindep', - help='An additional file to combine with collection bindep requirements.' - ) - introspect_parser.add_argument( - '--write-pip', dest='write_pip', - help='Write the combined pip requirements file to this location.' - ) - introspect_parser.add_argument( - '--write-bindep', dest='write_bindep', - help='Write the combined bindep requirements file to this location.' - ) - - return introspect_parser - - -EXCLUDE_REQUIREMENTS = frozenset(( - # obviously already satisfied or unwanted - 'ansible', 'ansible-base', 'python', 'ansible-core', - # general python test requirements - 'tox', 'pycodestyle', 'yamllint', 'pylint', - 'flake8', 'pytest', 'pytest-xdist', 'coverage', 'mock', 'testinfra', - # test requirements highly specific to Ansible testing - 'ansible-lint', 'molecule', 'galaxy-importer', 'voluptuous', - # already present in image for py3 environments - 'yaml', 'pyyaml', 'json', -)) - - -def sanitize_requirements(collection_py_reqs): - """ - Cleanup Python requirements by removing duplicates and excluded packages. - - The user requirements file will go through the deduplication process, but - skips the special package exclusion process. - - :param dict collection_py_reqs: A dict of lists of Python requirements, keyed - by fully qualified collection name. The special key `user` holds requirements - from the user specified requirements file from the ``--user-pip`` CLI option. - - :returns: A finalized list of sanitized Python requirements. - """ - # de-duplication - consolidated = [] - seen_pkgs = set() - - for collection, lines in collection_py_reqs.items(): - try: - for req in requirements.parse('\n'.join(lines)): - if req.specifier: - req.name = importlib.metadata.Prepared(req.name).normalized - req.collections = [collection] # add backref for later - if req.name is None: - consolidated.append(req) - continue - if req.name in seen_pkgs: - for prior_req in consolidated: - if req.name == prior_req.name: - prior_req.specs.extend(req.specs) - prior_req.collections.append(collection) - break - continue - consolidated.append(req) - seen_pkgs.add(req.name) - except Exception as e: - logger.warning('Warning: failed to parse requirements from %s, error: %s', collection, e) - - # removal of unwanted packages - sanitized = [] - for req in consolidated: - # Exclude packages, unless it was present in the user supplied requirements. - if req.name and req.name.lower() in EXCLUDE_REQUIREMENTS and 'user' not in req.collections: - logger.debug('# Excluding requirement %s from %s', req.name, req.collections) - continue - if req.vcs or req.uri: - # Requirement like git+ or http return as-is - new_line = req.line - elif req.name: - specs = ['{0}{1}'.format(cmp, ver) for cmp, ver in req.specs] - new_line = req.name + ','.join(specs) - else: - raise RuntimeError('Could not process {0}'.format(req.line)) - - sanitized.append(new_line + ' # from collection {}'.format(','.join(req.collections))) - - return sanitized - - -def write_file(filename: str, lines: list) -> bool: - parent_dir = os.path.dirname(filename) - if parent_dir and not os.path.exists(parent_dir): - logger.warning('Creating parent directory for %s', filename) - os.makedirs(parent_dir) - new_text = '\n'.join(lines) - if os.path.exists(filename): - with open(filename, 'r') as f: - if f.read() == new_text: - logger.debug("File %s is already up-to-date.", filename) - return False - else: - logger.warning('File %s had modifications and will be rewritten', filename) - with open(filename, 'w') as f: - f.write(new_text) - return True - - -def main(): - args = parse_args() - - if args.action == 'introspect': - run_introspect(args, logger) - - logger.error("An error has occurred.") - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/images/awx-ee/context/azure-cli.repo b/images/awx-ee/context/azure-cli.repo deleted file mode 100644 index 0fee27460..000000000 --- a/images/awx-ee/context/azure-cli.repo +++ /dev/null @@ -1,6 +0,0 @@ -[azure-cli] -name=Azure CLI -baseurl=https://packages.microsoft.com/yumrepos/azure-cli -enabled=1 -gpgcheck=1 -gpgkey=https://packages.microsoft.com/keys/microsoft.asc diff --git a/images/awx-ee/docker-bake.hcl b/images/awx-ee/docker-bake.hcl new file mode 100644 index 000000000..038066213 --- /dev/null +++ b/images/awx-ee/docker-bake.hcl @@ -0,0 +1,25 @@ +variable "GHCR" { + default = "ghcr.io" +} + +variable "IMAGE_TAG" { + default = "v3" +} + +group "default" { + targets = ["ee"] +} + +target "ee" { + context = "./context" + dockerfile = "Dockerfile" + platforms = ["linux/amd64", "linux/arm64"] + tags = [ + // "singledigital/awx-ee:${IMAGE_TAG}", + "${GHCR}/dpc-sdp/bay/awx-ee:${IMAGE_TAG}" + ] + args = { + PYCMD = "/usr/local/bin/python3" + PKGMGR = "/usr/bin/apt-get" + } +} diff --git a/images/awx-ee/execution-environment.yml b/images/awx-ee/execution-environment.yml index 59f5c9ade..8569235eb 100644 --- a/images/awx-ee/execution-environment.yml +++ b/images/awx-ee/execution-environment.yml @@ -1,21 +1,58 @@ --- -version: 1 +version: 3 + dependencies: + ansible_core: + package_pip: ansible-core==2.13.6 + ansible_runner: + package_pip: ansible-runner galaxy: requirements.yml python: requirements.txt - system: bindep.txt + +images: + base_image: + name: python:3.10 additional_build_steps: - prepend: + # Source for those scripts are at https://github.com/ansible/ansible-builder/tree/devel/src/ansible_builder/_target_scripts + prepend_base: [] + append_base: [] + + prepend_final: - LABEL org.opencontainers.image.authors="Digital Victoria" - LABEL org.opencontainers.image.description="Provides an AWX execution environment image optimised for use with SDP." - LABEL org.opencontainers.image.source="https://github.com/dpc-sdp/bay/blob/5.x/images/awx-ee/context/Dockerfile" - ARG LAGOON_CLI_VERSION=v0.15.4 - ARG NVM_INSTALL_VERSION=v0.39.1 - - ARG NVM_DIR=$HOME/.nvm - ARG NODE_VERSION=v14.15.1 - - RUN dnf module enable php:7.4 -y - append: + + append_final: + - | # Required dependencies. + RUN set -eux; \ + apt-get update && apt-get install -y \ + git git-lfs \ + jq \ + rsync \ + zip unzip; \ + rm -rf /var/lib/apt/lists/*; + + - | # Install php & composer. + RUN set -eux; \ + curl -sSL https://packages.sury.org/php/README.txt | bash -x; \ + apt-get update && apt-get install -y \ + php8.1-cli \ + php8.1-gd \ + php8.1-zip; \ + rm -rf /var/lib/apt/lists/*; + + - | # Install cli tools. + SHELL ["/bin/bash", "-c"] + RUN set -eux; \ + arch_linux=$(uname -m) && arch_linux="${arch_linux/aarch64/arm64}" && arch_linux="${arch_linux/x86_64/amd64}"; \ + curl -LO "https://dl.k8s.io/release/v1.25.12/bin/linux/${arch_linux}/kubectl"; \ + chmod +x kubectl; \ + mv ./kubectl /usr/local/bin/kubectl; + - RUN curl -L "https://github.com/uselagoon/lagoon-cli/releases/download/$LAGOON_CLI_VERSION/lagoon-cli-$LAGOON_CLI_VERSION-linux-amd64" -o /usr/local/bin/lagoon - RUN chmod +x /usr/local/bin/lagoon - RUN curl -L "https://github.com/github/hub/releases/download/v2.14.2/hub-linux-amd64-2.14.2.tgz" -o /tmp/hub && tar -xvf /tmp/hub -C /tmp && mv /tmp/hub-linux-amd64-2.14.2/bin/hub /usr/local/bin @@ -29,9 +66,7 @@ additional_build_steps: - RUN tar -C /tmp -xvf /tmp/gojq_v0.12.4_linux_amd64.tar.gz - RUN chmod +x /tmp/gojq_v0.12.4_linux_amd64/gojq - RUN mv /tmp/gojq_v0.12.4_linux_amd64/gojq /usr/local/bin - - RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc - - ADD azure-cli.repo /etc/yum.repos.d/azure-cli.repo - - RUN dnf --assumeyes install azure-cli - - RUN az aks install-cli - RUN touch $HOME/.bashrc && chmod +x $HOME/.bashrc - RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/$NVM_INSTALL_VERSION/install.sh | bash + - RUN curl -L "https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz" -o /tmp/helm && tar -xvf /tmp/helm -C /tmp && mv /tmp/linux-amd64/helm /usr/local/bin + - RUN chmod +x /usr/local/bin/helm diff --git a/images/awx-ee/requirements.txt b/images/awx-ee/requirements.txt index c6ee55285..f025ae31c 100644 --- a/images/awx-ee/requirements.txt +++ b/images/awx-ee/requirements.txt @@ -5,5 +5,13 @@ requests-oauthlib ansible-lint flake8 yamllint +kubernetes +lxml +gql +# Dependencies for the gql requests transport. +requests-toolbelt<1,>=0.9.1 +urllib3>=1.26 +requests<3,>=2.26 ruamel.yaml -pygithub \ No newline at end of file +pygithub +azure-cli \ No newline at end of file