diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..80de5bf --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +notebooks \ No newline at end of file diff --git a/.github/workflows/dea-intertidal-image.yml b/.github/workflows/dea-intertidal-image.yml new file mode 100644 index 0000000..2e29428 --- /dev/null +++ b/.github/workflows/dea-intertidal-image.yml @@ -0,0 +1,76 @@ +name: DEA Intertidal Image Push + +env: + IMAGE_NAME: geoscienceaustralia/dea-intertidal + +on: + push: + branches: + - main + paths: + - 'intertidal/**' + - 'data/**' + - '.github/workflows/dea-intertidal-image.yml' + - 'Dockerfile' + - 'requirements.in' + + release: + types: [created, edited, published] + + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Build DEA Intertidal image + timeout-minutes: 20 + shell: bash + run: | + docker-compose build + + push_ecr: + needs: [test] + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::538673716275:role/github-actions-role + aws-region: ap-southeast-2 + + - name: Get tag for this build if it exists + if: github.event_name == 'release' + run: | + echo "RELEASE=${GITHUB_REF/refs\/tags\/}" >> $GITHUB_ENV + + - name: Push release image to ECR + uses: whoan/docker-build-with-cache-action@master + if: github.event_name == 'release' + with: + registry: 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com + image_name: ${{ env.IMAGE_NAME }} + image_tag: ${{ env.RELEASE }} + + - name: Get git commit hash for push to main + if: github.event_name != 'release' + run: | + echo "TAG=dev$(git rev-parse --short HEAD)" \ + >> $GITHUB_ENV + + - name: Push unstable image to ECR + uses: whoan/docker-build-with-cache-action@master + if: github.event_name != 'release' + with: + registry: 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com + image_name: ${{ env.IMAGE_NAME }} + image_tag: latest,${{ env.TAG }} diff --git a/.gitignore b/.gitignore index 54ff8d3..b06333e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,8 @@ !*.ipynb !*.sh !*.yaml -!*.gitignore \ No newline at end of file +!*.yml +!*.in +!*.gitignore +!*.dockerignore +!*Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..63e43c2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +FROM osgeo/gdal:ubuntu-small-3.4.1 + +ENV DEBIAN_FRONTEND=noninteractive \ + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 + +# Apt installation +RUN apt-get update && \ + apt-get install -y \ + build-essential \ + fish \ + git \ + vim \ + htop \ + wget \ + unzip \ + python3-pip \ + libpq-dev python-dev \ + && apt-get autoclean && \ + apt-get autoremove && \ + rm -rf /var/lib/{apt,dpkg,cache,log} + +# Install pip-tools +RUN pip install pip-tools + +# Pip installation +RUN mkdir -p /conf +COPY requirements.in /conf/ +RUN pip-compile --extra-index-url=https://packages.dea.ga.gov.au/ --output-file=/conf/requirements.txt /conf/requirements.in +RUN pip install -r /conf/requirements.txt + +# Copy source code and install it +RUN mkdir -p /code +WORKDIR /code +ADD . /code + +RUN echo "Installing dea-intertidal through the Dockerfile." +RUN pip install --extra-index-url="https://packages.dea.ga.gov.au" . + +RUN pip freeze && pip check + +# Make sure it's working +RUN dea-intertidal --help diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..42840a5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +version: "3.6" + +services: + public_odc: + image: postgres:11.5-alpine + ports: + - "5436:5432" + environment: + - POSTGRES_DB=public-odc + - POSTGRES_PASSWORD=opendatacubepassword + - POSTGRES_USER=public-odc + expose: + - 5432 + restart: always + + dea_intertidal: + build: + context: . + args: + - ENVIRONMENT=dev + environment: + - ODC_DB_HOSTNAME=public_odc + - ODC_DB_USERNAME=public-odc + - ODC_DB_PASSWORD=opendatacubepassword + - ODC_DB_DATABASE=public-odc + - AWS_DEFAULT_REGION=ap-southeast-2 + - AWS_ACCESS_KEY_ID=fake_id + - AWS_SECRET_ACCESS_KEY=fake_key + - AWS_NO_SIGN_REQUEST=YES + - SQLALCHEMY_SILENCE_UBER_WARNING=1 + - GDAL_HTTP_MAX_RETRY=10 + volumes: + - .:/code + - ./tide_models/:/var/share/tide_models + depends_on: + - public_odc + command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 180 ; done"] + + public_index: + image: opendatacube/datacube-index:latest + environment: + - DB_HOSTNAME=public_odc + - DB_USERNAME=public-odc + - DB_PASSWORD=opendatacubepassword + - DB_DATABASE=public-odc + - DB_PORT=5432 + - AWS_DEFAULT_REGION=ap-southeast-2 + - SQLALCHEMY_SILENCE_UBER_WARNING=1 + depends_on: + - public_odc + entrypoint: bash -c 'sleep infinity' diff --git a/requirements.in b/requirements.in index 3eef6e2..8c819b6 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,6 @@ --extra-index-url=https://packages.dea.ga.gov.au/ aiohttp -bokeh==3.2.2 # temporary fix for Dask issue +# bokeh==3.2.2 # temporary fix for Dask issue botocore click==8.1.3 datacube[s3,performance]==1.8.13 @@ -21,4 +21,5 @@ scipy==1.10.1 sunriset==1.0 Shapely==2.0.1 tqdm==4.65.0 -xarray==2023.1.0 \ No newline at end of file +xarray==2023.1.0 +rioxarray