Support for loading data from csv files #242
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI build | |
| on: | |
| schedule: | |
| - cron: '0 10 * * *' # every day at 10am | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| jobs: | |
| py-lint: | |
| name: Lint | |
| continue-on-error: true | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Run pylint | |
| run: pylint --recursive=y . | |
| unit-tests: | |
| name: Unit tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Run unit tests | |
| run: make test | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| flags: unit | |
| integration-tests: | |
| needs: unit-tests | |
| name: Integration tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Run integration tests | |
| run: make validate | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| flags: integration | |
| build-image: | |
| name: Docker image build | |
| needs: [ integration-tests ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out Git repository | |
| uses: actions/checkout@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build image | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| tags: ghcr.io/bbmri-cz/fhir-module:latest | |
| outputs: type=docker,dest=/tmp/fhir-module.tar | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Upload FHIR-Module Image | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: fhir-module-image | |
| path: /tmp/fhir-module.tar | |
| system-test: | |
| name: System test | |
| needs: [ build-image ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out Git repository | |
| uses: actions/checkout@v3 | |
| - name: Install jq | |
| run: sudo apt-get install -y jq | |
| - name: Download artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: fhir-module-image | |
| path: /tmp | |
| - name: Load FHIR-Module Image | |
| run: docker load --input /tmp/fhir-module.tar | |
| - name: Create docker network | |
| run: docker network create system-test | |
| - name: Run sandbox in dev env | |
| run: docker compose --profile dev up -d | |
| - name: Wait for Blaze | |
| run: .github/scripts/wait-for-url.sh http://localhost:8080/health | |
| - name: Docker Logs | |
| run: docker logs blaze | |
| - name: Docker Logs | |
| run: sleep 20 && docker logs fhir-module | |
| - name: Verify upload succeeded | |
| run: .github/scripts/check-patient-exists.sh | |
| system-test-csv: | |
| name: System test csv files | |
| needs: [ system-test ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out Git repository | |
| uses: actions/checkout@v3 | |
| - name: Install jq | |
| run: sudo apt-get install -y jq | |
| - name: Download artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: fhir-module-image | |
| path: /tmp | |
| - name: Load FHIR-Module Image | |
| run: docker load --input /tmp/fhir-module.tar | |
| - name: Create docker network | |
| run: docker network create system-test | |
| - name: Generate compose.override.yaml file | |
| run: | | |
| echo "version: '3.9'" > compose.override.yaml | |
| echo "services:" >> compose.override.yaml | |
| echo " fhir-module:" >> compose.override.yaml | |
| echo " environment:" >> compose.override.yaml | |
| echo " PARSING_MAP_PATH: \"/opt/fhir-module/util/default_csv_map.json\"" >> compose.override.yaml | |
| echo " MATERIAL_TYPE_MAP_PATH: \"/opt/fhir-module/util/default_csv_material_type_map.json\"" >> compose.override.yaml | |
| echo " RECORDS_FILE_TYPE: \"csv\"" >> compose.override.yaml | |
| echo " volumes:" >> compose.override.yaml | |
| echo " - type: bind" >> compose.override.yaml | |
| echo " source: \"test/csv_data\"" >> compose.override.yaml | |
| echo " target: \"/opt/records\"" >> compose.override.yaml | |
| - name: Run sandbox in dev env | |
| run: docker compose --profile dev up -d | |
| - name: Wait for Blaze | |
| run: .github/scripts/wait-for-url.sh http://localhost:8080/health | |
| - name: Docker Logs | |
| run: docker logs blaze | |
| - name: Docker Logs | |
| run: sleep 20 && docker logs fhir-module | |
| - name: Verify upload succeeded | |
| run: .github/scripts/check-patient-exists.sh | |
| build-push-image: | |
| name: Publish docker image | |
| needs: [ system-test-csv ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out Git repository | |
| uses: actions/checkout@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Docker meta | |
| id: docker-meta | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: | | |
| ghcr.io/${{ github.repository_owner }}/fhir-module | |
| tags: | | |
| type=schedule | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | |
| - name: Build and push image | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: ${{ steps.docker-meta.outputs.tags }} | |
| labels: ${{ steps.docker-meta.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max |