ARCHE-Ingest #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ARCHE-Ingest | |
| on: | |
| workflow_dispatch | |
| jobs: | |
| build_pages: | |
| name: Ingest Data into ARCHE | |
| runs-on: ubuntu-latest | |
| env: | |
| ANT_OPTS: -Xmx5g | |
| REDMINE_ID: 20352 | |
| ARCHE_INSTANCE: arche-curation.acdh-dev | |
| ARCHE_INSTANCE_NAME: Curation | |
| PROJECT_NAME : auden-musulin-papers | |
| OUTPUT_DIR: rdf | |
| DATA_DIR: data/editions | |
| META_DIR: data/metadata | |
| INDICES_DIR: data/indexes | |
| steps: | |
| - name: Perform Checkout | |
| uses: actions/checkout@v3 | |
| - name: ingestion dependencies | |
| run: | | |
| composer require "acdh-oeaw/arche-ingest:^1" | |
| - name: Fetch WF-URL | |
| run: | | |
| echo "RUN_URL=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_ENV | |
| - name: ingest arche metadata topcol, col, res | |
| run: | | |
| echo 'STEP=Upload AIP to ${ARCHE_INSTANCE_NAME} Instance' >> $GITHUB_ENV | |
| vendor/bin/arche-import-metadata ${OUTPUT_DIR}/arche.ttl https://${ARCHE_INSTANCE}.oeaw.ac.at/api ${{secrets.ARCHE_LOGIN}} ${{secrets.ARCHE_PASSWORD}} --retriesOnConflict 25 | |
| # vendor/bin/arche-update-redmine --token ${{ secrets.REDMINE_TOKEN }} --append "$RUN_URL" $REDMINE_ID 'Upload AIP to ${ARCHE_INSTANCE_NAME} Instance' | |
| - name: Copy XML Files to to_ingest | |
| run: | | |
| mkdir $PWD/to_ingest | |
| find -path "./${DATA_DIR}/*.xml" -exec cp -prv '{}' './to_ingest' ';' | |
| find -path "./${INDICES_DIR}/*.xml" -exec cp -prv '{}' './to_ingest' ';' | |
| find -path "./${META_DIR}/*.odd" -exec cp -prv '{}' './to_ingest' ';' | |
| find -path "./${META_DIR}/*.rng" -exec cp -prv '{}' './to_ingest' ';' | |
| find -path "./data/amp-logo.png" -exec cp -prv '{}' './to_ingest' ';' | |
| - name: cache AV database | |
| id: avdb | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cvdupdate | |
| key: constant | |
| - name: refresh AV database | |
| run: | | |
| python3 -m pip install --user cvdupdate && cvd update | |
| - name: run repo-file-checker | |
| run: | | |
| echo 'STEP=Run repo-file-checker' >> $GITHUB_ENV | |
| docker run --rm -d --name filechecker -v `pwd`/filechecker/reports:/reports -v `pwd`/to_ingest:/data -v ~/.cvdupdate/database/:/var/lib/clamav -e DAEMONIZE=1 acdhch/arche-filechecker | |
| # vendor/bin/arche-update-redmine --token ${{ secrets.REDMINE_TOKEN }} --append "$RUN_URL" $REDMINE_ID 'Virus scan' | |
| # vendor/bin/arche-update-redmine --token ${{ secrets.REDMINE_TOKEN }} --append "$RUN_URL" $REDMINE_ID 'Run repo-file-checker' | |
| - name: ingest binaries | |
| run: | | |
| echo 'STEP=Upload Binaries' >> $GITHUB_ENV | |
| vendor/bin/arche-import-binary to_ingest https://id.acdh.oeaw.ac.at/${PROJECT_NAME} https://${ARCHE_INSTANCE}.oeaw.ac.at/api ${{secrets.ARCHE_LOGIN}} ${{secrets.ARCHE_PASSWORD}} --skip not_exist | |
| # vendor/bin/arche-update-redmine --token ${{ secrets.REDMINE_TOKEN }} --append "$RUN_URL" $REDMINE_ID 'Upload AIP to ${ARCHE_INSTANCE_NAME} Instance' | |
| - name: on failure | |
| if: ${{ failure() }} | |
| run: | | |
| vendor/bin/arche-update-redmine --token ${{ secrets.REDMINE_TOKEN }} --append "$RUN_URL" --statusCode 1 $REDMINE_ID "$STEP" |