Upstream dev to inference

.github/workflows/aws-apl.yaml

@@ -1,6 +1,9 @@
-# This workflow will build and push a new container image to Amazon ECR The following information will need to be included in your Github secrets: - AWS_ACCESS_KEY_ID: dedicated 
-# user AWS key ID - AWS_SECRET_ACCESS_KEY: dedicated user AWS access key - AWS_REGION: your AWS region, ex: us-east-1, us-west-2... - ECR_REPOSITORY_NAME: name of the ECR 
-# repository, ex: myrepo
+# This workflow will build and push a new container image to Amazon ECR
+# The following information will need to be included in your Github secrets:
+# - AWS_ACCESS_KEY_ID_APL: dedicated  user AWS key ID
+# - AWS_SECRET_ACCESS_KEY_APL: dedicated user AWS access key
+# - AWS_REGION_APL: your AWS region, ex: us-east-1, us-west-2...
+# - ECR_REPOSITORY_NAME_APL: name of the ECR repository, ex: myrepo
 on:
   push:
     branches:
@@ -9,10 +12,16 @@ on:
       - dataseed
     paths:
       - 'requirements.txt'
-      - 'packages.R'
       - 'local_install.R'
       - 'Dockerfile'
-      - 'R/pkgs/**' name: Deploy to Amazon ECR APL jobs:
+      - 'R/pkgs/**'
+      - '.Rprofile'
+      - 'packrat/**'
+      - '.github/workflows/aws-apl.yaml'
+
+name: Deploy to Amazon ECR APL
+
+jobs:
   deploy:
     name: Deploy
     runs-on: ubuntu-latest
@@ -37,11 +46,15 @@ on:
         IMAGE_TAG_RUN: ${{ github.run_number }}
         IMAGE_TAG_REF: ${{ github.ref }}
       run: |
-        # Build a docker container and push it to ECR so that it can be deployed to ECS.
+        # Build a docker container and
+        # push it to ECR so that it can
+        # be deployed to ECS.
         docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA .
+
         # Add new tags
         docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA $ECR_REGISTRY/$ECR_REPOSITORY:$(git log -1 --pretty=%h)
         docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA $ECR_REGISTRY/$ECR_REPOSITORY:${IMAGE_TAG_REF##*/}-r$IMAGE_TAG_RUN
+
         if [[ $IMAGE_TAG_REF == *master* ]]; then
           docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA $ECR_REGISTRY/$ECR_REPOSITORY:latest
           docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA $ECR_REGISTRY/$ECR_REPOSITORY:latest-master
@@ -53,8 +66,10 @@ on:
           echo "Error, branch not recognized"
           exit 1
         fi
+
         docker push $ECR_REGISTRY/$ECR_REPOSITORY
         echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY"
+
     - name: Logout of Amazon ECR
       if: always()
       run: docker logout ${{ steps.login-ecr.outputs.registry }}

.github/workflows/aws.yaml

@@ -19,6 +19,7 @@ on:
       - 'R/pkgs/**'
       - '.Rprofile'
       - 'packrat/**'
+      - '.github/workflows/aws.yaml'
 
 name: Deploy to Amazon ECR
 

.github/workflows/ci.yml

@@ -41,9 +41,9 @@ jobs:
           setwd("R/pkgs/hospitalization")
           devtools::test(stop_on_failure=TRUE)
         shell: Rscript {0}
-      - name: Run report_generation tests
+      - name: Run report.generation tests
         run: |
-          setwd("R/pkgs/report_generation")
+          setwd("R/pkgs/report.generation")
           devtools::test(stop_on_failure=TRUE)
         shell: Rscript {0}
       - name: Run integration tests

.github/workflows/dockerhub.yaml

@@ -17,6 +17,7 @@ on:
       - 'R/pkgs/**'
       - '.Rprofile'
       - 'packrat/**'
+      - '.github/workflows/dockerhub*'
 
 name: Deploy to DockerHub
 
@@ -29,6 +30,19 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v2
 
+    -
+      name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    -
+      name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        Username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+
     - name: Generate Tags
       id: tags
       env:
@@ -39,29 +53,24 @@ jobs:
         echo "::set-output name=reftag::${IMAGE_TAG_REF##*/}-r$IMAGE_TAG_RUN"
 
     - name: Build Push Master
-      uses: docker/build-push-action@v1
+      uses: docker/build-push-action@v2
       if: contains( github.ref, 'master') || contains( github.base_ref, 'master' )
       with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-        repository: hopkinsidd/covidscenariopipeline
-        tags: latest,latest-master,${{steps.tags.outputs.shashort}},${{steps.tags.outputs.reftag}},${{ github.sha }}
+        push: true
+        tags: |
+          hopkinsidd/covidscenariopipeline:latest
+          hopkinsidd/covidscenariopipeline:latest-master
+          hopkinsidd/covidscenariopipeline:${{steps.tags.outputs.shashort}}
+          hopkinsidd/covidscenariopipeline:${{steps.tags.outputs.reftag}}
+          hopkinsidd/covidscenariopipeline:${{ github.sha }}
 
     - name: Build Push Dev
-      uses: docker/build-push-action@v1
+      uses: docker/build-push-action@v2
       with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-        repository: hopkinsidd/covidscenariopipeline
-        tags: latest-dev,${{steps.tags.outputs.shashort}},${{steps.tags.outputs.reftag}},${{ github.sha }}
+        push: true
+        tags: |
+          hopkinsidd/covidscenariopipeline:latest-dev
+          hopkinsidd/covidscenariopipeline:${{steps.tags.outputs.shashort}}
+          hopkinsidd/covidscenariopipeline:${{steps.tags.outputs.reftag}}
+          hopkinsidd/covidscenariopipeline:${{ github.sha }}
       if: contains( github.ref , 'dev') || contains( github.base_ref, 'dev' )
-
-
-    - name: Build Push Dataseed
-      uses: docker/build-push-action@v1
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-        repository: hopkinsidd/covidscenariopipeline
-        tags: latest-dataseed,${{steps.tags.outputs.shashort}},${{steps.tags.outputs.reftag}},${{ github.sha }}
-      if: contains( github.ref , 'dataseed') || contains( github.base_ref, 'dataseed' )

Dockerfile

@@ -12,7 +12,6 @@ ENV LC_ALL en_US.UTF-8
 # set noninteractive installation
 ENV DEBIAN_FRONTEND noninteractive
 ENV R_VERSION 3.6.3-1bionic
-ENV RSTUDIO_VERSION 1.2.5033
 
 # see https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04
 # https://cran.r-project.org/bin/linux/debian/
@@ -70,7 +69,6 @@ RUN apt-get update && \
     supervisor \
     awscli \
     r-base-dev=$R_VERSION \
-    openjdk-8-jdk \
     # make sure we have up-to-date CA certs or curling some https endpoints (like python.org) may fail
     ca-certificates \
     # app user creation
@@ -96,14 +94,9 @@ RUN Rscript -e "install.packages('packrat',repos='https://cloud.r-project.org/')
 COPY --chown=app:app packrat $HOME/packrat
 COPY --chown=app:app Docker.Rprofile $HOME/.Rprofile
 COPY --chown=app:app R/pkgs $HOME/R/pkgs
-RUN Rscript -e 'packrat::restore()' \
-    && curl -O https://download2.rstudio.org/server/bionic/amd64/rstudio-server-$RSTUDIO_VERSION-amd64.deb \
-    && sudo apt-get install -f -y ./rstudio-server-$RSTUDIO_VERSION-amd64.deb \
-    && rm -f ./rstudio-server-$RSTUDIO_VERSION-amd64.deb
-RUN Rscript -e 'install.packages(list.files("R/pkgs",full.names=TRUE),type="source",repos=NULL)' \
+RUN Rscript -e 'packrat::restore()'
+RUN Rscript -e 'install.packages(list.files("R/pkgs",full.names=TRUE,recursive=TRUE),type="source",repos=NULL)'
 
-# expose Rstudio port
-EXPOSE 8787
 
 #####
 # Python (managed via pyenv)
@@ -134,15 +127,4 @@ RUN . $PYTHON_VENV_DIR/bin/activate \
     && pip install --upgrade pip setuptools \
     && pip install -r $HOME/requirements.txt
 
-
-#####
-# Spark
-#####
-
-ENV SPARK_VERSION 2.4.5
-
-RUN cd /opt \
-    && curl -L http://mirrors.ocf.berkeley.edu/apache/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop2.7.tgz | sudo tar xvfz - \
-    && sudo ln -s spark-$SPARK_VERSION-bin-hadoop2.7 spark
-
 CMD ["/bin/bash"]

R/pkgs/covidcommon/R/config.R

@@ -27,8 +27,15 @@ load_config <- function(fname) {
     fname <- Sys.getenv("CONFIG_PATH")
   }
   if (!missing(fname)) {
-    handlers <- list(map=function(x) { class(x) <- "config"; return(x) })
-    return(tryCatch(yaml.load_file(fname, handlers=handlers), error = function(e) { stop(paste("Could not find file: ", fname)) }))
+
+    if(!file.exists(fname)){
+      stop(paste("Could not find file:", fname))
+    } else{
+      handlers <- list(map=function(x) { class(x) <- "config"; return(x) })
+      return(tryCatch(yaml.load_file(fname, handlers=handlers), error = function(e) { stop(paste("The config", fname, "has an error. Run `yaml::read_yaml(", fname, ")` to identify the line where the error exists.")) }))
+    } 
+
+
   } else {
     return(NA)
   }

R/pkgs/covidcommon/tests/testthat/test-load_config.R

@@ -1,6 +1,8 @@
 test_that("load_config works", {
   fname <- tempfile()
   cat("yaml: TRUE\n",file=fname)
+  fname_bad <- tempfile()
+  cat("yaml: TRUE\n yaml2: FALSE\n",file=fname_bad)
 
   expect_equal(
     load_config(fname)$yaml,
@@ -9,7 +11,7 @@ test_that("load_config works", {
 
   expect_error(
     load_config(";lkdjaoijdsfjoasidjfaoiwerfj q2fu8ja8erfasdiofj aewr;fj aff409a urfa8rf a';j 38i a0fuadf "),
-    "file"
+    "Could not find"
   )
 
   expect_error(
@@ -21,6 +23,11 @@ test_that("load_config works", {
     load_config(fname)$missing$badkey,
     "missing"
   )
+
+  expect_error(
+    load_config(fname_bad),
+    "yaml::read_yaml"
+    )
 })
 
 test_that("as_evaled_expression works", {

R/pkgs/hospitalization/DESCRIPTION

@@ -23,3 +23,4 @@ Description: Generate hospitalization scenarios corresponding to the infection s
 License: What license it uses
 Encoding: UTF-8
 LazyData: true
+RoxygenNote: 7.1.0

R/pkgs/hospitalization/NAMESPACE

@@ -1,4 +1,5 @@
-# Generated by roxygen2: fake comment so roxygen2 overwrites silently.
-exportPattern("^[^\\.]")
+# Generated by roxygen2: do not edit by hand
 
-importFrom(foreach,"%dopar%")
+export(build_hospdeath_geoid_fixedIFR_par)
+export(build_hospdeath_par)
+export(create_delay_frame)

R/pkgs/hospitalization/R/hospdeath.R

@@ -108,7 +108,7 @@ create_delay_frame <- function(data, name, local_config){
       geoid %in% all_geoids
     ) %>%
     dplyr::arrange(geoid,time) %>%
-    ungroup()
+    dplyr::ungroup()
 
   data <- dplyr::arrange(data,geoid,time)
 
@@ -120,6 +120,9 @@ create_delay_frame <- function(data, name, local_config){
   return(data)
 }
 
+
+
+
 hosp_create_delay_frame <- function(X, p_X, data_, X_pars, varname) {
     X_ <- rbinom(length(data_[[X]]),data_[[X]],p_X)
     rc <- data.table::data.table(
@@ -388,6 +391,11 @@ build_hospdeath_geoid_fixedIFR_par <- function(
       left_join(prob_dat, by="geoid")
 
     # Add time things
+    dat_Mild <- hosp_create_delay_frame('incidI',
+                                     dat_$p_mild_inf,
+                                     dat_,
+                                     c(-Inf, 0), # we dont want a delay here, so this is the easiest way
+                                     "Mild") 
     dat_H <- hosp_create_delay_frame('incidI',
                                      dat_$p_hosp_inf_scaled,
                                      dat_,
@@ -408,13 +416,14 @@ build_hospdeath_geoid_fixedIFR_par <- function(
     ICU_dur_ <- round(exp(time_ICUdur_pars[1]))
     Vent_dur_ <- round(exp(time_ventdur_pars[1]))
 
-    stopifnot(is.data.table(dat_I) && is.data.table(dat_H) && is.data.table(data_ICU) && is.data.table(data_Vent) && is.data.table(data_D))
+    stopifnot(is.data.table(dat_I) && is.data.table(dat_Mild) && is.data.table(dat_H) && is.data.table(data_ICU) && is.data.table(data_Vent) && is.data.table(data_D))
 
     # Using `merge` instead of full_join for performance reasons
     res <- Reduce(function(x, y, ...) merge(x, y, all = TRUE, ...),
-                  list(dat_I, dat_H, data_ICU, data_Vent, data_D)) %>%
+                  list(dat_I, dat_Mild, dat_H, data_ICU, data_Vent, data_D)) %>%
       replace_na(
         list(incidI = 0,
+             incidMild = 0,
              incidH = 0,
              incidICU = 0,
              incidVent = 0,

R/pkgs/report_generation/NAMESPACE → R/pkgs/report.generation/NAMESPACE

@@ -24,6 +24,7 @@ export(make_scn_time_summary_table)
 export(make_scn_time_summary_table_withVent)
 export(plot_event_time_by_geoid)
 export(plot_geounit_attack_rate_map)
+export(plot_geounit_map)
 export(plot_hist_incidHosp_state)
 export(plot_line_hospPeak_time_county)
 export(plot_model_vs_obs)
@@ -35,4 +36,3 @@ export(plot_ts_incid_inf_state_sample)
 export(print_pretty_date)
 export(print_pretty_date_short)
 export(reference_chunk)
-export(setup_testing_environment)

R/pkgs/report_generation/R/DataLoadFuncs.R → R/pkgs/report.generation/R/DataLoadFuncs.R

@@ -118,7 +118,7 @@ load_scenario_sims_filtered <- function(scenario_dir,
 ##' with pre and post filters
 ##' 
 ##' @param scenario_dir the subdirectory containing this scenario
-##' @param name_filter function that 
+##' @param name_filter string that indicates which pdeath level to import (from the hosp filename) 
 ##' @param post_process function that does processing after 
 ##' @param geoid_len in defined, this we want to make geoids all the same length
 ##' @param padding_char character to add to the front of geoids if fixed length
@@ -172,8 +172,8 @@ load_hosp_sims_filtered <- function(scenario_dir,
 
     read_file(files[i]) %>%
       padfn %>%
-      post_process(...) %>%
-      mutate(sim_num = i)
+      mutate(sim_num = i) %>%
+      post_process(...) 
   }
 
   rc<- dplyr::bind_rows(rc)