v6 docker-compose project

popHealthDocker_v6/PH1/docker-compose.yml

@@ -0,0 +1,45 @@
+version: "3"
+services:
+  mongo:
+    image: mongo:3.4.23
+    restart: always
+    ports:
+      - 27017:27017
+
+  rabbitmq:
+    image: rabbitmq:3.7.14
+#   environment:
+#      RABBITMQ_DEFAULT_VHOST: /
+    restart: always
+    ports:
+      - 5672:5672
+      - 15672:15672
+
+  ecqm:
+    build:
+      dockerfile: Dockerfile
+      context: ../js-ecqm-engine-docker
+    restart: always
+    depends_on:
+      - mongo
+      - rabbitmq
+    environment:
+      RABBITMQ_HOST: rabbitmq
+      MONGODB_HOST: mongo
+
+  pophealthweb:
+    build:
+      dockerfile: Dockerfile
+      context: ../pophealthweb
+    depends_on:
+      - mongo
+      - rabbitmq
+      - ecqm
+    environment:
+      MONGODB_HOST: mongo
+      MONGODB_PORT: 27017
+      NLM_USERNAME: username
+      NLM_PASSWORD: password
+      RABBITMQ_URL: 'amqp://rabbitmq'
+    ports:
+      - 8082:80

popHealthDocker_v6/PH2/docker-compose.yml

@@ -0,0 +1,45 @@
+version: "3"
+services:
+  mongo:
+    image: mongo:3.4.23
+    restart: always
+    ports:
+      - 27018:27017
+
+  rabbitmq:
+    image: rabbitmq:3.7.14
+#   environment:
+#      RABBITMQ_DEFAULT_VHOST: /
+    restart: always
+#    ports:
+#      - 5672:5672
+#      - 15672:15672
+
+  ecqm:
+    build:
+      dockerfile: Dockerfile
+      context: ../js-ecqm-engine-docker
+    restart: always
+    depends_on:
+      - mongo
+      - rabbitmq
+    environment:
+      RABBITMQ_HOST: rabbitmq
+      MONGODB_HOST: mongo
+
+  pophealthweb:
+    build:
+      dockerfile: Dockerfile
+      context: ../pophealthweb
+    depends_on:
+      - mongo
+      - rabbitmq
+      - ecqm
+    environment:
+      MONGODB_HOST: mongo
+      MONGODB_PORT: 27017
+      NLM_USERNAME: starklogic
+      NLM_PASSWORD: password
+      RABBITMQ_URL: 'amqp://rabbitmq'
+    ports:
+      - 8083:80

popHealthDocker_v6/docker ReadMe.md

@@ -0,0 +1,20 @@
+Some quick notes on popHealth v6 docker-compose:
+
+1) I have used this project on both Windows Desktop and a couple different Ubuntu server hosts using the CE versions of docker.  I had a bit of trouble downloading the 'static measure' section on one server.  The work around was to simply pull the content manually and commented out the fetch section in pophealth.rake.  This hack (removing the cql_measure_json.zip fetch inside pophealth.rake) is NOT included here.  However, I do include a very similar hack to use a local instance of the bundle-2018.zip.  In this case, I figured it was easier to grab it once and re-use it rather than downloading every time.
+
+2)  Some optional hacks are available as comments inside the pophealthweb Dockerfile, including both #1 above and using a pre-prepped popHealth.yml. Use what works best for you.
+
+3) This version works -- but I have not gone back to optimize it.  Meaning, some of the image fetches are very specific -- for example the mongo:2.4.23 and the rabbitmq:3.7.14.  I have not gone back to complete test how flexible some of these things are -- or what kinds of code updates would be necessary to upgrade.  You'll also notice I create a new js-ecqm-engine-docker image.  I tried to use the public, available js-ecqm_engine docker image from docker hub.  But was not successful, so I ended up copying their docker file to build my own image.
+
+4) In a similar vein as above (#3), I'm not sure if the #fix JSON Parse bit in the pophealthweb\Dockerfile is necessary.  This was an early experiment that was never revisited.  It doesn't seem to hurt... but I don't think it is needed.
+
+5)  You'll notice the PH1 and PH2 folder instances.  These function as independent running stacks.  To create more iterations (PHn) simply copy PH1 and update the pophealthweb port (8082:80) to use a new, unique port (for example: 8084:80).  
+
+6) The start up the running stack by:
+     PS C:\docker\popHealthDocker - Public\PH1>docker-compose build
+        -- and --
+     PS C:\docker\popHealthDocker - Public\PH1>docker-compose up
+
+   You can also start it up in detached mode.
+
+7) pophealth v6 is already obsolete.  I expect this will have a limited shelf-life.  But it might find some use.

popHealthDocker_v6/js-ecqm-engine-docker/Dockerfile

@@ -0,0 +1,20 @@
+FROM node:10.0-slim
+
+ENV NODE_ENV production
+
+RUN apt-get update
+RUN apt-get install -y git
+
+WORKDIR /tmp
+RUN git clone https://github.com/WorldVistA/js-ecqm-engine.git
+RUN cp -R js-ecqm-engine /usr/src/app
+RUN rm -rf /tmp/js-ecqm-engine
+RUN ls /usr/src/app
+
+WORKDIR /usr/src/app
+
+RUN yarn install --only=production
+
+RUN chmod 755 bin/rabbit_worker.js
+
+CMD [ "bin/rabbit_worker.js"]

popHealthDocker_v6/pophealthweb/Dockerfile

@@ -0,0 +1,73 @@
+FROM ruby:2.3.7
+
+RUN gem install bundler -v '1.17.1'
+
+# install
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends apt-utils
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update
+RUN apt-get install -y nodejs
+RUN apt-get install -y npm
+RUN npm install -g bower
+RUN apt-get install -y apache2=2.4.*
+RUN apt-get install -y libcurl4-openssl-dev apache2-dev libapr1-dev libaprutil1-dev libssl-dev
+RUN apt autoremove -y
+
+RUN gem install passenger -v '5.3.7'
+
+
+# Pull popHealth
+WORKDIR /
+RUN git clone https://github.com/WorldVistA/popHealth.git -b v6 popHealth
+WORKDIR /popHealth
+
+# fix fontawesome reference
+RUN sed -i 's/"font-awesome": "^4.7.0",/"font-awesome": "~4.7.0",/g' /popHealth/bower.json
+
+# fix JSON Parse
+RUN sed -i 's/collection = JSON.parse(File.read(json_path)),/collection = JSON.parse(File.read(json_path), :max_nesting => false)/g' /popHealth/lib/measures/baseline_loader.rb
+
+# Use custom config yml
+#COPY popHealth.yml /popHealth/config/popHealth.yml
+
+# custom bundle installer using LOCAL bundle.zip
+#COPY pophealth.rake /popHealth/lib/tasks/pophealth.rake
+
+# fixes to prevent halt of measure import on error (exception handling added)
+COPY cql_bundle_importer.rb /popHealth/lib/cql_bundle_importer.rb
+
+# User local pre-downloaded bundle-2018.zip
+#COPY bundle-2018.zip /popHealth/bundles/bundle-2018.zip
+
+# Add a mongoid.yml config file that supports environment variables to specify the host and port
+COPY mongoid.yml config/mongoid.yml
+
+# finally install popHealth
+RUN bundle install
+
+# bower must be run as a user, not root
+RUN bower install --allow-root
+
+# apache configuration
+
+RUN passenger-install-apache2-module
+
+ADD apache/sites-available/pophealth /etc/apache2/sites-available/pophealth
+RUN rm /etc/apache2/sites-enabled/000-default*
+RUN ln -s /etc/apache2/sites-available/pophealth /etc/apache2/sites-enabled/000-default.conf
+
+ADD apache/conf-available/pophealth /etc/apache2/conf-available/pophealth
+RUN ln -s /etc/apache2/conf-available/pophealth /etc/apache2/conf-enabled/pophealth.conf
+
+ADD apache/mods-available/pophealth /etc/apache2/mods-available/pophealth
+RUN ln -f -s /etc/apache2/mods-available/pophealth /etc/apache2/mods-enabled/pophealth.conf
+
+
+EXPOSE 80
+
+# add the startup script
+ADD start.sh /popHealth/start.sh
+RUN chmod 777 /popHealth/start.sh
+
+ENTRYPOINT [ "/popHealth/start.sh" ]

popHealthDocker_v6/pophealthweb/apache/conf-available/pophealth

@@ -0,0 +1 @@
+SetEnv SECRET_KEY_BASE 149be94cda719a98e6428578ea63b6db90a9c58347058bca7ceaeb795ca56b4a

popHealthDocker_v6/pophealthweb/apache/mods-available/pophealth

@@ -0,0 +1,5 @@
+LoadModule passenger_module /usr/local/bundle/gems/passenger-5.3.7/buildout/apache2/mod_passenger.so
+<IfModule mod_passenger.c>
+   PassengerRoot /usr/local/bundle/gems/passenger-5.3.7
+   PassengerDefaultRuby /usr/local/bin/ruby
+</IfModule>

popHealthDocker_v6/pophealthweb/apache/sites-available/pophealth

@@ -0,0 +1,10 @@
+<VirtualHost *:80>
+   PassengerRuby /usr/local/bin/ruby
+   DocumentRoot /popHealth/public
+   TimeOut 1200
+   <Directory /popHealth/public>
+      AllowOverride all
+      Options -MultiViews
+      Require all granted
+   </Directory>
+</VirtualHost>

popHealthDocker_v6/pophealthweb/cql_bundle_importer.rb

@@ -0,0 +1,152 @@
+require 'id.rb'
+module CqlBundle
+  class CqlBundleImporter
+    SOURCE_ROOTS = { bundle: 'bundle.json',
+                     measures: 'measures', results: 'results',
+                     valuesets: File.join('value_sets', 'json', '*.json'),
+                     patients: 'patients' }.freeze
+    COLLECTION_NAMES = ['bundles', 'records', 'measures', 'individual_results', 'system.js'].freeze
+    DEFAULTS = { type: nil,
+                 update_measures: true,
+                 clear_collections: COLLECTION_NAMES }.freeze
+
+    # Import a quality bundle into the database. This includes metadata, measures, test patients, supporting JS libraries, and expected results.
+    #
+    # @param [File] zip The bundle zip file.
+    # @param [String] Type of measures to import, either 'ep', 'eh' or nil for all
+    # @param [Boolean] keep_existing If true, delete all current collections related to patients and measures.
+
+    def self.import(zip, options = {})
+      options = DEFAULTS.merge(options)
+      @measure_id_hash = {}
+      @patient_id_hash = {}
+
+      bundle = nil
+      Zip::ZipFile.open(zip.path) do |zip_file|
+        bundle = unpack_bundle(zip_file)
+        check_bundle_versions(bundle)
+
+        # Store the bundle metadata.
+        raise bundle.errors.full_messages.join(',') unless bundle.save
+        puts 'bundle metadata unpacked...'
+
+        unpack_and_store_valuesets(zip_file, bundle)
+        unpack_and_store_measures(zip_file, options[:type], bundle)
+        unpack_and_store_qdm_patients(zip_file, options[:type], bundle)
+        unpack_and_store_results(zip_file, options[:type], bundle)
+      end
+
+      bundle
+    ensure
+      # If the bundle is nil or the bundle has never been saved then do not set done_importing or run save.
+      if bundle&.created_at
+        bundle.done_importing = true
+        bundle.save
+      end
+    end
+
+    def self.check_bundle_versions(bundle)
+      bundle_versions = Hash[* HealthDataStandards::CQM::Bundle.where(deprecated: false).collect { |b| [b.version, b.id] }.flatten]
+
+      # no bundles before 2018 and no non-deprecated bundles with same year
+      old_year_err = 'Please use bundles for year 2018 or later.'
+      raise old_year_err if bundle.version[0..3].to_i < 2018
+      same_year_err = "A non-deprecated bundle with year #{bundle.version[0..3]} already exists in the database. Please deprecate previous bundles."
+      raise same_year_err unless bundle_versions.select { |vers, _id| vers[0..3] == bundle.version[0..3] }.empty?
+    end
+
+    def self.unpack_bundle(zip)
+      HealthDataStandards::CQM::Bundle.new(JSON.parse(zip.read(SOURCE_ROOTS[:bundle]), max_nesting: 100))
+    end
+
+    def self.unpack_and_store_valuesets(zip, bundle)
+      entries = zip.glob(SOURCE_ROOTS[:valuesets])
+      entries.each_with_index do |entry, index|
+        vs = HealthDataStandards::SVS::ValueSet.new(unpack_json(entry))
+        vs['bundle_id'] = bundle.id
+        HealthDataStandards::SVS::ValueSet.collection.insert_one(vs.as_document)
+        report_progress('Value Sets', (index * 100 / entries.length)) if (index % 10).zero?
+      end
+      puts "\rLoading: Value Sets Complete          "
+    end
+
+    def self.unpack_and_store_measures(zip, type, bundle)
+      entries = zip.glob(File.join(SOURCE_ROOTS[:measures], type || '**', '*.json'))
+      entries.each_with_index do |entry, index|
+        # puts "Processing measure #{entry}"
+        source_measure = unpack_json(entry)
+        # we clone so that we have a source without a bundle id
+        measure = source_measure.clone
+        measure['bundle_id'] = bundle.id
+        value_sets = []
+        measure['value_set_oid_version_objects'].each do |vsv|
+          value_sets << HealthDataStandards::SVS::ValueSet.where(oid: vsv['oid'], version: vsv['version']).first.id
+        end
+        measure['value_sets'] = value_sets
+        begin
+          mes = Mongoid.default_client['measures'].insert_one(measure)
+        rescue
+          puts "Error Processing measure #{entry}"
+          next
+        end
+        @measure_id_hash[measure['bonnie_measure_id']] = mes.inserted_id
+        report_progress('measures', (index * 100 / entries.length)) if (index % 10).zero?
+      end
+      puts "\rLoading: Measures Complete          "
+    end
+
+    def self.unpack_and_store_qdm_patients(zip, type, bundle)
+      entries = zip.glob(File.join(SOURCE_ROOTS[:patients], type || '**', 'json', '*.json'))
+      entries.each_with_index do |entry, index|
+        patient = QDM::Patient.new(unpack_json(entry))
+        patient['bundleId'] = bundle.id
+
+        reconnect_references(patient)
+
+        @patient_id_hash[patient['extendedData.master_patient_id']] = patient['id']
+        patient.save
+        report_progress('patients', (index * 100 / entries.length)) if (index % 10).zero?
+      end
+      puts "\rLoading: Patients Complete          "
+    end
+
+    def self.reconnect_references(patient)
+      patient.dataElements.each do |data_element|
+        next unless data_element[:relatedTo]
+        ref_array = []
+        oid_hash = {}
+        patient.dataElements.each do |de|
+          oid_hash[{ 'codes' => de['dataElementCodes'].map { |dec| dec['code'] }.flatten, 'start_time' => de['authorDatetime'].to_i }.hash] = de.id
+        end
+        data_element[:relatedTo].each do |ref|
+          ref_array << oid_hash[{ 'codes' => ref['codes'], 'start_time' => ref['start_time'] }.hash]
+        end
+        data_element.relatedTo = ref_array
+      end
+    end
+
+    def self.unpack_and_store_results(zip, _type, bundle)
+      zip.glob(File.join(SOURCE_ROOTS[:results], '*.json')).each do |entry|
+        contents = unpack_json(entry)
+        contents.each do |document|
+          # Replace ids in bundle, with ids created during import
+          document['patient_id'] = @patient_id_hash[document['patient_id']]
+          document['measure_id'] = @measure_id_hash[document['measure_id']]
+          document['extendedData'] = {}
+          document['extendedData']['correlation_id'] = bundle.id.to_s
+          Mongoid.default_client['qdm_individual_results'].insert_one(document)
+        end
+      end
+      puts "\rLoading: Results Complete          "
+    end
+
+    def self.unpack_json(entry)
+      JSON.parse(entry.get_input_stream.read, max_nesting: false)
+    end
+
+    def self.report_progress(label, percent)
+      print "\rLoading: #{label} #{percent}% complete"
+      STDOUT.flush
+    end
+  end
+end