merged updates from demo

.github/workflows/test.yml

@@ -0,0 +1,22 @@
+on:
+  push:
+    branches:
+    - test
+    - demo
+    - prod
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/extractor/Dockerfile-test
+          tags: extractor-test
+          platforms: linux/arm64

.gitignore

@@ -6,3 +6,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+/.idea/

.ruby-version

@@ -0,0 +1 @@
+3.1.2

Dockerfile

@@ -1,32 +1,29 @@
 # N.B.: this must match the Ruby version in the Gemfile, and /.ruby-version.
-FROM ruby:2.7.2
+FROM ruby:3.1.2
 
-ENV RAILS_ENV=production
+ENV RUBY_ENV=prod
 ENV RAILS_LOG_TO_STDOUT=true
 ENV RAILS_SERVE_STATIC_FILES=true
+ENV RUBY_HOME=/extractor
 
 RUN apt-get update && apt-get install -y \
   build-essential \
   git \
   libpq-dev \
   libarchive-dev
 
-RUN mkdir app
-WORKDIR app
 
+RUN mkdir extractor
+WORKDIR extractor
 
 # Copy the Gemfile as well as the Gemfile.lock and install gems.
 # This is a separate step so the dependencies will be cached.
 COPY Gemfile Gemfile.lock  ./
-RUN gem install bundler && bundle install --without development test --jobs 20 --retry 5
+RUN gem install bundler && bundle install
 
 # Copy the main application, except whatever is listed in .dockerignore.
 COPY . ./
 
-#RUN bin/rails assets:precompile
-
-EXPOSE 3000
-
 # This is the web server entry point. It will need to be overridden when
 # running the workers.
 CMD ["echo", "Error running task, please check the container override command!"]

Gemfile

@@ -1,16 +1,16 @@
 source "https://rubygems.org"
 git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
-ruby '2.7.2'
-
 # Use mime-types to determine mimetypes based on extension
 gem 'mime-types', require: 'mime/types/full'
 
 # Use rubyzip to read zip files
 gem 'rubyzip'
 
+gem 'config'
+
 # Use archive for non-zip archive files
-gem 'libarchive'
+gem 'ffi-libarchive'
 
 # Use os to interact with operating system
 gem 'os'
@@ -24,3 +24,7 @@ gem "aws-sdk-s3"
 
 gem "aws-sdk-sqs"
 
+gem 'minitest'
+
+gem 'simplecov'
+

Gemfile.lock

@@ -1,53 +1,103 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-eventstream (1.1.0)
-    aws-partitions (1.416.0)
-    aws-sdk-core (3.111.0)
-      aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
-      aws-sigv4 (~> 1.1)
-      jmespath (~> 1.0)
-    aws-sdk-kms (1.41.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-eventstream (1.3.0)
+    aws-partitions (1.862.0)
+    aws-sdk-core (3.190.0)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.8)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.74.0)
+      aws-sdk-core (~> 3, >= 3.188.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.87.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-s3 (1.141.0)
+      aws-sdk-core (~> 3, >= 3.189.0)
       aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.8)
+    aws-sdk-sqs (1.69.0)
+      aws-sdk-core (~> 3, >= 3.188.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.35.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
-      aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.2)
+    aws-sigv4 (1.8.0)
       aws-eventstream (~> 1, >= 1.0.2)
-    jmespath (1.4.0)
-    mime-types (3.3.1)
+    concurrent-ruby (1.2.2)
+    config (5.0.0)
+      deep_merge (~> 1.2, >= 1.2.1)
+      dry-validation (~> 1.0, >= 1.0.0)
+    deep_merge (1.2.2)
+    docile (1.4.0)
+    dry-configurable (1.1.0)
+      dry-core (~> 1.0, < 2)
+      zeitwerk (~> 2.6)
+    dry-core (1.0.1)
+      concurrent-ruby (~> 1.0)
+      zeitwerk (~> 2.6)
+    dry-inflector (1.0.0)
+    dry-initializer (3.1.1)
+    dry-logic (1.5.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0, < 2)
+      zeitwerk (~> 2.6)
+    dry-schema (1.13.3)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 1.0, >= 1.0.1)
+      dry-core (~> 1.0, < 2)
+      dry-initializer (~> 3.0)
+      dry-logic (>= 1.4, < 2)
+      dry-types (>= 1.7, < 2)
+      zeitwerk (~> 2.6)
+    dry-types (1.7.1)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0)
+      dry-inflector (~> 1.0)
+      dry-logic (~> 1.4)
+      zeitwerk (~> 2.6)
+    dry-validation (1.10.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0, < 2)
+      dry-initializer (~> 3.0)
+      dry-schema (>= 1.12, < 2)
+      zeitwerk (~> 2.6)
+    ffi (1.16.3)
+    ffi-libarchive (1.1.13)
+      ffi (~> 1.0)
+    jmespath (1.6.2)
+    mime-types (3.5.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.1104)
+    mime-types-data (3.2023.1003)
     mimemagic (0.3.10)
       nokogiri (~> 1)
       rake
-    nokogiri (1.11.2-x86_64-darwin)
+    minitest (5.20.0)
+    nokogiri (1.15.5-x86_64-darwin)
       racc (~> 1.4)
-    os (1.1.1)
-    racc (1.5.2)
-    rake (13.0.3)
-    rubyzip (2.3.0)
+    os (1.1.4)
+    racc (1.7.3)
+    rake (13.1.0)
+    rubyzip (2.3.2)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    zeitwerk (2.6.12)
 
 PLATFORMS
-  x86_64-darwin-19
+  x86_64-darwin-21
 
 DEPENDENCIES
   aws-sdk-s3
   aws-sdk-sqs
+  config
+  ffi-libarchive
   mime-types
   mimemagic (~> 0.3.6)
+  minitest
   os
   rake (~> 13.0)
   rubyzip
-
-RUBY VERSION
-   ruby 2.7.2p137
+  simplecov
 
 BUNDLED WITH
-   2.2.4
+   2.3.22

Rakefile

@@ -1,4 +1,11 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
-task default: %i[]
+require 'rake/testtask'
+require 'simplecov'
+require_relative 'bin/set-test-vars'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib' << 'test'
+  # t.libs << 'lib'
+  t.test_files = FileList['test/*_test.rb']
+end

bin/set-test-vars.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+ENV['RUBY_ENV'] = 'test'
+ENV['RUBY_HOME'] = ENV['IS_DOCKER'] == 'true' ? '/extractor' : '/Users/gschmitt/workspace/databank-archive-extractor'
+ENV['RUBY_TEST_HOME'] = "#{ENV['RUBY_HOME']}/test"

config/settings.yml

@@ -0,0 +1,3 @@
+aws:
+    region: "us-east-2"
+

config/settings/demo.yml

@@ -0,0 +1,8 @@
+aws:
+    efs:
+        mount_point: "/mnt/efs/"
+    sqs:
+        queue_name: "extractor-to-databank-demo"
+        queue_url: "https://sqs.us-east-2.amazonaws.com/721945215539/extractor-to-databank-demo"
+    s3:
+        json_bucket: "databank-demo-main"

config/settings/prod.yml

@@ -0,0 +1,8 @@
+aws:
+    efs:
+        mount_point: "/mnt/efs/"
+    sqs:
+        queue_name: "extractor-to-databank-prod"
+        queue_url: "https://sqs.us-east-2.amazonaws.com/721945215539/extractor-to-databank-prod"
+    s3:
+        json_bucket: "databank-main"