Merge pull request #451 from cul/2.6.x-s3-fcr-docker

2.6.x s3 fcr docker

.gitignore

@@ -24,6 +24,7 @@
 
 # Docker ignores (so these can be customized per-development-environment if needed)
 /docker/docker-compose.*.yml
+/docker/fedora.*.env
 
 # Ignore config files with sensitive data
 master.key

Gemfile.lock

@@ -292,7 +292,7 @@ GEM
     multi_json (1.15.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
-    mysql2 (0.5.5)
+    mysql2 (0.5.6)
     net-http (0.4.0)
       uri
     net-http-persistent (2.9.4)

docker/fedora/Dockerfile

@@ -11,11 +11,13 @@ FROM amazoncorretto:8
 ENV FEDORA_HOME /opt/fedora
 ENV CATALINA_HOME /opt/fedora/tomcat
 ENV PATH $CATALINA_HOME/bin:$PATH
-
 WORKDIR /opt
 
 COPY ./install.properties /opt/install.properties
 COPY ./setup-and-start.sh /opt/setup-and-start.sh
+COPY ./fedora.delegating-external.fcfg /opt/fedora.delegating-external.fcfg
+COPY ./permit-all-s3-resolution.xml /opt/permit-all-s3-resolution.xml
+COPY ./deny-unallowed-file-resolution.xml /opt/deny-unallowed-file-resolution.xml
 
 RUN curl -L https://github.com/fcrepo3/fcrepo/releases/download/v3.8.1/fcrepo-installer-3.8.1.jar -o /opt/fcrepo-installer.jar
 

docker/fedora/awsconfig/config

@@ -0,0 +1,3 @@
+[default]
+region=us-west-2
+output=json

docker/fedora/deny-unallowed-file-resolution.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Policy xmlns="urn:oasis:names:tc:xacml:1.0:policy"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        PolicyId="deny-file-resolve-if-not-allowed-dir"
+        RuleCombiningAlgId="urn:oasis:names:tc:xacml:1.0:rule-combining-algorithm:first-applicable">
+  <Description>deny any file datastream resolution if not in FEDORA_HOME/demo/succeed</Description>
+  <Target>
+    <Subjects>
+      <AnySubject/>
+    </Subjects>
+    <Resources>
+      <Resource>
+        <ResourceMatch MatchId="urn:oasis:names:tc:xacml:1.0:function:regexp-string-match">
+        <AttributeValue DataType="http://www.w3.org/2001/XMLSchema#string">^file:/.*$</AttributeValue>
+          <ResourceAttributeDesignator AttributeId="urn:fedora:names:fedora:2.1:resource:datastream:fileUri"
+            DataType="http://www.w3.org/2001/XMLSchema#string" MustBePresent="false" />
+        </ResourceMatch>
+      </Resource>
+    </Resources>
+    <Actions>
+      <Action>
+        <ActionMatch MatchId="urn:oasis:names:tc:xacml:1.0:function:string-equal">
+          <AttributeValue DataType="http://www.w3.org/2001/XMLSchema#string">urn:fedora:names:fedora:2.1:action:id-retrieveFile</AttributeValue>
+          <ActionAttributeDesignator DataType="http://www.w3.org/2001/XMLSchema#string" AttributeId="urn:fedora:names:fedora:2.1:action:id"/>
+        </ActionMatch>
+      </Action>
+    </Actions>
+  </Target>
+  <Rule RuleId="1" Effect="Permit">
+    <Condition FunctionId="urn:oasis:names:tc:xacml:1.0:function:or">
+      <Apply FunctionId="urn:oasis:names:tc:xacml:1.0:function:regexp-string-match">
+       <AttributeValue DataType="http://www.w3.org/2001/XMLSchema#string">^file:/opt/fixtures/.*$</AttributeValue>
+       <Apply FunctionId="urn:oasis:names:tc:xacml:1.0:function:string-one-and-only">s
+        <ResourceAttributeDesignator AttributeId="urn:fedora:names:fedora:2.1:resource:datastream:fileUri"
+            DataType="http://www.w3.org/2001/XMLSchema#string"/>
+       </Apply>
+     </Apply>
+    </Condition>
+  </Rule>
+  <Rule RuleId="2" Effect="Deny">
+  </Rule>
+</Policy>

docker/fedora/permit-all-s3-resolution.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Policy xmlns="urn:oasis:names:tc:xacml:1.0:policy"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        PolicyId="deny-file-resolve-if-not-allowed-dir"
+        RuleCombiningAlgId="urn:oasis:names:tc:xacml:1.0:rule-combining-algorithm:first-applicable">
+  <Description>deny any s3 datastream resolution if not in allowed URI patterns</Description>
+  <Target>
+    <Subjects>
+        <AnySubject/>
+    </Subjects>
+    <Resources>
+      <Resource>
+        <ResourceMatch MatchId="urn:oasis:names:tc:xacml:1.0:function:regexp-string-match">
+        <AttributeValue DataType="http://www.w3.org/2001/XMLSchema#string">^s3://.*$</AttributeValue>
+          <ResourceAttributeDesignator AttributeId="urn:fedora:names:fedora:2.1:resource:datastream:fileUri"
+            DataType="http://www.w3.org/2001/XMLSchema#string" MustBePresent="false" />
+        </ResourceMatch>
+      </Resource>
+    </Resources>
+    <Actions>
+      <Action>
+        <ActionMatch MatchId="urn:oasis:names:tc:xacml:1.0:function:string-equal">
+          <AttributeValue DataType="http://www.w3.org/2001/XMLSchema#string">urn:fedora:names:fedora:2.1:action:id-retrieveFile</AttributeValue>
+          <ActionAttributeDesignator DataType="http://www.w3.org/2001/XMLSchema#string" AttributeId="urn:fedora:names:fedora:2.1:action:id"/>
+        </ActionMatch>
+      </Action>
+    </Actions>
+  </Target>
+  <Rule RuleId="1" Effect="Deny">
+    <Condition FunctionId="urn:oasis:names:tc:xacml:1.0:function:not">
+      <Apply FunctionId="urn:oasis:names:tc:xacml:1.0:function:string-is-in">
+        <AttributeValue DataType="http://www.w3.org/2001/XMLSchema#string">administrator</AttributeValue>
+        <SubjectAttributeDesignator AttributeId="fedoraRole" DataType="http://www.w3.org/2001/XMLSchema#string"/>
+      </Apply>
+    </Condition>
+  </Rule>
+  <Rule RuleId="2" Effect="Permit">
+  </Rule>
+</Policy>

docker/fedora/setup-and-start.sh

@@ -5,6 +5,54 @@
 if [ ! -f /opt/fedora/tomcat/bin/catalina.sh ]; then
   java -jar /opt/fcrepo-installer.jar /opt/install.properties
   echo 'Done running installer jar...'
+
+  echo "Manually unpacking WAR to override libraries"
+  mkdir -p /opt/fedora/webapp-tmp/fedora
+  mv /opt/fedora/tomcat/webapps/fedora.war /opt/fedora/webapp-tmp/fedora/
+  cd /opt/fedora/webapp-tmp/fedora
+  jar -xvf fedora.war
+  cd /opt
+  mv /opt/fedora/webapp-tmp/fedora /opt/fedora/tomcat/webapps/
+
+  # # Temporarily switch Fedora port 8080 to 8081 so that the CI task process
+  # # monitoring Fedora startup doesn't think that Fedora is ready yet.
+  sed -i.bak 's/port="8080"/port="8081"/' /opt/fedora/tomcat/conf/server.xml
+
+  # # Start Fedora up for the first time (which sets up various files and directories)
+  /opt/fedora/tomcat/bin/catalina.sh start
+
+  # # Give Fedora some time to start up (and create various first-time startup files)
+  sleep 10
+
+  # Stop Fedora so that we can apply some overrides
+  /opt/fedora/tomcat/bin/catalina.sh stop
+
+  # Give Fedora some time to stop
+  sleep 10
+
+  # Revert server.xml change so that Fedora will run on port 8080 the next time we start it up.
+  rm /opt/fedora/tomcat/conf/server.xml
+  mv /opt/fedora/tomcat/conf/server.xml.bak /opt/fedora/tomcat/conf/server.xml
+
+  echo "Overriding Fedora 3 libraries"
+  rm /opt/fedora/tomcat/webapps/fedora/WEB-INF/lib/httpclient-4*.jar
+  rm /opt/fedora/tomcat/webapps/fedora/WEB-INF/lib/httpcore-4*.jar
+  rm /opt/fedora/tomcat/webapps/fedora/WEB-INF/lib/httpclient-4*.jar
+  cp /opt/jars/apache-http/*.jar /opt/fedora/tomcat/webapps/fedora/WEB-INF/lib/
+  rm /opt/fedora/tomcat/webapps/fedora/WEB-INF/lib/s3-url-protocol-*.jar
+  cp /opt/jars/cul/s3-url-protocol-*.jar /opt/fedora/tomcat/webapps/fedora/WEB-INF/lib/
+  cp /opt/jars/awssdk/*.jar /opt/fedora/tomcat/webapps/fedora/WEB-INF/lib/
+  rm /opt/fedora/tomcat/webapps/fedora/WEB-INF/lib/fcrepo3-s3-server-*.jar
+  cp /opt/jars/cul/fcrepo3-s3-server-*.jar /opt/fedora/tomcat/webapps/fedora/WEB-INF/lib/
+  echo "Done overriding Fedora 3 libraries; setting new FCFG config"
+  cp /opt/fedora.delegating-external.fcfg /opt/fedora/server/config/fedora.fcfg
+
+  # NOTE: The /opt/fedora/data/fedora-xacml-policies/repository-policies/ directory and the default content
+  # inside of it doesn't exist immediately after Fedora installation. This content is created only after
+  # Fedora starts up for the first time.
+  cp /opt/permit-all-s3-resolution.xml /opt/fedora/data/fedora-xacml-policies/repository-policies/default/permit-all-s3-resolution.xml
+  cp /opt/deny-unallowed-file-resolution.xml /opt/fedora/data/fedora-xacml-policies/repository-policies/default/deny-unallowed-file-resolution.xml
 fi
 
+# Start Fedora in the foreground
 /opt/fedora/tomcat/bin/catalina.sh run

docker/templates/docker-compose.development.yml

@@ -26,7 +26,18 @@ services:
   fedora:
     build: ./fedora
     restart: always
+    env_file:
+      - ./fedora.development.env
     ports:
       - '8080:8080'
     volumes:
       - fedora-install-dir:/opt/fedora
+      - type: bind
+        source: ./fedora/apache-http
+        target: /opt/jars/apache-http
+      - type: bind
+        source: ./fedora/awssdk
+        target: /opt/jars/awssdk
+      - type: bind
+        source: ./fedora/cul
+        target: /opt/jars/cul

docker/templates/docker-compose.test.yml

@@ -26,7 +26,21 @@ services:
   fedora:
     build: ./fedora
     restart: always
+    env_file:
+      - ./fedora.test.env
     ports:
       - '9080:8080'
     volumes:
       - fedora-install-dir:/opt/fedora
+      - type: bind
+        source: ../spec/fixtures
+        target: /opt/fixtures
+      - type: bind
+        source: ./fedora/apache-http
+        target: /opt/jars/apache-http
+      - type: bind
+        source: ./fedora/awssdk
+        target: /opt/jars/awssdk
+      - type: bind
+        source: ./fedora/cul
+        target: /opt/jars/cul

docker/templates/fedora.development.env

@@ -0,0 +1,2 @@
+AWS_REGION=us-west-2
+AWS_OUTPUT=json

docker/templates/fedora.test.env

@@ -0,0 +1,2 @@
+AWS_REGION=us-west-2
+AWS_OUTPUT=json

lib/tasks/hyacinth/docker.rake

@@ -35,7 +35,7 @@ namespace :hyacinth do
       expected_port = docker_compose_config['services']['fedora']['ports'][0].split(':')[0]
       url_to_check = "http://localhost:#{expected_port}/fedora/describe"
       puts "Waiting for Fedora to become available (at #{url_to_check})..."
-      Timeout.timeout(20, Timeout::Error, 'Timed out during Fedora startup check.') do
+      Timeout.timeout(60, Timeout::Error, 'Timed out during Fedora startup check.') do
         loop do
           begin
             sleep 0.25
@@ -61,7 +61,7 @@ namespace :hyacinth do
       docker_compose_template_dir = Rails.root.join('docker/templates')
       docker_compose_dest_dir = Rails.root.join('docker')
       Dir.foreach(docker_compose_template_dir) do |entry|
-        next unless entry.end_with?('.yml')
+        next unless entry.end_with?('.yml') || entry.end_with?('.env')
         src_path = File.join(docker_compose_template_dir, entry)
         dst_path = File.join(docker_compose_dest_dir, entry.gsub('.template', ''))
         if File.exist?(dst_path)

spec/features/digital_object_editor_ui_spec.rb

@@ -1,14 +1,14 @@
 require 'rails_helper'
 
 describe "Digital Object Editor UI" do
-  
+
   before(:each) do
     feature_spec_sign_in_admin_user
     wait_for_ajax
   end
 
   it "can create a new Digital Object", :js => true do
     expect(page).to have_content 'New Digital Object'
-    
+
   end
 end

spec/integration/fedora_read_spec.rb

@@ -0,0 +1,40 @@
+require 'rails_helper'
+
+describe "Fedora content ds read tests" do
+  describe "creating an Asset and attempting to download the bytes directly from Fedora" do
+    let(:pid) { 'sample:123' }
+    let(:docker_mounted_fixture_file_location) { '/opt/fixtures/files/lincoln.jpg' }
+
+    it "works" do
+      generic_resource = GenericResource.new(pid: pid)
+      content_ds = generic_resource.create_datastream(
+        ActiveFedora::Datastream,
+        'content',
+        controlGroup: 'E',
+        mimeType: BestType.mime_type.for_file_name(docker_mounted_fixture_file_location),
+        dsLabel: File.basename(docker_mounted_fixture_file_location),
+        versionable: true
+      )
+      content_ds.dsLocation = "file://#{docker_mounted_fixture_file_location}"
+      generic_resource.add_datastream(content_ds)
+
+      generic_resource.save
+
+      expect(generic_resource.datastreams['content'].dsLocation).to eq("file://#{docker_mounted_fixture_file_location}")
+      expect(generic_resource.datastreams['content'].content.length).to be_positive
+    end
+
+    # Uncomment this "test" when debugging Fedora errors:
+    # it "reads the fedora log to find out what went wrong", focus: true do
+    #   puts 'reading fedora log now...'
+    #   sleep 20
+    #   puts `docker container list`
+    #   fedora_container_id = `docker container list | grep fedora | awk '{print $1}'`.strip
+    #   puts "fedora_container_id: #{fedora_container_id}"
+    #   puts `docker exec #{fedora_container_id} ls -la /opt/fedora/server/logs/fedora.log`
+    #   fedora_log_content = `docker exec #{fedora_container_id} cat /opt/fedora/server/logs/fedora.log`
+    #   puts "fedora_log_content: #{fedora_log_content}"
+    #   expect(fedora_log_content).to eq('')
+    # end
+  end
+end