From 283f525cb3e859ea5a0795add224a901a935d0ef Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Feb 2022 12:10:02 -0500 Subject: [PATCH 001/171] Test with solr 8 config in solr/conf Update solr_wrapper configs to use solr 8 --- .circleci/config.yml | 13 +- .solr_wrapper | 6 +- config/solr_wrapper_test.yml | 4 +- solr/conf/_rest_managed.json | 3 + solr/conf/admin-extra.html | 31 +++ solr/conf/elevate.xml | 36 +++ solr/conf/mapping-ISOLatin1Accent.txt | 246 +++++++++++++++++ solr/conf/protwords.txt | 21 ++ solr/conf/schema.xml | 367 ++++++++++++++++++++++++++ solr/conf/scripts.conf | 24 ++ solr/conf/solrconfig.xml | 319 ++++++++++++++++++++++ solr/conf/spellings.txt | 2 + solr/conf/stopwords.txt | 58 ++++ solr/conf/stopwords_en.txt | 58 ++++ solr/conf/synonyms.txt | 31 +++ solr/conf/xslt/example.xsl | 132 +++++++++ solr/conf/xslt/example_atom.xsl | 67 +++++ solr/conf/xslt/example_rss.xsl | 66 +++++ solr/conf/xslt/luke.xsl | 337 +++++++++++++++++++++++ 19 files changed, 1812 insertions(+), 9 deletions(-) create mode 100644 solr/conf/_rest_managed.json create mode 100644 solr/conf/admin-extra.html create mode 100644 solr/conf/elevate.xml create mode 100644 solr/conf/mapping-ISOLatin1Accent.txt create mode 100644 solr/conf/protwords.txt create mode 100644 solr/conf/schema.xml create mode 100644 solr/conf/scripts.conf create mode 100644 solr/conf/solrconfig.xml create mode 100644 solr/conf/spellings.txt create mode 100644 solr/conf/stopwords.txt create mode 100644 solr/conf/stopwords_en.txt create mode 100644 solr/conf/synonyms.txt create mode 100644 solr/conf/xslt/example.xsl create mode 100644 solr/conf/xslt/example_atom.xsl create mode 100644 solr/conf/xslt/example_rss.xsl create mode 100644 solr/conf/xslt/luke.xsl diff --git a/.circleci/config.yml b/.circleci/config.yml index 20770c877e..8190f4e026 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2.1 orbs: - samvera: samvera/circleci-orb@1.0.0 + samvera: samvera/circleci-orb@1 jobs: build: docker: @@ -20,8 +20,12 @@ jobs: - image: ualbertalib/docker-fcrepo4:4.7 environment: CATALINA_OPTS: '-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC' - - image: solr:7-alpine - command: bin/solr -cloud -noprompt -f -p <> + - image: zookeeper:3.4 + - image: solr:8-slim + environment: + VERBOSE: yes + SECURITY_JSON: '{"authentication":{"blockUnknown": false, "class":"solr.BasicAuthPlugin", "credentials":{"solr":"IV0EHq1OnNrj6gvRCwvFwTrZ1+z1oBbnQdiVC3otuq0= Ndd7LKvVBAaZIF0QAVi1ekCfAJXr1GGfLtRUXhgrF8c="}, "realm":"My Solr users", "forwardCredentials": false}, "authorization":{ "class":"solr.RuleBasedAuthorizationPlugin", "permissions":[{"name":"security-edit", "role":"admin"}], "user-role":{"solr":"admin"}}}' + command: sh -c "server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd put /security.json \"${SECURITY_JSON}\" && solr-fg -cloud -noprompt -p << parameters.solr_port >> -z localhost:2181" - image: redis:alpine parameters: @@ -66,7 +70,8 @@ jobs: paths: - .cache/yarn - - samvera/install_solr_core + - samvera/install_solr_core: + solr_config_path: 'solr/conf' - run: command: | diff --git a/.solr_wrapper b/.solr_wrapper index a4225eabc6..8e84adf1f6 100644 --- a/.solr_wrapper +++ b/.solr_wrapper @@ -1,8 +1,8 @@ # Place any default configuration for solr_wrapper here -version: 7.7.2 -# port: 8983 +version: 8.11.1 +port: 8983 instance_dir: tmp/solr-development collection: persist: true - dir: solr/config/ + dir: solr/conf/ name: hydra-development diff --git a/config/solr_wrapper_test.yml b/config/solr_wrapper_test.yml index 36c63dadeb..943a7dd33b 100644 --- a/config/solr_wrapper_test.yml +++ b/config/solr_wrapper_test.yml @@ -1,8 +1,8 @@ #config/solr_wrapper_test.yml -# version: 6.0.0 +version: 8.11.1 port: 8985 instance_dir: tmp/solr-test collection: persist: false - dir: solr/config + dir: solr/conf name: hydra-test diff --git a/solr/conf/_rest_managed.json b/solr/conf/_rest_managed.json new file mode 100644 index 0000000000..e7ada3f6e4 --- /dev/null +++ b/solr/conf/_rest_managed.json @@ -0,0 +1,3 @@ +{ + "initArgs":{}, + "managedList":[]} \ No newline at end of file diff --git a/solr/conf/admin-extra.html b/solr/conf/admin-extra.html new file mode 100644 index 0000000000..21b50901cb --- /dev/null +++ b/solr/conf/admin-extra.html @@ -0,0 +1,31 @@ + + + diff --git a/solr/conf/elevate.xml b/solr/conf/elevate.xml new file mode 100644 index 0000000000..b91e75cec9 --- /dev/null +++ b/solr/conf/elevate.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + diff --git a/solr/conf/mapping-ISOLatin1Accent.txt b/solr/conf/mapping-ISOLatin1Accent.txt new file mode 100644 index 0000000000..186ca313dc --- /dev/null +++ b/solr/conf/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "??" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "??" => "ss" +# "\t" => " " +# "\n" => "" + +# ?? => A +"\u00C0" => "A" + +# ?? => A +"\u00C1" => "A" + +# ?? => A +"\u00C2" => "A" + +# ?? => A +"\u00C3" => "A" + +# ?? => A +"\u00C4" => "A" + +# ?? => A +"\u00C5" => "A" + +# ?? => AE +"\u00C6" => "AE" + +# ?? => C +"\u00C7" => "C" + +# ?? => E +"\u00C8" => "E" + +# ?? => E +"\u00C9" => "E" + +# ?? => E +"\u00CA" => "E" + +# ?? => E +"\u00CB" => "E" + +# ?? => I +"\u00CC" => "I" + +# ?? => I +"\u00CD" => "I" + +# ?? => I +"\u00CE" => "I" + +# ?? => I +"\u00CF" => "I" + +# ?? => IJ +"\u0132" => "IJ" + +# ?? => D +"\u00D0" => "D" + +# ?? => N +"\u00D1" => "N" + +# ?? => O +"\u00D2" => "O" + +# ?? => O +"\u00D3" => "O" + +# ?? => O +"\u00D4" => "O" + +# ?? => O +"\u00D5" => "O" + +# ?? => O +"\u00D6" => "O" + +# ?? => O +"\u00D8" => "O" + +# ?? => OE +"\u0152" => "OE" + +# ?? +"\u00DE" => "TH" + +# ?? => U +"\u00D9" => "U" + +# ?? => U +"\u00DA" => "U" + +# ?? => U +"\u00DB" => "U" + +# ?? => U +"\u00DC" => "U" + +# ?? => Y +"\u00DD" => "Y" + +# ?? => Y +"\u0178" => "Y" + +# ?? => a +"\u00E0" => "a" + +# ?? => a +"\u00E1" => "a" + +# ?? => a +"\u00E2" => "a" + +# ?? => a +"\u00E3" => "a" + +# ?? => a +"\u00E4" => "a" + +# ?? => a +"\u00E5" => "a" + +# ?? => ae +"\u00E6" => "ae" + +# ?? => c +"\u00E7" => "c" + +# ?? => e +"\u00E8" => "e" + +# ?? => e +"\u00E9" => "e" + +# ?? => e +"\u00EA" => "e" + +# ?? => e +"\u00EB" => "e" + +# ?? => i +"\u00EC" => "i" + +# ?? => i +"\u00ED" => "i" + +# ?? => i +"\u00EE" => "i" + +# ?? => i +"\u00EF" => "i" + +# ?? => ij +"\u0133" => "ij" + +# ?? => d +"\u00F0" => "d" + +# ?? => n +"\u00F1" => "n" + +# ?? => o +"\u00F2" => "o" + +# ?? => o +"\u00F3" => "o" + +# ?? => o +"\u00F4" => "o" + +# ?? => o +"\u00F5" => "o" + +# ?? => o +"\u00F6" => "o" + +# ?? => o +"\u00F8" => "o" + +# ?? => oe +"\u0153" => "oe" + +# ?? => ss +"\u00DF" => "ss" + +# ?? => th +"\u00FE" => "th" + +# ?? => u +"\u00F9" => "u" + +# ?? => u +"\u00FA" => "u" + +# ?? => u +"\u00FB" => "u" + +# ?? => u +"\u00FC" => "u" + +# ?? => y +"\u00FD" => "y" + +# ?? => y +"\u00FF" => "y" + +# ??? => ff +"\uFB00" => "ff" + +# ??? => fi +"\uFB01" => "fi" + +# ??? => fl +"\uFB02" => "fl" + +# ??? => ffi +"\uFB03" => "ffi" + +# ??? => ffl +"\uFB04" => "ffl" + +# ??? => ft +"\uFB05" => "ft" + +# ??? => st +"\uFB06" => "st" diff --git a/solr/conf/protwords.txt b/solr/conf/protwords.txt new file mode 100644 index 0000000000..5a32e50321 --- /dev/null +++ b/solr/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/solr/conf/schema.xml b/solr/conf/schema.xml new file mode 100644 index 0000000000..19697ecd05 --- /dev/null +++ b/solr/conf/schema.xml @@ -0,0 +1,367 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + diff --git a/solr/conf/scripts.conf b/solr/conf/scripts.conf new file mode 100644 index 0000000000..f58b262ae0 --- /dev/null +++ b/solr/conf/scripts.conf @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +user= +solr_hostname=localhost +solr_port=8983 +rsyncd_port=18983 +data_dir= +webapp_name=solr +master_host= +master_data_dir= +master_status_dir= diff --git a/solr/conf/solrconfig.xml b/solr/conf/solrconfig.xml new file mode 100644 index 0000000000..904379ce51 --- /dev/null +++ b/solr/conf/solrconfig.xml @@ -0,0 +1,319 @@ + + + + + + + + 5.0.0 + + + + + + + + + + + + + + + ${solr.blacklight-core.data.dir:} + + + + + + + + + + *:* + + + + + ${solr.ulog.dir:} + + + + ${solr.autoCommit.maxTime:15000} + false + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + edismax + explicit + *:* + OR + 2<-1 5<-2 6<90% + 1 + 2 + 0.01 + + + id + title_tesi^10.0 + section_label_tesim^5.0 + creator_ssim^3.0 + date_sim^3.0 + all_text_timv^1.0 + active_fedora_model_ssi + object_type_si + + + title_tesi^10.0 + section_label_tesim^5.0 + creator_ssim^3.0 + date_sim^3.0 + all_text_timv^1.0 + + + + author_tesim + + + + + title_tesim + + + + + subject_tesim + + + + + + *, + score + + + true + 1 + + true + default + true + true + false + 5 + + + + spellcheck + + + + + + off + all + 1 + {!raw f=id v=$id} + + id, + access_ssim, + discover_access_group_ssim,discover_access_person_ssim, + read_access_group_ssim,read_access_person_ssim, + edit_access_group_ssim,edit_access_person_ssim, + depositor_ti, + embargo_release_date_dtsi + inheritable_access_ssim, + inheritable_discover_access_group_ssim,inheritable_discover_access_person_ssim, + inheritable_read_access_group_ssim,inheritable_read_access_person_ssim, + inheritable_edit_access_group_ssim,inheritable_edit_access_person_ssim, + inheritable_embargo_release_date_dtsi + + + + + + + explicit + lucene + + + + + + + all + * + 1 + {!term f=id v=$id} + + + + + + + textSpell + + + + + + default + spell + ./spell + true + + + author + author_spell + ./spell_author + 0.7 + true + + + subject + subject_spell + ./spell_subject + 0.7 + true + + + title + title_spell + ./spell_title + 0.7 + true + + + + + + + + + + + + + + + + + last_modified + ignored_ + + + + + + + + + diff --git a/solr/conf/spellings.txt b/solr/conf/spellings.txt new file mode 100644 index 0000000000..765190ae55 --- /dev/null +++ b/solr/conf/spellings.txt @@ -0,0 +1,2 @@ +pizza +history diff --git a/solr/conf/stopwords.txt b/solr/conf/stopwords.txt new file mode 100644 index 0000000000..22f277fe0b --- /dev/null +++ b/solr/conf/stopwords.txt @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + diff --git a/solr/conf/stopwords_en.txt b/solr/conf/stopwords_en.txt new file mode 100644 index 0000000000..22f277fe0b --- /dev/null +++ b/solr/conf/stopwords_en.txt @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + diff --git a/solr/conf/synonyms.txt b/solr/conf/synonyms.txt new file mode 100644 index 0000000000..453eb3139e --- /dev/null +++ b/solr/conf/synonyms.txt @@ -0,0 +1,31 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaa => aaaa +bbb => bbbb1 bbbb2 +ccc => cccc1,cccc2 +a\=>a => b\=>b +a\,a => b\,b +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/solr/conf/xslt/example.xsl b/solr/conf/xslt/example.xsl new file mode 100644 index 0000000000..ff7cae746e --- /dev/null +++ b/solr/conf/xslt/example.xsl @@ -0,0 +1,132 @@ + + + + + + + + + + + + + + + <xsl:value-of select="$title"/> + + + +

+
+ This has been formatted by the sample "example.xsl" transform - + use your own XSLT to get a nicer page +
+ + + + + + + +
+ + + + +
+
+ + + + + + + + + + + + + + + javascript:toggle("");? +
+ + exp + + + + + + + + + + + + + + + + +
    + +
    • +     +
+ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/conf/xslt/example_atom.xsl b/solr/conf/xslt/example_atom.xsl new file mode 100644 index 0000000000..dbc7afa3b7 --- /dev/null +++ b/solr/conf/xslt/example_atom.xsl @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + Example Solr Atom 1.0 Feed + + This has been formatted by the sample "example_atom.xsl" transform - + use your own XSLT to get a nicer Atom feed. + + + Apache Solr + solr-user@lucene.apache.org + + + + + + tag:localhost,2007:example + + + + + + + + + <xsl:value-of select="str[@name='name']"/> + + tag:localhost,2007: + + + + + + diff --git a/solr/conf/xslt/example_rss.xsl b/solr/conf/xslt/example_rss.xsl new file mode 100644 index 0000000000..b5bd0cf9fe --- /dev/null +++ b/solr/conf/xslt/example_rss.xsl @@ -0,0 +1,66 @@ + + + + + + + + + + + + + Example Solr RSS 2.0 Feed + http://localhost:8983/solr + + This has been formatted by the sample "example_rss.xsl" transform - + use your own XSLT to get a nicer RSS feed. + + en-us + http://localhost:8983/solr + + + + + + + + + + + <xsl:value-of select="str[@name='name']"/> + + http://localhost:8983/solr/select?q=id: + + + + + + + http://localhost:8983/solr/select?q=id: + + + + diff --git a/solr/conf/xslt/luke.xsl b/solr/conf/xslt/luke.xsl new file mode 100644 index 0000000000..d3f71c6d82 --- /dev/null +++ b/solr/conf/xslt/luke.xsl @@ -0,0 +1,337 @@ + + + + + + + + + Solr Luke Request Handler Response + + + + + + + + + <xsl:value-of select="$title"/> + + + + + +

+ +

+
+ +
+ +

Index Statistics

+ + + +

Field Statistics

+ + + +

Document statistics

+ + + + + + + + + + +
+ +
+ + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + +
+

+ +

+ +
+ +
+ + + + +
+ + 50 + 800 + 160 + blue + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ background-color: ; width: px; height: px; +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
  • + +
    • +     +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + - + + - + + - + + - + + - + + - + + - + + - + + - + + - + + - + + - + + - + + + + + + + + + + + + + + + + + + From e1770d051bbb1d71e3240252137f86b84a7182f7 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 28 Feb 2022 12:30:16 -0500 Subject: [PATCH 002/171] Copy custom step from unreleased version of orb --- .circleci/config.yml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8190f4e026..2fdb079897 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,6 +39,12 @@ jobs: parallelism: type: integer default: 4 + solr_config_path: + type: string + default: 'solr/conf' + core_name: + type: string + default: 'hydra-test' working_directory: /home/app/avalon @@ -70,8 +76,29 @@ jobs: paths: - .cache/yarn - - samvera/install_solr_core: - solr_config_path: 'solr/conf' + # Copy solr 8 compliant step from unreleased version of orb + #- samvera/install_solr_core: + # solr_config_path: << parameters.solr_config_path >> + - run: + name: Wait for Solr + command: dockerize -wait tcp://localhost:<< parameters.solr_port >> -timeout 1m + - run: + name: Create solr core + command: | + if [ -d << parameters.solr_config_path >> ] + then + cd << parameters.solr_config_path >> + else + if [ -d "$(bundle show active-fedora)/lib/generators/active_fedora/config/solr/templates/solr/conf" ] + then + cd "$(bundle show active-fedora)/lib/generators/active_fedora/config/solr/templates/solr/conf" + else + cd "$(bundle show active-fedora)/lib/generators/active_fedora/config/solr/templates/solr/config" + fi + fi + zip -1 -r solr_conf.zip ./* + curl -H "Content-type:application/octet-stream" --data-binary @solr_conf.zip "http://solr:SolrRocks@127.0.0.1:<< parameters.solr_port >>/solr/admin/configs?action=UPLOAD&name=solrconfig" + curl -H 'Content-type: application/json' http://solr:SolrRocks@127.0.0.1:<< parameters.solr_port >>/api/collections/ -d '{create: {name: << parameters.core_name >>, config: solrconfig, numShards: 1}}' - run: command: | From 8d1d94a13283e76bd6376cc8b4627cfc7ab2214d Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 28 Feb 2022 11:25:21 -0500 Subject: [PATCH 003/171] Use solr 8 and latest postgres in docker-compose --- docker-compose.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e4d688fa1b..c116258aea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ networks: services: db: &db-avalon - image: postgres:10-alpine + image: postgres:14-alpine volumes: - database:/data environment: @@ -42,12 +42,11 @@ services: volumes: [] solr: &solr - image: solr:6.6.4-alpine + image: solr:8-slim volumes: - - ./solr/config:/opt/solr/avalon_conf + - ./solr/conf:/opt/solr/avalon_conf - solr:/opt/solr/server/solr/mycores - entrypoint: - - docker-entrypoint.sh + command: - solr-precreate - avalon - /opt/solr/avalon_conf @@ -56,7 +55,7 @@ services: solr-test: <<: *solr volumes: - - ./solr/config:/opt/solr/avalon_conf + - ./solr/conf:/opt/solr/avalon_conf hls: image: avalonmediasystem/nginx:minio From 2a4b3e7a2145286dd215725956ca4359249e2d2c Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Wed, 16 Feb 2022 11:37:47 -0500 Subject: [PATCH 004/171] Quick upgrade of dependencies to unlock rails 6.0 for testing --- Gemfile | 16 +- Gemfile.lock | 425 ++++++++++++++++++++++++++------------------------- 2 files changed, 227 insertions(+), 214 deletions(-) diff --git a/Gemfile b/Gemfile index 6e7e3cd05f..6d62c3ddfd 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,7 @@ source 'https://rubygems.org' # Core rails gem 'bootsnap', require: false gem 'listen' -gem 'rails', '=5.2.6.3' +gem 'rails', '=6.0.4.6' gem 'sprockets', '~>3.7.2' gem 'sqlite3' @@ -13,7 +13,7 @@ gem 'jquery-datatables' gem 'jquery-rails' gem 'jquery-ui-rails' gem 'react-rails' -gem 'sass-rails', '~> 5.0' +gem 'sass-rails', '>= 6' # Use the last known good version of sass gem 'sass', '3.4.22' gem 'sprockets-es6' @@ -22,22 +22,22 @@ gem 'webpacker' # Core Samvera gem 'active-fedora', '~> 13.2', '>= 13.2.5' -gem 'active_fedora-datastreams', '~> 0.3' -gem 'fedora-migrate', git: 'https://github.com/avalonmediasystem/fedora-migrate.git', tag: 'avalon-r6.5' -gem 'hydra-head', '~> 11.0' +gem 'active_fedora-datastreams', git: 'https://github.com/samvera-labs/active_fedora-datastreams', branch: 'rails6' +#gem 'fedora-migrate', git: 'https://github.com/avalonmediasystem/fedora-migrate.git', tag: 'avalon-r6.5' +gem 'hydra-head', '~> 12.0' gem 'ldp', '~> 1.0.3' gem 'noid-rails', '~> 3.0.1' gem 'rdf-rdfxml' gem 'rdf-vocab', '< 3.1.5' # Samvera version pins -gem 'blacklight', '< 7.0' +gem 'blacklight', '< 8' gem 'rdf', '~> 3.1' gem 'rsolr', '~> 1.0' # Rails & Samvera Plugins gem 'about_page', git: 'https://github.com/avalonmediasystem/about_page.git', tag: 'avalon-r6.5' -gem 'active_annotations', '~> 0.3' +gem 'active_annotations', git: 'https://github.com/avalonmediasystem/active_annotations.git', branch: 'the_future' gem 'activerecord-session_store', '>= 2.0.0' gem 'acts_as_list' gem 'api-pagination' @@ -50,7 +50,7 @@ gem 'rack-cors', require: 'rack/cors' gem 'rails_same_site_cookie' gem 'recaptcha', require: 'recaptcha/rails' gem 'samvera-persona', '~> 0.3' -gem 'speedy-af', '~> 0.1.3' +gem 'speedy-af', git: 'https://github.com/samvera-labs/speedy_af.git', branch: 'rails6' # Avalon Components gem 'avalon-workflow', git: "https://github.com/avalonmediasystem/avalon-workflow.git", tag: 'avalon-r6.5' diff --git a/Gemfile.lock b/Gemfile.lock index 10355c06af..627c838b09 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,6 +6,16 @@ GIT about_page (0.3.1) rails (>= 3.2) +GIT + remote: https://github.com/avalonmediasystem/active_annotations.git + revision: 0b8f5a8b4c319e6bc2c4764d9d0a3a7f351f1501 + branch: the_future + specs: + active_annotations (0.3.0) + json-ld + rails (>= 5.2, < 7.1) + rdf-vocab (>= 2.1.0) + GIT remote: https://github.com/avalonmediasystem/avalon-about.git revision: 8ae33197219cb6cb8b6fee1f616950c29f2b94ce @@ -43,16 +53,6 @@ GIT thor (~> 0.20) typhoeus -GIT - remote: https://github.com/avalonmediasystem/fedora-migrate.git - revision: 170a6e8aacb4413d133c9bd5635742d64ec855bc - tag: avalon-r6.5 - specs: - fedora-migrate (0.5.0) - rchardet - rdf-rdfxml - rubydora (~> 1.8) - GIT remote: https://github.com/avalonmediasystem/media-element-add-to-playlist.git revision: dfde0084f6c17ca16fa675086eb025440f99a02d @@ -86,40 +86,75 @@ GIT fugit (~> 1.1) sidekiq (>= 4.2.1) +GIT + remote: https://github.com/samvera-labs/active_fedora-datastreams + revision: 13eefc7bdc879fd6c3ad607a1ecc087898777d3d + branch: rails6 + specs: + active_fedora-datastreams (0.3.0) + active-fedora (>= 11.0.0.pre, < 14) + activemodel (< 6.1) + nom-xml (>= 0.5.1) + om (~> 3.1) + rdf (< 3.2) + rdf-rdfxml (~> 2.0) + +GIT + remote: https://github.com/samvera-labs/speedy_af.git + revision: acffb24d405316fb2da5b67f16d3c8dae71fb57e + branch: rails6 + specs: + speedy-af (0.2.0) + active-fedora (>= 11.0.0) + activesupport (> 5.2, < 6.1) + GIT remote: https://github.com/tawan/active-elastic-job.git - revision: 7d210bdbcc9c465cc9704bc6b130882fe4c8eb9f + revision: e1735bb48373dc398f704cffea52c73564fc1dcb specs: - active_elastic_job (3.1.0) + active_elastic_job (3.2.0) aws-sdk-sqs (~> 1) - rails (>= 5.0, < 7) + rails (>= 5.2.6, < 7.1) GEM remote: https://rubygems.org/ specs: - actioncable (5.2.6.3) - actionpack (= 5.2.6.3) + actioncable (6.0.4.6) + actionpack (= 6.0.4.6) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.6.3) - actionpack (= 5.2.6.3) - actionview (= 5.2.6.3) - activejob (= 5.2.6.3) + actionmailbox (6.0.4.6) + actionpack (= 6.0.4.6) + activejob (= 6.0.4.6) + activerecord (= 6.0.4.6) + activestorage (= 6.0.4.6) + activesupport (= 6.0.4.6) + mail (>= 2.7.1) + actionmailer (6.0.4.6) + actionpack (= 6.0.4.6) + actionview (= 6.0.4.6) + activejob (= 6.0.4.6) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.6.3) - actionview (= 5.2.6.3) - activesupport (= 5.2.6.3) + actionpack (6.0.4.6) + actionview (= 6.0.4.6) + activesupport (= 6.0.4.6) rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.6.3) - activesupport (= 5.2.6.3) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (6.0.4.6) + actionpack (= 6.0.4.6) + activerecord (= 6.0.4.6) + activestorage (= 6.0.4.6) + activesupport (= 6.0.4.6) + nokogiri (>= 1.8.5) + actionview (6.0.4.6) + activesupport (= 6.0.4.6) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.3) + rails-html-sanitizer (~> 1.1, >= 1.2.0) active-fedora (13.2.5) active-triples (>= 0.11.0, < 2.0.0) activemodel (>= 5.1) @@ -135,22 +170,11 @@ GEM activesupport (>= 3.0.0) rdf (>= 2.0.2, < 4.0) rdf-vocab (>= 2.0, < 4.0) - active_annotations (0.3.0) - json-ld - rails (~> 5.2) - rdf-vocab (>= 2.1.0) active_encode (0.8.2) rails sprockets (< 4) - active_fedora-datastreams (0.3.0) - active-fedora (>= 11.0.0.pre, < 14) - activemodel (< 6.0) - nom-xml (>= 0.5.1) - om (~> 3.1) - rdf (< 3.2) - rdf-rdfxml (~> 2.0) - activejob (5.2.6.3) - activesupport (= 5.2.6.3) + activejob (6.0.4.6) + activesupport (= 6.0.4.6) globalid (>= 0.3.6) activejob-traffic_control (0.1.3) activejob (>= 4.2) @@ -159,27 +183,28 @@ GEM activejob-uniqueness (0.2.2) activejob (>= 4.2, < 7) redlock (>= 1.2, < 2) - activemodel (5.2.6.3) - activesupport (= 5.2.6.3) - activerecord (5.2.6.3) - activemodel (= 5.2.6.3) - activesupport (= 5.2.6.3) - arel (>= 9.0) + activemodel (6.0.4.6) + activesupport (= 6.0.4.6) + activerecord (6.0.4.6) + activemodel (= 6.0.4.6) + activesupport (= 6.0.4.6) activerecord-session_store (2.0.0) actionpack (>= 5.2.4.1) activerecord (>= 5.2.4.1) multi_json (~> 1.11, >= 1.11.2) rack (>= 2.0.8, < 3) railties (>= 5.2.4.1) - activestorage (5.2.6.3) - actionpack (= 5.2.6.3) - activerecord (= 5.2.6.3) + activestorage (6.0.4.6) + actionpack (= 6.0.4.6) + activejob (= 6.0.4.6) + activerecord (= 6.0.4.6) marcel (~> 1.0.0) - activesupport (5.2.6.3) + activesupport (6.0.4.6) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) + zeitwerk (~> 2.2, >= 2.2.2) acts_as_list (1.0.4) activerecord (>= 4.2) addressable (2.8.0) @@ -187,33 +212,32 @@ GEM airbrussh (1.4.0) sshkit (>= 1.6.1, != 1.7.0) amazing_print (1.4.0) - api-pagination (4.8.2) - arel (9.0.0) + api-pagination (5.0.0) ast (2.4.2) audio_waveform-ruby (1.0.7) json (~> 2.3) autoprefixer-rails (10.4.2.0) execjs (~> 2) aws-eventstream (1.2.0) - aws-partitions (1.551.0) - aws-record (2.6.1) + aws-partitions (1.555.0) + aws-record (2.7.0) aws-sdk-dynamodb (~> 1.18) - aws-sdk-cloudfront (1.57.0) - aws-sdk-core (~> 3, >= 3.121.2) + aws-sdk-cloudfront (1.62.0) + aws-sdk-core (~> 3, >= 3.126.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.125.5) + aws-sdk-core (3.126.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.525.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-dynamodb (1.64.0) - aws-sdk-core (~> 3, >= 3.121.2) + aws-sdk-dynamodb (1.72.0) + aws-sdk-core (~> 3, >= 3.126.0) aws-sigv4 (~> 1.1) - aws-sdk-elastictranscoder (1.33.0) - aws-sdk-core (~> 3, >= 3.121.2) + aws-sdk-elastictranscoder (1.37.0) + aws-sdk-core (~> 3, >= 3.126.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.53.0) - aws-sdk-core (~> 3, >= 3.125.0) + aws-sdk-kms (1.54.0) + aws-sdk-core (~> 3, >= 3.126.0) aws-sigv4 (~> 1.1) aws-sdk-rails (3.6.1) aws-record (~> 2) @@ -222,15 +246,15 @@ GEM aws-sessionstore-dynamodb (~> 2) concurrent-ruby (~> 1) railties (>= 5.2.0) - aws-sdk-s3 (1.111.3) - aws-sdk-core (~> 3, >= 3.125.0) + aws-sdk-s3 (1.112.0) + aws-sdk-core (~> 3, >= 3.126.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.4) - aws-sdk-ses (1.42.0) - aws-sdk-core (~> 3, >= 3.121.2) + aws-sdk-ses (1.46.0) + aws-sdk-core (~> 3, >= 3.126.0) aws-sigv4 (~> 1.1) - aws-sdk-sqs (1.45.0) - aws-sdk-core (~> 3, >= 3.121.2) + aws-sdk-sqs (1.50.0) + aws-sdk-core (~> 3, >= 3.126.0) aws-sigv4 (~> 1.1) aws-sessionstore-dynamodb (2.0.1) aws-sdk-dynamodb (~> 1) @@ -249,27 +273,26 @@ GEM rubocop-performance rubocop-rails rubocop-rspec - blacklight (6.24.0) - bootstrap-sass (~> 3.2) + blacklight (7.22.2) deprecation globalid + i18n (>= 1.7.0) jbuilder (~> 2.7) kaminari (>= 0.15) - nokogiri (~> 1.6) - rails (>= 4.2, < 6) - rsolr (>= 1.0.6, < 3) - twitter-typeahead-rails (= 0.11.1.pre.corejavascript) - blacklight-access_controls (0.6.2) - blacklight (~> 6.0) + ostruct (>= 0.3.2) + rails (>= 5.1, < 7) + view_component (~> 2.42) + blacklight-access_controls (6.0.0) + blacklight (> 6.0, < 8) cancancan (~> 1.8) deprecation (~> 1.0) - bootsnap (1.9.1) - msgpack (~> 1.0) + bootsnap (1.10.3) + msgpack (~> 1.2) bootstrap-sass (3.4.0) autoprefixer-rails (>= 5.2.1) sassc (>= 2.0.0) bootstrap-toggle-rails (2.2.1.0) - bootstrap_form (4.5.0) + bootstrap_form (5.0.0) actionpack (>= 5.2) activemodel (>= 5.2) builder (3.2.4) @@ -318,7 +341,7 @@ GEM execjs coffee-script-source (1.12.2) concurrent-ruby (1.1.9) - config (3.1.0) + config (4.0.0) deep_merge (~> 1.2, >= 1.2.1) dry-validation (~> 1.0, >= 1.0.0) connection_pool (2.2.5) @@ -326,7 +349,7 @@ GEM rexml crass (1.0.6) daemons (1.4.1) - dalli (3.0.3) + dalli (3.2.1) database_cleaner (2.0.1) database_cleaner-active_record (~> 2.0.0) database_cleaner-active_record (2.0.1) @@ -334,7 +357,7 @@ GEM database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) declarative (0.0.20) - deep_merge (1.2.1) + deep_merge (1.2.2) deprecation (1.1.0) activesupport devise (4.8.1) @@ -343,10 +366,10 @@ GEM railties (>= 4.1.0) responders warden (~> 1.2.3) - devise_invitable (2.0.5) + devise_invitable (2.0.6) actionmailer (>= 5.0) devise (>= 4.6) - diff-lcs (1.4.4) + diff-lcs (1.5.0) docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) @@ -357,7 +380,7 @@ GEM dropbox_api (0.1.18) faraday (<= 1.0) oauth2 (~> 1.1) - dry-configurable (0.13.0) + dry-configurable (0.14.0) concurrent-ruby (~> 1.0) dry-core (~> 0.6) dry-container (0.9.0) @@ -366,11 +389,11 @@ GEM dry-core (0.7.1) concurrent-ruby (~> 1.0) dry-inflector (0.2.1) - dry-initializer (3.0.4) + dry-initializer (3.1.1) dry-logic (1.2.0) concurrent-ruby (~> 1.0) dry-core (~> 0.5, >= 0.5) - dry-schema (1.8.0) + dry-schema (1.9.1) concurrent-ruby (~> 1.0) dry-configurable (~> 0.13, >= 0.13.0) dry-core (~> 0.5, >= 0.5) @@ -383,12 +406,12 @@ GEM dry-core (~> 0.5, >= 0.5) dry-inflector (~> 0.1, >= 0.1.2) dry-logic (~> 1.0, >= 1.0.2) - dry-validation (1.7.0) + dry-validation (1.8.0) concurrent-ruby (~> 1.0) dry-container (~> 0.7, >= 0.7.1) dry-core (~> 0.5, >= 0.5) dry-initializer (~> 3.0) - dry-schema (~> 1.8, >= 1.8.0) + dry-schema (~> 1.9, >= 1.9.1) ebnf (2.2.1) amazing_print (~> 1.2) htmlentities (~> 4.3) @@ -396,8 +419,8 @@ GEM scanf (~> 1.0) sxp (~> 1.1) unicode-types (~> 1.6) - edtf (3.0.6) - activesupport (>= 3.0, < 7.0) + edtf (3.0.8) + activesupport (>= 3.0, < 8.0) email_spec (2.2.0) htmlentities (~> 4.3.3) launchy (~> 2.1) @@ -415,14 +438,14 @@ GEM factory_bot_rails (6.2.0) factory_bot (~> 6.2.0) railties (>= 5.0.0) - fakefs (1.4.0) + fakefs (1.4.1) faker (2.19.0) i18n (>= 1.6, < 2) faraday (0.17.4) multipart-post (>= 1.2, < 3) faraday-encoding (0.0.5) faraday - fastimage (2.2.5) + fastimage (2.2.6) fcrepo_wrapper (0.9.0) ruby-progressbar ffi (1.15.5) @@ -478,20 +501,19 @@ GEM http-form_data (2.3.0) http_logger (0.7.0) httpclient (2.8.3) - hydra-access-controls (11.0.7) + hydra-access-controls (12.0.1) active-fedora (>= 10.0.0) - activesupport (>= 4, < 6) - blacklight (>= 5.16) - blacklight-access_controls (~> 0.6.0) - cancancan (~> 1.8) + activesupport (>= 5.2, < 7) + blacklight-access_controls (~> 6.0) + cancancan (>= 1.8, < 4) deprecation (~> 1.0) - hydra-core (11.0.7) - hydra-access-controls (= 11.0.7) - railties (>= 4.0.0, < 6) - hydra-head (11.0.7) - hydra-access-controls (= 11.0.7) - hydra-core (= 11.0.7) - rails (>= 5.2, < 6.1) + hydra-core (12.0.1) + hydra-access-controls (= 12.0.1) + railties (>= 5.2, < 7) + hydra-head (12.0.1) + hydra-access-controls (= 12.0.1) + hydra-core (= 12.0.1) + rails (>= 5.2, < 7) i18n (1.10.0) concurrent-ruby (~> 1.0) iconv (1.0.8) @@ -500,9 +522,10 @@ GEM ims-lti (1.1.13) builder oauth (>= 0.4.5, < 0.6) - jbuilder (2.11.2) + jbuilder (2.11.5) + actionview (>= 5.0.0) activesupport (>= 5.0.0) - jmespath (1.5.0) + jmespath (1.6.0) jquery-datatables (1.10.20) jquery-rails (4.4.0) rails-dom-testing (>= 1, < 3) @@ -520,18 +543,18 @@ GEM rack (~> 2.0) rdf (~> 3.1) jwt (2.3.0) - kaminari (1.2.1) + kaminari (1.2.2) activesupport (>= 4.1.0) - kaminari-actionview (= 1.2.1) - kaminari-activerecord (= 1.2.1) - kaminari-core (= 1.2.1) - kaminari-actionview (1.2.1) + kaminari-actionview (= 1.2.2) + kaminari-activerecord (= 1.2.2) + kaminari-core (= 1.2.2) + kaminari-actionview (1.2.2) actionview - kaminari-core (= 1.2.1) - kaminari-activerecord (1.2.1) + kaminari-core (= 1.2.2) + kaminari-activerecord (1.2.2) activerecord - kaminari-core (= 1.2.1) - kaminari-core (1.2.1) + kaminari-core (= 1.2.2) + kaminari-core (1.2.2) launchy (2.5.0) addressable (~> 2.7) ldp (1.0.3) @@ -545,7 +568,7 @@ GEM rdf-vocab (>= 0.8) slop link_header (0.0.8) - listen (3.7.0) + listen (3.7.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) little-plugger (1.1.4) @@ -573,13 +596,13 @@ GEM matrix (0.4.2) memoist (0.16.2) method_source (1.0.0) - mime-types (3.3.1) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0901) + mime-types-data (3.2022.0105) mini_mime (1.1.2) mini_portile2 (2.8.0) minitest (5.15.0) - msgpack (1.4.2) + msgpack (1.4.5) multi_json (1.15.0) multi_xml (0.6.0) multipart-post (2.1.1) @@ -597,13 +620,12 @@ GEM nokogiri (1.13.3) mini_portile2 (~> 2.8.0) racc (~> 1.4) - nom-xml (1.1.0) - activesupport (>= 3.2.18) + nom-xml (1.2.0) i18n nokogiri - oauth (0.5.6) - oauth2 (1.4.7) - faraday (>= 0.8, < 2.0) + oauth (0.5.8) + oauth2 (1.4.8) + faraday (>= 0.8, < 3.0) jwt (>= 1.0, < 3.0) multi_json (~> 1.3) multi_xml (~> 0.5) @@ -626,14 +648,15 @@ GEM ruby-saml (~> 1.9) orm_adapter (0.5.0) os (0.9.6) + ostruct (0.5.3) parallel (1.21.0) - paranoia (2.5.0) + paranoia (2.5.2) activerecord (>= 5.1, < 7.1) - parser (3.0.2.0) + parser (3.1.0.0) ast (~> 2.4.1) - pg (1.2.3) - pretender (0.3.4) - actionpack (>= 4.2) + pg (1.3.2) + pretender (0.4.0) + actionpack (>= 5.2) pry (0.13.1) coderay (~> 1.1) method_source (~> 1.0) @@ -650,24 +673,26 @@ GEM rack (2.2.3) rack-cors (1.1.1) rack (>= 2.0.0) - rack-protection (2.1.0) + rack-protection (2.2.0) rack - rack-proxy (0.7.0) + rack-proxy (0.7.2) rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.2.6.3) - actioncable (= 5.2.6.3) - actionmailer (= 5.2.6.3) - actionpack (= 5.2.6.3) - actionview (= 5.2.6.3) - activejob (= 5.2.6.3) - activemodel (= 5.2.6.3) - activerecord (= 5.2.6.3) - activestorage (= 5.2.6.3) - activesupport (= 5.2.6.3) + rails (6.0.4.6) + actioncable (= 6.0.4.6) + actionmailbox (= 6.0.4.6) + actionmailer (= 6.0.4.6) + actionpack (= 6.0.4.6) + actiontext (= 6.0.4.6) + actionview (= 6.0.4.6) + activejob (= 6.0.4.6) + activemodel (= 6.0.4.6) + activerecord (= 6.0.4.6) + activestorage (= 6.0.4.6) + activesupport (= 6.0.4.6) bundler (>= 1.3.0) - railties (= 5.2.6.3) + railties (= 6.0.4.6) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) @@ -681,19 +706,18 @@ GEM rails_same_site_cookie (0.1.9) rack (>= 1.5) user_agent_parser (~> 2.6) - railties (5.2.6.3) - actionpack (= 5.2.6.3) - activesupport (= 5.2.6.3) + railties (6.0.4.6) + actionpack (= 6.0.4.6) + activesupport (= 6.0.4.6) method_source rake (>= 0.8.7) - thor (>= 0.19.0, < 2.0) - rainbow (3.0.0) + thor (>= 0.20.3, < 2.0) + rainbow (3.1.1) rake (13.0.6) - rb-fsevent (0.11.0) + rb-fsevent (0.11.1) rb-inotify (0.10.1) ffi (~> 1.0) rb-readline (0.5.5) - rchardet (1.8.0) rdf (3.1.15) hamster (~> 3.0) link_header (~> 0.0, >= 0.0.8) @@ -729,30 +753,30 @@ GEM recaptcha (5.8.1) json redis (4.6.0) - redis-actionpack (5.2.0) - actionpack (>= 5, < 7) + redis-actionpack (5.3.0) + actionpack (>= 5, < 8) redis-rack (>= 2.1.0, < 3) redis-store (>= 1.1.0, < 2) - redis-activesupport (5.2.1) - activesupport (>= 3, < 7) + redis-activesupport (5.3.0) + activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) - redis-rack (2.1.3) + redis-rack (2.1.4) rack (>= 2.0.8, < 3) redis-store (>= 1.2, < 2) redis-rails (5.0.2) redis-actionpack (>= 5.0, < 6) redis-activesupport (>= 5.0, < 6) redis-store (>= 1.2, < 2) - redis-store (1.9.0) + redis-store (1.9.1) redis (>= 4, < 5) redlock (1.2.2) redis (>= 3.0.0, < 5.0) - regexp_parser (2.1.1) + regexp_parser (2.2.1) representable (3.1.1) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) - request_store (1.5.0) + request_store (1.5.1) rack (>= 1.4) responders (3.0.1) actionpack (>= 5.0) @@ -769,18 +793,18 @@ GEM rubyzip (>= 1.3.0, < 3.0.0) rsolr (1.1.2) builder (>= 2.1.2) - rspec-core (3.10.1) - rspec-support (~> 3.10.0) - rspec-expectations (3.10.1) + rspec-core (3.11.0) + rspec-support (~> 3.11.0) + rspec-expectations (3.11.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) + rspec-support (~> 3.11.0) rspec-its (1.3.0) rspec-core (>= 3.0.0) rspec-expectations (>= 3.0.0) - rspec-mocks (3.10.2) + rspec-mocks (3.11.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-rails (5.0.2) + rspec-support (~> 3.11.0) + rspec-rails (5.1.0) actionpack (>= 5.2) activesupport (>= 5.2) railties (>= 5.2) @@ -790,8 +814,8 @@ GEM rspec-support (~> 3.10) rspec-retry (0.6.2) rspec-core (> 3.3) - rspec-support (3.10.3) - rspec_junit_formatter (0.4.1) + rspec-support (3.11.0) + rspec_junit_formatter (0.5.1) rspec-core (>= 2, < 4, != 2.12.0) rubocop (0.85.1) parallel (~> 1.10) @@ -818,18 +842,9 @@ GEM multipart-post oauth2 ruby-progressbar (1.11.0) - ruby-saml (1.13.0) + ruby-saml (1.14.0) nokogiri (>= 1.10.5) rexml - rubydora (1.9.1) - activemodel - activesupport - deprecation - equivalent-xml - hooks (~> 0.3) - mime-types - nokogiri - rest-client rubyzip (1.3.0) samvera-persona (0.3.0) devise (~> 4.6) @@ -838,21 +853,23 @@ GEM pretender rails (>= 5.2.4.3, < 6.1) sass (3.4.22) - sass-rails (5.1.0) - railties (>= 5.2.0) - sass (~> 3.1) - sprockets (>= 2.8, < 4.0) - sprockets-rails (>= 2.0, < 4.0) - tilt (>= 1.1, < 3) + sass-rails (6.0.0) + sassc-rails (~> 2.1, >= 2.1.1) sassc (2.4.0) ffi (~> 1.9) + sassc-rails (2.1.2) + railties (>= 4.0.0) + sassc (>= 2.0) + sprockets (> 3.0) + sprockets-rails + tilt scanf (1.0.0) scrub_rb (1.0.1) selenium-webdriver (3.142.7) childprocess (>= 0.5, < 4.0) rubyzip (>= 1.2.2) semantic_range (3.0.0) - shoulda-matchers (5.0.0) + shoulda-matchers (5.1.0) activesupport (>= 5.2.0) sidekiq (6.4.1) connection_pool (>= 2.2.2) @@ -868,7 +885,7 @@ GEM simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) - simplecov_json_formatter (0.1.3) + simplecov_json_formatter (0.1.4) slop (4.9.1) solr_wrapper (3.1.2) http @@ -881,9 +898,6 @@ GEM nokogiri stomp xml-simple - speedy-af (0.1.3) - active-fedora (>= 11.0.0) - activesupport (< 5.3) sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -911,10 +925,6 @@ GEM thread_safe (0.3.6) tilt (2.0.10) trailblazer-option (0.1.2) - twitter-typeahead-rails (0.11.1.pre.corejavascript) - actionpack (>= 3.1) - jquery-rails - railties (>= 3.1) typhoeus (1.4.0) ethon (>= 0.9.0) tzinfo (1.2.9) @@ -927,15 +937,18 @@ GEM unf_ext (0.0.8) unicode-display_width (1.8.0) unicode-types (1.7.0) - user_agent_parser (2.7.0) + user_agent_parser (2.9.0) + view_component (2.49.0) + activesupport (>= 5.0.0, < 8.0) + method_source (~> 1.0) warden (1.2.9) rack (>= 2.0.9) wavefile (1.0.1) - web-console (3.7.0) - actionview (>= 5.0) - activemodel (>= 5.0) + web-console (4.2.0) + actionview (>= 6.0.0) + activemodel (>= 6.0.0) bindex (>= 0.4.0) - railties (>= 5.0) + railties (>= 6.0.0) webdrivers (3.9.4) nokogiri (~> 1.6) rubyzip (~> 1.0) @@ -959,9 +972,10 @@ GEM nokogiri (~> 1.8) xray-rails (0.3.2) rails (>= 3.1.0) + zeitwerk (2.5.4) zk (1.10.0) zookeeper (~> 1.5.0) - zookeeper (1.5.0) + zookeeper (1.5.1) zoom (0.5.0) PLATFORMS @@ -970,10 +984,10 @@ PLATFORMS DEPENDENCIES about_page! active-fedora (~> 13.2, >= 13.2.5) - active_annotations (~> 0.3) + active_annotations! active_elastic_job! active_encode (~> 0.8.2) - active_fedora-datastreams (~> 0.3) + active_fedora-datastreams! activejob-traffic_control activejob-uniqueness activerecord-session_store (>= 2.0.0) @@ -991,7 +1005,7 @@ DEPENDENCIES aws-sdk-sqs aws-sigv4 bixby - blacklight (< 7.0) + blacklight (< 8) bootsnap bootstrap-sass (< 3.4.1) bootstrap-toggle-rails @@ -1021,11 +1035,10 @@ DEPENDENCIES faker fastimage fcrepo_wrapper - fedora-migrate! google-analytics-rails (= 1.1.0) hashdiff (>= 1.0) hooks - hydra-head (~> 11.0) + hydra-head (~> 12.0) iconv (~> 1.0.6) iiif_manifest (~> 0.6) ims-lti (~> 1.1.13) @@ -1053,7 +1066,7 @@ DEPENDENCIES pry-rails puma (>= 4.3.8) rack-cors - rails (= 5.2.6.3) + rails (= 6.0.4.6) rails-controller-testing rails_same_site_cookie rb-readline @@ -1072,14 +1085,14 @@ DEPENDENCIES rspec_junit_formatter samvera-persona (~> 0.3) sass (= 3.4.22) - sass-rails (~> 5.0) + sass-rails (>= 6) selenium-webdriver shoulda-matchers sidekiq (~> 6.2) sidekiq-cron (~> 1.2)! simplecov solr_wrapper (>= 0.16) - speedy-af (~> 0.1.3) + speedy-af! sprockets (~> 3.7.2) sprockets-es6 sqlite3 From 275c96ff3152def789a688e6d144b570170286e5 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Fri, 18 Feb 2022 14:11:58 -0500 Subject: [PATCH 005/171] Apply new defaults for Rails 6 --- app/controllers/application_controller.rb | 1 - bin/setup | 15 ++- bin/yarn | 6 +- config/application.rb | 2 +- config/boot.rb | 2 +- config/cable.yml | 10 ++ config/environments/development.rb | 41 ++++---- config/environments/production.rb | 96 ++++++++++--------- config/environments/test.rb | 24 ++--- .../initializers/content_security_policy.rb | 30 ++++++ config/initializers/fedora_migrate.rb | 12 --- .../new_framework_defaults_6_0.rb | 45 +++++++++ ..._attachments_for_blob_id.active_storage.rb | 10 ++ 13 files changed, 189 insertions(+), 105 deletions(-) create mode 100644 config/cable.yml create mode 100644 config/initializers/content_security_policy.rb delete mode 100644 config/initializers/fedora_migrate.rb create mode 100644 config/initializers/new_framework_defaults_6_0.rb create mode 100644 db/migrate/20220218184645_add_foreign_key_constraint_to_active_storage_attachments_for_blob_id.active_storage.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index f6f4a20a96..a4c6a5e185 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -33,7 +33,6 @@ class ApplicationController < ActionController::Base before_action :rewrite_v4_ids, if: proc{|c| request.method_symbol == :get && [params[:id], params[:content]].compact.any? { |i| i =~ /^[a-z]+:[0-9]+$/}} before_action :set_no_cache_headers, if: proc{|c| request.xhr? } prepend_before_action :remove_zero_width_chars - skip_after_action :discard_flash_if_xhr # Suppress overwhelming Blacklight deprecation warning def set_no_cache_headers response.headers["Cache-Control"] = "no-cache, no-store" diff --git a/bin/setup b/bin/setup index ca842c1167..5853b5ea87 100755 --- a/bin/setup +++ b/bin/setup @@ -1,33 +1,32 @@ #!/usr/bin/env ruby -require 'pathname' require 'fileutils' -include FileUtils # path to your application root. -APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) +APP_ROOT = File.expand_path('..', __dir__) def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") end -chdir APP_ROOT do - # This script is a starting point to setup your application. +FileUtils.chdir APP_ROOT do + # This script is a way to setup or update your development environment automatically. + # This script is idempotent, so that you can run it at anytime and get an expectable outcome. # Add necessary setup steps to this file. puts '== Installing dependencies ==' system! 'gem install bundler --conservative' system('bundle check') || system!('bundle install') - # Install JavaScript dependencies if using Yarn + # Install JavaScript dependencies # system('bin/yarn') # puts "\n== Copying sample files ==" # unless File.exist?('config/database.yml') - # cp 'config/database.yml.sample', 'config/database.yml' + # FileUtils.cp 'config/database.yml.sample', 'config/database.yml' # end puts "\n== Preparing database ==" - system! 'bin/rails db:setup' + system! 'bin/rails db:prepare' puts "\n== Removing old logs and tempfiles ==" system! 'bin/rails log:clear tmp:clear' diff --git a/bin/yarn b/bin/yarn index c2bacef836..460dd565b4 100755 --- a/bin/yarn +++ b/bin/yarn @@ -1,8 +1,8 @@ #!/usr/bin/env ruby -VENDOR_PATH = File.expand_path('..', __dir__) -Dir.chdir(VENDOR_PATH) do +APP_ROOT = File.expand_path('..', __dir__) +Dir.chdir(APP_ROOT) do begin - exec "yarnpkg #{ARGV.join(" ")}" + exec "yarnpkg", *ARGV rescue Errno::ENOENT $stderr.puts "Yarn executable was not detected in the system." $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install" diff --git a/config/application.rb b/config/application.rb index 5a6e389085..e0c2fffda2 100644 --- a/config/application.rb +++ b/config/application.rb @@ -18,7 +18,7 @@ class Application < Rails::Application end # Initialize configuration defaults for originally generated Rails version. - config.load_defaults 5.1 + config.load_defaults 6.0 # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers diff --git a/config/boot.rb b/config/boot.rb index da8896b7b2..b9e460cef3 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -1,4 +1,4 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) require 'bundler/setup' # Set up gems listed in the Gemfile. -require 'bootsnap/setup' +require 'bootsnap/setup' # Speed up boot time by caching expensive operations. diff --git a/config/cable.yml b/config/cable.yml new file mode 100644 index 0000000000..1f20902804 --- /dev/null +++ b/config/cable.yml @@ -0,0 +1,10 @@ +development: + adapter: async + +test: + adapter: test + +production: + adapter: redis + url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> + channel_prefix: avalon_production diff --git a/config/environments/development.rb b/config/environments/development.rb index eb76029c61..66df51f6fc 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,37 +1,36 @@ -optimized = [1,'yes','true'].include?(ENV['OPTIMIZED_DEV']) - Rails.application.configure do - # Verifies that versions and hashed value of the package contents in the project's package.json -config.webpacker.check_yarn_integrity = false - # Settings specified here will take precedence over those in config/application.rb. # In the development environment your application's code is reloaded on # every request. This slows down response time but is perfect for development # since you don't have to restart the web server when you make code changes. - config.cache_classes = optimized + config.cache_classes = false # Do not eager load code on boot. - config.eager_load = optimized + config.eager_load = false # Show full error reports. config.consider_all_requests_local = true - # Disable Whiny Rails in Dev - config.web_console.whiny_requests = false - # Enable/disable caching. By default caching is disabled. - if optimized + # Run rails dev:cache to toggle caching. + if Rails.root.join('tmp', 'caching-dev.txt').exist? config.action_controller.perform_caching = true + config.action_controller.enable_fragment_cache_logging = true + config.cache_store = :memory_store config.public_file_server.headers = { - 'Cache-Control' => "public, max-age=#{2.days.seconds.to_i}" + 'Cache-Control' => "public, max-age=#{2.days.to_i}" } else config.action_controller.perform_caching = false + config.cache_store = :null_store end + # Store uploaded files on the local file system (see config/storage.yml for options). + config.active_storage.service = :local + # Don't care if the mailer can't send. config.action_mailer.raise_delivery_errors = false @@ -43,25 +42,19 @@ # Raise an error on page load if there are pending migrations. config.active_record.migration_error = :page_load + # Highlight code that triggered database queries in logs. + config.active_record.verbose_query_logs = true + # Debug mode disables concatenation and preprocessing of assets. # This option may cause significant delays in view rendering with a large # number of complex assets. - config.assets.debug = !optimized - - # Asset digests allow you to set far-future HTTP expiration dates on all assets, - # yet still be able to expire them through the digest params. - config.assets.digest = true - - # Adds additional error checking when serving assets at runtime. - # Checks for improperly declared sprockets dependencies. - # Raises helpful error messages. - config.assets.raise_runtime_errors = true + config.assets.debug = true # Suppress logger output for asset requests. config.assets.quiet = true - # Raises error for missing translations - config.action_view.raise_on_missing_translations = true + # Raises error for missing translations. + # config.action_view.raise_on_missing_translations = true # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. diff --git a/config/environments/production.rb b/config/environments/production.rb index da8df52ad5..ce735d9d25 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -1,9 +1,9 @@ Rails.application.configure do - # Verifies that versions and hashed value of the package contents in the project's package.json -config.webpacker.check_yarn_integrity = false - # Settings specified here will take precedence over those in config/application.rb. + # Verifies that versions and hashed value of the package contents in the project's package.json + config.webpacker.check_yarn_integrity = false + # Code is not reloaded between requests. config.cache_classes = true @@ -17,16 +17,9 @@ config.consider_all_requests_local = false config.action_controller.perform_caching = true - # Enable Rack::Cache to put a simple HTTP cache in front of your application - # Add `rack-cache` to your Gemfile before enabling this. - # For large-scale production use, consider using a caching reverse proxy like - # NGINX, varnish or squid. - # config.action_dispatch.rack_cache = true - - # Attempt to read encrypted secrets from `config/secrets.yml.enc`. - # Requires an encryption key in `ENV["RAILS_MASTER_KEY"]` or - # `config/secrets.yml.key`. - config.read_encrypted_secrets = true + # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] + # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). + # config.require_master_key = true # Disable serving static files from the `/public` folder by default since # Apache or NGINX already handles this. @@ -34,17 +27,13 @@ # Compress JavaScripts and CSS. config.assets.js_compressor = Uglifier.new(:harmony => true) + + # Compress CSS using a preprocessor. # config.assets.css_compressor = :sass # Do not fallback to assets pipeline if a precompiled asset is missed. config.assets.compile = true - # Asset digests allow you to set far-future HTTP expiration dates on all assets, - # yet still be able to expire them through the digest params. - config.assets.digest = true - - # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb - # Enable serving of images, stylesheets, and JavaScripts from an asset server. # config.action_controller.asset_host = 'http://assets.example.com' @@ -52,7 +41,10 @@ # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX - # Mount Action Cable outside main process or domain + # Store uploaded files on the local file system (see config/storage.yml for options). + #config.active_storage.service = :local + + # Mount Action Cable outside main process or domain. # config.action_cable.mount_path = nil # config.action_cable.url = 'wss://example.com/cable' # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] @@ -60,40 +52,25 @@ # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true - # Use default logging formatter so that PID and timestamp are not suppressed. - config.log_formatter = ::Logger::Formatter.new - - # Enable logging to both stdout and file, in more compact format - if ENV["RAILS_LOG_TO_STDOUT"].present? - logger = ActiveSupport::Logger.new(STDOUT) - logger.formatter = config.log_formatter - config.logger = ActiveSupport::TaggedLogging.new(logger) - else - config.lograge.enabled = true - config.lograge.custom_options = lambda do |event| - {:time => event.time} - end - end - # Use the lowest log level to ensure availability of diagnostic information # when problems arise. config.log_level = :info # Suppress logger output for asset requests. config.assets.quiet = true - - # Prepend all log lines with the following tags. - # config.log_tags = [ :subdomain, :uuid ] - # Use a different logger for distributed setups. - # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) + # Prepend all log lines with the following tags. + config.log_tags = [ :request_id ] # Use a different cache store in production. # config.cache_store = :mem_cache_store - # Use a real queuing backend for Active Job (and separate queues per environment) + # Use a real queuing backend for Active Job (and separate queues per environment). # config.active_job.queue_adapter = :resque - # config.active_job.queue_name_prefix = "avalon_#{Rails.env}" + # config.active_job.queue_name_prefix = "avalon_production" + config.active_job.queue_adapter = Settings&.active_job&.queue_adapter || :sidekiq + require 'active_job/queue_adapters/better_active_elastic_job_adapter' if config.active_job.queue_adapter == :active_elastic_job + config.action_mailer.perform_caching = false # Ignore bad email addresses and do not raise email delivery errors. @@ -107,11 +84,42 @@ # Send deprecation notices to registered listeners. config.active_support.deprecation = :notify + # Use default logging formatter so that PID and timestamp are not suppressed. + config.log_formatter = ::Logger::Formatter.new + + # Use a different logger for distributed setups. + # require 'syslog/logger' + # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') + + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new(STDOUT) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + end + # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false - config.active_job.queue_adapter = Settings&.active_job&.queue_adapter || :sidekiq - require 'active_job/queue_adapters/better_active_elastic_job_adapter' if config.active_job.queue_adapter == :active_elastic_job + # Inserts middleware to perform automatic connection switching. + # The `database_selector` hash is used to pass options to the DatabaseSelector + # middleware. The `delay` is used to determine how long to wait after a write + # to send a subsequent read to the primary. + # + # The `database_resolver` class is used by the middleware to determine which + # database is appropriate to use based on the time delay. + # + # The `database_resolver_context` class is used by the middleware to set + # timestamps for the last write to the primary. The resolver uses the context + # class timestamps to determine how long to wait before reading from the + # replica. + # + # By default Rails will store a last write timestamp in the session. The + # DatabaseSelector middleware is designed as such you can define your own + # strategy for connection switching and pass that into the middleware through + # these configuration options. + # config.active_record.database_selector = { delay: 2.seconds } + # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver + # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session # Additional production specific initializers Dir["config/environments/production/*.rb"].each {|file| load file } diff --git a/config/environments/test.rb b/config/environments/test.rb index b54a5abde4..470dee4bef 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,10 +1,11 @@ +# The test environment is used exclusively to run your application's +# test suite. You never need to work with it otherwise. Remember that +# your test database is "scratch space" for the test suite and is wiped +# and recreated between test runs. Don't rely on the data there! + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. - # The test environment is used exclusively to run your application's - # test suite. You never need to work with it otherwise. Remember that - # your test database is "scratch space" for the test suite and is wiped - # and recreated between test runs. Don't rely on the data there! config.cache_classes = true # Do not eager load code on boot. This avoids loading your whole application @@ -15,18 +16,23 @@ # Configure public file server for tests with Cache-Control for performance. config.public_file_server.enabled = true config.public_file_server.headers = { - 'Cache-Control' => "public, max-age=#{1.hour.seconds.to_i}" + 'Cache-Control' => "public, max-age=#{1.hour.to_i}" } # Show full error reports and disable caching. config.consider_all_requests_local = true config.action_controller.perform_caching = false + config.cache_store = :null_store # Raise exceptions instead of rendering exception templates. config.action_dispatch.show_exceptions = false # Disable request forgery protection in test environment. config.action_controller.allow_forgery_protection = false + + # Store uploaded files on the local file system in a temporary directory. + config.active_storage.service = :test + config.action_mailer.perform_caching = false # Tell Action Mailer not to deliver emails to the real world. @@ -37,10 +43,6 @@ # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr - # Raises error for missing translations - config.action_view.raise_on_missing_translations = true - - config.active_record.sqlite3.represent_boolean_as_integer = true - - config.active_storage.service = :test + # Raises error for missing translations. + # config.action_view.raise_on_missing_translations = true end diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb new file mode 100644 index 0000000000..35d0f26fcd --- /dev/null +++ b/config/initializers/content_security_policy.rb @@ -0,0 +1,30 @@ +# Be sure to restart your server when you modify this file. + +# Define an application-wide content security policy +# For further information see the following documentation +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy + +# Rails.application.config.content_security_policy do |policy| +# policy.default_src :self, :https +# policy.font_src :self, :https, :data +# policy.img_src :self, :https, :data +# policy.object_src :none +# policy.script_src :self, :https +# policy.style_src :self, :https +# # If you are using webpack-dev-server then specify webpack-dev-server host +# policy.connect_src :self, :https, "http://localhost:3035", "ws://localhost:3035" if Rails.env.development? + +# # Specify URI for violation reports +# # policy.report_uri "/csp-violation-report-endpoint" +# end + +# If you are using UJS then enable automatic nonce generation +# Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) } + +# Set the nonce only to specific directives +# Rails.application.config.content_security_policy_nonce_directives = %w(script-src) + +# Report CSP violations to a specified URI +# For further information see the following documentation: +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only +# Rails.application.config.content_security_policy_report_only = true diff --git a/config/initializers/fedora_migrate.rb b/config/initializers/fedora_migrate.rb deleted file mode 100644 index f4ea13779f..0000000000 --- a/config/initializers/fedora_migrate.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'fedora-migrate' - -FedoraMigrate::ContentMover.class_eval do - def move_content - target.content = StringIO.new(source.content) - # Comment out the next line assuming that we are setting it explicitly on target before getting here - # target.original_name = source.label.try(:gsub, /"/, '\"') - target.mime_type = source.mimeType - save - report.error = "Failed checksum" unless valid? - end -end diff --git a/config/initializers/new_framework_defaults_6_0.rb b/config/initializers/new_framework_defaults_6_0.rb new file mode 100644 index 0000000000..92240ef5f5 --- /dev/null +++ b/config/initializers/new_framework_defaults_6_0.rb @@ -0,0 +1,45 @@ +# Be sure to restart your server when you modify this file. +# +# This file contains migration options to ease your Rails 6.0 upgrade. +# +# Once upgraded flip defaults one by one to migrate to the new default. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. + +# Don't force requests from old versions of IE to be UTF-8 encoded. +# Rails.application.config.action_view.default_enforce_utf8 = false + +# Embed purpose and expiry metadata inside signed and encrypted +# cookies for increased security. +# +# This option is not backwards compatible with earlier Rails versions. +# It's best enabled when your entire app is migrated and stable on 6.0. +# Rails.application.config.action_dispatch.use_cookies_with_metadata = true + +# Change the return value of `ActionDispatch::Response#content_type` to Content-Type header without modification. +# Rails.application.config.action_dispatch.return_only_media_type_on_content_type = false + +# Return false instead of self when enqueuing is aborted from a callback. +# Rails.application.config.active_job.return_false_on_aborted_enqueue = true + +# Send Active Storage analysis and purge jobs to dedicated queues. +# Rails.application.config.active_storage.queues.analysis = :active_storage_analysis +# Rails.application.config.active_storage.queues.purge = :active_storage_purge + +# When assigning to a collection of attachments declared via `has_many_attached`, replace existing +# attachments instead of appending. Use #attach to add new attachments without replacing existing ones. +# Rails.application.config.active_storage.replace_on_assign_to_many = true + +# Use ActionMailer::MailDeliveryJob for sending parameterized and normal mail. +# +# The default delivery jobs (ActionMailer::Parameterized::DeliveryJob, ActionMailer::DeliveryJob), +# will be removed in Rails 6.1. This setting is not backwards compatible with earlier Rails versions. +# If you send mail in the background, job workers need to have a copy of +# MailDeliveryJob to ensure all delivery jobs are processed properly. +# Make sure your entire app is migrated and stable on 6.0 before using this setting. +# Rails.application.config.action_mailer.delivery_job = "ActionMailer::MailDeliveryJob" + +# Enable the same cache key to be reused when the object being cached of type +# `ActiveRecord::Relation` changes by moving the volatile information (max updated at and count) +# of the relation's cache key into the cache version to support recycling cache key. +# Rails.application.config.active_record.collection_cache_versioning = true diff --git a/db/migrate/20220218184645_add_foreign_key_constraint_to_active_storage_attachments_for_blob_id.active_storage.rb b/db/migrate/20220218184645_add_foreign_key_constraint_to_active_storage_attachments_for_blob_id.active_storage.rb new file mode 100644 index 0000000000..ff5d72c7ea --- /dev/null +++ b/db/migrate/20220218184645_add_foreign_key_constraint_to_active_storage_attachments_for_blob_id.active_storage.rb @@ -0,0 +1,10 @@ +# This migration comes from active_storage (originally 20180723000244) +class AddForeignKeyConstraintToActiveStorageAttachmentsForBlobId < ActiveRecord::Migration[6.0] + def up + return if foreign_key_exists?(:active_storage_attachments, column: :blob_id) + + if table_exists?(:active_storage_blobs) + add_foreign_key :active_storage_attachments, :active_storage_blobs, column: :blob_id + end + end +end From 7873b28afa996b474ed9bd5234c6400e9f242034 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Fri, 18 Feb 2022 17:00:05 -0500 Subject: [PATCH 006/171] Making guesses about quick upgrade to bootstrap 4 to get the assets to compile. Also dropping our fork of browse-everything for now. --- Gemfile | 10 +- Gemfile.lock | 103 ++++++++++--------- app/assets/javascripts/application.js | 6 +- app/assets/stylesheets/application.scss | 1 - app/assets/stylesheets/avalon.scss | 12 +-- app/assets/stylesheets/avalon/_footer.scss | 2 +- app/assets/stylesheets/avalon/_form.scss | 3 + app/assets/stylesheets/avalon/_header.scss | 6 +- app/assets/stylesheets/avalon/_homepage.scss | 12 +-- app/assets/stylesheets/avalon/_nav.scss | 22 +++- app/assets/stylesheets/blacklight.scss | 4 +- 11 files changed, 105 insertions(+), 76 deletions(-) diff --git a/Gemfile b/Gemfile index 6d62c3ddfd..9669afc21c 100644 --- a/Gemfile +++ b/Gemfile @@ -5,18 +5,22 @@ gem 'bootsnap', require: false gem 'listen' gem 'rails', '=6.0.4.6' gem 'sprockets', '~>3.7.2' +#gem 'sprockets-rails', require: 'sprockets/railtie' gem 'sqlite3' # Assets +gem 'bootstrap', '~> 4.0' gem 'coffee-rails', '~> 4.2.0' +gem "font-awesome-rails" gem 'jquery-datatables' gem 'jquery-rails' gem 'jquery-ui-rails' gem 'react-rails' -gem 'sass-rails', '>= 6' +#gem 'sass-rails', '>= 6' # Use the last known good version of sass gem 'sass', '3.4.22' gem 'sprockets-es6' +gem 'twitter-typeahead-rails', '0.11.1.pre.corejavascript' gem 'uglifier', '>= 1.3.0' gem 'webpacker' @@ -42,7 +46,7 @@ gem 'activerecord-session_store', '>= 2.0.0' gem 'acts_as_list' gem 'api-pagination' gem 'avalon-about', git: 'https://github.com/avalonmediasystem/avalon-about.git', tag: 'avalon-r6.4' -gem 'bootstrap-sass', '< 3.4.1' # Pin to less than 3.4.1 due to change in behavior with popovers +#gem 'bootstrap-sass', '< 3.4.1' # Pin to less than 3.4.1 due to change in behavior with popovers gem 'bootstrap-toggle-rails' gem 'bootstrap_form' gem 'iiif_manifest', '~> 0.6' @@ -68,7 +72,7 @@ gem "omniauth-saml", "~> 2.0" # Media Access & Transcoding gem 'active_encode', '~> 0.8.2' gem 'audio_waveform-ruby', '~> 1.0.7', require: 'audio_waveform' -gem 'browse-everything', git: "https://github.com/avalonmediasystem/browse-everything.git", branch: 'v0.16.1-gdrive-fixes-plus' +gem 'browse-everything'#, git: "https://github.com/avalonmediasystem/browse-everything.git", branch: 'v0.16.1-gdrive-fixes-plus' gem 'fastimage' gem 'media_element_add_to_playlist', git: 'https://github.com/avalonmediasystem/media-element-add-to-playlist.git', tag: 'avalon-r6.5' gem 'mediainfo', git: "https://github.com/avalonmediasystem/mediainfo.git", branch: 'avalon_fixes' diff --git a/Gemfile.lock b/Gemfile.lock index 627c838b09..9550c9f424 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -32,27 +32,6 @@ GIT specs: avalon-workflow (0.0.3) -GIT - remote: https://github.com/avalonmediasystem/browse-everything.git - revision: 7dddc2bbafe9e30c5caa9d88e57d136b8bca521a - branch: v0.16.1-gdrive-fixes-plus - specs: - browse-everything (0.16.1) - addressable (~> 2.5) - aws-sdk-s3 - bootstrap-sass (~> 3.2) - dropbox_api (>= 0.1.10) - font-awesome-rails - google-api-client (~> 0.21) - google_drive (~> 2.1) - googleauth (= 0.6.2) - rails (>= 4.2) - ruby-box - sass-rails - signet (~> 0.8) - thor (~> 0.20) - typhoeus - GIT remote: https://github.com/avalonmediasystem/media-element-add-to-playlist.git revision: dfde0084f6c17ca16fa675086eb025440f99a02d @@ -288,13 +267,25 @@ GEM deprecation (~> 1.0) bootsnap (1.10.3) msgpack (~> 1.2) - bootstrap-sass (3.4.0) - autoprefixer-rails (>= 5.2.1) - sassc (>= 2.0.0) + bootstrap (4.6.1) + autoprefixer-rails (>= 9.1.0) + popper_js (>= 1.14.3, < 2) + sassc-rails (>= 2.0.0) bootstrap-toggle-rails (2.2.1.0) bootstrap_form (5.0.0) actionpack (>= 5.2) activemodel (>= 5.2) + browse-everything (1.1.2) + addressable (~> 2.5) + aws-sdk-s3 + dropbox_api (>= 0.1.10) + google-api-client (~> 0.23) + google_drive (>= 2.1, < 4) + googleauth (>= 0.6.6, < 1.0) + rails (>= 4.2, < 7.0) + ruby-box + signet (~> 0.8) + typhoeus builder (3.2.4) byebug (11.1.3) cancancan (1.17.0) @@ -457,29 +448,46 @@ GEM fugit (1.5.2) et-orbi (~> 1.1, >= 1.1.8) raabro (~> 1.4) + gems (1.2.0) globalid (1.0.0) activesupport (>= 5.0) google-analytics-rails (1.1.0) - google-api-client (0.32.1) + google-api-client (0.53.0) + google-apis-core (~> 0.1) + google-apis-generator (~> 0.1) + google-apis-core (0.4.2) addressable (~> 2.5, >= 2.5.1) - googleauth (>= 0.5, < 0.10.0) - httpclient (>= 2.8.1, < 3.0) + googleauth (>= 0.16.2, < 2.a) + httpclient (>= 2.8.1, < 3.a) mini_mime (~> 1.0) representable (~> 3.0) - retriable (>= 2.0, < 4.0) - signet (~> 0.10) - google_drive (2.1.3) - google-api-client (>= 0.11.0, < 1.0.0) + retriable (>= 2.0, < 4.a) + rexml + webrick + google-apis-discovery_v1 (0.8.0) + google-apis-core (>= 0.4, < 2.a) + google-apis-drive_v3 (0.18.0) + google-apis-core (>= 0.4, < 2.a) + google-apis-generator (0.4.1) + activesupport (>= 5.0) + gems (~> 1.2) + google-apis-core (>= 0.4, < 2.a) + google-apis-discovery_v1 (~> 0.5) + thor (>= 0.20, < 2.a) + google-apis-sheets_v4 (0.11.0) + google-apis-core (>= 0.4, < 2.a) + google_drive (3.0.7) + google-apis-drive_v3 (>= 0.5.0, < 1.0.0) + google-apis-sheets_v4 (>= 0.4.0, < 1.0.0) googleauth (>= 0.5.0, < 1.0.0) nokogiri (>= 1.5.3, < 2.0.0) - googleauth (0.6.2) - faraday (~> 0.12) + googleauth (0.17.1) + faraday (>= 0.17.3, < 2.0) jwt (>= 1.4, < 3.0) - logging (~> 2.0) - memoist (~> 0.12) + memoist (~> 0.16) multi_json (~> 1.11) - os (~> 0.9) - signet (~> 0.7) + os (>= 0.9, < 2.0) + signet (~> 0.15) haml (5.2.2) temple (>= 0.8.0) tilt @@ -571,13 +579,9 @@ GEM listen (3.7.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) - little-plugger (1.1.4) llhttp-ffi (0.4.0) ffi-compiler (~> 1.0) rake (~> 13.0) - logging (2.3.0) - little-plugger (~> 1.1) - multi_json (~> 1.14) lograge (0.11.2) actionpack (>= 4) activesupport (>= 4) @@ -647,7 +651,7 @@ GEM omniauth (~> 2.0) ruby-saml (~> 1.9) orm_adapter (0.5.0) - os (0.9.6) + os (1.1.4) ostruct (0.5.3) parallel (1.21.0) paranoia (2.5.2) @@ -655,6 +659,7 @@ GEM parser (3.1.0.0) ast (~> 2.4.1) pg (1.3.2) + popper_js (1.16.0) pretender (0.4.0) actionpack (>= 5.2) pry (0.13.1) @@ -853,8 +858,6 @@ GEM pretender rails (>= 5.2.4.3, < 6.1) sass (3.4.22) - sass-rails (6.0.0) - sassc-rails (~> 2.1, >= 2.1.1) sassc (2.4.0) ffi (~> 1.9) sassc-rails (2.1.2) @@ -925,6 +928,10 @@ GEM thread_safe (0.3.6) tilt (2.0.10) trailblazer-option (0.1.2) + twitter-typeahead-rails (0.11.1.pre.corejavascript) + actionpack (>= 3.1) + jquery-rails + railties (>= 3.1) typhoeus (1.4.0) ethon (>= 0.9.0) tzinfo (1.2.9) @@ -962,6 +969,7 @@ GEM rack-proxy (>= 0.6.1) railties (>= 5.2) semantic_range (>= 2.3.0) + webrick (1.7.0) websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -1007,10 +1015,10 @@ DEPENDENCIES bixby blacklight (< 8) bootsnap - bootstrap-sass (< 3.4.1) + bootstrap (~> 4.0) bootstrap-toggle-rails bootstrap_form - browse-everything! + browse-everything byebug capistrano (~> 3.6) capistrano-passenger @@ -1035,6 +1043,7 @@ DEPENDENCIES faker fastimage fcrepo_wrapper + font-awesome-rails google-analytics-rails (= 1.1.0) hashdiff (>= 1.0) hooks @@ -1085,7 +1094,6 @@ DEPENDENCIES rspec_junit_formatter samvera-persona (~> 0.3) sass (= 3.4.22) - sass-rails (>= 6) selenium-webdriver shoulda-matchers sidekiq (~> 6.2) @@ -1096,6 +1104,7 @@ DEPENDENCIES sprockets (~> 3.7.2) sprockets-es6 sqlite3 + twitter-typeahead-rails (= 0.11.1.pre.corejavascript) uglifier (>= 1.3.0) wavefile (~> 1.0.1) web-console diff --git a/app/assets/javascripts/application.js b/app/assets/javascripts/application.js index 81c8bb6e65..44c41b882e 100644 --- a/app/assets/javascripts/application.js +++ b/app/assets/javascripts/application.js @@ -26,10 +26,14 @@ // Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details // about supported directives. // -//= require jquery +//= require jquery3 //= require jquery_ujs +//= require rails-ujs // Required by Blacklight +//= require popper +//= require twitter/typeahead +//= require bootstrap //= require jquery-ui //= require blacklight/blacklight //= require browse_everything diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index bdc9014537..24b81d5085 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -43,7 +43,6 @@ *= require_self */ -@import 'bootstrap-sprockets'; @import 'branding'; @import 'bootstrap'; @import 'bootstrap-toggle'; diff --git a/app/assets/stylesheets/avalon.scss b/app/assets/stylesheets/avalon.scss index c3ecd65ccd..e93c6e8da5 100644 --- a/app/assets/stylesheets/avalon.scss +++ b/app/assets/stylesheets/avalon.scss @@ -140,19 +140,19 @@ div.alert-danger { } .mobile-hidden { - @media (max-width: $screen-xs-max) { + @include media-breakpoint-down(xs) { display: none !important; } } .sm-hidden { - @media (max-width: $screen-sm-max) { + @include media-breakpoint-down(sm) { display: none !important; } } .desktop-hidden { - @media (min-width: $screen-sm-min) { + @include media-breakpoint-up(sm) { display: none !important; } } @@ -412,7 +412,7 @@ a[data-trigger='submit'] { } } -@media (max-width: $screen-xs-max) { +@include media-breakpoint-down(xs) { #content { .btn { margin-top: 2px; @@ -1073,14 +1073,14 @@ h5.panel-title { text-align: right; } -@media (min-width: $screen-xs-max) { +@include media-breakpoint-up(xs) { .dataTables_length { float: left; } } // Make table body scrollable in mobile devices -@media (max-width: $screen-xs-max) { +@include media-breakpoint-down(xs) { .dataTableBody { width: 100%; overflow-x: scroll; diff --git a/app/assets/stylesheets/avalon/_footer.scss b/app/assets/stylesheets/avalon/_footer.scss index 46ba5772af..4efb552f48 100644 --- a/app/assets/stylesheets/avalon/_footer.scss +++ b/app/assets/stylesheets/avalon/_footer.scss @@ -42,7 +42,7 @@ footer { float: right; } - @media screen and (max-width: $screen-xs-max) { + @include media-breakpoint-down(xs) { font-size: 12px; ul { diff --git a/app/assets/stylesheets/avalon/_form.scss b/app/assets/stylesheets/avalon/_form.scss index 1ce6edcd87..6ee02a1de1 100644 --- a/app/assets/stylesheets/avalon/_form.scss +++ b/app/assets/stylesheets/avalon/_form.scss @@ -14,6 +14,9 @@ * --- END LICENSE_HEADER BLOCK --- */ +// Copied in from bootstrap-sass +$border-radius-base: 4px !default; + // Form-group input fields .form-group .input-group { width: 100%; diff --git a/app/assets/stylesheets/avalon/_header.scss b/app/assets/stylesheets/avalon/_header.scss index 0ac6c89b0b..06869e701b 100644 --- a/app/assets/stylesheets/avalon/_header.scss +++ b/app/assets/stylesheets/avalon/_header.scss @@ -15,7 +15,7 @@ */ .logo { - @media screen and (max-width: $screen-sm-max) { + @include media-breakpoint-down(sm) { max-width: 120px; } } @@ -28,7 +28,7 @@ .header-logo { flex-grow: 1; - @media screen and (max-width: $screen-sm-max) { + @inclue media-breakpoint-down(sm) { img { max-width: 120px; margin-right: 10px; @@ -69,7 +69,7 @@ header div { border: 0; } - @media screen and (max-width: $screen-sm-max) { + @include media-breakpoint-down(sm) { width: inherit; margin: 7px 7px 0 7px; margin-bottom: 1rem; diff --git a/app/assets/stylesheets/avalon/_homepage.scss b/app/assets/stylesheets/avalon/_homepage.scss index 11796399a4..77fc9741ca 100644 --- a/app/assets/stylesheets/avalon/_homepage.scss +++ b/app/assets/stylesheets/avalon/_homepage.scss @@ -32,7 +32,7 @@ main.homepage-main { height: 500px; position: relative; - @media screen and (max-width: $screen-xs-max) { + @include media-breakpoint-down(xs) { height: 350px; } } @@ -50,7 +50,7 @@ main.homepage-main { .global-search-wrapper { width: 400px; - @media screen and (max-width: $screen-xs-max) { + @include media-breakpoint-down(xs) { width: 300px; margin-bottom: 100px; } @@ -73,20 +73,20 @@ main.homepage-main { .homepage-splash-featured-collection { display: flex; - @media screen and (max-width: $screen-xs-max) { + @include media-breakpoint-down(xs) { padding-bottom: 2rem; flex-direction: column; } .homepage-featured-collection-image { width: 100%; - @media screen and (min-width: $screen-sm-min) { + @include media-breakpoint-up(sm) { width: 400px; } - @media screen and (min-width: $screen-md-min) { + @include media-breakpoint-up(md) { width: 500px; } - @media screen and (min-width: $screen-lg-min) { + @include media-breakpoint-up(lg) { width: 600px; } } diff --git a/app/assets/stylesheets/avalon/_nav.scss b/app/assets/stylesheets/avalon/_nav.scss index e12d143220..3c2369637f 100644 --- a/app/assets/stylesheets/avalon/_nav.scss +++ b/app/assets/stylesheets/avalon/_nav.scss @@ -14,6 +14,18 @@ * --- END LICENSE_HEADER BLOCK --- */ +// Copied from bootstrap-sass +$line-height-computed: floor(($font-size-base * $line-height-base)) !default; // ~20px +$gray-base: #000 !default; +$gray-lighter: lighten($gray-base, 93.5%) !default; // #eee +$gray-darker: lighten($gray-base, 13.5%) !default; // #222 +$state-success-text: #3c763d !default; +$state-success-bg: #dff0d8 !default; +$state-success-border: darken(adjust-hue($state-success-bg, -10), 5%) !default; +$state-info-text: #31708f !default; +$state-info-bg: #d9edf7 !default; +$state-info-border: darken(adjust-hue($state-info-bg, -10), 7%) !default; + /** * Overridding Blacklight's brand logo */ @@ -26,7 +38,7 @@ } .navbar-toggle { - @media screen and (max-width: $screen-xs-max) { + @include media-breakpoint-down(xs) { background: $primaryDark; } } @@ -48,7 +60,7 @@ .navbar-nav { > li > a { - @media screen and (max-width: $screen-sm-max) { + @include media-breakpoint-down(sm) { padding-left: 10px; padding-right: 10px; } @@ -56,7 +68,7 @@ } .navbar-default { - @media screen and (max-width: $screen-xs-max) { + @include media-breakpoint-down(xs) { background: $primaryDark; } } @@ -73,7 +85,7 @@ margin-right: 6px; } - @media screen and (max-width: $screen-xs-max) { + @include media-breakpoint-down(xs) { background: white; box-shadow: 0 0 5px; } @@ -186,7 +198,7 @@ width: 100%; } -@media (max-width: $screen-xs-max) { +@include media-breakpoint-down(xs) { .manage-btn { height: 40px; } diff --git a/app/assets/stylesheets/blacklight.scss b/app/assets/stylesheets/blacklight.scss index 01b25bfdf6..855f5e245a 100644 --- a/app/assets/stylesheets/blacklight.scss +++ b/app/assets/stylesheets/blacklight.scss @@ -14,8 +14,6 @@ * --- END LICENSE_HEADER BLOCK --- */ -@import 'bootstrap-sprockets'; - @import 'bootstrap'; @import 'blacklight/blacklight'; @@ -34,4 +32,4 @@ .clear-bookmarks { margin-left: 0px; -} \ No newline at end of file +} From 1f402b940df14a5b29d96f7fec2488cfa303a615 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Feb 2022 08:46:12 -0500 Subject: [PATCH 007/171] Remove ActiveModel::Dirty monkey-patch --- config/initializers/active_fedora_general.rb | 21 +++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/config/initializers/active_fedora_general.rb b/config/initializers/active_fedora_general.rb index 95449eb944..dbbbf09274 100644 --- a/config/initializers/active_fedora_general.rb +++ b/config/initializers/active_fedora_general.rb @@ -8,15 +8,18 @@ end ActiveFedora::Base.logger = Rails.logger -# Monkey-patch to short circuit ActiveModel::Dirty which attempts to load the whole master files ordered list when calling nodes_will_change! -# This leads to a stack level too deep exception when attempting to delete a master file from a media object on the manage files step. -# See https://github.com/samvera/active_fedora/pull/1312/commits/7c8bbbefdacefd655a2ca653f5950c991e1dc999#diff-28356c4daa0d55cbaf97e4269869f510R100-R103 -ActiveFedora::Aggregation::ListSource.class_eval do - def attribute_will_change!(attr) - return super unless attr == 'nodes' - attributes_changed_by_setter[:nodes] = true - end -end +# ActiveModel::Dirty's internals were substantially rewritten in Rails 6 making the following monkey-patch potentially unnecessary. +# We will need to test this throroughly to ensure it is safe to remove. +# +## Monkey-patch to short circuit ActiveModel::Dirty which attempts to load the whole master files ordered list when calling nodes_will_change! +## This leads to a stack level too deep exception when attempting to delete a master file from a media object on the manage files step. +## See https://github.com/samvera/active_fedora/pull/1312/commits/7c8bbbefdacefd655a2ca653f5950c991e1dc999#diff-28356c4daa0d55cbaf97e4269869f510R100-R103 +#ActiveFedora::Aggregation::ListSource.class_eval do +# def attribute_will_change!(attr) +# return super unless attr == 'nodes' +# attributes_changed_by_setter[:nodes] = true +# end +#end # Override to avoid deprecation warning. Remove this monkey-patch whenever Avalon upgrades to a version of LDP which has this fix. Ldp::Response.class_eval do From 23905460165199b61fb72fc6c6d5e71aa2c1b4c8 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Feb 2022 08:48:42 -0500 Subject: [PATCH 008/171] add_show_tools_partial is no longer a helper but method on the blacklight config --- app/controllers/bookmarks_controller.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/bookmarks_controller.rb b/app/controllers/bookmarks_controller.rb index 9ded5552f0..f6cb6cde39 100644 --- a/app/controllers/bookmarks_controller.rb +++ b/app/controllers/bookmarks_controller.rb @@ -25,21 +25,21 @@ class BookmarksController < CatalogController blacklight_config.show.document_actions[:email].if = false if blacklight_config.show.document_actions[:email] blacklight_config.show.document_actions[:citation].if = false if blacklight_config.show.document_actions[:citation] - add_show_tools_partial( :update_access_control, callback: :access_control_action, if: Proc.new { |context, config, options| context.user_can? :update_access_control } ) + blacklight_config.add_show_tools_partial( :update_access_control, callback: :access_control_action, if: Proc.new { |context, config, options| context.user_can? :update_access_control } ) - add_show_tools_partial( :move, callback: :move_action, if: Proc.new { |context, config, options| context.user_can? :move } ) + blacklight_config.add_show_tools_partial( :move, callback: :move_action, if: Proc.new { |context, config, options| context.user_can? :move } ) - add_show_tools_partial( :publish, callback: :status_action, modal: false, partial: 'formless_document_action', if: Proc.new { |context, config, options| context.user_can? :publish } ) + blacklight_config.add_show_tools_partial( :publish, callback: :status_action, modal: false, partial: 'formless_document_action', if: Proc.new { |context, config, options| context.user_can? :publish } ) - add_show_tools_partial( :unpublish, callback: :status_action, modal: false, partial: 'formless_document_action', if: Proc.new { |context, config, options| context.user_can? :unpublish } ) + blacklight_config.add_show_tools_partial( :unpublish, callback: :status_action, modal: false, partial: 'formless_document_action', if: Proc.new { |context, config, options| context.user_can? :unpublish } ) - add_show_tools_partial( :delete, callback: :delete_action, if: Proc.new { |context, config, options| context.user_can? :delete } ) + blacklight_config.add_show_tools_partial( :delete, callback: :delete_action, if: Proc.new { |context, config, options| context.user_can? :delete } ) - add_show_tools_partial( :add_to_playlist, callback: :add_to_playlist_action ) + blacklight_config.add_show_tools_partial( :add_to_playlist, callback: :add_to_playlist_action ) - add_show_tools_partial( :intercom_push, callback: :intercom_push_action, if: Proc.new { |context, config, options| context.user_can? :intercom_push } ) + blacklight_config.add_show_tools_partial( :intercom_push, callback: :intercom_push_action, if: Proc.new { |context, config, options| context.user_can? :intercom_push } ) - add_show_tools_partial( :merge, callback: :merge_action, if: Proc.new { |context, config, options| context.user_can? :merge } ) + blacklight_config.add_show_tools_partial( :merge, callback: :merge_action, if: Proc.new { |context, config, options| context.user_can? :merge } ) before_action :verify_permissions, only: :index From a89e2f56df61a6f4877d864c21ea1796b3ac6e4f Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Feb 2022 09:09:10 -0500 Subject: [PATCH 009/171] Mail job class changed in Rails 6 --- config/initializers/action_mailer.rb | 2 +- spec/controllers/admin_collections_controller_spec.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/initializers/action_mailer.rb b/config/initializers/action_mailer.rb index ba36b8c31a..60b87d4ecb 100644 --- a/config/initializers/action_mailer.rb +++ b/config/initializers/action_mailer.rb @@ -1,4 +1,4 @@ -ActionMailer::DeliveryJob.rescue_from(StandardError) do |exception| +ActionMailer::MailDeliveryJob.rescue_from(StandardError) do |exception| Rails.logger.error "Error delivering mail: #{exception}" end diff --git a/spec/controllers/admin_collections_controller_spec.rb b/spec/controllers/admin_collections_controller_spec.rb index 432f51dfc0..2cdccac9f6 100644 --- a/spec/controllers/admin_collections_controller_spec.rb +++ b/spec/controllers/admin_collections_controller_spec.rb @@ -281,7 +281,7 @@ # allow(mock_email).to receive(:deliver_later) # expect(NotificationsMailer).to receive(:new_collection).and_return(mock_email) # FIXME: This delivers two instead of one for some reason - expect {post 'create', params: { format:'json', admin_collection: {name: collection.name, description: collection.description, unit: collection.unit, managers: collection.managers} }}.to have_enqueued_job(ActionMailer::DeliveryJob).twice + expect {post 'create', params: { format:'json', admin_collection: {name: collection.name, description: collection.description, unit: collection.unit, managers: collection.managers} }}.to have_enqueued_job(ActionMailer::MailDeliveryJob).twice # post 'create', format:'json', admin_collection: {name: collection.name, description: collection.description, unit: collection.unit, managers: collection.managers} end it "should create a new collection" do @@ -317,7 +317,7 @@ # expect(mock_delay).to receive(:update_collection) @collection = FactoryBot.create(:collection) # put 'update', id: @collection.id, admin_collection: {name: "#{@collection.name}-new", description: @collection.description, unit: @collection.unit} - expect {put 'update', params: { id: @collection.id, admin_collection: {name: "#{@collection.name}-new", description: @collection.description, unit: @collection.unit} }}.to have_enqueued_job(ActionMailer::DeliveryJob).once + expect {put 'update', params: { id: @collection.id, admin_collection: {name: "#{@collection.name}-new", description: @collection.description, unit: @collection.unit} }}.to have_enqueued_job(ActionMailer::MailDeliveryJob).once end context "update REST API" do From 948b2e41b68c607d997156e871d7156a022b76d5 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Feb 2022 09:45:52 -0500 Subject: [PATCH 010/171] ContentType headers now include charset by default --- spec/controllers/admin_collections_controller_spec.rb | 2 +- spec/controllers/catalog_controller_spec.rb | 4 ++-- spec/controllers/collections_controller_spec.rb | 6 +++--- spec/controllers/master_files_controller_spec.rb | 6 +++--- spec/controllers/media_objects_controller_spec.rb | 6 +++--- spec/controllers/omniauth_callbacks_controller_spec.rb | 2 +- spec/controllers/timelines_controller_spec.rb | 8 ++++---- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/spec/controllers/admin_collections_controller_spec.rb b/spec/controllers/admin_collections_controller_spec.rb index 2cdccac9f6..6dea45f794 100644 --- a/spec/controllers/admin_collections_controller_spec.rb +++ b/spec/controllers/admin_collections_controller_spec.rb @@ -542,7 +542,7 @@ it 'returns the poster' do get :poster, params: { id: collection.id } expect(response).to have_http_status(:ok) - expect(response.content_type).to eq "image/png" + expect(response.content_type).to eq "image/png; charset=utf-8" expect(response.body).not_to be_blank end end diff --git a/spec/controllers/catalog_controller_spec.rb b/spec/controllers/catalog_controller_spec.rb index 49ce1aa264..48bb235c5d 100644 --- a/spec/controllers/catalog_controller_spec.rb +++ b/spec/controllers/catalog_controller_spec.rb @@ -262,7 +262,7 @@ get 'index', params: { q: "", format: 'atom' } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(response.content_type).to eq "application/atom+xml" + expect(response.content_type).to eq "application/atom+xml; charset=utf-8" expect(assigns(:document_list).count).to eq 2 expect(assigns(:document_list).map(&:id)).to match_array [private_media_object.id, public_media_object.id] end @@ -271,7 +271,7 @@ it "should not show results" do get 'index', params: { q: "", format: 'atom' } expect(response).to be_successful - expect(response.content_type).to eq "application/atom+xml" + expect(response.content_type).to eq "application/atom+xml; charset=utf-8" expect(response).to render_template('catalog/index') expect(assigns(:document_list).count).to eq 1 expect(assigns(:document_list).map(&:id)).to eq([public_media_object.id]) diff --git a/spec/controllers/collections_controller_spec.rb b/spec/controllers/collections_controller_spec.rb index 3f09bae56f..077752c34a 100644 --- a/spec/controllers/collections_controller_spec.rb +++ b/spec/controllers/collections_controller_spec.rb @@ -77,7 +77,7 @@ login_as :administrator get 'index', params: { format: :json } expect(response).to be_ok - expect(response.content_type).to eq "application/json" + expect(response.content_type).to eq "application/json; charset=utf-8" expect(assigns(:doc_presenters).count).to eql(1) expect(assigns(:doc_presenters).map(&:id)).to match_array([collection.id]) end @@ -131,7 +131,7 @@ login_as :administrator get 'show', params: { id: collection.id, format: :json } expect(response).to be_ok - expect(response.content_type).to eq "application/json" + expect(response.content_type).to eq "application/json; charset=utf-8" expect(assigns(:doc_presenter).id).to eq collection.id end end @@ -147,7 +147,7 @@ it 'returns the poster image' do get 'poster', params: { id: collection.id } expect(response).to be_ok - expect(response.content_type).to eq 'image/png' + expect(response.content_type).to eq 'image/png; charset=utf-8' end context 'when the collection does not have a poster image' do diff --git a/spec/controllers/master_files_controller_spec.rb b/spec/controllers/master_files_controller_spec.rb index 4ccb996013..673133926b 100644 --- a/spec/controllers/master_files_controller_spec.rb +++ b/spec/controllers/master_files_controller_spec.rb @@ -532,7 +532,7 @@ class << file get('waveform', params: { id: master_file.id, empty: true }) expect(response).to have_http_status(:ok) expect(response.content_type).to eq('application/json') - expect(response['Content-Disposition']).to eq('attachment; filename="empty_waveform.json"') + expect(response['Content-Disposition']).to eq("attachment; filename=\"empty_waveform.json\"; filename*=UTF-8''empty_waveform.json") end end end @@ -593,13 +593,13 @@ class << file it 'returns the dynamic bitrate HLS manifest' do login_as :administrator expect(get('hls_manifest', params: { id: master_file.id, quality: 'auto' })).to have_http_status(:ok) - expect(response.content_type).to eq 'application/x-mpegURL' + expect(response.content_type).to eq 'application/x-mpegURL; charset=utf-8' end it 'returns a single quality HLS manifest' do login_as :administrator expect(get('hls_manifest', params: { id: master_file.id, quality: 'high' })).to have_http_status(:ok) - expect(response.content_type).to eq 'application/x-mpegURL' + expect(response.content_type).to eq 'application/x-mpegURL; charset=utf-8' end it 'returns a manifest if public' do diff --git a/spec/controllers/media_objects_controller_spec.rb b/spec/controllers/media_objects_controller_spec.rb index fec54cb87c..9ab83d9cc7 100644 --- a/spec/controllers/media_objects_controller_spec.rb +++ b/spec/controllers/media_objects_controller_spec.rb @@ -1374,7 +1374,7 @@ it 'returns descMetadata' do get :deliver_content, params: { id: media_object.id, file: 'descMetadata' } expect(response.status).to eq 200 - expect(response.content_type).to eq 'text/xml' + expect(response.content_type).to eq 'text/xml; charset=utf-8' expect(response.body).to eq media_object.descMetadata.content end end @@ -1389,7 +1389,7 @@ it 'returns a json preview of the media object' do get :move_preview, params: { id: media_object.id, format: 'json' } expect(response.status).to eq 200 - expect(response.content_type).to eq 'application/json' + expect(response.content_type).to eq 'application/json; charset=utf-8' json_preview = JSON.parse(response.body) expect(json_preview.keys).to eq ['id', 'title', 'collection', 'main_contributors', 'publication_date', 'published_by', 'published'] end @@ -1404,7 +1404,7 @@ it 'returns a json preview of the media object' do get :move_preview, params: { id: media_object.id, format: 'json' } expect(response.status).to eq 200 - expect(response.content_type).to eq 'application/json' + expect(response.content_type).to eq 'application/json; charset=utf-8' json_preview = JSON.parse(response.body) expect(json_preview.keys).to eq ['id', 'title', 'collection', 'main_contributors', 'publication_date', 'published_by', 'published'] end diff --git a/spec/controllers/omniauth_callbacks_controller_spec.rb b/spec/controllers/omniauth_callbacks_controller_spec.rb index 70c9cd4b02..87030e6eec 100644 --- a/spec/controllers/omniauth_callbacks_controller_spec.rb +++ b/spec/controllers/omniauth_callbacks_controller_spec.rb @@ -56,7 +56,7 @@ it 'returns self-closing page' do post :identity, params: params - expect(response.content_type).to eq 'text/html' + expect(response.content_type).to eq 'text/html; charset=utf-8' expect(response.body).to eq self_closing_html end end diff --git a/spec/controllers/timelines_controller_spec.rb b/spec/controllers/timelines_controller_spec.rb index 925b3857e4..e84a8d6fae 100644 --- a/spec/controllers/timelines_controller_spec.rb +++ b/spec/controllers/timelines_controller_spec.rb @@ -303,7 +303,7 @@ it "returns the created timeline as json" do post :create, params: { timeline: valid_attributes, format: :json }, session: valid_session expect(response).to be_created - expect(response.content_type).to eq 'application/json' + expect(response.content_type).to eq 'application/json; charset=utf-8' expect(response.location).to eq "http://test.host/timelines/1" response_json = JSON.parse(response.body) new_timeline = Timeline.last @@ -314,7 +314,7 @@ it 'generates a token if visibility is private-with-token' do post :create, params: { timeline: valid_attributes.merge(visibility: Timeline::PRIVATE_WITH_TOKEN), format: :json }, session: valid_session expect(response).to be_successful - expect(response.content_type).to eq 'application/json' + expect(response.content_type).to eq 'application/json; charset=utf-8' response_json = JSON.parse(response.body) expect(response_json["access_token"]).not_to be_blank end @@ -328,7 +328,7 @@ it "returns an unprocessable_entity response with the errors" do post :create, params: { timeline: invalid_attributes, format: :json }, session: valid_session expect(response).to be_unprocessable - expect(response.content_type).to eq 'application/json' + expect(response.content_type).to eq 'application/json; charset=utf-8' expect(JSON.parse(response.body)).not_to be_blank end end @@ -402,7 +402,7 @@ post :create, params: { format: :json }, body: submitted_manifest.to_json, session: valid_session new_timeline = Timeline.last expect(response).to be_created - expect(response.content_type).to eq 'application/json' + expect(response.content_type).to eq 'application/json; charset=utf-8' expect(response.location).to eq "http://test.host/timelines/#{new_timeline.id}" response_json = JSON.parse(response.body) # Ensure that stored timeline, response timeline, and submitted manifest all match From 61cfa3e88b846f4829d50d9abd0ad0bd0abd1030 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Feb 2022 09:54:48 -0500 Subject: [PATCH 011/171] Remove Fedora 3 -> 4 migration code --- Gemfile | 1 - .../javascripts/report_refresh.js.coffee | 24 --- app/controllers/application_controller.rb | 2 - .../migration_status_controller.rb | 83 -------- app/helpers/migration_report_helper.rb | 35 ---- .../default_rights_datastream_mover.rb | 32 --- .../desc_metadata_datastream_mover.rb | 39 ---- .../admin_collection/object_mover.rb | 58 ------ .../class_ordered_repository_migrator.rb | 195 ------------------ .../desc_metadata_datastream_mover.rb | 28 --- .../derivative/encoding_datastream_mover.rb | 55 ----- .../fedora_migrate/derivative/object_mover.rb | 60 ------ .../dublin_core_datastream_mover.rb | 37 ---- .../inherited_rights_datastream_mover.rb | 28 --- .../lease/desc_metadata_datastream_mover.rb | 25 --- .../fedora_migrate/lease/object_mover.rb | 43 ---- .../desc_metadata_datastream_mover.rb | 31 --- .../mh_metadata_datastream_mover.rb | 23 --- .../master_file/object_mover.rb | 103 --------- .../display_metadata_datastream_mover.rb | 25 --- .../dublin_core_datastream_mover.rb | 26 --- .../master_file_aggregation_mover.rb | 61 ------ .../media_object/object_mover.rb | 99 --------- .../reassign_id_object_mover.rb | 95 --------- .../reassign_id_rels_ext_datastream_mover.rb | 72 ------- .../reassign_id_target_constructor.rb | 22 -- .../simple_xml_datastream_mover.rb | 52 ----- .../status_tracking_datastream_mover.rb | 48 ----- app/views/layouts/migration_report.html.erb | 19 -- app/views/migration_status/_fedora3.html.erb | 38 ---- app/views/migration_status/_fedora4.html.erb | 50 ----- app/views/migration_status/detail.html.erb | 89 -------- app/views/migration_status/index.html.erb | 48 ----- app/views/migration_status/show.html.erb | 39 ---- lib/tasks/avalon.rake | 138 ------------- .../migration_status_controller_spec.rb | 66 ------ .../admin_collection_object_mover_spec.rb | 42 ---- spec/migration/derivative_object_mover.rb | 42 ---- spec/migration/lease_object_mover.rb | 42 ---- spec/migration/master_file_object_mover.rb | 42 ---- spec/migration/media_object_object_mover.rb | 44 ---- spec/rails_helper.rb | 1 - spec/requests/redirect_spec.rb | 4 +- 43 files changed, 2 insertions(+), 2104 deletions(-) delete mode 100644 app/assets/javascripts/report_refresh.js.coffee delete mode 100644 app/controllers/migration_status_controller.rb delete mode 100644 app/helpers/migration_report_helper.rb delete mode 100644 app/migration/fedora_migrate/admin_collection/default_rights_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/admin_collection/desc_metadata_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/admin_collection/object_mover.rb delete mode 100644 app/migration/fedora_migrate/class_ordered_repository_migrator.rb delete mode 100644 app/migration/fedora_migrate/derivative/desc_metadata_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/derivative/encoding_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/derivative/object_mover.rb delete mode 100644 app/migration/fedora_migrate/dublin_core_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/inherited_rights_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/lease/desc_metadata_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/lease/object_mover.rb delete mode 100644 app/migration/fedora_migrate/master_file/desc_metadata_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/master_file/mh_metadata_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/master_file/object_mover.rb delete mode 100644 app/migration/fedora_migrate/media_object/display_metadata_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/media_object/dublin_core_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/media_object/master_file_aggregation_mover.rb delete mode 100644 app/migration/fedora_migrate/media_object/object_mover.rb delete mode 100644 app/migration/fedora_migrate/reassign_id_object_mover.rb delete mode 100644 app/migration/fedora_migrate/reassign_id_rels_ext_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/reassign_id_target_constructor.rb delete mode 100644 app/migration/fedora_migrate/simple_xml_datastream_mover.rb delete mode 100644 app/migration/fedora_migrate/status_tracking_datastream_mover.rb delete mode 100644 app/views/layouts/migration_report.html.erb delete mode 100644 app/views/migration_status/_fedora3.html.erb delete mode 100644 app/views/migration_status/_fedora4.html.erb delete mode 100644 app/views/migration_status/detail.html.erb delete mode 100644 app/views/migration_status/index.html.erb delete mode 100644 app/views/migration_status/show.html.erb delete mode 100644 spec/controllers/migration_status_controller_spec.rb delete mode 100644 spec/migration/admin_collection_object_mover_spec.rb delete mode 100644 spec/migration/derivative_object_mover.rb delete mode 100644 spec/migration/lease_object_mover.rb delete mode 100644 spec/migration/master_file_object_mover.rb delete mode 100644 spec/migration/media_object_object_mover.rb diff --git a/Gemfile b/Gemfile index 9669afc21c..ad3709f361 100644 --- a/Gemfile +++ b/Gemfile @@ -27,7 +27,6 @@ gem 'webpacker' # Core Samvera gem 'active-fedora', '~> 13.2', '>= 13.2.5' gem 'active_fedora-datastreams', git: 'https://github.com/samvera-labs/active_fedora-datastreams', branch: 'rails6' -#gem 'fedora-migrate', git: 'https://github.com/avalonmediasystem/fedora-migrate.git', tag: 'avalon-r6.5' gem 'hydra-head', '~> 12.0' gem 'ldp', '~> 1.0.3' gem 'noid-rails', '~> 3.0.1' diff --git a/app/assets/javascripts/report_refresh.js.coffee b/app/assets/javascripts/report_refresh.js.coffee deleted file mode 100644 index 6169eda5c1..0000000000 --- a/app/assets/javascripts/report_refresh.js.coffee +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -$ -> - if $('.migration_report').length > 0 - refresh = -> - if $('#live-update').is(':checked') - $.get document.location.href - .done (data) -> - $('.migration_report').html(data) - setTimeout(refresh, 5000) - setTimeout(refresh, 5000) - $('#live-update').change -> refresh() diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index a4c6a5e185..4cb213aac0 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -47,8 +47,6 @@ def mejs end def rewrite_v4_ids - return if params[:controller] =~ /migration/ - params.permit! query_result = ActiveFedora::SolrService.query(%{identifier_ssim:"#{params[:id]}"}, rows: 1, fl: 'id') diff --git a/app/controllers/migration_status_controller.rb b/app/controllers/migration_status_controller.rb deleted file mode 100644 index 4b3edcb4ab..0000000000 --- a/app/controllers/migration_status_controller.rb +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -class MigrationStatusController < ApplicationController - - before_action :auth - layout 'migration_report' - - def index - @counts = MigrationStatus.summary - render without_layout_if_xhr - end - - def show - criteria = { source_class: params[:class], datastream: nil } - if params[:status].present? - criteria[:status] = "migrate" if params[:status] == "in progress" - criteria[:status] ||= params[:status] - end - @statuses = MigrationStatus.where(criteria) - .order(sanitize_order(params[:order]) || :id) - .page(params[:page]) - .per(params[:per]) - render without_layout_if_xhr - end - - def detail - @statuses = MigrationStatus.where(f3_pid: params[:id]) - @statuses = MigrationStatus.where(f4_pid: params[:id]) if @statuses.empty? - raise ActiveRecord::RecordNotFound if @statuses.blank? - @class = @statuses.first.source_class - @f4_pid = @statuses.first.f4_pid - if @f4_pid - @f4_obj = ActiveFedora::Base.find(@f4_pid) rescue nil - end - @f3_pid = @statuses.first.f3_pid - if @f3_pid - @f3_obj = FedoraMigrate.source.connection.find(@f3_pid) rescue nil - end - render without_layout_if_xhr - end - - def report - requested_filename = "#{params[:id].sub(/:/, '_')}.json" - migration_report_directory = Rails.root.join("migration_report") - raise ActionController::RoutingError, 'Not Found' unless Dir.exist? migration_report_directory - found_filename = Dir.entries(migration_report_directory).find { |filename| filename == requested_filename } - file = Rails.root.join("migration_report", found_filename) if found_filename - raise ActionController::RoutingError, 'Not Found' unless file && File.exist?(file) - send_file file, type: 'application/json', disposition: 'inline' - end - - def auth - authorize! :read, MigrationStatus - end - - def without_layout_if_xhr - request.xhr? ? { layout: false } : {} - end - -private - - # Avoid SQL injection attack on ActiveRecord order method - # Input must be in format "column asc" or "column desc" - def sanitize_order(order_param) - if order_param.present? - { order_param.split.first => order_param.split.second } - else - nil - end - end -end diff --git a/app/helpers/migration_report_helper.rb b/app/helpers/migration_report_helper.rb deleted file mode 100644 index 39ff370d8c..0000000000 --- a/app/helpers/migration_report_helper.rb +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module MigrationReportHelper - def status_link(klass, display_name, status_name=display_name) - params = { class: klass, status: status_name }.reject { |k,v| v.nil? } - params.delete(:status) if status_name == 'total' - link_to @counts[klass][status_name].to_i, admin_migration_report_by_class_path(params) - end - - def sort_params(col_name, default='id') - new_params = params.reject { |k,v| k =~ /^(controller|action)$/ } - (current_col,current_order) = new_params[:order].to_s.split - current_col ||= default - current_order ||= 'ASC' - order = (current_col == col_name && current_order == 'ASC') ? 'DESC' : 'ASC' - new_params[:order] = "#{col_name} #{order}" - new_params.permit(:order, :class) - end - - def status_string(status) - status == "migrate" ? "In Progress" : status.titleize - end -end diff --git a/app/migration/fedora_migrate/admin_collection/default_rights_datastream_mover.rb b/app/migration/fedora_migrate/admin_collection/default_rights_datastream_mover.rb deleted file mode 100644 index f04bdcab9e..0000000000 --- a/app/migration/fedora_migrate/admin_collection/default_rights_datastream_mover.rb +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module AdminCollection - class DefaultRightsDatastreamMover < PermissionsMover - def migrate - [:read_groups, :read_users].each do |permission| - next unless target.respond_to?("default_" + permission.to_s + "=") - report << "default_#{permission} = #{send(permission)}" - target.send("default_" + permission.to_s + "=", send(permission)) - end - target.default_hidden = discover_groups.include?("nobody") if target.respond_to?("default_hidden=") - report << "default_hidden = #{target.default_hidden}" - # save - # super - report - end - end - end -end diff --git a/app/migration/fedora_migrate/admin_collection/desc_metadata_datastream_mover.rb b/app/migration/fedora_migrate/admin_collection/desc_metadata_datastream_mover.rb deleted file mode 100644 index b5b89ec09b..0000000000 --- a/app/migration/fedora_migrate/admin_collection/desc_metadata_datastream_mover.rb +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module AdminCollection - class DescMetadataDatastreamMover < FedoraMigrate::SimpleXmlDatastreamMover - - def fields_to_copy - %w(name unit description dropbox_directory_name) - end - - def migrate - super - add_unit_to_controlled_vocabulary(target.unit) - end - - private - - def add_unit_to_controlled_vocabulary(unit) - v = Avalon::ControlledVocabulary.vocabulary - unless v[:units].include? unit - v[:units] |= Array(unit) - Avalon::ControlledVocabulary.vocabulary = v - end - end - end - end -end diff --git a/app/migration/fedora_migrate/admin_collection/object_mover.rb b/app/migration/fedora_migrate/admin_collection/object_mover.rb deleted file mode 100644 index 84565f00b0..0000000000 --- a/app/migration/fedora_migrate/admin_collection/object_mover.rb +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module AdminCollection - class ObjectMover < ReassignIdObjectMover - DESC_METADATA_DATASTREAM = "descMetadata".freeze - - def migrate_datastreams - migrate_desc_metadata - migrate_dublin_core - # Make sure to migrate permissions before any of the other rights - # datastreams since it creates the permissions report - migrate_permissions - migrate_inherited_rights - migrate_default_rights - # migrate_dates #skip because it doesn't do anything for us - save - migrate_relationships - # super - end - - def migrate_desc_metadata - return unless source.datastreams.keys.include?(DESC_METADATA_DATASTREAM) - mover = FedoraMigrate::AdminCollection::DescMetadataDatastreamMover.new(source.datastreams[DESC_METADATA_DATASTREAM], target) - mover.migrate - # report.descMetadata = mover.migrate - end - - def migrate_default_rights - return unless source.datastreams.keys.include?('defaultRights') - mover = FedoraMigrate::AdminCollection::DefaultRightsDatastreamMover.new(source.datastreams['defaultRights'], target) - mover.migrate - report.permissions += mover.migrate - end - - def self.wipeout!(collection) - collection.default_permissions.destroy_all - super - end - - def self.empty?(collection) - collection.default_permissions.blank? && super - end - end - end -end diff --git a/app/migration/fedora_migrate/class_ordered_repository_migrator.rb b/app/migration/fedora_migrate/class_ordered_repository_migrator.rb deleted file mode 100644 index 9d44552691..0000000000 --- a/app/migration/fedora_migrate/class_ordered_repository_migrator.rb +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - class ClassOrderedRepositoryMigrator < RepositoryMigrator - - def migrate_objects(pids = nil, overwrite = false, skip_completed = false) - @pids_whitelist = pids - @overwrite = overwrite - @skip_completed = skip_completed - class_order.each do |klass| - ::MediaObject.skip_callback(:save, :before, :update_dependent_properties!) if klass == ::MediaObject - Parallel.map_with_index(gather_pids_for_class(klass), in_processes: parallel_processes, progress: "Migrating #{klass.to_s}") do |pid, i| - next unless qualifying_pid?(pid, klass) - # Let solr catch up - ActiveFedora::SolrService.instance.conn.commit if (i % 100 == 0) - ActiveFedora::SolrService.instance.conn.optimize if (i % 1000 == 0) - remove_object(pid, klass) unless overwrite? - migrate_object(source_object(pid), klass) - end - ::MediaObject.set_callback(:save, :before, :update_dependent_properties!) if klass == ::MediaObject - end - class_order.each do |klass| - if second_pass_needed?(klass) - Parallel.map_with_index(gather_pids_for_class(klass), in_processes: parallel_processes, progress: "Migrating #{klass.to_s} (second pass)") do |pid, i| - next unless qualifying_pid?(pid, klass, :second_pass) - # Let solr catch up - ActiveFedora::SolrService.instance.conn.commit if (i % 100 == 0) - ActiveFedora::SolrService.instance.conn.optimize if (i % 1000 == 0) - migrate_object(source_object(pid), klass, :second_pass) - end - end - end - @report.reload - end - - def migration_required?(pid, klass, method=:migrate) - status_report = MigrationStatus.find_by(source_class: klass, f3_pid: pid, datastream: nil) - status_report.nil? || - (status_report.status != 'completed' && status_report.status != 'waiting' && method == :migrate) || - (status_report.status != 'completed' && method == :second_pass) || - (!skip_completed? && overwrite? && method == :second_pass) || - (!skip_completed? && overwrite? && method == :migrate && status_report.status != 'waiting') - end - - private - - def gather_pids_for_class(klass) - # TODO make this faster by querying for @pids_whitelist instead of all objects - query = "SELECT ?pid WHERE { ?pid <#{class_to_model_name(klass)}> }" - # Query and filter using pids whitelist - pids = FedoraMigrate.source.connection.sparql(query)["pid"].collect {|pid| pid.split('/').last} - gathered_pids = @pids_whitelist.blank? ? pids : pids & @pids_whitelist - - if skip_completed? - completed_pids = MigrationStatus.where(source_class: klass, status: 'completed', datastream: nil).pluck(:f3_pid) - gathered_pids - completed_pids - else - gathered_pids - end - end - - def source_object(pid) - FedoraMigrate.source.connection.find(pid) - end - - def initialize_report(source) - result = SingleObjectReport.new - result.status = false - @report.save(source.pid, result) - result - end - - def remove_object(pid, klass) - target = klass.where(migrated_from_ssim: construct_migrate_from_uri(pid).to_s).first - target.delete unless target.nil? - end - - def cleanout_object!(target, klass) - return nil unless target - #target_id = target.id - #target_class = target.class - #success = target.delete.eradicate - success = object_mover(klass).wipeout!(target) - raise RuntimeError("Failed to cleanout object: #{target_id}") unless success - #target_class.new(id: target_id) - target - end - - def overwrite? - !!@overwrite - end - - def skip_completed? - !!@skip_completed - end - - def migrate_object(source, klass, method=:migrate) - result = initialize_report(source) - status_record = MigrationStatus.find_or_create_by(source_class: klass.name, f3_pid: source.pid, datastream: nil) - unless (status_record.status == 'failed') && (method == :second_pass) - begin - target = klass.where(migrated_from_ssim: construct_migrate_from_uri(source.pid).to_s).first - if overwrite? && (method != :second_pass) - target = cleanout_object!(target, klass) - unless target.nil? - MigrationStatus.where(f3_pid: status_record.f3_pid).delete_all - status_record = MigrationStatus.find_or_create_by(source_class: klass.name, f3_pid: source.pid, datastream: nil) - end - end - status_record.update_attributes status: method.to_s, log: nil - options[:report] = @report.results[source.pid] = reload_single_item_report(source.pid) if method == :second_pass - result.object = object_mover(klass).new(source, target, options).send(method) - status_record.reload - if status_record.status == "failed" - result.status = false - else - status_record.update_attribute :f4_pid, result.object.id unless method == :second_pass - result.status = true - end - rescue StandardError => e - result.object = {exception: e.class.name, message: e.message, backtrace: e.backtrace[0..15]} - status_record.update_attribute :log, %{#{e.class.name}: "#{e.message}"} - result.status = false - ensure - status_record.update_attribute :status, end_status(result, method, klass) - remove_object(source.pid, klass) if status_record.status == "failed" - @report.save(source.pid, result) - end - end - end - - def end_status(result, method, klass) - if result.status - if method == :migrate and second_pass_needed?(klass) - return 'waiting' - else - return 'completed' - end - end - return 'failed' - end - - def second_pass_needed?(klass) - object_mover(klass).instance_methods.include?(:second_pass) - end - - def object_mover(klass) - ("FedoraMigrate::" + klass.name.gsub(/::/,'') + "::ObjectMover").constantize - end - - def class_order - @options[:class_order] - end - - def parallel_processes - (@options[:parallel_processes] || (Parallel.processor_count - 2)).to_i - end - - def qualifying_pid?(pid, klass, method=:migrate) - name = pid.split(/:/).first - name.match(namespace) && migration_required?(pid, klass, method) - end - - def parse_model_name(object) - model_uri = object.models.find {|m| m.start_with? "info:fedora/afmodel"} - model_uri.nil? ? nil : model_uri[/afmodel:(.+?)$/, 1].gsub(/_/,'::') - end - - def class_to_model_name(klass) - "info:fedora/afmodel:#{klass.name.gsub(/(::)/, '_')}" - end - - def construct_migrate_from_uri(pid) - RDF::URI.new(FedoraMigrate.fedora_config.credentials[:url]) / "/objects/#{pid}" - end - - def reload_single_item_report(pid) - path = @report.path - file = File.join(path, pid.tr(':', "_") + ".json") - JSON.parse(File.read(file)) - end - end -end diff --git a/app/migration/fedora_migrate/derivative/desc_metadata_datastream_mover.rb b/app/migration/fedora_migrate/derivative/desc_metadata_datastream_mover.rb deleted file mode 100644 index 26935c60ab..0000000000 --- a/app/migration/fedora_migrate/derivative/desc_metadata_datastream_mover.rb +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module Derivative - class DescMetadataDatastreamMover < FedoraMigrate::SimpleXmlDatastreamMover - - def fields_to_copy - %w(location_url hls_url duration track_id hls_track_id managed) - end - - def migrate - super - end - end - end -end diff --git a/app/migration/fedora_migrate/derivative/encoding_datastream_mover.rb b/app/migration/fedora_migrate/derivative/encoding_datastream_mover.rb deleted file mode 100644 index 1f49b52abc..0000000000 --- a/app/migration/fedora_migrate/derivative/encoding_datastream_mover.rb +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module Derivative - class EncodingDatastreamMover < Mover - attr_accessor :encodingProfile - - def post_initialize - @encodingProfile = xml_from_content - end - - def migrate - target.quality = present_or_nil(encodingProfile.xpath('//quality').text) - target.mime_type = present_or_nil(encodingProfile.xpath('//mime_type').text) - target.audio_bitrate = present_or_nil(encodingProfile.xpath('//audio/bitrate').text) - target.audio_codec = present_or_nil(encodingProfile.xpath('//audio/codec').text) - target.video_bitrate = present_or_nil(encodingProfile.xpath('//video/bitrate').text) - target.video_codec = present_or_nil(encodingProfile.xpath('//video/codec').text) - resolution = present_or_nil(encodingProfile.xpath('//video/resolution').text) - res_width,res_height = resolution.present? ? resolution.split('x') : [nil,nil] - width = present_or_nil(encodingProfile.xpath('//video/resolution/width').text) - height = present_or_nil(encodingProfile.xpath('//video/resolution/height').text) - if (res_width.present? and width.present? and res_width!=width) or (res_height.present? and height.present? and res_height!=height) - raise FedoraMigrate::Errors::MigrationError, "Derivative resolution and height/width do not match" - end - target.width = width || res_width - target.height = height || res_height - super - end - - private - def xml_from_content - Nokogiri::XML(source.content) - end - - def present_or_nil(value) - return nil unless value.present? - value - end - - end - end -end diff --git a/app/migration/fedora_migrate/derivative/object_mover.rb b/app/migration/fedora_migrate/derivative/object_mover.rb deleted file mode 100644 index 5a1d000369..0000000000 --- a/app/migration/fedora_migrate/derivative/object_mover.rb +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module Derivative - class ObjectMover < ReassignIdObjectMover - DESC_METADATA_DATASTREAM = "descMetadata".freeze - DERIVATIVE_DATASTREAM = "derivativeFile".freeze - ENCODING_DATASTREAM = "encoding".freeze - - def migrate - #find master file pid to see if it's already failed - rels_rdf = RDF::Graph.new { |g| g.from_rdfxml(source.datastreams["RELS-EXT"].content) } - master_file_pid = rels_rdf.statements.find {|s| s.predicate == ActiveFedora::RDF::Fcrepo::RelsExt.isDerivationOf }.object.to_s.split('/').last - mf_status = MigrationStatus.where(f3_pid: master_file_pid).first.status - raise FedoraMigrate::Errors::MigrationError, "Parent master file (#{master_file_pid}) failed to migrate" if mf_status == "failed" - super - end - - def migrate_datastreams - migrate_desc_metadata - migrate_file_location - migrate_transcoding_metadata - migrate_relationships - save - # super - end - - def migrate_desc_metadata - return unless source.datastreams.keys.include?(DESC_METADATA_DATASTREAM) - mover = FedoraMigrate::Derivative::DescMetadataDatastreamMover.new(source.datastreams[DESC_METADATA_DATASTREAM], target) - mover.migrate - # report.descMetadata = mover.migrate - end - - def migrate_transcoding_metadata - return unless source.datastreams.keys.include?(ENCODING_DATASTREAM) - mover = FedoraMigrate::Derivative::EncodingDatastreamMover.new(source.datastreams[ENCODING_DATASTREAM], target) - mover.migrate - end - - def migrate_file_location - return unless source.datastreams.keys.include?(DERIVATIVE_DATASTREAM) - target.derivativeFile = source.datastreams[DERIVATIVE_DATASTREAM].content - end - - end - end -end diff --git a/app/migration/fedora_migrate/dublin_core_datastream_mover.rb b/app/migration/fedora_migrate/dublin_core_datastream_mover.rb deleted file mode 100644 index 218fba2f3a..0000000000 --- a/app/migration/fedora_migrate/dublin_core_datastream_mover.rb +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - class DublinCoreDatastreamMover < Mover - - attr_accessor :dc - - def post_initialize - @dc = xml_from_content - dc.remove_namespaces! - end - - def migrate - target.identifier += dc.xpath('//identifier').map(&:text) - target.identifier += [source.pid] - super - end - - private - - def xml_from_content - Nokogiri::XML(source.content) - end - end -end diff --git a/app/migration/fedora_migrate/inherited_rights_datastream_mover.rb b/app/migration/fedora_migrate/inherited_rights_datastream_mover.rb deleted file mode 100644 index 526b7eb8b7..0000000000 --- a/app/migration/fedora_migrate/inherited_rights_datastream_mover.rb +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - class InheritedRightsDatastreamMover < PermissionsMover - def migrate - FedoraMigrate::Permissions.instance_methods.each do |permission| - next unless target.respond_to?("inherited_" + permission.to_s + "=") - report << "inherited_#{permission} = #{send(permission)}" - target.send("inherited_" + permission.to_s + "=", send(permission)) - end - # save - # super - report - end - end -end diff --git a/app/migration/fedora_migrate/lease/desc_metadata_datastream_mover.rb b/app/migration/fedora_migrate/lease/desc_metadata_datastream_mover.rb deleted file mode 100644 index 6e28783af1..0000000000 --- a/app/migration/fedora_migrate/lease/desc_metadata_datastream_mover.rb +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module Lease - class DescMetadataDatastreamMover < FedoraMigrate::SimpleXmlDatastreamMover - - def fields_to_copy - %w(begin_time end_time) - end - - end - end -end diff --git a/app/migration/fedora_migrate/lease/object_mover.rb b/app/migration/fedora_migrate/lease/object_mover.rb deleted file mode 100644 index 3f0d3989a8..0000000000 --- a/app/migration/fedora_migrate/lease/object_mover.rb +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module Lease - class ObjectMover < ReassignIdObjectMover - DESC_METADATA_DATASTREAM = "descMetadata".freeze - - def migrate - #find mediaobject pid to see if it's already failed -# media_object_pids = FedoraMigrate.source.connection.find_by_sparql("SELECT ?pid FROM <#ri> WHERE { ?pid <#{source.uri}> }").collect(&:pid) -# raise FedoraMigrate::Errors::MigrationError, "Parent media object(s) (#{media_object_pids}) failed to migrate" if media_object_pids.all? {|mo_pid| MigrationStatus.where(f3_pid: mo_pid).first.status == "failed" } - super - end - - def migrate_datastreams - migrate_desc_metadata - migrate_permissions - migrate_inherited_rights - save - migrate_relationships - end - - def migrate_desc_metadata - return unless source.datastreams.keys.include?(DESC_METADATA_DATASTREAM) - mover = FedoraMigrate::Lease::DescMetadataDatastreamMover.new(source.datastreams[DESC_METADATA_DATASTREAM], target) - mover.migrate - # report.descMetadata = mover.migrate - end - end - end -end diff --git a/app/migration/fedora_migrate/master_file/desc_metadata_datastream_mover.rb b/app/migration/fedora_migrate/master_file/desc_metadata_datastream_mover.rb deleted file mode 100644 index daca131730..0000000000 --- a/app/migration/fedora_migrate/master_file/desc_metadata_datastream_mover.rb +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module MasterFile - class DescMetadataDatastreamMover < FedoraMigrate::SimpleXmlDatastreamMover - - def fields_to_copy - %w(file_checksum file_size duration display_aspect_ratio original_frame_size date_digitized physical_description file_location file_format) - end - - def migrate - super - ['poster_offset','thumbnail_offset'].each do |field| - copy_field(field, &:to_i) - end - end - end - end -end diff --git a/app/migration/fedora_migrate/master_file/mh_metadata_datastream_mover.rb b/app/migration/fedora_migrate/master_file/mh_metadata_datastream_mover.rb deleted file mode 100644 index f2983b93fc..0000000000 --- a/app/migration/fedora_migrate/master_file/mh_metadata_datastream_mover.rb +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module MasterFile - class MhMetadataDatastreamMover < FedoraMigrate::SimpleXmlDatastreamMover - def fields_to_copy - %w(workflow_id workflow_name percent_complete percent_succeeded percent_failed status_code operation error encoder_classname) - end - end - end -end diff --git a/app/migration/fedora_migrate/master_file/object_mover.rb b/app/migration/fedora_migrate/master_file/object_mover.rb deleted file mode 100644 index ffe589fcae..0000000000 --- a/app/migration/fedora_migrate/master_file/object_mover.rb +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module MasterFile - class ObjectMover < ReassignIdObjectMover - DESC_METADATA_DATASTREAM = "descMetadata".freeze - POSTER_DATASTREAM = "poster".freeze - THUMBNAIL_DATASTREAM = "thumbnail".freeze - STRUCTURAL_METADATA_DATASTREAM = "structuralMetadata".freeze - CAPTIONS_DATASTREAM = "captions".freeze - MASTERFILE_DATASTREAM = "masterFile".freeze - MH_METADATA_DATASTREAM = "mhMetadata".freeze - - def migrate - #find mediaobject pid to see if it's already failed - rels_rdf = RDF::Graph.new { |g| g.from_rdfxml(source.datastreams["RELS-EXT"].content) } - media_object_pid = rels_rdf.statements.find {|s| s.predicate == ActiveFedora::RDF::Fcrepo::RelsExt.isPartOf }.object.to_s.split('/').last - mo_status = MigrationStatus.where(f3_pid: media_object_pid).first.status - raise FedoraMigrate::Errors::MigrationError, "Parent media object (#{media_object_pid}) failed to migrate" if mo_status == "failed" - super - end - - def migrate_datastreams - migrate_dublin_core - migrate_desc_metadata - migrate_transcoding_metadata - migrate_title - migrate_file_location - save # save before creating contained files - migrate_poster_and_thumbnail - migrate_structural_metadata - migrate_captions - save - migrate_relationships - migrate_permalink - target.ldp_source.save #Because save isn't persisting the isPartOf relationship - # super - end - - def migrate_desc_metadata - return unless source.datastreams.keys.include?(DESC_METADATA_DATASTREAM) - mover = FedoraMigrate::MasterFile::DescMetadataDatastreamMover.new(source.datastreams[DESC_METADATA_DATASTREAM], target) - mover.migrate - # report.descMetadata = mover.migrate - end - - def migrate_transcoding_metadata - return unless source.datastreams.keys.include?(MH_METADATA_DATASTREAM) - mover = FedoraMigrate::MasterFile::MhMetadataDatastreamMover.new(source.datastreams[MH_METADATA_DATASTREAM], target) - result = mover.migrate - if target.workflow_name.nil? || (not ::MasterFile::WORKFLOWS.include?(target.workflow_name)) - target.workflow_name = target.file_format == 'Sound' ? 'fullaudio' : 'avalon' - end - result - end - - def migrate_poster_and_thumbnail - migrate_content_datastream(POSTER_DATASTREAM, target.poster, "#{POSTER_DATASTREAM}.jpg") - migrate_content_datastream(THUMBNAIL_DATASTREAM, target.thumbnail, "#{THUMBNAIL_DATASTREAM}.jpg") - end - - def migrate_structural_metadata - migrate_content_datastream(STRUCTURAL_METADATA_DATASTREAM, target.structuralMetadata, "#{STRUCTURAL_METADATA_DATASTREAM}.xml") - end - - def migrate_captions - migrate_content_datastream(CAPTIONS_DATASTREAM, target.captions, source.datastreams[CAPTIONS_DATASTREAM].label.try(:gsub, /"/, '\"')) - end - - def migrate_file_location - return unless source.datastreams.keys.include?(MASTERFILE_DATASTREAM) - target.masterFile = source.datastreams[MASTERFILE_DATASTREAM].content.to_s.force_encoding(Encoding.default_external) - end - - def migrate_title - return unless source.label - target.title = source.label - end - - private - def migrate_content_datastream(ds_name, target_file, filename) - return unless source.datastreams.keys.include?(ds_name) - # Manually set ebucore:filename before the file gets persisted - target_file.original_name = filename - mover = FedoraMigrate::DatastreamMover.new(source.datastreams[ds_name], target_file) - mover.migrate - #report.content_datastreams << ContentDatastreamReport.new(target.attached_files[ds_name], mover.migrate) - end - end - end -end diff --git a/app/migration/fedora_migrate/media_object/display_metadata_datastream_mover.rb b/app/migration/fedora_migrate/media_object/display_metadata_datastream_mover.rb deleted file mode 100644 index 89b5b94ba6..0000000000 --- a/app/migration/fedora_migrate/media_object/display_metadata_datastream_mover.rb +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module MediaObject - class DisplayMetadataDatastreamMover < FedoraMigrate::SimpleXmlDatastreamMover - - def migrate - copy_field('duration', &:to_i) - copy_field('avalon_resource_type', nil, true) - end - end - end -end diff --git a/app/migration/fedora_migrate/media_object/dublin_core_datastream_mover.rb b/app/migration/fedora_migrate/media_object/dublin_core_datastream_mover.rb deleted file mode 100644 index ee52f32710..0000000000 --- a/app/migration/fedora_migrate/media_object/dublin_core_datastream_mover.rb +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module MediaObject - class DublinCoreDatastreamMover < FedoraMigrate::DublinCoreDatastreamMover - - def migrate - target.avalon_uploader = dc.xpath('//creator').text - target.avalon_publisher = dc.xpath('//publisher').text - super - end - end - end -end diff --git a/app/migration/fedora_migrate/media_object/master_file_aggregation_mover.rb b/app/migration/fedora_migrate/media_object/master_file_aggregation_mover.rb deleted file mode 100644 index 73ebf96085..0000000000 --- a/app/migration/fedora_migrate/media_object/master_file_aggregation_mover.rb +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - module MediaObject - class MasterFileAggregationMover < ObjectMover - def migrate - return false unless target.migrated_from.present? - master_files = ::MasterFile.where(isPartOf_ssim: target.id) - if source.datastreams.has_key?('sectionsMetadata') - sections_md = Nokogiri::XML(source.datastreams['sectionsMetadata'].content) - old_pid_order = sections_md.xpath('fields/section_pid').collect(&:text) - unless lists_equivalent?(old_pid_order, master_files.collect {|mf| pid_from_obj(mf)}) - fail_dependent_objects(master_files) - raise FedoraMigrate::Errors::MigrationError, "Master files found don't match media object expectations." - end - target.ordered_master_files = master_files.sort do |a,b| - old_pid_order.index(pid_from_obj(a)) <=> old_pid_order.index(pid_from_obj(b)) - end - else - target.ordered_master_files = master_files - end - target.save - master_files.collect(&:id) - end - - private - def pid_from_obj(obj) - obj.migrated_from.first.rdf_subject.to_s.split('/').last - end - - def lists_equivalent?(a,b) - a.size == b.size && ((a-b) + (b-a)).blank? - end - - def fail_dependent_objects(master_files) - #fail master_files - master_files.each do |mf| - fail_object(mf, source.pid) - ::Derivative.where(isDerivationOf_ssim: mf.id).each {|d| fail_object(d, pid_from_obj(mf))} - end - end - def fail_object(obj, parent_pid) - status_record = MigrationStatus.where(source_class: obj.class.name, f4_pid: obj.id).first - return unless status_record - status_record.update_attributes status: 'failed', log: "Parent object (#{parent_pid}) failed to migrate" - end - end - end -end diff --git a/app/migration/fedora_migrate/media_object/object_mover.rb b/app/migration/fedora_migrate/media_object/object_mover.rb deleted file mode 100644 index ae71104242..0000000000 --- a/app/migration/fedora_migrate/media_object/object_mover.rb +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -require 'ostruct' - -module FedoraMigrate - module MediaObject - class ObjectMover < ReassignIdObjectMover - DESC_METADATA_DATASTREAM = "descMetadata".freeze - WORKFLOW_DATASTREAM = "workflow".freeze - DISPLAY_METADATA_DATASTREAM = "displayMetadata".freeze - - def migrate_datastreams - migrate_dublin_core - migrate_relationships - migrate_permissions - migrate_desc_metadata #Need to do this after target is saved - migrate_workflow - migrate_display_metadata - migrate_permalink - # migrate_dates #skip because it doesn't do anything for us - save - # super - end - - def migrate_dublin_core - return unless source.datastreams.keys.include?('DC') - mover = FedoraMigrate::MediaObject::DublinCoreDatastreamMover.new(source.datastreams['DC'], target) - mover.migrate - # report.dc = mover.migrate - end - - def migrate_desc_metadata - return unless source.datastreams.keys.include?(DESC_METADATA_DATASTREAM) - mover = FedoraMigrate::StatusTrackingDatastreamMover.new(source.datastreams[DESC_METADATA_DATASTREAM], target.attached_files[DESC_METADATA_DATASTREAM], options) - report.content_datastreams << ContentDatastreamReport.new(target.attached_files[DESC_METADATA_DATASTREAM], mover.migrate) - # add MODS recordIdentifier for new fedora noid id - add_fedora4_record_info - end - - def add_fedora4_record_info - doc = Nokogiri::XML(target.attached_files[DESC_METADATA_DATASTREAM].content) - f3_recid = doc.xpath('//mods:recordIdentifier', mods: "http://www.loc.gov/mods/v3").first - f4_recid = f3_recid.dup - f4_recid.content = target.uri - f4_recid["source"] = "Fedora4" - f3_recid.add_next_sibling f4_recid - # I don't know why setting original_name needs to be here instead of before calling the mover, but it seems to work here now and not before - # Manually set ebucore:filename before the file gets persisted - target.attached_files[DESC_METADATA_DATASTREAM].original_name = "#{DESC_METADATA_DATASTREAM}.xml" - target.attached_files[DESC_METADATA_DATASTREAM].content = doc.to_xml - target.attached_files[DESC_METADATA_DATASTREAM].save - end - - def migrate_workflow - return unless source.datastreams.keys.include?(WORKFLOW_DATASTREAM) - # Manually set ebucore:filename before the file gets persisted - target.attached_files[WORKFLOW_DATASTREAM].original_name = "#{WORKFLOW_DATASTREAM}.xml" - FedoraMigrate::StatusTrackingDatastreamMover.new(source.datastreams[WORKFLOW_DATASTREAM], target.workflow).migrate - end - - def migrate_display_metadata - return unless source.datastreams.keys.include?(DISPLAY_METADATA_DATASTREAM) - FedoraMigrate::MediaObject::DisplayMetadataDatastreamMover.new(source.datastreams[DISPLAY_METADATA_DATASTREAM], target).migrate - end - - def second_pass - @report = OpenStruct.new(options[:report]) if options[:report].present? - mover = FedoraMigrate::MediaObject::MasterFileAggregationMover.new(source, target) - report.master_file_order = mover.migrate - report - end - - def self.wipeout!(media_object) - media_object.ordered_master_files = [] - media_object.master_files = [] - media_object.save - super - end - - def self.empty?(media_object) - media_object.ordered_master_files.to_a.blank? && - media_object.master_files.blank? && - super - end - end - end -end diff --git a/app/migration/fedora_migrate/reassign_id_object_mover.rb b/app/migration/fedora_migrate/reassign_id_object_mover.rb deleted file mode 100644 index b770febcf7..0000000000 --- a/app/migration/fedora_migrate/reassign_id_object_mover.rb +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - class ReassignIdObjectMover < ObjectMover - SinglePassReport = Struct.new(:id, :class, :content_datastreams, :rdf_datastreams, :permissions, :dates, :relationships) - - def results_report - SinglePassReport.new.tap do |report| - report.content_datastreams = [] - report.rdf_datastreams = [] - end - end - - def prepare_target - target.migrated_from = [construct_migrate_from_uri(source)] - super - end - - def complete_target - after_object_migration - save - complete_report - end - - def complete_report - report.id = target.id - end - - def target - @target ||= FedoraMigrate::ReassignIdTargetConstructor.new(source).build - end - - def self.wipeout!(obj) - return false if obj.new_record? - obj.access_control.destroy if obj.respond_to?(:access_control) - obj.attached_files.values.each do |file| - next if file.new_record? - file.destroy - file.eradicate - end - obj.reload - obj.resource.clear - self.empty?(obj) - end - - def self.empty?(obj) - obj.resource.blank? && - (!obj.respond_to?(:access_control) || obj.access_control.blank?) && - obj.attached_files.values.all?(&:blank?) - end - - private - - def migrate_dublin_core - return unless source.datastreams.keys.include?('DC') - mover = FedoraMigrate::DublinCoreDatastreamMover.new(source.datastreams['DC'], target) - mover.migrate - # report.dc = mover.migrate - end - - def migrate_inherited_rights - return unless source.datastreams.keys.include?('inheritedRights') - mover = FedoraMigrate::InheritedRightsDatastreamMover.new(source.datastreams['inheritedRights'], target) - mover.migrate - report.permissions ||= [] - report.permissions += mover.migrate - end - - def migrate_relationships - report.relationships = FedoraMigrate::ReassignIdRelsExtDatastreamMover.new(source, target).migrate - end - - def migrate_permalink - permalink_value = target.ldp_source.graph.find{|stmt| stmt.predicate == "http://projecthydra.org/ns/relations#hasPermalink"}.object.to_s rescue nil - return unless permalink_value - target.permalink = permalink_value - end - - def construct_migrate_from_uri(source) - RDF::URI.new(FedoraMigrate.fedora_config.credentials[:url]) / "/objects/#{source.pid}" - end - end -end diff --git a/app/migration/fedora_migrate/reassign_id_rels_ext_datastream_mover.rb b/app/migration/fedora_migrate/reassign_id_rels_ext_datastream_mover.rb deleted file mode 100644 index e65a639bc6..0000000000 --- a/app/migration/fedora_migrate/reassign_id_rels_ext_datastream_mover.rb +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - class ReassignIdRelsExtDatastreamMover < RelsExtDatastreamMover - # def post_initialize - # @target = ActiveFedora::Base.find(target.id) - # rescue ActiveFedora::ObjectNotFoundError - # raise FedoraMigrate::Errors::MigrationError, "Target object was not found in Fedora 4. Did you migrate it?" - # end - - def migrate - migrate_statements - migrate_whitelist - # target.save - report - end - - def migrate_whitelist - graph.statements.each do |stmt| - if predicate_whitelist.include?(stmt.predicate) - triple = [target.rdf_subject, stmt.predicate, stmt.object] - target.ldp_source.graph << triple - report << triple.join("--") - end - end - end - - private - - def locate_object_id(id) - return target if source.pid == id - ActiveFedora::Base.where(identifier_ssim: id.downcase).first.try(:id) - end - - def migrate_object(fc3_uri) - obj_id = locate_object_id(fc3_uri.to_s.split('/').last) - #FIXME raise error or return if obj_id.nil? - RDF::URI.new(ActiveFedora::Base.id_to_uri(obj_id)) - end - - def predicate_blacklist - [ActiveFedora::RDF::Fcrepo::Model.hasModel, "http://projecthydra.org/ns/relations#hasModelVersion"] - end - - def predicate_whitelist - ['http://projecthydra.org/ns/relations#hasPermalink'] - end - - def missing_object?(statement) - return false if locate_object_id(statement.object.to_s.split('/').last).present? - report << "could not migrate relationship #{statement.predicate} because #{statement.object} doesn't exist in Fedora 4" unless predicate_whitelist.include?(statement.predicate) - true - end - - # All the graph statements except hasModel and those with missing objects - def statements - graph.statements.reject { |stmt| predicate_blacklist.include?(stmt.predicate) || missing_object?(stmt) } - end - end -end diff --git a/app/migration/fedora_migrate/reassign_id_target_constructor.rb b/app/migration/fedora_migrate/reassign_id_target_constructor.rb deleted file mode 100644 index d93047ed51..0000000000 --- a/app/migration/fedora_migrate/reassign_id_target_constructor.rb +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - class ReassignIdTargetConstructor < TargetConstructor - def build - raise FedoraMigrate::Errors::MigrationError, "No qualified targets found in #{source.pid}" if target.nil? - target.new - end - end -end diff --git a/app/migration/fedora_migrate/simple_xml_datastream_mover.rb b/app/migration/fedora_migrate/simple_xml_datastream_mover.rb deleted file mode 100644 index 31850a7b2b..0000000000 --- a/app/migration/fedora_migrate/simple_xml_datastream_mover.rb +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - class SimpleXmlDatastreamMover < FedoraMigrate::Mover - - attr_accessor :content - - def post_initialize - @content = xml_from_content - end - - def fields_to_copy - [] - end - - def migrate - fields_to_copy.each { |field| copy_field(field) } - super - end - - private - def copy_field(source_field, target_field=nil, multiple=false, &block) - target_field = source_field if target_field.nil? - target.send("#{source_field}=".to_sym, present_or_nil(fetch_field(target_field.to_s, multiple), &block)) - end - - def fetch_field(name, multiple=false) - multiple ? @content.xpath("fields/#{name}").map(&:text) : @content.xpath("fields/#{name}").text - end - - def present_or_nil(value) - return nil unless value.present? - block_given? ? yield(value) : value - end - - def xml_from_content - Nokogiri::XML(source.content) - end - end -end diff --git a/app/migration/fedora_migrate/status_tracking_datastream_mover.rb b/app/migration/fedora_migrate/status_tracking_datastream_mover.rb deleted file mode 100644 index 29bfe346de..0000000000 --- a/app/migration/fedora_migrate/status_tracking_datastream_mover.rb +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -module FedoraMigrate - class StatusTrackingDatastreamMover < DatastreamMover - DIGEST_CLASS = Digest::SHA2 - - def migrate - ds_name = source.dsid - source_class = source.digital_object.models.find {|m| /afmodel/ =~ m}.scan(/afmodel:(.+)$/).flatten.last rescue source.class.name - status = MigrationStatus.create source_class: source_class, f3_pid: source.pid, f4_pid: target.id.split(/\//).first, datastream: ds_name, status: 'migrate' - begin - super - checksums = { - source: generate_checksum { source.content }, - target: generate_checksum { target.content } - } - success = xml? ? EquivalentXml.equivalent?(source.content,target.content) : checksums[:source] == checksums[:target] - log_message = success ? nil : (xml? ? 'XML fails equivalency test' : 'Checksums do not match') - status.update_attributes checksum: checksums[:target], status: (success ? 'completed' : 'failed'), log: log_message - rescue Exception => e - status.update_attributes status: 'failed', log: e.message - ensure - status.save - end - end - - private - def generate_checksum - (DIGEST_CLASS.new << yield).hexdigest - end - - def xml? - !!(source.mimeType =~ %r{[/+]xml$}) - end - end -end diff --git a/app/views/layouts/migration_report.html.erb b/app/views/layouts/migration_report.html.erb deleted file mode 100644 index 7e2e1109df..0000000000 --- a/app/views/layouts/migration_report.html.erb +++ /dev/null @@ -1,19 +0,0 @@ -<%# -Copyright 2011-2022, The Trustees of Indiana University and Northwestern - University. Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed - under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - CONDITIONS OF ANY KIND, either express or implied. See the License for the - specific language governing permissions and limitations under the License. ---- END LICENSE_HEADER BLOCK --- -%> -
- <%= yield %> -
-<% parent_layout 'avalon' %> diff --git a/app/views/migration_status/_fedora3.html.erb b/app/views/migration_status/_fedora3.html.erb deleted file mode 100644 index f51f3d7d38..0000000000 --- a/app/views/migration_status/_fedora3.html.erb +++ /dev/null @@ -1,38 +0,0 @@ -<%# -Copyright 2011-2022, The Trustees of Indiana University and Northwestern - University. Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed - under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - CONDITIONS OF ANY KIND, either express or implied. See the License for the - specific language governing permissions and limitations under the License. ---- END LICENSE_HEADER BLOCK --- -%> -

Fedora 3

-
-
-<% if @f3_obj.present? %> - <% @f3_obj.datastreams.each do |name, ds| %> -
-

- - <%= name.titleize %> - -

-
-
- <% if ds.mimeType.start_with?("image") %> - <%= ('' % Base64.encode64(ds.content)).html_safe %> - <% else %> - 
<%= ds.content.body.force_encoding(Encoding::UTF_8).strip %>
- <% end %> -
- <% end %> -<% end %> -
-
diff --git a/app/views/migration_status/_fedora4.html.erb b/app/views/migration_status/_fedora4.html.erb deleted file mode 100644 index 2356273561..0000000000 --- a/app/views/migration_status/_fedora4.html.erb +++ /dev/null @@ -1,50 +0,0 @@ -<%# -Copyright 2011-2022, The Trustees of Indiana University and Northwestern - University. Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed - under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - CONDITIONS OF ANY KIND, either express or implied. See the License for the - specific language governing permissions and limitations under the License. ---- END LICENSE_HEADER BLOCK --- -%> -

Fedora 4

-
- <% if @f4_obj.present? %> -
-
-

- fcr:metadata -

-
-
- 
<%= @f4_obj.resource.dump(:ttl) %>
-
- - <% @f4_obj.serialize_attached_files.each do |label, file| %> - <% next unless file.content.present? %> -
-

- - <%= label %> - -

-
-
- <% if file.mime_type.start_with?('image') %> - <%= ('' % Base64.encode64(file.content)).html_safe %> - <% else %> - 

-				<%= file.content %>
-
- <% end %> -
- <% end %> -
- <% end %> -
diff --git a/app/views/migration_status/detail.html.erb b/app/views/migration_status/detail.html.erb deleted file mode 100644 index 2195b0dd31..0000000000 --- a/app/views/migration_status/detail.html.erb +++ /dev/null @@ -1,89 +0,0 @@ -<%# -Copyright 2011-2022, The Trustees of Indiana University and Northwestern - University. Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed - under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - CONDITIONS OF ANY KIND, either express or implied. See the License for the - specific language governing permissions and limitations under the License. ---- END LICENSE_HEADER BLOCK --- -%> - - - - - -
    -
  1. <%= link_to "Migration Report", admin_migration_report_path %>
    2. -
  2. <%= link_to @class, admin_migration_report_by_class_path(class: @class) %>
    3. -
  3. <%= params[:id]%>
    4. -
- -

-<% if @f3_pid.present? %> - Migrated <%=@f3_pid%> -<% else %> - Migrated ??? -<% end %> -<% if @f4_pid.present? %> - to - <% unless ['Derivative', 'Lease'].include? @class %> - <%= link_to objects_url(@f4_pid), objects_path(@f4_pid) %> - <% else %> - <%= @f4_pid %> - <% end %> -<% else %> -<% end %> - -

-
- -<% unless ['Bookmark', 'AvalonClip', 'AvalonMarker'].include? @class %> - Fedora 3 Datastreams | - Fedora 4 Properties | - <%= link_to 'Full Report', admin_migration_report_report_path(id: params[:id]), target: 'migration_json'%> -<% end %> - -
-
- - - - - - - - <% @statuses.each do |status| %> - - - - - - - <% end %> -
DatastreamChecksumStatusMessage
<%= status.datastream || '[main object]' %><%= status.checksum || 'Not Applicable' %><%= status.status %><%= status.log %>
- -<% unless ['Bookmark', 'AvalonClip', 'AvalonMarker'].include? @class %> - <%= render 'fedora3' %> - <%= render 'fedora4' %> -<% end %> diff --git a/app/views/migration_status/index.html.erb b/app/views/migration_status/index.html.erb deleted file mode 100644 index 5bfa1e2d4b..0000000000 --- a/app/views/migration_status/index.html.erb +++ /dev/null @@ -1,48 +0,0 @@ -<%# -Copyright 2011-2022, The Trustees of Indiana University and Northwestern - University. Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed - under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - CONDITIONS OF ANY KIND, either express or implied. See the License for the - specific language governing permissions and limitations under the License. ---- END LICENSE_HEADER BLOCK --- -%> -
    -
  1. Migration Report
    2. -
- - - - - - - - - - <% @counts.keys.each do |klass| %> - - - - - - - - - <% end %> - <% totals = @counts.values.inject(Hash.new(0)) { |h,l| l.each_pair { |k,v| h[k] += v }; h } %> - - - <% ['completed','failed','waiting','in progress','total'].each do |status| %> - - <% end %> - -
Class NameCompletedFailedWaitingIn ProgressTotal
<%= link_to klass, admin_migration_report_by_class_path(class: klass) %><%= status_link(klass, 'completed') %><%= status_link(klass, 'failed') %><%= status_link(klass, 'waiting') %><%= status_link(klass, 'in progress') %><%= status_link(klass, 'total') %>
Total<%= totals[status] %>
-
- Live Update -
diff --git a/app/views/migration_status/show.html.erb b/app/views/migration_status/show.html.erb deleted file mode 100644 index c3c4de10ec..0000000000 --- a/app/views/migration_status/show.html.erb +++ /dev/null @@ -1,39 +0,0 @@ -<%# -Copyright 2011-2022, The Trustees of Indiana University and Northwestern - University. Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed - under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - CONDITIONS OF ANY KIND, either express or implied. See the License for the - specific language governing permissions and limitations under the License. ---- END LICENSE_HEADER BLOCK --- -%> -
    -
  1. <%= link_to "Migration Report", admin_migration_report_path %>
    2. -
  2. <%= params[:class] %>
    3. -
- - - - - - - - <% @statuses.each do |status| %> - - - - - - - <% end %> -
<%= link_to 'Avalon 5 ID', admin_migration_report_by_class_path(sort_params('f3_pid','f3_pid')) %><%= link_to 'Avalon 6 ID', admin_migration_report_by_class_path(sort_params('f4_pid','f3_pid')) %><%= link_to 'Status', admin_migration_report_by_class_path(sort_params('status','f3_pid')) %>Message
<%= link_to status.f3_pid, admin_migration_report_detail_path(id: status.f3_pid) %><%= status.f4_pid %><%= status_string(status.status) %><%= status.log %>
-<%= paginate @statuses %> -
- Live Update -
diff --git a/lib/tasks/avalon.rake b/lib/tasks/avalon.rake index 4aad116455..e3269983e7 100644 --- a/lib/tasks/avalon.rake +++ b/lib/tasks/avalon.rake @@ -18,144 +18,6 @@ namespace :avalon do ActiveFedora::Cleaner.clean! end - desc "Migrate Avalon 5.x to 6.x" - task migrate: :environment do - Rake::Task['avalon:migrate:repo'].invoke - Rake::Task['avalon:migrate:db'].invoke - end - - namespace :migrate do - desc "Migrate all my objects" - task repo: :environment do - unless ENV['CONFIRM'] == 'yes' - $stderr.puts <<-EOC -WARNING: This migration task currently has known issues. - For example, some metadata is not migrated or is migrated incorrectly. - -This migration task is part of a larger migration process. More info can be found at: -https://wiki.dlib.indiana.edu/display/VarVideo/Avalon+5+to+6+Database+Migration - -Please run `rake avalon:migrate:repo CONFIRM=yes` to confirm. -EOC - exit 1 - end - ids = ENV['pids'].split(',') unless ENV['pids'].nil? - ids = Array(ids) | File.readlines(ENV['pidfile']).map(&:strip) unless ENV['pidfile'].nil? - parallel_processes = ENV['parallel_processes'] - overwrite = !!ENV['overwrite'] - skip_completed = !!ENV['skip_completed'] - namespace = ENV['namespace'] || Settings&.fedora&.namespace || 'avalon' - - #disable callbacks - Admin::Collection.skip_callback(:save, :around, :reindex_members) - # Don't accidentally delete derivatives - ::Derivative.skip_callback(:destroy, :before, :retract_distributed_files!) - #::MediaObject.skip_callback(:save, :before, :update_dependent_properties!) - - models = [Admin::Collection, ::Lease, ::MediaObject, ::MasterFile, ::Derivative] - migrator = FedoraMigrate::ClassOrderedRepositoryMigrator.new(namespace, class_order: models, parallel_processes: parallel_processes) - migrator.migrate_objects(ids, overwrite, skip_completed) - migrator - end - - desc "Migrate my database" - task db: :environment do - Bookmark.find_each do |b| - status_record = MigrationStatus.find_or_create_by(source_class: Bookmark.name, f3_pid: "Bookmark:#{b.id}") - next if status_record.status == "completed" - status_record.update_attributes status: "migrate", log: nil - begin - obj = MediaObject.where("identifier_ssim:\"#{b.document_id.downcase}\"").first - obj ||= MediaObject.where(id: b.document_id).first - raise FedoraMigrate::Errors::MigrationError, "Media Object with Avalon 5 ID #{b.document_id} could not be found" unless obj - b.document_id = obj.id - b.save! - status_record.update_attribute :status, "completed" - rescue StandardError => e - status_record.update_attributes status: "failed", log: %{#{e.class.name}: "#{e.message}"} - end - end - AvalonClip.find_each do |anno| - status_record = MigrationStatus.find_or_create_by(source_class: AvalonClip.name, f3_pid: "AvalonClip:#{anno.id}") - next if status_record.status == "completed" - status_record.update_attributes status: "migrate", log: nil - begin - old_id = anno.source.split('/').last - mf = MasterFile.where("identifier_ssim:\"#{old_id.downcase}\"").first - mf ||= MasterFile.where(id: old_id).first - raise FedoraMigrate::Errors::MigrationError, "Master File with Avalon 5 ID #{old_id} could not be found" unless mf - anno.master_file = mf - anno.save! - status_record.update_attribute :status, "completed" - rescue StandardError => e - status_record.update_attributes status: "failed", log: %{#{e.class.name}: "#{e.message}"} - end - end - AvalonMarker.find_each do |anno| - status_record = MigrationStatus.find_or_create_by(source_class: AvalonMarker.name, f3_pid: "AvalonMarker:#{anno.id}") - next if status_record.status == "completed" - status_record.update_attributes status: "migrate", log: nil - begin - old_id = anno.source.split('/').last - mf = MasterFile.where("identifier_ssim:\"#{old_id.downcase}\"").first - mf ||= MasterFile.where(id: old_id).first - raise FedoraMigrate::Errors::MigrationError, "Master File with Avalon 5 ID #{old_id} could not be found" unless mf - anno.master_file = mf - anno.save! - status_record.update_attribute :status, "completed" - rescue StandardError => e - status_record.update_attributes status: "failed", log: %{#{e.class.name}: "#{e.message}"} - end - end - end - - desc "Cleanup failed bookmarks" - task bookmark_cleanup: :environment do - deleted_count = 0 - passed_count = 0 - failed_count = 0 - - Bookmark.all.each do |b| - if MediaObject.where(id: b.document_id).count > 0 - passed_count = passed_count + 1 - else - begin - b.destroy - deleted_count = deleted_count + 1 - rescue Exception => e - puts "Failed to delete #{b.id}" - failed_count = failed_count + 1 - puts e.message - end - end - end - puts "Deleted: #{deleted_count} Passed: #{passed_count} Failed: #{failed_count}" - end - - desc "Migrate related items for Avalon 6.0 to 6.1" - task related_item: :environment do - MediaObject.find_each({},{batch_size:5}) do |mo| - doc = Nokogiri::XML(mo.descMetadata.content) - doc.xpath('//mods:relatedItem/mods:location/mods:url[@displayLabel]', mods: "http://www.loc.gov/mods/v3").each do |url| - label = url['displayLabel'] - relatedItem = url.ancestors('relatedItem').first - relatedItem.set_attribute('displayLabel',label) - url.remove_attribute('displayLabel') - end - mo.descMetadata.content = doc.to_xml - mo.descMetadata.save - end - end - - desc "Add Fedora4 identifier as mods record_identifier" - task record_identifier: :environment do - MediaObject.find_each({},{batch_size:5}) do |mo| - mo.save if mo.descMetadata.record_identifier.empty? - end - end - - end - desc 'migrate databases for the rails app and the active annotations gem' task :db_migrate do `rake db:migrate` diff --git a/spec/controllers/migration_status_controller_spec.rb b/spec/controllers/migration_status_controller_spec.rb deleted file mode 100644 index 21a4caec13..0000000000 --- a/spec/controllers/migration_status_controller_spec.rb +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -require 'rails_helper' - -describe MigrationStatusController do - let(:obj) { FactoryBot.create(:master_file, migrated_from: [fedora3_pid]) } - let(:fedora3_pid) { 'avalon:12345' } - let(:avalon_noid) { obj.id } - - describe 'security' do - context 'not logged in' do - it "all routes should redirect to restricted content page" do - expect(get :index).to render_template('errors/restricted_pid') - expect(get :show, params: { class: 'MediaObject' }).to render_template('errors/restricted_pid') - expect(get :detail, params: { id: 'avalon:12345' }).to render_template('errors/restricted_pid') - expect(get :report, params: { id: 'avalon:12345' }).to render_template('errors/restricted_pid') - end - end - - context 'with end-user' do - before do - login_as :user - end - - it "all routes should redirect to restricted content page" do - expect(get :index).to render_template('errors/restricted_pid') - expect(get :show, params: { class: 'MediaObject' }).to render_template('errors/restricted_pid') - expect(get :detail, params: { id: 'avalon:12345' }).to render_template('errors/restricted_pid') - expect(get :report, params: { id: 'avalon:12345' }).to render_template('errors/restricted_pid') - end - end - end - - describe '#detail' do - before do - login_as :administrator - MigrationStatus.create!(f3_pid: fedora3_pid, f4_pid: avalon_noid, source_class: obj.class) - end - - context 'with a Fedora 3 pid' do - it 'returns the details' do - get :detail, params: { id: fedora3_pid } - expect(response).to have_http_status(:ok) - end - end - - context 'with an Avalon noid' do - it 'returns the details' do - get :detail, params: { id: avalon_noid } - expect(response).to have_http_status(:ok) - end - end - end -end diff --git a/spec/migration/admin_collection_object_mover_spec.rb b/spec/migration/admin_collection_object_mover_spec.rb deleted file mode 100644 index b09f6c316c..0000000000 --- a/spec/migration/admin_collection_object_mover_spec.rb +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -require 'rails_helper' - -describe FedoraMigrate::AdminCollection::ObjectMover do - let(:collection) { FactoryBot.create(:collection) } - describe 'empty?' do - it 'returns true when the admin collection has been wiped' do - described_class.wipeout!(collection) - expect(described_class.empty?(collection)).to be_truthy - end - it 'returns false if the admin collection has any information' do - expect(described_class.empty?(collection)).to be_falsey - end - end - describe 'wipeout!' do - it 'wipes all of the data' do - resources = [:resource, :default_permissions, :access_control] - resources.each do |res| - expect(collection.send(res).blank?).to be_falsey - end - expect(described_class.empty?(collection)).to be_falsey - described_class.wipeout!(collection) - resources.each do |res| - expect(collection.send(res).blank?).to be_truthy - end - expect(described_class.empty?(collection)).to be_truthy - end - end -end diff --git a/spec/migration/derivative_object_mover.rb b/spec/migration/derivative_object_mover.rb deleted file mode 100644 index c32a1dd756..0000000000 --- a/spec/migration/derivative_object_mover.rb +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -require 'rails_helper' - -describe FedoraMigrate::Derivative::ObjectMover do - let(:derivative) { FactoryBot.create(:derivative) } - describe 'empty?' do - it 'returns true when the derivative has been wiped' do - described_class.wipeout!(derivative) - expect(described_class.empty?(derivative)).to be_truthy - end - it 'returns false if the derivative has any information' do - expect(described_class.empty?(derivative)).to be_falsey - end - end - describe 'wipeout!' do - it 'wipes all of the data' do - resources = [:resource] - resources.each do |res| - expect(derivative.send(res).blank?).to be_falsey - end - expect(described_class.empty?(derivative)).to be_falsey - described_class.wipeout!(derivative) - resources.each do |res| - expect(derivative.send(res).blank?).to be_truthy - end - expect(described_class.empty?(derivative)).to be_truthy - end - end -end diff --git a/spec/migration/lease_object_mover.rb b/spec/migration/lease_object_mover.rb deleted file mode 100644 index 7cb09c2da8..0000000000 --- a/spec/migration/lease_object_mover.rb +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -require 'rails_helper' - -describe FedoraMigrate::Lease::ObjectMover do - let(:lease) { FactoryBot.create(:lease, inherited_read_users: [ FactoryBot.create(:user).user_key ]) } - describe 'empty?' do - it 'returns true when the admin lease has been wiped' do - described_class.wipeout!(lease) - expect(described_class.empty?(lease)).to be_truthy - end - it 'returns false if the admin lease has any information' do - expect(described_class.empty?(lease)).to be_falsey - end - end - describe 'wipeout!' do - it 'wipes all of the data' do - resources = [:resource, :default_permissions] - resources.each do |res| - expect(lease.send(res).blank?).to be_falsey - end - expect(described_class.empty?(lease)).to be_falsey - described_class.wipeout!(lease) - resources.each do |res| - expect(lease.send(res).blank?).to be_truthy - end - expect(described_class.empty?(lease)).to be_truthy - end - end -end diff --git a/spec/migration/master_file_object_mover.rb b/spec/migration/master_file_object_mover.rb deleted file mode 100644 index 44c79501c3..0000000000 --- a/spec/migration/master_file_object_mover.rb +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -require 'rails_helper' - -describe FedoraMigrate::MasterFile::ObjectMover do - let(:master_file) { FactoryBot.create(:master_file, :with_derivative, :with_thumbnail, :with_poster, :with_structure, :with_captions) } - describe 'empty?' do - it 'returns true when the master file has been wiped' do - described_class.wipeout!(master_file) - expect(described_class.empty?(master_file)).to be_truthy - end - it 'returns false if the master file has any information' do - expect(described_class.empty?(master_file)).to be_falsey - end - end - describe 'wipeout!' do - it 'wipes all of the data' do - resources = [:resource, :thumbnail, :structuralMetadata, :captions, :poster] - resources.each do |res| - expect(master_file.send(res).blank?).to be_falsey - end - expect(described_class.empty?(master_file)).to be_falsey - described_class.wipeout!(master_file) - resources.each do |res| - expect(master_file.send(res).blank?).to be_truthy - end - expect(described_class.empty?(master_file)).to be_truthy - end - end -end diff --git a/spec/migration/media_object_object_mover.rb b/spec/migration/media_object_object_mover.rb deleted file mode 100644 index c449ea1c0a..0000000000 --- a/spec/migration/media_object_object_mover.rb +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2011-2022, The Trustees of Indiana University and Northwestern -# University. Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# --- END LICENSE_HEADER BLOCK --- - -require 'rails_helper' - -describe FedoraMigrate::MediaObject::ObjectMover do - let(:media_object) { FactoryBot.create(:media_object, :with_master_file, :with_completed_workflow) } - describe 'empty?' do - it 'returns true when the media object has been wiped' do - described_class.wipeout!(media_object) - expect(described_class.empty?(media_object)).to be_truthy - end - it 'returns false if the media object has any information' do - expect(described_class.empty?(media_object)).to be_falsey - end - end - describe 'wipeout!' do - it 'wipes all of the data' do - resources = [:resource, :access_control, :descMetadata, :workflow, :master_files] - resources.each do |res| - expect(media_object.send(res).blank?).to be_falsey - end - expect(media_object.ordered_master_files.to_a.blank?).to be_falsey - expect(described_class.empty?(media_object)).to be_falsey - described_class.wipeout!(media_object) - resources.each do |res| - expect(media_object.send(res).blank?).to be_truthy - end - expect(media_object.ordered_master_files.to_a.blank?).to be_truthy - expect(described_class.empty?(media_object)).to be_truthy - end - end -end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 8032ab97b1..0374274772 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -18,7 +18,6 @@ SimpleCov.start('rails') do add_filter '/spec' - add_filter '/app/migration' end SimpleCov.command_name 'spec' end diff --git a/spec/requests/redirect_spec.rb b/spec/requests/redirect_spec.rb index 33a86fffef..90d697b549 100644 --- a/spec/requests/redirect_spec.rb +++ b/spec/requests/redirect_spec.rb @@ -16,8 +16,8 @@ describe 'redirect', type: :request do it 'stores url to redirect to when unauthorized and needing to authenticate (#authorize!)' do - get '/admin/migration_report' - expect(request.env['rack.session']['previous_url']).to eq '/admin/migration_report' + get '/admin/collections' + expect(request.env['rack.session']['previous_url']).to eq '/admin/collections' expect(response).to render_template('errors/restricted_pid') end From dff7de1223f8bf571f19f5b0f1802f1ff0117ed5 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Feb 2022 10:02:19 -0500 Subject: [PATCH 012/171] Blacklight migration: include default component configuration and change partial path --- app/controllers/catalog_controller.rb | 9 +++++++++ app/views/layouts/avalon.html.erb | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 767242d875..6eeb3f55ea 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -26,6 +26,15 @@ class CatalogController < ApplicationController before_action :load_home_page_collections, only: :index, if: proc { helpers.current_page? root_path } configure_blacklight do |config| + + # Default component configuration + config.add_results_document_tool(:bookmark, partial: 'bookmark_control', if: :render_bookmarks_control?) + config.add_results_collection_tool(:sort_widget) + config.add_results_collection_tool(:per_page_widget) + config.add_results_collection_tool(:view_type_group) + config.add_show_tools_partial(:bookmark, partial: 'bookmark_control', if: :render_bookmarks_control?) + config.add_nav_action(:bookmark, partial: 'blacklight/nav/bookmark', if: :render_bookmarks_control?) + ## Class for sending and receiving requests from a search index # config.repository_class = Blacklight::Solr::Repository # diff --git a/app/views/layouts/avalon.html.erb b/app/views/layouts/avalon.html.erb index 94c4362bb0..ed10ea14cc 100644 --- a/app/views/layouts/avalon.html.erb +++ b/app/views/layouts/avalon.html.erb @@ -41,7 +41,7 @@ Unless required by applicable law or agreed to in writing, software distributed
- <%= render partial: 'shared/ajax_modal' %> + <%= render partial: 'shared/modal' %> <% if defined?(Samvera::Persona) %> <%= render partial: 'modules/become_message' %> <% end %> From fa112530bd2d0454233c66682f4b3ab0a59c5047 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Feb 2022 10:29:25 -0500 Subject: [PATCH 013/171] Apply new migration from Rails 6 --- db/schema.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index a1646d682f..1081f1f488 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -2,15 +2,15 @@ # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. # -# Note that this schema.rb definition is the authoritative source for your -# database schema. If you need to create the application database on another -# system, you should be using db:schema:load, not running all the migrations -# from scratch. The latter is a flawed and unsustainable approach (the more migrations -# you'll amass, the slower it'll run and the greater likelihood for issues). +# This file is the source Rails uses to define your schema when running `rails +# db:schema:load`. When creating a new database, `rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_02_02_160327) do +ActiveRecord::Schema.define(version: 2022_02_18_184645) do create_table "active_encode_encode_records", force: :cascade do |t| t.string "global_id" @@ -268,4 +268,5 @@ t.index ["username"], name: "index_users_on_username", unique: true end + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" end From f833413d59f0fcfd935c2cac2185c2cda9005894 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 14 Mar 2022 14:51:29 -0400 Subject: [PATCH 014/171] Add inflection to ensure Avalon::RDFVocab works with zeitwerk --- config/initializers/inflections.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index ac033bf9dc..05e3df93cc 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -11,6 +11,8 @@ # end # These inflection rules are supported but not enabled by default: -# ActiveSupport::Inflector.inflections(:en) do |inflect| -# inflect.acronym 'RESTful' -# end +ActiveSupport::Inflector.inflections(:en) do |inflect| + inflect.acronym 'RDF' +end + + From 374a2f30208271d7fa553bce566032644ed97a18 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 14 Mar 2022 15:10:09 -0400 Subject: [PATCH 015/171] Allowlist external postgres for docker-compose and CI testing --- spec/rails_helper.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 0374274772..f8448e9bc7 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -104,6 +104,8 @@ config.before :suite do WebMock.disable_net_connect!(allow: ['localhost', '127.0.0.1', 'fedora', 'fedora-test', 'solr', 'solr-test', 'matterhorn', 'https://chromedriver.storage.googleapis.com']) + DatabaseCleaner.allow_remote_database_url = true + DatabaseCleaner.url_allowlist = ['postgres://postgres:password@db/avalon', 'postgresql://postgres@localhost:5432/postgres'] DatabaseCleaner.clean_with(:truncation) ActiveFedora::Cleaner.clean! disable_production_minter! From 9b5cf3f40c344ee7c38a7cef1cc67aca61912461 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 14 Mar 2022 15:10:56 -0400 Subject: [PATCH 016/171] @document_list is deprecated so switch to @response.documents --- app/controllers/bookmarks_controller.rb | 2 +- app/controllers/collections_controller.rb | 4 +- app/views/bookmarks/index.html.erb | 4 +- app/views/catalog/_select_all.html.erb | 2 +- spec/controllers/catalog_controller_spec.rb | 88 ++++++++++----------- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/app/controllers/bookmarks_controller.rb b/app/controllers/bookmarks_controller.rb index f6cb6cde39..3e5103b5e1 100644 --- a/app/controllers/bookmarks_controller.rb +++ b/app/controllers/bookmarks_controller.rb @@ -85,7 +85,7 @@ def verify_permissions # @bookmarks = token_or_current_or_guest_user.bookmarks # bookmark_ids = @bookmarks.collect { |b| b.document_id.to_s } # - # @response, @document_list = get_solr_response_for_document_ids(bookmark_ids, defType: 'edismax') + # @response = get_solr_response_for_document_ids(bookmark_ids, defType: 'edismax') # # respond_to do |format| # format.html { } diff --git a/app/controllers/collections_controller.rb b/app/controllers/collections_controller.rb index 33138fc3b8..6137fe8d10 100644 --- a/app/controllers/collections_controller.rb +++ b/app/controllers/collections_controller.rb @@ -35,7 +35,7 @@ def index def show response = repository.search(CollectionSearchBuilder.new(self)) document = response.documents.find { |doc| doc.id == params[:id] } - # Only go on if params[:id] is in @document_list + # Only go on if params[:id] is in @response.documents raise CanCan::AccessDenied unless document @doc_presenter = CollectionPresenter.new(document, view_context) @@ -48,7 +48,7 @@ def show def poster response = repository.search(CollectionSearchBuilder.new(self)) document = response.documents.find { |doc| doc.id == params[:id] } - # Only go on if params[:id] is in @document_list + # Only go on if params[:id] is in @response.documents raise CanCan::AccessDenied unless document @collection = Admin::Collection.find(params['id']) diff --git a/app/views/bookmarks/index.html.erb b/app/views/bookmarks/index.html.erb index 020314884b..f70cb95046 100644 --- a/app/views/bookmarks/index.html.erb +++ b/app/views/bookmarks/index.html.erb @@ -20,14 +20,14 @@ Unless required by applicable law or agreed to in writing, software distributed

<%= t('blacklight.bookmarks.need_login') %>

- <%- elsif @document_list.blank? -%> + <%- elsif @response.documents.blank? -%> <% else %> <%# link_to t('blacklight.bookmarks.clear.action_title'), clear_bookmarks_path, :method => :delete, :data => { :confirm => t('blacklight.bookmarks.clear.action_confirm') }, :class => 'clear-bookmarks btn btn-danger pull-right' %> <%= render 'sort_and_per_page' %> - <%= render partial: 'tools', locals: { document_list: @document_list } %> + <%= render partial: 'tools', locals: { document_list: @response.documents } %> <%= render_document_index %> <%= render 'results_pagination' %> diff --git a/app/views/catalog/_select_all.html.erb b/app/views/catalog/_select_all.html.erb index 63abd6976a..bc9a6ee0db 100644 --- a/app/views/catalog/_select_all.html.erb +++ b/app/views/catalog/_select_all.html.erb @@ -17,7 +17,7 @@ Unless required by applicable law or agreed to in writing, software distributed <% if current_or_guest_user %>
- <% all_selected = @document_list.all? do |d| + <% all_selected = @response.documents.all? do |d| current_or_guest_user.bookmarks.where({ document_id: d.id }).exists? end %> <%= check_box_tag('bookmarks_selectall', '', all_selected) %> diff --git a/spec/controllers/catalog_controller_spec.rb b/spec/controllers/catalog_controller_spec.rb index 48bb235c5d..ee8de0b6ee 100644 --- a/spec/controllers/catalog_controller_spec.rb +++ b/spec/controllers/catalog_controller_spec.rb @@ -22,22 +22,22 @@ get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(1) - expect(assigns(:document_list).map(&:id)).to eq([mo.id]) + expect(assigns(:response).documents.count).to eql(1) + expect(assigns(:response).documents.map(&:id)).to eq([mo.id]) end it "should not show results for items that are not public" do mo = FactoryBot.create(:published_media_object, visibility: 'restricted') get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(0) + expect(assigns(:response).documents.count).to eql(0) end it "should not show results for items that are not published" do mo = FactoryBot.create(:media_object, visibility: 'public') get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(0) + expect(assigns(:response).documents.count).to eql(0) end end describe "as an authenticated user" do @@ -49,22 +49,22 @@ get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(1) - expect(assigns(:document_list).map(&:id)).to eq([mo.id]) + expect(assigns(:response).documents.count).to eql(1) + expect(assigns(:response).documents.map(&:id)).to eq([mo.id]) end it "should not show results for items that are not public or available to registered users" do mo = FactoryBot.create(:published_media_object, visibility: 'private') get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(0) + expect(assigns(:response).documents.count).to eql(0) end it "should not show results for items that are not published" do mo = FactoryBot.create(:media_object, visibility: 'public') get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(0) + expect(assigns(:response).documents.count).to eql(0) end end describe "as a manager" do @@ -76,16 +76,16 @@ get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(1) - expect(assigns(:document_list).map(&:id)).to eq([mo.id]) + expect(assigns(:response).documents.count).to eql(1) + expect(assigns(:response).documents.map(&:id)).to eq([mo.id]) end it "should show results for items that are hidden and belong to one of my collections" do mo = FactoryBot.create(:media_object, hidden: true, visibility: 'private', collection: collection) get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(1) - expect(assigns(:document_list).map(&:id)).to eq([mo.id]) + expect(assigns(:response).documents.count).to eql(1) + expect(assigns(:response).documents.map(&:id)).to eq([mo.id]) end it "should show results for items that are not hidden and do not belong to one of my collections along with hidden items that belong to my collections" do mo = FactoryBot.create(:media_object, hidden: true, visibility: 'private', collection: collection) @@ -93,22 +93,22 @@ get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(2) - expect(assigns(:document_list).map(&:id)).to match_array([mo.id, mo2.id]) + expect(assigns(:response).documents.count).to eql(2) + expect(assigns(:response).documents.map(&:id)).to match_array([mo.id, mo2.id]) end it "should not show results for items that do not belong to one of my collections" do mo = FactoryBot.create(:media_object, visibility: 'private') get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(0) + expect(assigns(:response).documents.count).to eql(0) end it "should not show results for hidden items that do not belong to one of my collections" do mo = FactoryBot.create(:media_object, hidden: true, visibility: 'private', read_users: [manager.user_key]) get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(0) + expect(assigns(:response).documents.count).to eql(0) end end describe "as an administrator" do @@ -119,8 +119,8 @@ get 'index', params: { :q => "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eq 1 - expect(assigns(:document_list).map(&:id)).to eq([mo.id]) + expect(assigns(:response).documents.count).to eq 1 + expect(assigns(:response).documents.map(&:id)).to eq([mo.id]) end end describe "as an lti user" do @@ -131,8 +131,8 @@ get 'index', params: { :q => "read_access_virtual_group_ssim:#{lti_group}" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eql(1) - expect(assigns(:document_list).map(&:id)).to eq([mo.id]) + expect(assigns(:response).documents.count).to eql(1) + expect(assigns(:response).documents.map(&:id)).to eq([mo.id]) end end @@ -144,29 +144,29 @@ end it "should show no results when no items are visible to the user's IP address" do get 'index', params: { :q => "" } - expect(assigns(:document_list).count).to eq 0 + expect(assigns(:response).documents.count).to eq 0 end it "should show results for items visible to the the user's IP address" do allow_any_instance_of(ActionDispatch::Request).to receive(:remote_ip).and_return(@ip_address1) get 'index', params: { :q => "" } - expect(assigns(:document_list).count).to eq 1 - expect(assigns(:document_list).map(&:id)).to include @mo.id + expect(assigns(:response).documents.count).to eq 1 + expect(assigns(:response).documents.map(&:id)).to include @mo.id end it "should show results for items visible to the the user's IPv4 subnet" do ip_address2 = Faker::Internet.ip_v4_address allow_any_instance_of(ActionDispatch::Request).to receive(:remote_ip).and_return(ip_address2) mo2 = FactoryBot.create(:published_media_object, visibility: 'private', read_groups: [ip_address2+'/30']) get 'index', params: { :q => "" } - expect(assigns(:document_list).count).to be >= 1 - expect(assigns(:document_list).map(&:id)).to include mo2.id + expect(assigns(:response).documents.count).to be >= 1 + expect(assigns(:response).documents.map(&:id)).to include mo2.id end it "should show results for items visible to the the user's IPv6 subnet" do ip_address3 = Faker::Internet.ip_v6_address allow_any_instance_of(ActionDispatch::Request).to receive(:remote_ip).and_return(ip_address3) mo3 = FactoryBot.create(:published_media_object, visibility: 'private', read_groups: [ip_address3+'/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ff00']) get 'index', params: { :q => "" } - expect(assigns(:document_list).count).to be >= 1 - expect(assigns(:document_list).map(&:id)).to include mo3.id + expect(assigns(:response).documents.count).to be >= 1 + expect(assigns(:response).documents.map(&:id)).to include mo3.id end end @@ -181,8 +181,8 @@ #since an incorrect one will lead to an empty query resulting in a false positive below expect(query).not_to be_empty get 'index', params: { :q => query } - expect(assigns(:document_list).count).to eq 1 - expect(assigns(:document_list).map(&:id)). to eq [media_object.id] + expect(assigns(:response).documents.count).to eq 1 + expect(assigns(:response).documents.map(&:id)). to eq [media_object.id] end end end @@ -195,19 +195,19 @@ end it "should find results based upon structure" do get 'index', params: { q: 'CD 1' } - expect(assigns(:document_list).count).to eq 1 - expect(assigns(:document_list).collect(&:id)). to eq [@media_object.id] + expect(assigns(:response).documents.count).to eq 1 + expect(assigns(:response).documents.collect(&:id)). to eq [@media_object.id] end it 'should find results based upon section labels' do get 'index', params: { q: 'Test Label' } - expect(assigns(:document_list).count).to eq 1 - expect(assigns(:document_list).collect(&:id)). to eq [@media_object.id] + expect(assigns(:response).documents.count).to eq 1 + expect(assigns(:response).documents.collect(&:id)). to eq [@media_object.id] end it 'should find items in correct relevancy order' do media_object_1 = FactoryBot.create(:fully_searchable_media_object, title: 'Test Label') get 'index', params: { q: 'Test Label' } - expect(assigns(:document_list).count).to eq 2 - expect(assigns(:document_list).collect(&:id)).to eq [media_object_1.id, @media_object.id] + expect(assigns(:response).documents.count).to eq 2 + expect(assigns(:response).documents.collect(&:id)).to eq [media_object_1.id, @media_object.id] end end @@ -218,15 +218,15 @@ it "should sort correctly by title" do get :index, params: { :sort => 'title_ssort asc, date_ssi desc' } - expect(assigns(:document_list).map(&:id)).to eq [m2.id, m3.id, m1.id] + expect(assigns(:response).documents.map(&:id)).to eq [m2.id, m3.id, m1.id] end it "should sort correctly by date" do get :index, params: { :sort => 'date_ssi desc, title_ssort asc' } - expect(assigns(:document_list).map(&:id)).to eq [m3.id, m2.id, m1.id] + expect(assigns(:response).documents.map(&:id)).to eq [m3.id, m2.id, m1.id] end it "should sort correctly by creator" do get :index, params: { :sort => 'creator_ssort asc, title_ssort asc' } - expect(assigns(:document_list).map(&:id)).to eq [m2.id, m1.id, m3.id] + expect(assigns(:response).documents.map(&:id)).to eq [m2.id, m1.id, m3.id] end end @@ -242,8 +242,8 @@ get :index, params: { q: "" } expect(response).to be_successful expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eq 1 - expect(assigns(:document_list).collect(&:id)). to eq [media_object.id] + expect(assigns(:response).documents.count).to eq 1 + expect(assigns(:response).documents.collect(&:id)). to eq [media_object.id] end end end @@ -263,8 +263,8 @@ expect(response).to be_successful expect(response).to render_template('catalog/index') expect(response.content_type).to eq "application/atom+xml; charset=utf-8" - expect(assigns(:document_list).count).to eq 2 - expect(assigns(:document_list).map(&:id)).to match_array [private_media_object.id, public_media_object.id] + expect(assigns(:response).documents.count).to eq 2 + expect(assigns(:response).documents.map(&:id)).to match_array [private_media_object.id, public_media_object.id] end end context "without api key" do @@ -273,8 +273,8 @@ expect(response).to be_successful expect(response.content_type).to eq "application/atom+xml; charset=utf-8" expect(response).to render_template('catalog/index') - expect(assigns(:document_list).count).to eq 1 - expect(assigns(:document_list).map(&:id)).to eq([public_media_object.id]) + expect(assigns(:response).documents.count).to eq 1 + expect(assigns(:response).documents.map(&:id)).to eq([public_media_object.id]) end end end From c03eed33198d056cfe2f17861218f7d071292eb7 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 14 Mar 2022 16:26:47 -0400 Subject: [PATCH 017/171] respository can be found in blacklight_config --- app/controllers/admin/collections_controller.rb | 2 +- app/controllers/catalog_controller.rb | 2 +- app/controllers/collections_controller.rb | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/admin/collections_controller.rb b/app/controllers/admin/collections_controller.rb index d4ddc1f1fd..58ace90470 100644 --- a/app/controllers/admin/collections_controller.rb +++ b/app/controllers/admin/collections_controller.rb @@ -22,7 +22,7 @@ class Admin::CollectionsController < ApplicationController def load_and_authorize_collections authorize!(params[:action].to_sym, Admin::Collection) - repository = CatalogController.new.repository + repository = CatalogController.new.blacklight_config.repository # Allow the number of collections to be greater than 100 blacklight_config.max_per_page = 100_000 builder = ::CollectionSearchBuilder.new([:add_access_controls_to_solr_params_if_not_admin, :only_wanted_models, :add_paging_to_solr], self).rows(100_000) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 6eeb3f55ea..a247c79a28 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -200,7 +200,7 @@ def load_home_page_collections featured_collections = Settings.home_page&.featured_collections if featured_collections.present? builder = ::CollectionSearchBuilder.new(self).rows(100_000) - response = repository.search(builder) + response = blacklight_config.repository.search(builder) collection = response.documents.select { |doc| featured_collections.include? doc.id }.sample @featured_collection = ::Admin::CollectionPresenter.new(collection) if collection end diff --git a/app/controllers/collections_controller.rb b/app/controllers/collections_controller.rb index 6137fe8d10..795753734f 100644 --- a/app/controllers/collections_controller.rb +++ b/app/controllers/collections_controller.rb @@ -16,7 +16,7 @@ class CollectionsController < CatalogController skip_before_action :enforce_show_permissions, only: :show def index - response = repository.search(CollectionSearchBuilder.new(self)) + response = blacklight_config.repository.search(CollectionSearchBuilder.new(self)) collections = response.documents if (params[:only] == 'carousel') collections = collections.select { |doc| Settings.home_page&.carousel_collections&.include? doc.id } @@ -33,7 +33,7 @@ def index end def show - response = repository.search(CollectionSearchBuilder.new(self)) + response = blacklight_config.repository.search(CollectionSearchBuilder.new(self)) document = response.documents.find { |doc| doc.id == params[:id] } # Only go on if params[:id] is in @response.documents raise CanCan::AccessDenied unless document @@ -46,7 +46,7 @@ def show end def poster - response = repository.search(CollectionSearchBuilder.new(self)) + response = blacklight_config.repository.search(CollectionSearchBuilder.new(self)) document = response.documents.find { |doc| doc.id == params[:id] } # Only go on if params[:id] is in @response.documents raise CanCan::AccessDenied unless document From 62e4656c31155c9d6405f5fae6ed8691dabdb747 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 14 Mar 2022 16:36:36 -0400 Subject: [PATCH 018/171] SearchBuilders need controller scope which supplies blacklight_config --- spec/models/search_builder_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/models/search_builder_spec.rb b/spec/models/search_builder_spec.rb index 0b38faa694..f0b3609b90 100644 --- a/spec/models/search_builder_spec.rb +++ b/spec/models/search_builder_spec.rb @@ -1,9 +1,10 @@ require 'rails_helper' RSpec.describe SearchBuilder do - subject(:builder) { described_class.new processor_chain } + subject(:builder) { described_class.new(processor_chain, scope) } let(:processor_chain) { [] } + let(:scope) { CatalogController.new } let(:manager) { FactoryBot.create(:manager) } let(:ability) { Ability.new(manager) } @@ -14,4 +15,4 @@ expect(subject.only_published_items({})).to eq ["test:clause OR workflow_published_sim:\"Published\""] end end -end \ No newline at end of file +end From 551062a0b368568a12eca57fe75b78451420924c Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 14 Mar 2022 16:44:55 -0400 Subject: [PATCH 019/171] fetch is available through search_service --- app/controllers/bookmarks_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/bookmarks_controller.rb b/app/controllers/bookmarks_controller.rb index 3e5103b5e1..e2226cf5ab 100644 --- a/app/controllers/bookmarks_controller.rb +++ b/app/controllers/bookmarks_controller.rb @@ -116,7 +116,7 @@ def count def action_documents bookmarks = token_or_current_or_guest_user.bookmarks bookmark_ids = bookmarks.collect { |b| b.document_id.to_s } - fetch(bookmark_ids, rows: bookmark_ids.count) + search_service.fetch(bookmark_ids, rows: bookmark_ids.count) end def access_control_action documents From d34f6c37548fadf6f6140446306e3d24fdf41084 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 14 Mar 2022 16:55:00 -0400 Subject: [PATCH 020/171] Upgrade to latest ActiveFedora --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9550c9f424..b981787b78 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -134,7 +134,7 @@ GEM erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - active-fedora (13.2.5) + active-fedora (13.2.7) active-triples (>= 0.11.0, < 2.0.0) activemodel (>= 5.1) activesupport (>= 5.1) @@ -432,7 +432,7 @@ GEM fakefs (1.4.1) faker (2.19.0) i18n (>= 1.6, < 2) - faraday (0.17.4) + faraday (0.17.5) multipart-post (>= 1.2, < 3) faraday-encoding (0.0.5) faraday From 3070d08b7e883bb9ccc56c315284036554377d14 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Tue, 15 Mar 2022 12:01:53 -0400 Subject: [PATCH 021/171] Update views to newer Blacklight methods --- app/views/catalog/_document_list.html.erb | 3 ++- .../catalog/_index_media_object.html.erb | 20 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/app/views/catalog/_document_list.html.erb b/app/views/catalog/_document_list.html.erb index 23aaf03844..e231c7773a 100644 --- a/app/views/catalog/_document_list.html.erb +++ b/app/views/catalog/_document_list.html.erb @@ -14,9 +14,10 @@ Unless required by applicable law or agreed to in writing, software distributed --- END LICENSE_HEADER BLOCK --- %> <% # container for all documents in index list view -%> +<% view_config = local_assigns[:view_config] || blacklight_config&.view_config(document_index_view_type) %>
<% if render_bookmarks_control? && documents.count>1 %> <%= render "select_all" %> <% end %> - <%= render documents, :as => :document %> + <%= render documents, as: :document, view_config: view_config %>
diff --git a/app/views/catalog/_index_media_object.html.erb b/app/views/catalog/_index_media_object.html.erb index ce71bb30d8..3ad123fcd4 100644 --- a/app/views/catalog/_index_media_object.html.erb +++ b/app/views/catalog/_index_media_object.html.erb @@ -13,16 +13,14 @@ Unless required by applicable law or agreed to in writing, software distributed specific language governing permissions and limitations under the License. --- END LICENSE_HEADER BLOCK --- %> - -<% doc_presenter = index_presenter(document) %> -<%# default partial to display solr document fields in catalog index view -%> +<%# TODO: Use new Blacklight::DocumentMetadataComponent instead of overriding this view %> +<% doc_presenter = document_presenter(document) %> +<%# default partial to display solr document fields in catalog index view %>
- - <% index_fields(document).each do |field_name, field| -%> - <% if should_render_index_field? document, field %> -
<%= render_index_field_label document, field: field_name %>
-
<%= doc_presenter.field_value field_name %>
- <% end -%> - <% end -%> - + <% doc_presenter.field_presenters.each do |field_presenter| %> + <% if field_presenter.render_field? %> +
<%= field_presenter.label %>
+
<%= field_presenter.values %>
+ <% end %> + <% end %>
From 545d4eaeed0859c6a00a9f989bc1b46d857be907 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Tue, 15 Mar 2022 12:12:48 -0400 Subject: [PATCH 022/171] with_format only accepts symbols now --- app/views/catalog/_document_media_object.atom.builder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/catalog/_document_media_object.atom.builder b/app/views/catalog/_document_media_object.atom.builder index 87ffd2547a..d68e748919 100644 --- a/app/views/catalog/_document_media_object.atom.builder +++ b/app/views/catalog/_document_media_object.atom.builder @@ -17,7 +17,7 @@ xml.entry do xml.author { xml.name(document.to_semantic_values[:author].first) } end - with_format("html") do + with_format(:html) do xml.summary "type" => "html" do xml.text! render_document_partial(document, :index, From 484042873b004d5d337726a524bc5c0abd11c656 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Tue, 15 Mar 2022 12:47:38 -0400 Subject: [PATCH 023/171] force_path_style config is only allowed on s3 service --- config/initializers/aws.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/initializers/aws.rb b/config/initializers/aws.rb index e46d944290..ff540a99fe 100644 --- a/config/initializers/aws.rb +++ b/config/initializers/aws.rb @@ -5,7 +5,9 @@ endpoint: Settings.minio.endpoint, access_key_id: Settings.minio.access, secret_access_key: Settings.minio.secret, - force_path_style: true, region: ENV["AWS_REGION"] - ) + ) + + # Service specific global configuration + Aws.config[:s3] = { force_path_style: true } end From acf9d8e4f17214fb6ccdd2307c50b7b9e6f15c72 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Tue, 15 Mar 2022 12:48:13 -0400 Subject: [PATCH 024/171] Don't use null cache store for tests since some are testing caching --- config/environments/test.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/config/environments/test.rb b/config/environments/test.rb index 470dee4bef..d301e54903 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -22,7 +22,6 @@ # Show full error reports and disable caching. config.consider_all_requests_local = true config.action_controller.perform_caching = false - config.cache_store = :null_store # Raise exceptions instead of rendering exception templates. config.action_dispatch.show_exceptions = false From d4ece8dea3d16d144083d33aca491167b8a72300 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Wed, 16 Mar 2022 10:33:30 -0400 Subject: [PATCH 025/171] Use blacklight-access_controls github for rails 6 support in cancancan --- Gemfile | 1 + Gemfile.lock | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Gemfile b/Gemfile index ad3709f361..56b92b5c8c 100644 --- a/Gemfile +++ b/Gemfile @@ -35,6 +35,7 @@ gem 'rdf-vocab', '< 3.1.5' # Samvera version pins gem 'blacklight', '< 8' +gem 'blacklight-access_controls', git: 'https://github.com/projectblacklight/blacklight-access_controls.git' gem 'rdf', '~> 3.1' gem 'rsolr', '~> 1.0' diff --git a/Gemfile.lock b/Gemfile.lock index b981787b78..79c043a7ef 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -65,6 +65,15 @@ GIT fugit (~> 1.1) sidekiq (>= 4.2.1) +GIT + remote: https://github.com/projectblacklight/blacklight-access_controls.git + revision: 21e04f53217b7d6a0c90faf6393286635cf525f0 + specs: + blacklight-access_controls (6.0.0) + blacklight (> 6.0, < 8) + cancancan (>= 1.8) + deprecation (~> 1.0) + GIT remote: https://github.com/samvera-labs/active_fedora-datastreams revision: 13eefc7bdc879fd6c3ad607a1ecc087898777d3d @@ -252,19 +261,15 @@ GEM rubocop-performance rubocop-rails rubocop-rspec - blacklight (7.22.2) + blacklight (7.24.0) deprecation globalid i18n (>= 1.7.0) jbuilder (~> 2.7) kaminari (>= 0.15) ostruct (>= 0.3.2) - rails (>= 5.1, < 7) - view_component (~> 2.42) - blacklight-access_controls (6.0.0) - blacklight (> 6.0, < 8) - cancancan (~> 1.8) - deprecation (~> 1.0) + rails (>= 5.1, < 7.1) + view_component (~> 2.43) bootsnap (1.10.3) msgpack (~> 1.2) bootstrap (4.6.1) @@ -288,7 +293,7 @@ GEM typhoeus builder (3.2.4) byebug (11.1.3) - cancancan (1.17.0) + cancancan (3.3.0) capistrano (3.16.0) airbrussh (>= 1.0.0) i18n @@ -587,7 +592,7 @@ GEM activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.14.0) + loofah (2.15.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -924,7 +929,7 @@ GEM sxp (1.1.0) rdf (~> 3.1) temple (0.8.2) - thor (0.20.3) + thor (1.2.1) thread_safe (0.3.6) tilt (2.0.10) trailblazer-option (0.1.2) @@ -945,7 +950,7 @@ GEM unicode-display_width (1.8.0) unicode-types (1.7.0) user_agent_parser (2.9.0) - view_component (2.49.0) + view_component (2.50.0) activesupport (>= 5.0.0, < 8.0) method_source (~> 1.0) warden (1.2.9) @@ -1014,6 +1019,7 @@ DEPENDENCIES aws-sigv4 bixby blacklight (< 8) + blacklight-access_controls! bootsnap bootstrap (~> 4.0) bootstrap-toggle-rails From 95105fa377abb3e8be4fcc4d42842ade552185a4 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Wed, 16 Mar 2022 10:33:49 -0400 Subject: [PATCH 026/171] Collapse uses show instead of in for visible by default Co-authored-by: Dananji Withana Co-authored-by: Michael Johnson --- app/views/playlists/_form.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/playlists/_form.html.erb b/app/views/playlists/_form.html.erb index 22f94e63c9..5a2c3774ab 100644 --- a/app/views/playlists/_form.html.erb +++ b/app/views/playlists/_form.html.erb @@ -13,7 +13,7 @@ Unless required by applicable law or agreed to in writing, software distributed specific language governing permissions and limitations under the License. --- END LICENSE_HEADER BLOCK --- %> -
+
<%= form_for(@playlist, html: { id: 'playlist_form', class: 'playlist_actions' }) do |f| %>
<%= f.label "Name" %> From 7fcdf03ad0a8738bfca45416f518012923df5555 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Wed, 16 Mar 2022 15:43:35 -0400 Subject: [PATCH 027/171] Upgrade to blackligtht-access_controls v6.0.1 --- Gemfile | 2 +- Gemfile.lock | 15 +++++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Gemfile b/Gemfile index 56b92b5c8c..c3df356b17 100644 --- a/Gemfile +++ b/Gemfile @@ -35,7 +35,7 @@ gem 'rdf-vocab', '< 3.1.5' # Samvera version pins gem 'blacklight', '< 8' -gem 'blacklight-access_controls', git: 'https://github.com/projectblacklight/blacklight-access_controls.git' +gem 'blacklight-access_controls', '>= 6.0.1' # ensure rails 6 support gem 'rdf', '~> 3.1' gem 'rsolr', '~> 1.0' diff --git a/Gemfile.lock b/Gemfile.lock index 79c043a7ef..b7d1caef90 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -65,15 +65,6 @@ GIT fugit (~> 1.1) sidekiq (>= 4.2.1) -GIT - remote: https://github.com/projectblacklight/blacklight-access_controls.git - revision: 21e04f53217b7d6a0c90faf6393286635cf525f0 - specs: - blacklight-access_controls (6.0.0) - blacklight (> 6.0, < 8) - cancancan (>= 1.8) - deprecation (~> 1.0) - GIT remote: https://github.com/samvera-labs/active_fedora-datastreams revision: 13eefc7bdc879fd6c3ad607a1ecc087898777d3d @@ -270,6 +261,10 @@ GEM ostruct (>= 0.3.2) rails (>= 5.1, < 7.1) view_component (~> 2.43) + blacklight-access_controls (6.0.1) + blacklight (> 6.0, < 8) + cancancan (>= 1.8) + deprecation (~> 1.0) bootsnap (1.10.3) msgpack (~> 1.2) bootstrap (4.6.1) @@ -1019,7 +1014,7 @@ DEPENDENCIES aws-sigv4 bixby blacklight (< 8) - blacklight-access_controls! + blacklight-access_controls (>= 6.0.1) bootsnap bootstrap (~> 4.0) bootstrap-toggle-rails From f1ffe09ef1ec52996ca5d943947bc153b27f646b Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Mar 2022 11:47:14 -0400 Subject: [PATCH 028/171] Force a larger browser size so the footer doesn't render over form controls --- spec/rails_helper.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index f8448e9bc7..8c8807402f 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -142,6 +142,11 @@ ActiveFedora::Cleaner.clean! end + # Remove this check to test on smaller window sizes? + config.before(:each, js: true) do + Capybara.page.driver.browser.manage.window.resize_to(1920,1080) # desktop size + end + # RSpec Rails can automatically mix in different behaviours to your tests # based on their file location, for example enabling you to call `get` and # `post` in specs under `spec/controllers`. From 7433c8e45893540dd9f289c32833c4eaf33e5b3c Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Mar 2022 15:30:58 -0400 Subject: [PATCH 029/171] Fix undefined error when this is run on pages without a progress bar div --- app/assets/javascripts/avalon_progress.js.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/avalon_progress.js.coffee b/app/assets/javascripts/avalon_progress.js.coffee index d334c59ef7..62f3f512f4 100644 --- a/app/assets/javascripts/avalon_progress.js.coffee +++ b/app/assets/javascripts/avalon_progress.js.coffee @@ -75,7 +75,7 @@ class AvalonProgress $('.avalon-player').html('
') $(document).ready -> - if $('.progress-bar').size() == 0 + if $('.progress-bar').length == 0 return progress_controller = new AvalonProgress() From 55f37ffdf895e3237e6a0838286c67c165cb9ef5 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Mar 2022 15:31:17 -0400 Subject: [PATCH 030/171] Load bootstrap4 css --- app/assets/javascripts/datatables.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/datatables.js b/app/assets/javascripts/datatables.js index 1d61fbc922..049fac90b4 100644 --- a/app/assets/javascripts/datatables.js +++ b/app/assets/javascripts/datatables.js @@ -32,7 +32,7 @@ // require datatables/extensions/Scroller/dataTables.scroller // require datatables/extensions/Select/dataTables.select -//= require datatables/dataTables.bootstrap +//= require datatables/dataTables.bootstrap4 // require datatables/extensions/AutoFill/autoFill.bootstrap // require datatables/extensions/Buttons/buttons.bootstrap // require datatables/extensions/Responsive/responsive.bootstrap From b69ce0eb7364bbc32b1f1d5e096544e353161415 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Mar 2022 15:33:54 -0400 Subject: [PATCH 031/171] Capybara tuning to increase browser window size and enable browser logging --- spec/rails_helper.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 8c8807402f..beaa19390e 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -72,12 +72,15 @@ Capybara.server = :webrick Capybara.register_driver :selenium_chrome_headless_docker_friendly do |app| Capybara::Selenium::Driver.load_selenium + caps = Selenium::WebDriver::Remote::Capabilities.chrome(loggingPrefs:{browser: 'ALL'}) browser_options = ::Selenium::WebDriver::Chrome::Options.new browser_options.args << '--headless' browser_options.args << '--disable-gpu' # Sandbox cannot be used inside unprivileged Docker container browser_options.args << '--no-sandbox' - Capybara::Selenium::Driver.new(app, browser: :chrome, options: browser_options) + # Next line commented out in favor of before(:each) resize_to that applies to all drivers + # browser_options.args << '--window-size=1920,1080' + Capybara::Selenium::Driver.new(app, browser: :chrome, options: browser_options, desired_capabilities: caps) end # eg `SHOW_BROWSER=true ./bin/rspec` will show you an actual chrome browser From 2a14a94e08d5fbba21c0c582265df65718e21e18 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Mar 2022 15:34:58 -0400 Subject: [PATCH 032/171] Ensure we install a version of chromedriver that is compatible with the version of chrome Using instructions from https://chromedriver.chromium.org/downloads/version-selection --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ed71759370..1e22d92787 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,9 +34,11 @@ FROM ruby:2.7-bullseye as download LABEL stage=build LABEL project=avalon RUN curl -L https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz | tar xvz -C /usr/bin/ -RUN curl https://chromedriver.storage.googleapis.com/2.46/chromedriver_linux64.zip -o /usr/local/bin/chromedriver \ - && chmod +x /usr/local/bin/chromedriver RUN curl https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -o /chrome.deb +RUN chrome_version=`dpkg-deb -f /chrome.deb Version | cut -d '.' -f 1-3` +RUN chromedriver_version=`curl https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${chrome_version}` +RUN curl https://chromedriver.storage.googleapis.com/index.html?path=${chromedriver_version} -o /usr/local/bin/chromedriver \ + && chmod +x /usr/local/bin/chromedriver RUN apt-get -y update && apt-get install -y ffmpeg From f32fb6c8ad5fc66bbc65b212cc3aa5cd738edc43 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Mon, 21 Mar 2022 15:59:25 -0400 Subject: [PATCH 033/171] Mark tests as pending --- spec/features/capybara_playlist_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/features/capybara_playlist_spec.rb b/spec/features/capybara_playlist_spec.rb index c37f33fa29..89b62c59bd 100644 --- a/spec/features/capybara_playlist_spec.rb +++ b/spec/features/capybara_playlist_spec.rb @@ -14,7 +14,7 @@ require 'rails_helper' -describe 'Playlist' do +describe 'Playlist', skip: "Datatables fails to load only in test" do after { Warden.test_reset! } it 'checks navigation when create new playlist is accessed' do user = FactoryBot.create(:administrator) From af872f2bf26bdf5bd9f9504095a0284ed55dacad Mon Sep 17 00:00:00 2001 From: dananji Date: Thu, 10 Mar 2022 17:09:21 -0500 Subject: [PATCH 034/171] Navbar, datatable updates --- app/assets/javascripts/avalon.js | 20 -- app/assets/javascripts/datatables.js | 10 +- .../mejs4_plugin_hd_toggle.es6 | 179 ------------------ app/assets/stylesheets/application.scss | 3 +- app/assets/stylesheets/avalon.scss | 15 +- app/assets/stylesheets/avalon/_nav.scss | 27 ++- app/assets/stylesheets/branding.scss | 34 ++-- .../{datatables.css => datatables.scss} | 38 ++-- app/views/_user_util_links.html.erb | 42 ++-- app/views/admin/groups/index.html.erb | 18 +- app/views/encode_records/index.html.erb | 109 ++++++----- .../modules/_avalon_search_form.html.erb | 2 +- app/views/modules/_global_navigation.html.erb | 9 +- 13 files changed, 167 insertions(+), 339 deletions(-) delete mode 100644 app/assets/javascripts/media_player_wrapper/mejs4_plugin_hd_toggle.es6 rename app/assets/stylesheets/{datatables.css => datatables.scss} (67%) mode change 100644 => 100755 diff --git a/app/assets/javascripts/avalon.js b/app/assets/javascripts/avalon.js index 34aa56cf65..bf3866b980 100644 --- a/app/assets/javascripts/avalon.js +++ b/app/assets/javascripts/avalon.js @@ -105,19 +105,6 @@ $(document).ready(function() { } }); - /* Handle dropdown list for `Manage` in nav-bar */ - // Manage user access with mouse-point - $('#manage-dropdown').hover( - function() { - $(this).addClass('open'); - $(this).attr('aria-expanded', 'true'); - }, - function() { - $(this).removeClass('open'); - $(this).attr('aria-expanded', 'false'); - } - ); - // Close dropdown when Tab key is used with Shift key $('#manage-dropdown').keydown(function(e) { if (e.which == 9 && e.shiftKey) { @@ -126,11 +113,4 @@ $(document).ready(function() { } }); - // Close dropdown if Tab key is pressed when focused on last element - $('#manage-dropdown ul > li:last-child').keydown(function(e) { - if (e.which == 9) { - $('#manage-dropdown').removeClass('open'); - $('#manage-dropdown').attr('aria-expanded', 'false'); - } - }); }); diff --git a/app/assets/javascripts/datatables.js b/app/assets/javascripts/datatables.js index 049fac90b4..0464e222d0 100644 --- a/app/assets/javascripts/datatables.js +++ b/app/assets/javascripts/datatables.js @@ -16,7 +16,8 @@ //= require datatables/jquery.dataTables -//optional add '=' enable +// optional change '//' --> '//=' to enable + // require datatables/extensions/AutoFill/dataTables.autoFill // require datatables/extensions/Buttons/dataTables.buttons // require datatables/extensions/Buttons/buttons.html5 @@ -28,11 +29,12 @@ // require datatables/extensions/FixedHeader/dataTables.fixedHeader // require datatables/extensions/KeyTable/dataTables.keyTable // require datatables/extensions/Responsive/dataTables.responsive +// require datatables/extensions/RowGroup/dataTables.rowGroup // require datatables/extensions/RowReorder/dataTables.rowReorder // require datatables/extensions/Scroller/dataTables.scroller // require datatables/extensions/Select/dataTables.select //= require datatables/dataTables.bootstrap4 -// require datatables/extensions/AutoFill/autoFill.bootstrap -// require datatables/extensions/Buttons/buttons.bootstrap -// require datatables/extensions/Responsive/responsive.bootstrap +// require datatables/extensions/AutoFill/autoFill.bootstrap4 +// require datatables/extensions/Buttons/buttons.bootstrap4 +// require datatables/extensions/Responsive/responsive.bootstrap4 diff --git a/app/assets/javascripts/media_player_wrapper/mejs4_plugin_hd_toggle.es6 b/app/assets/javascripts/media_player_wrapper/mejs4_plugin_hd_toggle.es6 deleted file mode 100644 index c90b9a63f8..0000000000 --- a/app/assets/javascripts/media_player_wrapper/mejs4_plugin_hd_toggle.es6 +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2011-2022, The Trustees of Indiana University and Northwestern -// University. Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -'use strict'; - -/** - * HD Toggle plugin - * - * A custom Avalon MediaElement 4 plugin for toggling between 2 sources - */ - -// Translations (English required) -mejs.i18n.en['mejs.hd-toggle'] = 'Toggle between HD and SD quality'; - -// Feature configuration -Object.assign(mejs.MepDefaults, { - /** - * @type {String} - */ - hdToggleLabel: 'HD', - /** - * @type {String} - */ - hdToggleText: null, - /** - * @type {String} - */ - hdToggleOn: true, - /** - * @type {String} - */ - hdToggleBetween: ['high', 'medium'] -}); - -Object.assign(MediaElementPlayer.prototype, { - // Public variables (also documented according to JSDoc specifications) - - /** - * Feature constructor. - * - * Always has to be prefixed with `build` and the name that will be used in MepDefaults.features list - * @param {MediaElementPlayer} player - * @param {HTMLElement} controls - * @param {HTMLElement} layers - * @param {HTMLElement} media - */ - /* eslint-disable complexity */ - buildhdToggle(player, controls, layers, media) { - const t = this; - const hdToggleText = mejs.Utils.isString(t.options.hdToggleText) - ? t.options.hdToggleText - : mejs.i18n.t('mejs.hd-toggle'); - player.qualities = []; - - player.sources = $(player.domNode).find('source'); - for (var i = 0; i < player.sources.length; i++) { - var src = player.sources[i]; - for (var j = 0; j < player.options.hdToggleBetween.length; j++) { - if ( - src.getAttribute('data-quality') === player.options.hdToggleBetween[j] - ) { - player.qualities[j] = src.getAttribute('src'); - } - } - } - - player.hdtoggleButton = document.createElement('div'); - player.hdtoggleButton.className = - t.options.classPrefix + - 'button ' + - t.options.classPrefix + - 'hd-toggle-button'; - player.hdtoggleButton.innerHTML = ``; - - player.hdtoggleButton.addEventListener('click', t.toggleQuality.bind(t)); - - // Add control button to player - t.addControlElement(player.hdtoggleButton, 'hdToggle'); - - if (player.options.hdToggleOn && player.qualities[0] !== null) { - player.hdtoggleButton.className += ' mejs__hdtoggle-on'; - player.switchStream(player.qualities[0]); - } else if (player.qualities[1] !== null) { - player.switchStream(player.qualities[1]); - } else { - // Fixme: Ideally we should display this message in a player overlay - alert( - 'Did not find ' + - hdToggleBetween[0] + - ' and ' + - hdToggleBetween[1] + - ' streams' - ); - } - }, - /* eslint-enable complexity */ - - /** - * Feature destructor. - * - * Always has to be prefixed with `clean` and the name that was used in MepDefaults.features list - * @param {MediaElementPlayer} player - * @param {HTMLElement} controls - * @param {HTMLElement} layers - * @param {HTMLElement} media - */ - cleanhdToggle(player, controls, layers, media) { - if (player && player.hdtoggleButton) { - player.hdtoggleButton.remove(); - } - }, - - /** - * Toggle HD button state and change media source - */ - toggleQuality: function() { - let $btn = $(this.hdtoggleButton); - if ($btn.hasClass('mejs__hdtoggle-on')) { - $btn.removeClass('mejs__hdtoggle-on'); - this.switchStream(this.qualities[1]); - } else { - $btn.addClass('mejs__hdtoggle-on'); - this.switchStream(this.qualities[0]); - } - }, - - /** - * Switch the currently playing file - * - * @param src new media SRC - */ - switchStream: function(src) { - let media = this.media; - - // Do nothing if asked to to the same thing - if (media.currentSrc !== src) { - let currentTime = media.currentTime; - let paused = media.paused; - - media.pause(); - media.setSrc(src); - media.addEventListener( - 'loadedmetadata', - function(e) { - // Continue playing where we stopped - media.currentTime = currentTime; - }, - true - ); - - var canPlayAfterSourceSwitchHandler = function(e) { - if (!paused) { - media.play(); - } - media.removeEventListener( - 'canplay', - canPlayAfterSourceSwitchHandler, - true - ); - }; - media.addEventListener('canplay', canPlayAfterSourceSwitchHandler, true); - - media.load(); - } - } -}); diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index 24b81d5085..4d1fbac3e9 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -30,7 +30,6 @@ * Require stylesheets for our gems * *= require jquery-ui/datepicker - *= require datatables * * Require all of our vendored stylesheets *= require_tree ../../../vendor/assets/stylesheets/. @@ -59,3 +58,5 @@ @import 'nestable'; @import 'avalon'; + +@import "datatables"; diff --git a/app/assets/stylesheets/avalon.scss b/app/assets/stylesheets/avalon.scss index e93c6e8da5..291ce6bf79 100644 --- a/app/assets/stylesheets/avalon.scss +++ b/app/assets/stylesheets/avalon.scss @@ -39,6 +39,17 @@ body { } /* Override Bootstrap CSS */ + +/* + Bootstrap 4 has increased global font-size from 14px to 1rem. + This font size of 1rem, gets computed as 16px by browsers, increasing + font-size for text elements. Reset this to 14px. + See https://getbootstrap.com/docs/4.0/migration/#global-changes. +*/ +html { + font-size: 14px; +} + .alert { margin-bottom: 10px; } @@ -119,7 +130,7 @@ div.alert-danger { // Dotted line separator from branding guide .separator { - border-bottom: 2px dotted $primaryDark; + border-bottom: 2px dotted $secondary; background: white; height: 1px; } @@ -130,7 +141,7 @@ div.alert-danger { font-family: $museoSlab; } .headline { - color: $primaryDark; + color: $secondary; font-size: 29.3px; font-family: $museoSlab; } diff --git a/app/assets/stylesheets/avalon/_nav.scss b/app/assets/stylesheets/avalon/_nav.scss index 3c2369637f..9efed96162 100644 --- a/app/assets/stylesheets/avalon/_nav.scss +++ b/app/assets/stylesheets/avalon/_nav.scss @@ -39,7 +39,7 @@ $state-info-border: darken(adjust-hue($state-info-bg, -10), 7%) !de .navbar-toggle { @include media-breakpoint-down(xs) { - background: $primaryDark; + background: $secondary; } } } @@ -59,7 +59,7 @@ $state-info-border: darken(adjust-hue($state-info-bg, -10), 7%) !de } .navbar-nav { - > li > a { + > .nav-item > .nav-link { @include media-breakpoint-down(sm) { padding-left: 10px; padding-right: 10px; @@ -67,9 +67,9 @@ $state-info-border: darken(adjust-hue($state-info-bg, -10), 7%) !de } } -.navbar-default { +.navbar-light { @include media-breakpoint-down(xs) { - background: $primaryDark; + background: $secondary; } } @@ -91,9 +91,9 @@ $state-info-border: darken(adjust-hue($state-info-bg, -10), 7%) !de } } -.container > .navbar-header { - height: auto; -} +// .container > .navbar-header { +// height: auto; +// } .navbar-inner { padding-right: 12px; @@ -178,12 +178,17 @@ $state-info-border: darken(adjust-hue($state-info-bg, -10), 7%) !de .manage-btn:hover { background-color: $yellow!important; - color: $primaryDark; + color: $secondary; +} + +#manage-dropdown:hover .manage-dropdown-menu { + display: block; + margin-top: 0; } .manage-btn:focus { background-color: $yellow; - color: $primaryDark; + color: $secondary; } .manage-dropdown-menu { @@ -192,6 +197,10 @@ $state-info-border: darken(adjust-hue($state-info-bg, -10), 7%) !de li { width: 100%; } + + .dropdown-item:hover { + background-color: $yellow; + } } .manage-dropdown { diff --git a/app/assets/stylesheets/branding.scss b/app/assets/stylesheets/branding.scss index 919efc36f5..6bcdf2cf96 100644 --- a/app/assets/stylesheets/branding.scss +++ b/app/assets/stylesheets/branding.scss @@ -31,7 +31,7 @@ $museoSlab: 'MuseoSlab'; // Primary colors $primary: #80a590; -$primaryDark: #2a5459; +$secondary: #2a5459; $dark: #0a2326; // Secondary colors @@ -62,7 +62,7 @@ $lightblue: #31708f; * https://github.com/twbs/bootstrap-sass/blob/master/assets/stylesheets/bootstrap/_variables.scss */ /* Branding colors */ -$brand-primary: $primaryDark; +$brand-primary: $secondary; $brand-success: $success; $brand-info: $info; $brand-warning: $warning; @@ -79,7 +79,7 @@ $padding-base-vertical: 4px; // $headings-font-family: $museoSlab; /* Links */ -$link-color: $primaryDark; +$link-color: $secondary; $link-hover-color: $primary; /* Alerts */ @@ -97,7 +97,7 @@ $state-danger-text: $danger; /* Header */ $login-text-color: $gray; -$login-link-color: $primaryDark; +$login-link-color: $secondary; $login-link-background-color: $orange; /* Sidenav */ @@ -106,17 +106,17 @@ $login-link-background-color: $orange; $navCollapseWidth: 650px; /* Navbar */ -$navbar-default-bg: $primaryDark; -$navbar-default-color: $white; -$navbar-default-toggle-hover-bg: $yellow; -$navbar-default-link-color: $white; -$navbar-default-link-hover-color: $primaryDark; -$navbar-default-link-hover-bg: $yellow; -$navbar-default-link-active-color: $primaryDark; -$navbar-default-link-active-bg: $yellow; +// $navbar-light-bg: $secondary; +$navbar-light-color: $white; +$navbar-light-toggle-hover-bg: $yellow; +$navbar-light-link-color: $white; +$navbar-light-link-hover-color: $secondary; +// $navbar-light-link-hover-bg: $yellow; +$navbar-light-link-active-color: $secondary; +// $navbar-light-link-active-bg: $yellow; $navbar-border-radius: 0; -$navbar-default-toggle-icon-bar-bg: $white; -$navbar-default-toggle-border-color: $white; +$navbar-light-toggle-icon-bar-bg: $white; +$navbar-light-toggle-border-color: $white; // Panels $panel-primary-text: red; @@ -135,12 +135,12 @@ $btn-default-color: $dark; $btn-default-bg: $white; $btn-default-border: $gray; $btn-primary-color: $white; -$btn-primary-bg: $primaryDark; -$btn-primary-border: $primaryDark; +$btn-primary-bg: $secondary; +$btn-primary-border: $secondary; $btn-success-color: $white; $btn-success-bg: $success; $btn-success-border: $success; -$btn-info-color: $primaryDark; +$btn-info-color: $secondary; $btn-info-bg: $info; $btn-info-border: $info; $btn-danger-color: $white; diff --git a/app/assets/stylesheets/datatables.css b/app/assets/stylesheets/datatables.scss old mode 100644 new mode 100755 similarity index 67% rename from app/assets/stylesheets/datatables.css rename to app/assets/stylesheets/datatables.scss index e2f532eccb..3cd6111740 --- a/app/assets/stylesheets/datatables.css +++ b/app/assets/stylesheets/datatables.scss @@ -14,21 +14,21 @@ * --- END LICENSE_HEADER BLOCK --- */ -/* -*= require datatables/dataTables.bootstrap -* -*optional add '=' enable -* require datatables/extensions/AutoFill/autoFill.bootstrap -* require datatables/extensions/Buttons/buttons.bootstrap -* require datatables/extensions/ColReorder/colReorder.bootstrap -* require datatables/extensions/FixedColumns/fixedColumns.bootstrap -* require datatables/extensions/FixedHeader/fixedHeader.bootstrap -* require datatables/extensions/KeyTable/keyTable.bootstrap -* require datatables/extensions/Responsive/responsive.bootstrap -* require datatables/extensions/RowReorder/rowReorder.bootstrap -* require datatables/extensions/Scroller/scroller.bootstrap -* require datatables/extensions/Select/select.bootstrap -*/ +@import 'datatables/dataTables.bootstrap4'; + +// optional remove '//=' to enable + +//@import 'datatables/extensions/AutoFill/autoFill.bootstrap4'; +//@import 'datatables/extensions/Buttons/buttons.bootstrap4'; +//@import 'datatables/extensions/ColReorder/colReorder.bootstrap4'; +//@import 'datatables/extensions/FixedColumns/fixedColumns.bootstrap4'; +//@import 'datatables/extensions/FixedHeader/fixedHeader.bootstrap4'; +//@import 'datatables/extensions/KeyTable/keyTable.bootstrap4'; +//@import 'datatables/extensions/Responsive/responsive.bootstrap4'; +//@import 'datatables/extensions/RowGroup/rowGroup.bootstrap4'; +//@import 'datatables/extensions/RowReorder/rowReorder.bootstrap4'; +//@import 'datatables/extensions/Scroller/scroller.bootstrap4'; +//@import 'datatables/extensions/Select/select.bootstrap4'; table.dataTable { border-collapse: collapse !important; @@ -58,6 +58,14 @@ table.dataTable thead .sorting_desc_disabled { *cursor: hand; } +table.dataTable thead .sorting:before, +table.dataTable thead .sorting_asc:before, +table.dataTable thead .sorting_desc:before, +table.dataTable thead .sorting_asc_disabled:before, +table.dataTable thead .sorting_desc_disabled:before { + display: none; +} + table.dataTable thead .sorting:after, table.dataTable thead .sorting_asc:after, table.dataTable thead .sorting_desc:after, diff --git a/app/views/_user_util_links.html.erb b/app/views/_user_util_links.html.erb index 23b5fc2f67..cf924b6cea 100644 --- a/app/views/_user_util_links.html.erb +++ b/app/views/_user_util_links.html.erb @@ -13,61 +13,61 @@ Unless required by applicable law or agreed to in writing, software distributed specific language governing permissions and limitations under the License. --- END LICENSE_HEADER BLOCK --- %> -
    -
  • <%= link_to 'Browse', main_app.search_catalog_path(search_field: 'all_fields', utf8: '✓', q: '') %>
    • -
  • ><%= link_to 'Collections', main_app.collections_path %>
    • +
      +
    • <%= link_to 'Browse', main_app.search_catalog_path(search_field: 'all_fields', utf8: '✓', q: ''), class: 'nav-link' %>
      • +
    • ><%= link_to 'Collections', main_app.collections_path, class: 'nav-link' %>
      • <% if user_session && user_session[:lti_group] %> -
    • - <%= link_to(vgroup_display(user_session[:lti_group]), main_app.search_catalog_path('f[read_access_virtual_group_ssim][]' => user_session[:lti_group])) %> -
      • +
    • + <%= link_to(vgroup_display(user_session[:lti_group]), main_app.search_catalog_path('f[read_access_virtual_group_ssim][]' => user_session[:lti_group])) %> +
      • <% end %> <% if current_ability.can? :create, Playlist %>
    • > - <%= link_to 'Playlists', main_app.playlists_path, id:'playlists_nav' %> + <%= link_to 'Playlists', main_app.playlists_path, id:'playlists_nav', class: 'nav-link' %>
      • <% end %> <% if current_ability.can? :create, Timeline %>
    • > - <%= link_to 'Timelines', main_app.timelines_path, id:'timelines_nav' %> + <%= link_to 'Timelines', main_app.timelines_path, id:'timelines_nav', class: 'nav-link' %>
      • <% end %> <% if render_bookmarks_control? %> -
    • > - <%= link_to main_app.bookmarks_path, id:'bookmarks_nav' do %> - <%= t('blacklight.header_links.bookmarks') %> - (<%= current_or_guest_user.bookmarks.count %>) +
    • > + <%= link_to main_app.bookmarks_path, id:'bookmarks_nav', class: 'nav-link' do %> + <%= t('blacklight.header_links.bookmarks') %> + (<%= current_or_guest_user.bookmarks.count %>) <% end %>
      • <% end %> <% if can?(:read, Admin::Collection) || can?(:manage, Admin::Group) || (defined?(Samvera::Persona) && can?(:manage, User)) %> -
    • -
      - -
        +
          <% if can? :read, Admin::Collection %>
        • > - <%= link_to 'Manage Content', main_app.admin_collections_path %>
          • + <%= link_to 'Manage Content', main_app.admin_collections_path, class: 'dropdown-item' %> <% end %> <% if can? :read, :encode_dashboard %>
        • > - <%= link_to 'Manage Encoding Jobs', main_app.encode_records_path %>
          • + <%= link_to 'Manage Encoding Jobs', main_app.encode_records_path, class: 'dropdown-item' %> <% end %> <% if can? :manage, Admin::Group %>
        • > - <%= link_to 'Manage Groups', main_app.admin_groups_path %>
          • + <%= link_to 'Manage Groups', main_app.admin_groups_path, class: 'dropdown-item' %> <% end %> <% if defined?(Samvera::Persona) %> <% if can? :manage, User %>
        • > - <%= link_to 'Manage Users', main_app.persona_users_path %>
          • + <%= link_to 'Manage Users', main_app.persona_users_path, class: 'dropdown-item' %> <% end %> <% if can? :manage, :jobs %>
        • - <%= link_to(main_app.jobs_path, target: 'blank') do %> + <%= link_to(main_app.jobs_path, target: 'blank', class: 'dropdown-item') do %> Manage Worker Jobs <% end %>
          • diff --git a/app/views/admin/groups/index.html.erb b/app/views/admin/groups/index.html.erb index 8e9ea3b453..e0ff72f4e9 100644 --- a/app/views/admin/groups/index.html.erb +++ b/app/views/admin/groups/index.html.erb @@ -21,11 +21,11 @@ Unless required by applicable law or agreed to in writing, software distributed
          -
          -
          -

          System Groups

          +
          +
          +

          System Groups

          -
          +
          @@ -47,16 +47,16 @@ Unless required by applicable law or agreed to in writing, software distributed
          -
          -
          -

          Additional Groups

          +
          +
          +

          Additional Groups

          -
          +
          <%= form_tag(admin_groups_path) do -%>
          <%= text_field_tag 'admin_group', nil, :placeholder => 'Enter group name here...', class: 'form-control' %> - + <%= button_tag 'Create Group', type: 'submit', class: 'btn btn-primary' %> diff --git a/app/views/encode_records/index.html.erb b/app/views/encode_records/index.html.erb index d6aa0639f8..6704afbec2 100644 --- a/app/views/encode_records/index.html.erb +++ b/app/views/encode_records/index.html.erb @@ -40,65 +40,64 @@ Unless required by applicable law or agreed to in writing, software distributed
          <% content_for :page_scripts do %> - + $(document).ready( function () { + $('#encode-records').dataTable({ + stateSave: true, + processing: true, + serverSide: true, + dom:'<"dataTableToolsTop"Blf><"dataTableBody"t><"dataTableToolsBottom"ipr>', + autoWidth:true, + order: [[ 6, 'desc' ]], + lengthMenu: [10, 20, 50, 100], + pageLength: 20, + ajax: { + url: '/encode_records/paged_index', + type: 'POST' + } + }).on( 'draw.dt', function () { + clearTimeout(encodeProgressTimeout); + update_encode_progress(); + }) + }); + <% end %> diff --git a/app/views/modules/_avalon_search_form.html.erb b/app/views/modules/_avalon_search_form.html.erb index 27f64080e6..37fb1ba183 100644 --- a/app/views/modules/_avalon_search_form.html.erb +++ b/app/views/modules/_avalon_search_form.html.erb @@ -23,7 +23,7 @@ Unless required by applicable law or agreed to in writing, software distributed
          - + diff --git a/app/views/modules/_global_navigation.html.erb b/app/views/modules/_global_navigation.html.erb index 2583761f2a..d4d27d2beb 100644 --- a/app/views/modules/_global_navigation.html.erb +++ b/app/views/modules/_global_navigation.html.erb @@ -13,16 +13,13 @@ Unless required by applicable law or agreed to in writing, software distributed specific language governing permissions and limitations under the License. --- END LICENSE_HEADER BLOCK --- %> -
          Create Options: -
          - 
          <%= encode_presenter.create_options %>
          -
          + 
          <%= encode_presenter.create_options %>

          Raw Object: -

          - 
          <%= encode_presenter.raw_object %>
          -
          + 
          <%= encode_presenter.raw_object %>

          diff --git a/app/views/layouts/avalon.html.erb b/app/views/layouts/avalon.html.erb index ed10ea14cc..bee422c561 100644 --- a/app/views/layouts/avalon.html.erb +++ b/app/views/layouts/avalon.html.erb @@ -57,7 +57,7 @@ Unless required by applicable law or agreed to in writing, software distributed <% unless current_page?(main_app.root_path) %> -
          +
          <%= render 'modules/flash_messages' %> <%= yield %>
          diff --git a/app/views/modules/_global_navigation.html.erb b/app/views/modules/_global_navigation.html.erb index d4d27d2beb..cb1e74f2f2 100644 --- a/app/views/modules/_global_navigation.html.erb +++ b/app/views/modules/_global_navigation.html.erb @@ -13,8 +13,8 @@ Unless required by applicable law or agreed to in writing, software distributed specific language governing permissions and limitations under the License. --- END LICENSE_HEADER BLOCK --- %> -
          - - - - - - - - - - - - -
          NameDescriptionVisibilityLast UpdatedTagsActions
          -
          - <% end %> +
          +
          + <% unless timelines.empty? %> +
          + Filter: +
          + + + + + + + + + + + + + +
          NameDescriptionVisibilityLast UpdatedTagsActions
          +
          + <% end %> +
          <% unless timelines.empty? %> From a67268a0e2c3eec58e422685d333170224d7cc37 Mon Sep 17 00:00:00 2001 From: dananji Date: Thu, 17 Mar 2022 16:55:27 -0400 Subject: [PATCH 037/171] Mix of CSS classes and js changes --- app/assets/stylesheets/avalon.scss | 18 ++++- app/assets/stylesheets/avalon/_homepage.scss | 3 +- app/assets/stylesheets/avalon/_nav.scss | 2 +- app/assets/stylesheets/avalon/_playlists.scss | 8 +-- app/assets/stylesheets/branding.scss | 40 ++++++----- app/controllers/playlists_controller.rb | 6 +- app/controllers/timelines_controller.rb | 6 +- app/views/_user_util_links.html.erb | 2 +- app/views/playlists/_info.html.erb | 26 +++---- .../playlists/_show_playlist_details.html.erb | 38 +++++----- app/views/timelines/_form.html.erb | 24 +++---- .../timelines/_show_timeline_details.html.erb | 71 ++++++++++--------- 12 files changed, 129 insertions(+), 115 deletions(-) diff --git a/app/assets/stylesheets/avalon.scss b/app/assets/stylesheets/avalon.scss index f2d9c0f6f7..68c58f56b2 100644 --- a/app/assets/stylesheets/avalon.scss +++ b/app/assets/stylesheets/avalon.scss @@ -47,7 +47,7 @@ body { See https://getbootstrap.com/docs/4.0/migration/#global-changes. */ html { - font-size: 15px; + font-size: 14px; } // .row { @@ -67,6 +67,10 @@ h2 { font-family: $museoSlab; } +.btn-info { + color: $primary; +} + tr.active-false td { color: $gray; } @@ -152,7 +156,7 @@ div.alert-danger { font-family: $museoSlab; } .headline { - color: $secondary; + color: $primary; font-size: 29.3px; font-family: $museoSlab; } @@ -1134,3 +1138,13 @@ td { .btn-light { background-color: $btn-light-bg; } + +.btn-outline { + background-color: $btn-outline-bg-color; + color: $btn-outline-color; + border: 1px solid $btn-outline-border-color; + + &:hover { + background-color: $lightgray; + } +} \ No newline at end of file diff --git a/app/assets/stylesheets/avalon/_homepage.scss b/app/assets/stylesheets/avalon/_homepage.scss index 77fc9741ca..f1b55b89d1 100644 --- a/app/assets/stylesheets/avalon/_homepage.scss +++ b/app/assets/stylesheets/avalon/_homepage.scss @@ -16,6 +16,7 @@ main.homepage-main { padding-bottom: 0; + padding-top: 0.25rem; .alert { margin-bottom: 40px; @@ -63,7 +64,7 @@ main.homepage-main { color: white; width: 100%; background: rgba(42, 84, 89, 0.8); - padding: 2.5rem 0; + padding: 1.5rem 0; font-family: $museoSlab; .lead { diff --git a/app/assets/stylesheets/avalon/_nav.scss b/app/assets/stylesheets/avalon/_nav.scss index 17a511b30c..d175c83449 100644 --- a/app/assets/stylesheets/avalon/_nav.scss +++ b/app/assets/stylesheets/avalon/_nav.scss @@ -43,7 +43,7 @@ $state-info-border: darken(adjust-hue($state-info-bg, -10), 7%) !de } } - padding: 0; + padding: 0 1rem; } .navbar { diff --git a/app/assets/stylesheets/avalon/_playlists.scss b/app/assets/stylesheets/avalon/_playlists.scss index daf76a5db3..69131ea677 100644 --- a/app/assets/stylesheets/avalon/_playlists.scss +++ b/app/assets/stylesheets/avalon/_playlists.scss @@ -21,19 +21,13 @@ } .playlist-details-headline-wrapper { - margin: 3rem 0 1rem; + margin: 2rem 0 1rem; display: flex; align-items: center; .headline { margin: 0; } - - .playlist-details-edit-button { - padding: 0; - margin-left: 1rem; - font-size: 2rem; - } } .playlist-title { diff --git a/app/assets/stylesheets/branding.scss b/app/assets/stylesheets/branding.scss index 7a6b58abc5..3f0d8c819f 100644 --- a/app/assets/stylesheets/branding.scss +++ b/app/assets/stylesheets/branding.scss @@ -40,13 +40,6 @@ $orange: #fbb040; $white: rgb(255, 255, 255); $blueGreen: #477076; -// Generated palate colors from http://colormind.io/ for Avalon style guide -// Add new HEX values below to match your organization's flavor of info/success/warning/danger -$info: #fbb040; -$success: #429453; -$warning: #bf841b; -$danger: #f44336; - // Colors not from the Styleguide $greenBG: scale-color($secondary, $saturation: -40%, $lightness: 60%); $gray: #999; @@ -55,6 +48,13 @@ $red: #d9534f; $blue: #2a4d59; $lightblue: #31708f; +// Generated palate colors from http://colormind.io/ for Avalon style guide +// Add new HEX values below to match your organization's flavor of info/success/warning/danger +$info: #fbb040; +$success: #429453; +$warning: #bf841b; +$danger: #f44336; + /** * Bootstrap variable overrides * @@ -80,7 +80,7 @@ $padding-base-vertical: 4px; /* Links */ $link-color: $primary; -$link-hover-color: $primary; +$link-hover-color: $secondary; /* Alerts */ $alert-success-bg: $success; @@ -130,20 +130,22 @@ $navbar-inverse-toggle-border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) $grid-gutter-width: 16px; /* Buttons */ -$btn-border-radius-base: 2px; -$btn-default-color: $dark; -$btn-default-bg: $white; -$btn-default-border: $gray; +// $btn-border-radius-base: 2px; +// $btn-default-color: $dark; +// $btn-default-bg: $white; +// $btn-default-border: $gray; // $btn-primary-color: $white; // $btn-primary-bg: $secondary; // $btn-primary-border: $secondary; -$btn-success-color: $white; -$btn-success-bg: $success; -$btn-success-border: $success; -$btn-info-color: $primary; -$btn-info-bg: $info; -$btn-info-border: $info; -$btn-danger-color: $white; +// $btn-success-color: $white; +// $btn-success-bg: $success; +// $btn-success-border: $success; +// $btn-info-color: $primary; +// $btn-info-bg: $info; +// $btn-info-border: $info; +$btn-outline-color: $dark; +$btn-outline-border-color: $gray; +$btn-outline-bg-color: $white; $btn-light-bg: $lightgray; /* Pagination */ diff --git a/app/controllers/playlists_controller.rb b/app/controllers/playlists_controller.rb index 657de5310b..3ed1652e71 100644 --- a/app/controllers/playlists_controller.rb +++ b/app/controllers/playlists_controller.rb @@ -80,13 +80,13 @@ def paged_index "recordsFiltered": playlistsFilteredTotal, "data": @playlists.collect do |playlist| copy_button = view_context.button_tag( type: 'button', data: { playlist: playlist }, - class: 'copy-playlist-button btn btn-default btn-xs') do + class: 'copy-playlist-button btn btn-outline btn-sm') do " Copy".html_safe end - edit_button = view_context.link_to(edit_playlist_path(playlist), class: 'btn btn-default btn-xs') do + edit_button = view_context.link_to(edit_playlist_path(playlist), class: 'btn btn-outline btn-sm') do " Edit".html_safe end - delete_button = view_context.link_to(playlist_path(playlist), method: :delete, class: 'btn btn-xs btn-danger btn-confirmation', data: {placement: 'bottom'}) do + delete_button = view_context.link_to(playlist_path(playlist), method: :delete, class: 'btn btn-sm btn-danger btn-confirmation', data: {placement: 'bottom'}) do " Delete".html_safe end [ diff --git a/app/controllers/timelines_controller.rb b/app/controllers/timelines_controller.rb index 6459e070dd..dbff6d2565 100644 --- a/app/controllers/timelines_controller.rb +++ b/app/controllers/timelines_controller.rb @@ -67,13 +67,13 @@ def paged_index "data": @timelines.collect do |timeline| copy_button = view_context.button_tag(type: 'button', data: { timeline: timeline }, - class: 'copy-timeline-button btn btn-default btn-xs') do + class: 'copy-timeline-button btn btn-sm btn-outline') do " Copy".html_safe end - edit_button = view_context.link_to(edit_timeline_path(timeline), class: 'btn btn-default btn-xs') do + edit_button = view_context.link_to(edit_timeline_path(timeline), class: 'btn btn-sm btn-outline') do " Edit Details".html_safe end - delete_button = view_context.link_to(timeline_path(timeline), method: :delete, class: 'btn btn-xs btn-danger btn-confirmation', data: { placement: 'bottom' }) do + delete_button = view_context.link_to(timeline_path(timeline), method: :delete, class: 'btn btn-sm btn-danger btn-confirmation', data: { placement: 'bottom' }) do " Delete".html_safe end [ diff --git a/app/views/_user_util_links.html.erb b/app/views/_user_util_links.html.erb index 404a78445f..c39dc54547 100644 --- a/app/views/_user_util_links.html.erb +++ b/app/views/_user_util_links.html.erb @@ -13,7 +13,7 @@ Unless required by applicable law or agreed to in writing, software distributed specific language governing permissions and limitations under the License. --- END LICENSE_HEADER BLOCK --- %> -
            +
            • <%= link_to 'Browse', main_app.search_catalog_path(search_field: 'all_fields', utf8: '✓', q: ''), class: 'nav-link' %>
            • ><%= link_to 'Collections', main_app.collections_path, class: 'nav-link' %>
              • diff --git a/app/views/playlists/_info.html.erb b/app/views/playlists/_info.html.erb index ab3582d648..5b48b4636a 100644 --- a/app/views/playlists/_info.html.erb +++ b/app/views/playlists/_info.html.erb @@ -20,10 +20,10 @@ Unless required by applicable law or agreed to in writing, software distributed <% if can? :read, @current_masterfile %> -
              -
              -