From 9997c731a78986cf07bfde1f0a79987f9f48baa8 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 21 Oct 2019 14:45:06 -0400 Subject: [PATCH 01/77] schema refactor changes; change field type for full_text_link field --- app/controllers/catalog_controller.rb | 2 +- app/models/franklin_indexer.rb | 36 +++++++++++++-------------- app/models/hathi_indexer.rb | 4 +-- app/models/solr_document.rb | 2 +- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 452f2871..9fbf8b03 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -111,7 +111,7 @@ def expire_session alma_mms_id score format_a - full_text_link_text_a + full_text_link_a isbn_isxn language_a title diff --git a/app/models/franklin_indexer.rb b/app/models/franklin_indexer.rb index 452c3039..aaab1399 100644 --- a/app/models/franklin_indexer.rb +++ b/app/models/franklin_indexer.rb @@ -57,7 +57,7 @@ def initialize # append extra params to the Solr update URL for solr-side cross reference handling # and duplicate ID deletion - processors = [ 'xref-copyfield', 'fl-multiplex', 'shingles', 'id_hash' ] + processors = [ 'xref-copyfield', 'fl-multiplex', 'shingles', 'id_hash', 'content_hash', 'hex_to_numeric' ] if ENV['SOLR_USE_UID_DISTRIB_PROCESSOR'] # disable; handle deletion outside of solr, either permanently or pending bug fixes #processors << 'uid-distrib' @@ -110,10 +110,10 @@ def define_all_fields define_cluster_id - define_full_text_link_text_a + define_full_text_link_a # do NOT use *_xml_stored_single because it uses a Str (max 32k) for storage - to_field 'marcrecord_xml_stored_single_large', get_plain_marc_xml + to_field 'marcrecord_xml', get_plain_marc_xml # Our keyword searches use pf/qf to search multiple fields, so # we don't need this field; leaving it commented out here just in case. @@ -125,7 +125,7 @@ def define_all_fields define_access_facet - to_field 'format_f_stored' do |rec, acc| + to_field 'format_f' do |rec, acc| acc.concat(pennlibmarc.get_format(rec)) end @@ -150,19 +150,19 @@ def define_all_fields end # this is now automatically copied on the Solr side - # to_field 'subject_f_stored' do |rec, acc| + # to_field 'subject_f' do |rec, acc| # acc.concat(pennlibmarc.get_subject_facet_values(rec)) # end - to_field "db_type_f_stored" do |rec, acc| + to_field "db_type_f" do |rec, acc| acc.concat(pennlibmarc.get_db_types(rec)) end - to_field "db_category_f_stored" do |rec, acc| + to_field "db_category_f" do |rec, acc| acc.concat(pennlibmarc.get_db_categories(rec)) end - to_field "db_subcategory_f_stored" do |rec, acc| + to_field "db_subcategory_f" do |rec, acc| acc.concat(pennlibmarc.get_db_subcategories(rec)) end @@ -182,7 +182,7 @@ def define_all_fields acc.concat(pennlibmarc.get_call_number_xfacet_values(rec)) end - to_field "language_f_stored" do |rec, acc| + to_field "language_f" do |rec, acc| acc.concat(pennlibmarc.get_language_values(rec)) end @@ -190,19 +190,19 @@ def define_all_fields acc.concat(pennlibmarc.get_language_values(rec)) end - to_field "library_f_stored" do |rec, acc| + to_field "library_f" do |rec, acc| acc.concat(pennlibmarc.get_library_values(rec)) end - to_field "specific_location_f_stored" do |rec, acc| + to_field "specific_location_f" do |rec, acc| acc.concat(pennlibmarc.get_specific_location_values(rec)) end - to_field "classification_f_stored" do |rec, acc| + to_field "classification_f" do |rec, acc| acc.concat(pennlibmarc.get_classification_values(rec)) end - to_field "genre_f_stored" do |rec, acc| + to_field "genre_f" do |rec, acc| acc.concat(pennlibmarc.get_genre_values(rec)) end @@ -352,7 +352,7 @@ def define_all_fields acc << val if val end - to_field 'publication_date_f_stored' do |rec, acc, ctx| + to_field 'publication_date_f' do |rec, acc, ctx| val = ctx.clipboard.dig(:dates, :pub_date_decade) acc << val if val end @@ -367,7 +367,7 @@ def define_all_fields acc << val if val end - to_field "isbn_isxn_stored", extract_marc(%W{020az 022alz}, :separator=>nil) do |rec, acc| + to_field "isbn_isxn", extract_marc(%W{020az 022alz}, :separator=>nil) do |rec, acc| orig = acc.dup acc.map!{|x| StdNum::ISBN.allNormalizedValues(x)} acc << orig @@ -446,7 +446,7 @@ def define_mms_id end def define_access_facet - to_field "access_f_stored" do |rec, acc| + to_field "access_f" do |rec, acc| acc.concat(pennlibmarc.get_access_values(rec)) end end @@ -498,8 +498,8 @@ def define_record_source_facet end end - def define_full_text_link_text_a - to_field 'full_text_link_text_a' do |rec, acc| + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| result = pennlibmarc.get_full_text_link_values(rec) if result.present? acc << result.to_json diff --git a/app/models/hathi_indexer.rb b/app/models/hathi_indexer.rb index 0e006108..a65eff65 100644 --- a/app/models/hathi_indexer.rb +++ b/app/models/hathi_indexer.rb @@ -79,8 +79,8 @@ def hathi_link(id, type) [url, id, suffix].join end - def define_full_text_link_text_a - to_field 'full_text_link_text_a' do |rec, acc| + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| links = [] diff --git a/app/models/solr_document.rb b/app/models/solr_document.rb index 8dfed0d1..e6b8c36e 100644 --- a/app/models/solr_document.rb +++ b/app/models/solr_document.rb @@ -156,7 +156,7 @@ def all_doc_ids_for_cluster # returns the full text link field values for all the documents in the cluster def full_text_links_for_cluster_display structs = cluster_docs.map do |expanded_doc| - field_value = expanded_doc.fetch('full_text_link_text_a', []) + field_value = expanded_doc.fetch('full_text_link_a', []) if field_value.present? { id: expanded_doc.id, From 2910e8d744e86ffa0ab3b7482fce6f5e07547f3e Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Fri, 6 Dec 2019 14:33:22 -0500 Subject: [PATCH 02/77] add hooks for indexing other institutions records; and actual indexer for Duke --- app/models/base_indexer.rb | 6 +++ app/models/duke_indexer.rb | 101 +++++++++++++++++++++++++++++++++++++ index_solr_duke.sh | 3 ++ lib/tasks/penn_lib.rake | 24 +++++++++ 4 files changed, 134 insertions(+) create mode 100644 app/models/duke_indexer.rb create mode 100755 index_solr_duke.sh diff --git a/app/models/base_indexer.rb b/app/models/base_indexer.rb index 755e3166..7c46e7ad 100644 --- a/app/models/base_indexer.rb +++ b/app/models/base_indexer.rb @@ -5,6 +5,12 @@ module RecordSource PENN = 1 HATHI = 2 CRL = 3 + DUKE = 4 + CORNELL = 5 + BROWN = 6 + COLUMBIA = 7 + HARVARD = 8 + STANFORD = 9 end end diff --git a/app/models/duke_indexer.rb b/app/models/duke_indexer.rb new file mode 100644 index 00000000..9302f885 --- /dev/null +++ b/app/models/duke_indexer.rb @@ -0,0 +1,101 @@ + +class DukeIndexer < FranklinIndexer + + def define_record_source_id + to_field 'record_source_id' do |rec, acc| + acc << RecordSource::DUKE + end + end + + def define_record_source_facet + to_field 'record_source_f' do |rec, acc| + acc << 'Duke' + end + end + + def get_namespaced_id(rec) + id = get_001_id(rec) + id.blank? ? nil : id + end + + def link_to_source_context(rec) + system_id = get_001_id(rec) + "https://find.library.duke.edu/trln/#{system_id}" + end + + def define_mms_id + # no-op + end + + def define_id + to_field 'id' do |rec, acc, context| + id = get_namespaced_id(rec) + if id.nil? + context.skip!('Skipping institutional record with no 001') + end + acc.replace([id]) + end + end + + def get_001_id(rec) + id = rec.fields('001').first&.value&.strip + id.blank? ? nil : id + end + + def define_grouped_id + to_field 'grouped_id' do |rec, acc| + oclc_ids = get_oclc_id_values(rec) + if oclc_ids.size > 1 + puts 'Warning: Multiple OCLC IDs found, using the first one' + end + oclc_id = oclc_ids.first + id = get_namespaced_id(rec) + + prefix = oclc_id.present? ? "#{oclc_id}!" : '' + acc << "#{prefix}#{id}" + end + end + + def subfield_a_is_oclc(sf) + sf.code == 'a' && sf.value =~ /^(\(OCoLC\).*|[0-9]+$)/ + end + + def get_oclc_id_values(rec) + rec.fields('035') + .select { |f| f.any? { |sf| subfield_a_is_oclc(sf) } } + .take(1) + .flat_map do |field| + field.find_all { |sf| subfield_a_is_oclc(sf) }.map do |sf| + m = /^\s*(\(OCoLC\)[^1-9]*|0*)([0-9]+).*$/.match(sf.value) + if m + m[2] + end + end.compact + end + end + + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| + + links = [] + + links << { + linktext: 'View record in source context', + linkurl: link_to_source_context(rec) + } + + acc << links.to_json + + end + end + + def get_cluster_id(rec) + pennlibmarc.get_oclc_id_values(rec).first || begin + id = get_namespaced_id(rec) + digest = Digest::MD5.hexdigest(id) + # first 8 hex digits = first 4 bytes. construct an int out of that hex str. + digest[0,8].hex + end + end + +end diff --git a/index_solr_duke.sh b/index_solr_duke.sh new file mode 100755 index 00000000..6769ded0 --- /dev/null +++ b/index_solr_duke.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export MARC_SOURCE=DUKE +./index_solr.sh "$@" diff --git a/lib/tasks/penn_lib.rake b/lib/tasks/penn_lib.rake index 7171fb83..4737d6d9 100644 --- a/lib/tasks/penn_lib.rake +++ b/lib/tasks/penn_lib.rake @@ -27,6 +27,18 @@ namespace :pennlib do CrlIndexer.new when 'HATHI' HathiIndexer.new + when 'DUKE' + DukeIndexer.new + when 'CORNELL' + CornellIndexer.new + when 'BROWN' + BrownIndexer.new + when 'COLUMBIA' + ColumbiaIndexer.new + when 'HARVARD' + HarvardIndexer.new + when 'STANFORD' + StanfordIndexer.new else FranklinIndexer.new end @@ -44,6 +56,18 @@ namespace :pennlib do CrlIndexer.new when 'HATHI' HathiIndexer.new + when 'DUKE' + DukeIndexer.new + when 'CORNELL' + CornellIndexer.new + when 'BROWN' + BrownIndexer.new + when 'COLUMBIA' + ColumbiaIndexer.new + when 'HARVARD' + HarvardIndexer.new + when 'STANFORD' + StanfordIndexer.new else FranklinIndexer.new end From 99660773bd9a43eef9831b7e17bef5fc68de8fa2 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 10 Dec 2019 08:50:15 -0500 Subject: [PATCH 03/77] was using wrong oclc function for duke cluster_id --- app/models/duke_indexer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/duke_indexer.rb b/app/models/duke_indexer.rb index 9302f885..b99f4e99 100644 --- a/app/models/duke_indexer.rb +++ b/app/models/duke_indexer.rb @@ -90,7 +90,7 @@ def define_full_text_link_a end def get_cluster_id(rec) - pennlibmarc.get_oclc_id_values(rec).first || begin + get_oclc_id_values(rec).first || begin id = get_namespaced_id(rec) digest = Digest::MD5.hexdigest(id) # first 8 hex digits = first 4 bytes. construct an int out of that hex str. From 8757c016a3bae4ace0c708b1f613b22d202ff027 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 10 Dec 2019 08:50:36 -0500 Subject: [PATCH 04/77] added brown indexer, patterned after duke --- app/models/brown_indexer.rb | 103 ++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 app/models/brown_indexer.rb diff --git a/app/models/brown_indexer.rb b/app/models/brown_indexer.rb new file mode 100644 index 00000000..978f07d5 --- /dev/null +++ b/app/models/brown_indexer.rb @@ -0,0 +1,103 @@ + +class BrownIndexer < FranklinIndexer + + def define_record_source_id + to_field 'record_source_id' do |rec, acc| + acc << RecordSource::BROWN + end + end + + def define_record_source_facet + to_field 'record_source_f' do |rec, acc| + acc << 'Brown' + end + end + + def get_namespaced_id(rec) + id = get_local_system_id(rec) + id.blank? ? nil : "BROWN_#{id}" + end + + def link_to_source_context(rec) + system_id = get_local_system_id(rec) + "https://search.library.brown.edu/catalog/b#{system_id}" + end + + def define_mms_id + # no-op + end + + def define_id + to_field 'id' do |rec, acc, context| + id = get_namespaced_id(rec) + if id.nil? + context.skip!('Skipping institutional record with bad/no 907a') + end + acc.replace([id]) + end + end + + def get_001_id(rec) + id = rec.fields('001').first&.value&.strip + id.blank? ? nil : id + end + + def define_grouped_id + to_field 'grouped_id' do |rec, acc| + oclc_id = get_oclc_id(rec) + id = get_namespaced_id(rec) + + prefix = oclc_id.present? ? "#{oclc_id}!" : '' + acc << "#{prefix}#{id}" + end + end + + def get_oclc_id(rec) + candidate = get_001_id(rec) + m = /oc?[mn][^1-9]*([1-9][0-9]*)/.match(candidate) + m ? m[1] : nil + end + + def subfield_a_is_system_id(sf) + sf.code == 'a' && sf.value =~ /^\.b[0-9]+/ + end + + def get_local_system_id(rec) + rec.fields('907') + .select { |f| f.any? { |sf| subfield_a_is_system_id(sf) } } + .take(1) + .flat_map do |field| + field.find_all { |sf| subfield_a_is_system_id(sf) }.map do |sf| + m = /^\s*\.b([0-9]+).*$/.match(sf.value) + if m + m[1] + end + end.compact.first + end.compact.first + end + + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| + + links = [] + + links << { + linktext: 'View record in source context', + linkurl: link_to_source_context(rec) + } + + acc << links.to_json + + end + end + + def get_cluster_id(rec) + get_oclc_id(rec) || begin + id = get_namespaced_id(rec) + digest = Digest::MD5.hexdigest(id) + # first 8 hex digits = first 4 bytes. construct an int out of that hex str. + digest[0,8].hex + end + end + +end From 114d4cf0dc2f1cc2b9e35c3df5aeb13075abcad3 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 10 Dec 2019 09:13:03 -0500 Subject: [PATCH 05/77] add top-level indexing script for brown --- index_solr_brown.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 index_solr_brown.sh diff --git a/index_solr_brown.sh b/index_solr_brown.sh new file mode 100755 index 00000000..bee8258d --- /dev/null +++ b/index_solr_brown.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export MARC_SOURCE=BROWN +./index_solr.sh "$@" From 653f6bb6e820332eeef434d72b1729eb24d551c8 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 10 Dec 2019 10:53:39 -0500 Subject: [PATCH 06/77] add stanford indexer and top-level script --- app/models/stanford_indexer.rb | 104 +++++++++++++++++++++++++++++++++ index_solr_stanford.sh | 3 + 2 files changed, 107 insertions(+) create mode 100644 app/models/stanford_indexer.rb create mode 100755 index_solr_stanford.sh diff --git a/app/models/stanford_indexer.rb b/app/models/stanford_indexer.rb new file mode 100644 index 00000000..867da6cf --- /dev/null +++ b/app/models/stanford_indexer.rb @@ -0,0 +1,104 @@ + +class StanfordIndexer < FranklinIndexer + + def define_record_source_id + to_field 'record_source_id' do |rec, acc| + acc << RecordSource::STANFORD + end + end + + def define_record_source_facet + to_field 'record_source_f' do |rec, acc| + acc << 'Stanford' + end + end + + def get_namespaced_id(rec) + id = get_001_id(rec) + id.blank? ? nil : id + end + + def link_to_source_context(rec) + system_id = get_001_id(rec) + "https://searchworks.stanford.edu/view/#{system_id}" + end + + def define_mms_id + # no-op + end + + def define_id + to_field 'id' do |rec, acc, context| + id = get_namespaced_id(rec) + if id.nil? + context.skip!('Skipping institutional record with no 001') + end + if rec.fields('245').empty? + context.skip!('Skipping record with no 245; not a bib record?') + end + acc.replace([id]) + end + end + + def get_001_id(rec) + id = rec.fields('001').first&.value&.strip + id.blank? ? nil : id[1..-1] # strip the leading "a" present in Stanford 001 + end + + def define_grouped_id + to_field 'grouped_id' do |rec, acc| + oclc_ids = get_oclc_id_values(rec) + if oclc_ids.size > 1 + puts 'Warning: Multiple OCLC IDs found, using the first one' + end + oclc_id = oclc_ids.first + id = get_namespaced_id(rec) + + prefix = oclc_id.present? ? "#{oclc_id}!" : '' + acc << "#{prefix}#{id}" + end + end + + def subfield_a_is_oclc(sf) + sf.code == 'a' && sf.value =~ /^\(OCoLC(-M)?\).*/ + end + + def get_oclc_id_values(rec) + rec.fields('035') + .select { |f| f.any? { |sf| subfield_a_is_oclc(sf) } } + .take(1) + .flat_map do |field| + field.find_all { |sf| subfield_a_is_oclc(sf) }.map do |sf| + m = /^\s*\(OCoLC(-M)?\)[^1-9]*([1-9][0-9]*).*$/.match(sf.value) + if m + m[2] + end + end.compact + end + end + + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| + + links = [] + + links << { + linktext: 'View record in source context', + linkurl: link_to_source_context(rec) + } + + acc << links.to_json + + end + end + + def get_cluster_id(rec) + get_oclc_id_values(rec).first || begin + id = get_namespaced_id(rec) + digest = Digest::MD5.hexdigest(id) + # first 8 hex digits = first 4 bytes. construct an int out of that hex str. + digest[0,8].hex + end + end + +end diff --git a/index_solr_stanford.sh b/index_solr_stanford.sh new file mode 100755 index 00000000..fe785d9a --- /dev/null +++ b/index_solr_stanford.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export MARC_SOURCE=STANFORD +./index_solr.sh "$@" From 717852ece62708e3d220089c7f63dc6ae4b39a2e Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 10 Dec 2019 11:15:37 -0500 Subject: [PATCH 07/77] add cornell indexer and top-level script --- app/models/cornell_indexer.rb | 83 +++++++++++++++++++++++++++++++++++ index_solr_cornell.sh | 3 ++ 2 files changed, 86 insertions(+) create mode 100644 app/models/cornell_indexer.rb create mode 100755 index_solr_cornell.sh diff --git a/app/models/cornell_indexer.rb b/app/models/cornell_indexer.rb new file mode 100644 index 00000000..e408e6d3 --- /dev/null +++ b/app/models/cornell_indexer.rb @@ -0,0 +1,83 @@ + +class CornellIndexer < FranklinIndexer + + def define_record_source_id + to_field 'record_source_id' do |rec, acc| + acc << RecordSource::CORNELL + end + end + + def define_record_source_facet + to_field 'record_source_f' do |rec, acc| + acc << 'Cornell' + end + end + + def get_namespaced_id(rec) + id = get_001_id(rec) + id.blank? ? nil : "CORNELL_#{id}" + end + + def link_to_source_context(rec) + system_id = get_001_id(rec) + "https://newcatalog.library.cornell.edu/catalog/#{system_id}" + end + + def define_mms_id + # no-op + end + + def define_id + to_field 'id' do |rec, acc, context| + id = get_namespaced_id(rec) + if id.nil? + context.skip!('Skipping institutional record with no 001') + end + acc.replace([id]) + end + end + + def get_001_id(rec) + id = rec.fields('001').first&.value&.strip + id.blank? ? nil : id + end + + def define_grouped_id + to_field 'grouped_id' do |rec, acc| + oclc_ids = pennlibmarc.get_oclc_id_values(rec) + if oclc_ids.size > 1 + puts 'Warning: Multiple OCLC IDs found, using the first one' + end + oclc_id = oclc_ids.first + id = get_namespaced_id(rec) + + prefix = oclc_id.present? ? "#{oclc_id}!" : '' + acc << "#{prefix}#{id}" + end + end + + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| + + links = [] + + links << { + linktext: 'View record in source context', + linkurl: link_to_source_context(rec) + } + + acc << links.to_json + + end + end + + def get_cluster_id(rec) + pennlibmarc.get_oclc_id_values(rec).first || begin + id = get_namespaced_id(rec) + digest = Digest::MD5.hexdigest(id) + # first 8 hex digits = first 4 bytes. construct an int out of that hex str. + digest[0,8].hex + end + end + +end diff --git a/index_solr_cornell.sh b/index_solr_cornell.sh new file mode 100755 index 00000000..c7c8171e --- /dev/null +++ b/index_solr_cornell.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export MARC_SOURCE=CORNELL +./index_solr.sh "$@" From 92384123aa5d5663165cf36dab4d7f1d11ca2ae4 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 10 Dec 2019 11:29:08 -0500 Subject: [PATCH 08/77] add columbia indexer and top-level script --- app/models/columbia_indexer.rb | 83 ++++++++++++++++++++++++++++++++++ index_solr_columbia.sh | 3 ++ 2 files changed, 86 insertions(+) create mode 100644 app/models/columbia_indexer.rb create mode 100755 index_solr_columbia.sh diff --git a/app/models/columbia_indexer.rb b/app/models/columbia_indexer.rb new file mode 100644 index 00000000..89c09887 --- /dev/null +++ b/app/models/columbia_indexer.rb @@ -0,0 +1,83 @@ + +class ColumbiaIndexer < FranklinIndexer + + def define_record_source_id + to_field 'record_source_id' do |rec, acc| + acc << RecordSource::COLUMBIA + end + end + + def define_record_source_facet + to_field 'record_source_f' do |rec, acc| + acc << 'Columbia' + end + end + + def get_namespaced_id(rec) + id = get_001_id(rec) + id.blank? ? nil : "COLUMBIA_#{id}" + end + + def link_to_source_context(rec) + system_id = get_001_id(rec) + "https://clio.columbia.edu/catalog/#{system_id}" + end + + def define_mms_id + # no-op + end + + def define_id + to_field 'id' do |rec, acc, context| + id = get_namespaced_id(rec) + if id.nil? + context.skip!('Skipping institutional record with no 001') + end + acc.replace([id]) + end + end + + def get_001_id(rec) + id = rec.fields('001').first&.value&.strip + id.blank? ? nil : id + end + + def define_grouped_id + to_field 'grouped_id' do |rec, acc| + oclc_ids = pennlibmarc.get_oclc_id_values(rec) + if oclc_ids.size > 1 + puts 'Warning: Multiple OCLC IDs found, using the first one' + end + oclc_id = oclc_ids.first + id = get_namespaced_id(rec) + + prefix = oclc_id.present? ? "#{oclc_id}!" : '' + acc << "#{prefix}#{id}" + end + end + + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| + + links = [] + + links << { + linktext: 'View record in source context', + linkurl: link_to_source_context(rec) + } + + acc << links.to_json + + end + end + + def get_cluster_id(rec) + pennlibmarc.get_oclc_id_values(rec).first || begin + id = get_namespaced_id(rec) + digest = Digest::MD5.hexdigest(id) + # first 8 hex digits = first 4 bytes. construct an int out of that hex str. + digest[0,8].hex + end + end + +end diff --git a/index_solr_columbia.sh b/index_solr_columbia.sh new file mode 100755 index 00000000..f434dba3 --- /dev/null +++ b/index_solr_columbia.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export MARC_SOURCE=COLUMBIA +./index_solr.sh "$@" From f9c28f89254033b2f07208ebcf45bd5721ca1a98 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 10 Dec 2019 16:16:58 -0500 Subject: [PATCH 09/77] initial facet UI adaptation for 7 record sources --- app/controllers/catalog_controller.rb | 29 ++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 9fbf8b03..479d5c5b 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -134,7 +134,25 @@ def expire_session 'facet.mincount': 0, # fq: '{!tag=cluster}{!collapse field=cluster_id nullPolicy=expand size=5000000 min=record_source_id}', # this approach needs expand.field=cluster_id - fq: %q~{!tag=cluster}NOT ({!join from=cluster_id to=cluster_id v='record_source_f:"Penn"'} AND record_source_f:"HathiTrust")~, + fq: '{!bool tag=cluster must_not=$x1 must_not=$x2 must_not=$x3 must_not=$x4 must_not=$x5 must_not=$x6}', + x1: '{!bool filter=$j1 filter=$o1}', + x2: '{!bool filter=$j2 filter=$o2}', + x3: '{!bool filter=$j3 filter=$o3}', + x4: '{!bool filter=$j4 filter=$o4}', + x5: '{!bool filter=$j5 filter=$o5}', + x6: '{!bool filter=$j6 filter=$o6}', + j1: '{!join from=cluster_id to=cluster_id v=record_source_f:Brown}', + j2: '{!join from=cluster_id to=cluster_id v=record_source_f:Columbia}', + j3: '{!join from=cluster_id to=cluster_id v=record_source_f:Cornell}', + j4: '{!join from=cluster_id to=cluster_id v=record_source_f:Duke}', + j5: '{!join from=cluster_id to=cluster_id v=record_source_f:Penn}', + j6: '{!join from=cluster_id to=cluster_id v=record_source_f:Stanford}', + o1: 'record_source_f:(Columbia OR Cornell OR Duke OR Penn OR Stanford OR HathiTrust)', + o2: 'record_source_f:(Cornell OR Duke OR Penn OR Stanford OR HathiTrust)', + o3: 'record_source_f:(Duke OR Penn OR Stanford OR HathiTrust)', + o4: 'record_source_f:(Penn OR Stanford OR HathiTrust)', + o5: 'record_source_f:(Stanford OR HathiTrust)', + o6: 'record_source_f:HathiTrust', expand: 'true', 'expand.field': 'cluster_id', 'expand.q': '*:*', @@ -302,8 +320,13 @@ def expire_session 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} } config.add_facet_field 'record_source_f', label: 'Record Source', collapse: false, solr_params: @@MINCOUNT, query: { - 'HathiTrust' => { :label => 'HathiTrust', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'HathiTrust\\'}'}"}, - 'Penn' => { :label => 'Penn', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Penn\\'}'}"} + 'Brown' => { :label => 'Brown', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Brown\\'}'}"}, + 'Columbia' => { :label => 'Columbia', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Columbia\\'}'}"}, + 'Cornell' => { :label => 'Cornell', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Cornell\\'}'}"}, + 'Duke' => { :label => 'Duke', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Duke\\'}'}"}, + 'Penn' => { :label => 'Penn', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Penn\\'}'}"}, + 'Stanford' => { :label => 'Stanford', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Stanford\\'}'}"}, + 'HathiTrust' => { :label => 'HathiTrust', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'HathiTrust\\'}'}"} } config.add_facet_field 'format_f', label: 'Format', limit: 5, collapse: false, solr_params: @@MINCOUNT config.add_facet_field 'author_creator_f', label: 'Author/Creator', limit: 5, index_range: 'A'..'Z', collapse: false, solr_params: @@MINCOUNT From 972081d94806a1dc229bf55323c56eb3ff96e5ea Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 11 Dec 2019 12:19:54 -0500 Subject: [PATCH 10/77] TEMPORARY: query faceting (not field) for multi-source format_f we're getting a bunch of extra unique values from other institutions' records, blowing out the filterCache and killing performance. This is a temporary workaround until we can figure out how to fix format for other record sources --- app/controllers/catalog_controller.rb | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 479d5c5b..ed575fbf 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -328,7 +328,28 @@ def expire_session 'Stanford' => { :label => 'Stanford', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Stanford\\'}'}"}, 'HathiTrust' => { :label => 'HathiTrust', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'HathiTrust\\'}'}"} } - config.add_facet_field 'format_f', label: 'Format', limit: 5, collapse: false, solr_params: @@MINCOUNT + config.add_facet_field 'format_f', label: 'Format', limit: 5, collapse: false, solr_params: @@MINCOUNT, query: { + 'Book' => { :label => 'Book', :fq => "{!term f=format_f v='Book'}"}, + 'Government document' => { :label => 'Government document', :fq => "{!term f=format_f v='Government document'}"}, + 'Journal/Periodical' => { :label => 'Journal/Periodical', :fq => "{!term f=format_f v='Journal/Periodical'}"}, + 'Microformat' => { :label => 'Microformat', :fq => "{!term f=format_f v='Microformat'}"}, + 'Sound recording' => { :label => 'Sound recording', :fq => "{!term f=format_f v='Sound recording'}"}, + 'Musical score' => { :label => 'Musical score', :fq => "{!term f=format_f v='Musical score'}"}, + 'Video' => { :label => 'Video', :fq => "{!term f=format_f v='Video'}"}, + 'Conference/Event' => { :label => 'Conference/Event', :fq => "{!term f=format_f v='Conference/Event'}"}, + 'Manuscript' => { :label => 'Manuscript', :fq => "{!term f=format_f v='Manuscript'}"}, + 'Thesis/Dissertation' => { :label => 'Thesis/Dissertation', :fq => "{!term f=format_f v='Thesis/Dissertation'}"}, + 'Newspaper' => { :label => 'Newspaper', :fq => "{!term f=format_f v='Newspaper'}"}, + 'Datafile' => { :label => 'Datafile', :fq => "{!term f=format_f v='Datafile'}"}, + 'Image' => { :label => 'Image', :fq => "{!term f=format_f v='Image'}"}, + 'Website/Database' => { :label => 'Website/Database', :fq => "{!term f=format_f v='Website/Database'}"}, + 'Map/Atlas' => { :label => 'Map/Atlas', :fq => "{!term f=format_f v='Map/Atlas'}"}, + 'Archive' => { :label => 'Archive', :fq => "{!term f=format_f v='Archive'}"}, + 'Other' => { :label => 'Other', :fq => "{!term f=format_f v='Other'}"}, + 'Database & Article Index' => { :label => 'Database & Article Index', :fq => "{!term f=format_f v='Database & Article Index'}"}, + '3D object' => { :label => '3D object', :fq => "{!term f=format_f v='3D object'}"}, + 'Projected graphic' => { :label => 'Projected graphic', :fq => "{!term f=format_f v='Projected graphic'}"}, + } config.add_facet_field 'author_creator_f', label: 'Author/Creator', limit: 5, index_range: 'A'..'Z', collapse: false, solr_params: @@MINCOUNT #config.add_facet_field 'subject_taxonomy', label: 'Subject Taxonomy', collapse: false, :partial => 'blacklight/hierarchy/facet_hierarchy', :json_facet => @@SUBJECT_TAXONOMY, :top_level_field => 'toplevel_subject_f', :helper_method => :render_subcategories config.add_facet_field 'subject_f', label: 'Subject', limit: 5, index_range: 'A'..'Z', collapse: false, solr_params: @@MINCOUNT From e9ed6e9c5bac6181ed80e621333e4a7e6a5b9894 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 11 Dec 2019 12:47:31 -0500 Subject: [PATCH 11/77] add princeton indexer, facets, and top-level script --- app/controllers/catalog_controller.rb | 20 ++++--- app/models/base_indexer.rb | 1 + app/models/princeton_indexer.rb | 83 +++++++++++++++++++++++++++ index_solr_princeton.sh | 3 + lib/tasks/penn_lib.rake | 4 ++ 5 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 app/models/princeton_indexer.rb create mode 100755 index_solr_princeton.sh diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index ed575fbf..e6a7c9ef 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -134,25 +134,28 @@ def expire_session 'facet.mincount': 0, # fq: '{!tag=cluster}{!collapse field=cluster_id nullPolicy=expand size=5000000 min=record_source_id}', # this approach needs expand.field=cluster_id - fq: '{!bool tag=cluster must_not=$x1 must_not=$x2 must_not=$x3 must_not=$x4 must_not=$x5 must_not=$x6}', + fq: '{!bool tag=cluster must_not=$x1 must_not=$x2 must_not=$x3 must_not=$x4 must_not=$x5 must_not=$x6 must_not=$x7}', x1: '{!bool filter=$j1 filter=$o1}', x2: '{!bool filter=$j2 filter=$o2}', x3: '{!bool filter=$j3 filter=$o3}', x4: '{!bool filter=$j4 filter=$o4}', x5: '{!bool filter=$j5 filter=$o5}', x6: '{!bool filter=$j6 filter=$o6}', + x7: '{!bool filter=$j7 filter=$o7}', j1: '{!join from=cluster_id to=cluster_id v=record_source_f:Brown}', j2: '{!join from=cluster_id to=cluster_id v=record_source_f:Columbia}', j3: '{!join from=cluster_id to=cluster_id v=record_source_f:Cornell}', j4: '{!join from=cluster_id to=cluster_id v=record_source_f:Duke}', j5: '{!join from=cluster_id to=cluster_id v=record_source_f:Penn}', - j6: '{!join from=cluster_id to=cluster_id v=record_source_f:Stanford}', - o1: 'record_source_f:(Columbia OR Cornell OR Duke OR Penn OR Stanford OR HathiTrust)', - o2: 'record_source_f:(Cornell OR Duke OR Penn OR Stanford OR HathiTrust)', - o3: 'record_source_f:(Duke OR Penn OR Stanford OR HathiTrust)', - o4: 'record_source_f:(Penn OR Stanford OR HathiTrust)', - o5: 'record_source_f:(Stanford OR HathiTrust)', - o6: 'record_source_f:HathiTrust', + j6: '{!join from=cluster_id to=cluster_id v=record_source_f:Princeton}', + j7: '{!join from=cluster_id to=cluster_id v=record_source_f:Stanford}', + o1: 'record_source_f:(Columbia OR Cornell OR Duke OR Penn OR Princeton OR Stanford OR HathiTrust)', + o2: 'record_source_f:(Cornell OR Duke OR Penn OR Princeton OR Stanford OR HathiTrust)', + o3: 'record_source_f:(Duke OR Penn OR Princeton OR Stanford OR HathiTrust)', + o4: 'record_source_f:(Penn OR Princeton OR Stanford OR HathiTrust)', + o5: 'record_source_f:(Princeton OR Stanford OR HathiTrust)', + o6: 'record_source_f:(Stanford OR HathiTrust)', + o7: 'record_source_f:HathiTrust', expand: 'true', 'expand.field': 'cluster_id', 'expand.q': '*:*', @@ -325,6 +328,7 @@ def expire_session 'Cornell' => { :label => 'Cornell', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Cornell\\'}'}"}, 'Duke' => { :label => 'Duke', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Duke\\'}'}"}, 'Penn' => { :label => 'Penn', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Penn\\'}'}"}, + 'Princeton' => { :label => 'Princeton', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Princeton\\'}'}"}, 'Stanford' => { :label => 'Stanford', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Stanford\\'}'}"}, 'HathiTrust' => { :label => 'HathiTrust', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'HathiTrust\\'}'}"} } diff --git a/app/models/base_indexer.rb b/app/models/base_indexer.rb index 7c46e7ad..2c115718 100644 --- a/app/models/base_indexer.rb +++ b/app/models/base_indexer.rb @@ -11,6 +11,7 @@ module RecordSource COLUMBIA = 7 HARVARD = 8 STANFORD = 9 + PRINCETON = 10 end end diff --git a/app/models/princeton_indexer.rb b/app/models/princeton_indexer.rb new file mode 100644 index 00000000..79572f50 --- /dev/null +++ b/app/models/princeton_indexer.rb @@ -0,0 +1,83 @@ + +class PrincetonIndexer < FranklinIndexer + + def define_record_source_id + to_field 'record_source_id' do |rec, acc| + acc << RecordSource::PRINCETON + end + end + + def define_record_source_facet + to_field 'record_source_f' do |rec, acc| + acc << 'Princeton' + end + end + + def get_namespaced_id(rec) + id = get_001_id(rec) + id.blank? ? nil : "PRINCETON_#{id}" + end + + def link_to_source_context(rec) + system_id = get_001_id(rec) + "https://catalog.princeton.edu/catalog/#{system_id}" + end + + def define_mms_id + # no-op + end + + def define_id + to_field 'id' do |rec, acc, context| + id = get_namespaced_id(rec) + if id.nil? + context.skip!('Skipping institutional record with no 001') + end + acc.replace([id]) + end + end + + def get_001_id(rec) + id = rec.fields('001').first&.value&.strip + id.blank? ? nil : id + end + + def define_grouped_id + to_field 'grouped_id' do |rec, acc| + oclc_ids = pennlibmarc.get_oclc_id_values(rec) + if oclc_ids.size > 1 + puts 'Warning: Multiple OCLC IDs found, using the first one' + end + oclc_id = oclc_ids.first + id = get_namespaced_id(rec) + + prefix = oclc_id.present? ? "#{oclc_id}!" : '' + acc << "#{prefix}#{id}" + end + end + + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| + + links = [] + + links << { + linktext: 'View record in Princeton\'s catalog', + linkurl: link_to_source_context(rec) + } + + acc << links.to_json + + end + end + + def get_cluster_id(rec) + pennlibmarc.get_oclc_id_values(rec).first || begin + id = get_namespaced_id(rec) + digest = Digest::MD5.hexdigest(id) + # first 8 hex digits = first 4 bytes. construct an int out of that hex str. + digest[0,8].hex + end + end + +end diff --git a/index_solr_princeton.sh b/index_solr_princeton.sh new file mode 100755 index 00000000..25d3dcf4 --- /dev/null +++ b/index_solr_princeton.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export MARC_SOURCE=PRINCETON +./index_solr.sh "$@" diff --git a/lib/tasks/penn_lib.rake b/lib/tasks/penn_lib.rake index 4737d6d9..1cc65792 100644 --- a/lib/tasks/penn_lib.rake +++ b/lib/tasks/penn_lib.rake @@ -39,6 +39,8 @@ namespace :pennlib do HarvardIndexer.new when 'STANFORD' StanfordIndexer.new + when 'PRINCETON' + PrincetonIndexer.new else FranklinIndexer.new end @@ -68,6 +70,8 @@ namespace :pennlib do HarvardIndexer.new when 'STANFORD' StanfordIndexer.new + when 'PRINCETON' + PrincetonIndexer.new else FranklinIndexer.new end From 1478c0d65ec2071f8969af40d68462b38017cba4 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 11 Dec 2019 12:52:18 -0500 Subject: [PATCH 12/77] properly namespace Stanford's ids --- app/models/stanford_indexer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/stanford_indexer.rb b/app/models/stanford_indexer.rb index 867da6cf..9589195e 100644 --- a/app/models/stanford_indexer.rb +++ b/app/models/stanford_indexer.rb @@ -15,7 +15,7 @@ def define_record_source_facet def get_namespaced_id(rec) id = get_001_id(rec) - id.blank? ? nil : id + id.blank? ? nil : "STANFORD_#{id}" end def link_to_source_context(rec) From 90dc3f4fc6d5d40d293e65a746772f8281b258a6 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 11 Dec 2019 12:55:10 -0500 Subject: [PATCH 13/77] add better label for link to source context --- app/models/brown_indexer.rb | 2 +- app/models/columbia_indexer.rb | 2 +- app/models/cornell_indexer.rb | 2 +- app/models/duke_indexer.rb | 2 +- app/models/stanford_indexer.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/brown_indexer.rb b/app/models/brown_indexer.rb index 978f07d5..5727002e 100644 --- a/app/models/brown_indexer.rb +++ b/app/models/brown_indexer.rb @@ -82,7 +82,7 @@ def define_full_text_link_a links = [] links << { - linktext: 'View record in source context', + linktext: 'View record in Brown\'s catalog', linkurl: link_to_source_context(rec) } diff --git a/app/models/columbia_indexer.rb b/app/models/columbia_indexer.rb index 89c09887..198e2382 100644 --- a/app/models/columbia_indexer.rb +++ b/app/models/columbia_indexer.rb @@ -62,7 +62,7 @@ def define_full_text_link_a links = [] links << { - linktext: 'View record in source context', + linktext: 'View record in Columbia\'s catalog', linkurl: link_to_source_context(rec) } diff --git a/app/models/cornell_indexer.rb b/app/models/cornell_indexer.rb index e408e6d3..8c85e250 100644 --- a/app/models/cornell_indexer.rb +++ b/app/models/cornell_indexer.rb @@ -62,7 +62,7 @@ def define_full_text_link_a links = [] links << { - linktext: 'View record in source context', + linktext: 'View record in Cornell\'s catalog', linkurl: link_to_source_context(rec) } diff --git a/app/models/duke_indexer.rb b/app/models/duke_indexer.rb index b99f4e99..86f51f3d 100644 --- a/app/models/duke_indexer.rb +++ b/app/models/duke_indexer.rb @@ -80,7 +80,7 @@ def define_full_text_link_a links = [] links << { - linktext: 'View record in source context', + linktext: 'View record in Duke\'s catalog', linkurl: link_to_source_context(rec) } diff --git a/app/models/stanford_indexer.rb b/app/models/stanford_indexer.rb index 9589195e..c1baf3d7 100644 --- a/app/models/stanford_indexer.rb +++ b/app/models/stanford_indexer.rb @@ -83,7 +83,7 @@ def define_full_text_link_a links = [] links << { - linktext: 'View record in source context', + linktext: 'View record in Stanford\'s catalog', linkurl: link_to_source_context(rec) } From fb26988c1f35940f1d387ef1af745058850d9fad Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Thu, 12 Dec 2019 13:21:55 -0500 Subject: [PATCH 14/77] dynamic record source preferences, etc. --- app/controllers/catalog_controller.rb | 56 ++++++++++++++++----------- app/models/search_builder.rb | 25 +++++++++++- 2 files changed, 58 insertions(+), 23 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index e6a7c9ef..a8076f28 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -81,6 +81,18 @@ def expire_session end end + def self.generate_cluster_fq(index, limit) + return '*:*' if limit < 1 + ret = '{!bool tag=cluster ex=cluster' + i = 0 + loop do + ret += " must_not=$x#{i}_#{index}" + break if i >= limit + i += 1 + end + ret + '}' + end + configure_blacklight do |config| # default advanced config values config.advanced_search ||= Blacklight::OpenStructWithHashAccess.new @@ -134,28 +146,7 @@ def expire_session 'facet.mincount': 0, # fq: '{!tag=cluster}{!collapse field=cluster_id nullPolicy=expand size=5000000 min=record_source_id}', # this approach needs expand.field=cluster_id - fq: '{!bool tag=cluster must_not=$x1 must_not=$x2 must_not=$x3 must_not=$x4 must_not=$x5 must_not=$x6 must_not=$x7}', - x1: '{!bool filter=$j1 filter=$o1}', - x2: '{!bool filter=$j2 filter=$o2}', - x3: '{!bool filter=$j3 filter=$o3}', - x4: '{!bool filter=$j4 filter=$o4}', - x5: '{!bool filter=$j5 filter=$o5}', - x6: '{!bool filter=$j6 filter=$o6}', - x7: '{!bool filter=$j7 filter=$o7}', - j1: '{!join from=cluster_id to=cluster_id v=record_source_f:Brown}', - j2: '{!join from=cluster_id to=cluster_id v=record_source_f:Columbia}', - j3: '{!join from=cluster_id to=cluster_id v=record_source_f:Cornell}', - j4: '{!join from=cluster_id to=cluster_id v=record_source_f:Duke}', - j5: '{!join from=cluster_id to=cluster_id v=record_source_f:Penn}', - j6: '{!join from=cluster_id to=cluster_id v=record_source_f:Princeton}', - j7: '{!join from=cluster_id to=cluster_id v=record_source_f:Stanford}', - o1: 'record_source_f:(Columbia OR Cornell OR Duke OR Penn OR Princeton OR Stanford OR HathiTrust)', - o2: 'record_source_f:(Cornell OR Duke OR Penn OR Princeton OR Stanford OR HathiTrust)', - o3: 'record_source_f:(Duke OR Penn OR Princeton OR Stanford OR HathiTrust)', - o4: 'record_source_f:(Penn OR Princeton OR Stanford OR HathiTrust)', - o5: 'record_source_f:(Princeton OR Stanford OR HathiTrust)', - o6: 'record_source_f:(Stanford OR HathiTrust)', - o7: 'record_source_f:HathiTrust', + # MOVE TO FACET: fq: '{!bool tag=cluster must_not=$x1 must_not=$x2 must_not=$x3 must_not=$x4 must_not=$x5 must_not=$x6 must_not=$x7}', expand: 'true', 'expand.field': 'cluster_id', 'expand.q': '*:*', @@ -322,6 +313,17 @@ def expire_session 'Online' => { :label => 'Online', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'Online\\'}'}"}, 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} } + config.add_facet_field 'cluster', label: 'Cluster Prioritize', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { + 'Brown' => { :label => 'Brown', :fq => generate_cluster_fq(0, 6)}, + 'Columbia' => { :label => 'Columbia', :fq => generate_cluster_fq(1, 6)}, + 'Cornell' => { :label => 'Cornell', :fq => generate_cluster_fq(2, 6)}, + 'Duke' => { :label => 'Duke', :fq => generate_cluster_fq(3, 6)}, + 'Penn' => { :label => 'Penn', :fq => generate_cluster_fq(4, 6)}, + 'Princeton' => { :label => 'Princeton', :fq => generate_cluster_fq(5, 6)}, + 'Stanford' => { :label => 'Stanford', :fq => generate_cluster_fq(6, 6)}, + 'HathiTrust' => { :label => 'HathiTrust', :fq => generate_cluster_fq(7, 6)}, + 'Dynamic' => { :label => 'Dynamic', :fq => '*:*'} + } config.add_facet_field 'record_source_f', label: 'Record Source', collapse: false, solr_params: @@MINCOUNT, query: { 'Brown' => { :label => 'Brown', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Brown\\'}'}"}, 'Columbia' => { :label => 'Columbia', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Columbia\\'}'}"}, @@ -332,6 +334,16 @@ def expire_session 'Stanford' => { :label => 'Stanford', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Stanford\\'}'}"}, 'HathiTrust' => { :label => 'HathiTrust', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'HathiTrust\\'}'}"} } + config.add_facet_field 'record_source_exclusive', label: 'Record Source Exclusive', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { + 'Brown' => { :label => 'Brown', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Brown}'}, + 'Columbia' => { :label => 'Columbia', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Columbia}'}, + 'Cornell' => { :label => 'Cornell', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Cornell}'}, + 'Duke' => { :label => 'Duke', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Duke}'}, + 'Penn' => { :label => 'Penn', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Penn}'}, + 'Princeton' => { :label => 'Princeton', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Princeton}'}, + 'Stanford' => { :label => 'Stanford', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Stanford}'}, + 'HathiTrust' => { :label => 'HathiTrust', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=HathiTrust}'}, + } config.add_facet_field 'format_f', label: 'Format', limit: 5, collapse: false, solr_params: @@MINCOUNT, query: { 'Book' => { :label => 'Book', :fq => "{!term f=format_f v='Book'}"}, 'Government document' => { :label => 'Government document', :fq => "{!term f=format_f v='Government document'}"}, diff --git a/app/models/search_builder.rb b/app/models/search_builder.rb index d1d77863..5cbe0428 100644 --- a/app/models/search_builder.rb +++ b/app/models/search_builder.rb @@ -3,7 +3,7 @@ class SearchBuilder < Blacklight::SearchBuilder include Blacklight::Solr::SearchBuilderBehavior include BlacklightAdvancedSearch::AdvancedSearchBuilder self.default_processor_chain += [:add_advanced_search_to_solr, :override_sort_when_q_is_empty, :lowercase_expert_boolean_operators, - :add_left_anchored_title, :add_routing_hash] + :add_left_anchored_title, :add_routing_hash, :add_cluster_params] include BlacklightRangeLimit::RangeLimitBuilder include BlacklightSolrplugins::FacetFieldsQueryFilter @@ -68,6 +68,29 @@ def with(blacklight_params = {}) super(params_copy) end + @@record_sources = ['Brown', 'Columbia', 'Cornell', 'Duke', 'Penn', 'Princeton', 'Stanford', 'HathiTrust'] + + def add_cluster_params(solr_parameters) + if 'Dynamic' == blacklight_params.dig(:f, :cluster, 0) + solr_parameters[:fq] << '{!collapse tag=cluster ex=cluster field=cluster_id nullPolicy=expand size=3000000}' + end + source_idx = 0 + loop do + other_sources = @@record_sources.dup + cluster = other_sources.delete_at(source_idx) + clause = 0 + loop do + solr_parameters["x#{clause}_#{source_idx}"] = "{!bool filter=$j#{clause}_#{source_idx} filter=$o#{clause}_#{source_idx}}" + solr_parameters["j#{clause}_#{source_idx}"] = "{!join from=cluster_id to=cluster_id v=record_source_f:#{cluster}}" + solr_parameters["o#{clause}_#{source_idx}"] = "record_source_f:(#{other_sources.join(' OR ')})" + break if other_sources.length == 1 + cluster = other_sources.shift + clause += 1 + end + break unless (source_idx += 1) < @@record_sources.length + end + end + def add_left_anchored_title(solr_parameters) qq = solr_parameters[:qq] return if qq.nil? || !qq.present? From 36e3631c5d3d94674abfac39003c2c07affe20f0 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 16 Dec 2019 22:51:16 -0500 Subject: [PATCH 15/77] change join filter queries to match solr warming --- app/controllers/catalog_controller.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index a8076f28..cbad5bd0 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -310,7 +310,7 @@ def self.generate_cluster_fq(index, limit) 'Other' => { :label => 'Other', :fq => "{!tag=azlist ex=azlist}title_xfacet:/[ -`{-~].*/"} } config.add_facet_field 'access_f', label: 'Access', collapse: false, solr_params: @@MINCOUNT, query: { - 'Online' => { :label => 'Online', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'Online\\'}'}"}, + 'Online' => { :label => 'Online', :fq => "{!join from=cluster_id to=cluster_id v=access_f:Online}"}, 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} } config.add_facet_field 'cluster', label: 'Cluster Prioritize', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { @@ -325,14 +325,14 @@ def self.generate_cluster_fq(index, limit) 'Dynamic' => { :label => 'Dynamic', :fq => '*:*'} } config.add_facet_field 'record_source_f', label: 'Record Source', collapse: false, solr_params: @@MINCOUNT, query: { - 'Brown' => { :label => 'Brown', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Brown\\'}'}"}, - 'Columbia' => { :label => 'Columbia', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Columbia\\'}'}"}, - 'Cornell' => { :label => 'Cornell', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Cornell\\'}'}"}, - 'Duke' => { :label => 'Duke', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Duke\\'}'}"}, - 'Penn' => { :label => 'Penn', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Penn\\'}'}"}, - 'Princeton' => { :label => 'Princeton', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Princeton\\'}'}"}, - 'Stanford' => { :label => 'Stanford', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'Stanford\\'}'}"}, - 'HathiTrust' => { :label => 'HathiTrust', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=record_source_f v=\\'HathiTrust\\'}'}"} + 'Brown' => { :label => 'Brown', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Brown}"}, + 'Columbia' => { :label => 'Columbia', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Columbia}"}, + 'Cornell' => { :label => 'Cornell', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Cornell}"}, + 'Duke' => { :label => 'Duke', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Duke}"}, + 'Penn' => { :label => 'Penn', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Penn}"}, + 'Princeton' => { :label => 'Princeton', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Princeton}"}, + 'Stanford' => { :label => 'Stanford', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Stanford}"}, + 'HathiTrust' => { :label => 'HathiTrust', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:HathiTrust}"} } config.add_facet_field 'record_source_exclusive', label: 'Record Source Exclusive', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { 'Brown' => { :label => 'Brown', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Brown}'}, From e2336dbe827b0363952a8afe6f408bc2562db10e Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 18 Dec 2019 12:37:12 -0500 Subject: [PATCH 16/77] add tiered dedupe respecting asserted encoding level --- app/controllers/catalog_controller.rb | 39 ++++++++++++++++++++++----- app/models/search_builder.rb | 4 +++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index cbad5bd0..14c6e6e9 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -81,14 +81,29 @@ def expire_session end end - def self.generate_cluster_fq(index, limit) + def self.generate_cluster_fq(index, limit, elvl = false) return '*:*' if limit < 1 ret = '{!bool tag=cluster ex=cluster' - i = 0 - loop do - ret += " must_not=$x#{i}_#{index}" - break if i >= limit - i += 1 + if elvl + i = 0 + loop do + ret += " must_not=$y#{i}_#{index}" + break if i >= limit + i += 1 + end + i = 0 + loop do + ret += " must_not=$z#{i}_#{index}" + break if i >= limit + i += 1 + end + else + i = 0 + loop do + ret += " must_not=$x#{i}_#{index}" + break if i >= limit + i += 1 + end end ret + '}' end @@ -313,6 +328,10 @@ def self.generate_cluster_fq(index, limit) 'Online' => { :label => 'Online', :fq => "{!join from=cluster_id to=cluster_id v=access_f:Online}"}, 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} } + config.add_facet_field 'elvl', label: 'Encoding Level', collapse: false, solr_params: @@MINCOUNT, query: { + 'Full' => { :label => 'Full', :fq => "elvl_rank_isort:0"}, + 'Other' => { :label => 'Other', :fq => '{!bool must_not=elvl_rank_isort:0}'} + } config.add_facet_field 'cluster', label: 'Cluster Prioritize', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { 'Brown' => { :label => 'Brown', :fq => generate_cluster_fq(0, 6)}, 'Columbia' => { :label => 'Columbia', :fq => generate_cluster_fq(1, 6)}, @@ -322,6 +341,14 @@ def self.generate_cluster_fq(index, limit) 'Princeton' => { :label => 'Princeton', :fq => generate_cluster_fq(5, 6)}, 'Stanford' => { :label => 'Stanford', :fq => generate_cluster_fq(6, 6)}, 'HathiTrust' => { :label => 'HathiTrust', :fq => generate_cluster_fq(7, 6)}, + 'Brown-e' => { :label => 'Brown-e', :fq => generate_cluster_fq(0, 6, true)}, + 'Columbia-e' => { :label => 'Columbia-e', :fq => generate_cluster_fq(1, 6, true)}, + 'Cornell-e' => { :label => 'Cornell-e', :fq => generate_cluster_fq(2, 6, true)}, + 'Duke-e' => { :label => 'Duke-e', :fq => generate_cluster_fq(3, 6, true)}, + 'Penn-e' => { :label => 'Penn-e', :fq => generate_cluster_fq(4, 6, true)}, + 'Princeton-e' => { :label => 'Princeton-e', :fq => generate_cluster_fq(5, 6, true)}, + 'Stanford-e' => { :label => 'Stanford-e', :fq => generate_cluster_fq(6, 6, true)}, + 'HathiTrust-e' => { :label => 'HathiTrust-e', :fq => generate_cluster_fq(7, 6, true)}, 'Dynamic' => { :label => 'Dynamic', :fq => '*:*'} } config.add_facet_field 'record_source_f', label: 'Record Source', collapse: false, solr_params: @@MINCOUNT, query: { diff --git a/app/models/search_builder.rb b/app/models/search_builder.rb index 5cbe0428..3014f730 100644 --- a/app/models/search_builder.rb +++ b/app/models/search_builder.rb @@ -81,8 +81,12 @@ def add_cluster_params(solr_parameters) clause = 0 loop do solr_parameters["x#{clause}_#{source_idx}"] = "{!bool filter=$j#{clause}_#{source_idx} filter=$o#{clause}_#{source_idx}}" + solr_parameters["y#{clause}_#{source_idx}"] = "{!bool filter=$k#{clause}_#{source_idx} filter=$o#{clause}_#{source_idx}}" + solr_parameters["z#{clause}_#{source_idx}"] = "{!bool filter=$j#{clause}_#{source_idx} filter=$p#{clause}_#{source_idx}}" solr_parameters["j#{clause}_#{source_idx}"] = "{!join from=cluster_id to=cluster_id v=record_source_f:#{cluster}}" + solr_parameters["k#{clause}_#{source_idx}"] = "{!join from=cluster_id to=cluster_id v='{!bool filter=record_source_f:#{cluster} filter=elvl_rank_isort:0}'}" solr_parameters["o#{clause}_#{source_idx}"] = "record_source_f:(#{other_sources.join(' OR ')})" + solr_parameters["p#{clause}_#{source_idx}"] = "{!bool filter='record_source_f:(#{other_sources.join(' OR ')})' must_not=elvl_rank_isort:0}" break if other_sources.length == 1 cluster = other_sources.shift clause += 1 From cb777b0e5970cc5ade2ffc92bb98aa17f85af61f Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Thu, 19 Dec 2019 10:34:12 -0500 Subject: [PATCH 17/77] temporary(?) disable/reorder some facets for public demo --- app/controllers/catalog_controller.rb | 72 +++++++++++++-------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 14c6e6e9..f5fe1f44 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -324,15 +324,11 @@ def self.generate_cluster_fq(index, limit, elvl = false) 'Z' => { :label => 'Z', :fq => "{!prefix tag=azlist ex=azlist f=title_xfacet v='z'}"}, 'Other' => { :label => 'Other', :fq => "{!tag=azlist ex=azlist}title_xfacet:/[ -`{-~].*/"} } - config.add_facet_field 'access_f', label: 'Access', collapse: false, solr_params: @@MINCOUNT, query: { - 'Online' => { :label => 'Online', :fq => "{!join from=cluster_id to=cluster_id v=access_f:Online}"}, - 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} - } - config.add_facet_field 'elvl', label: 'Encoding Level', collapse: false, solr_params: @@MINCOUNT, query: { - 'Full' => { :label => 'Full', :fq => "elvl_rank_isort:0"}, - 'Other' => { :label => 'Other', :fq => '{!bool must_not=elvl_rank_isort:0}'} - } - config.add_facet_field 'cluster', label: 'Cluster Prioritize', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { +# config.add_facet_field 'access_f', label: 'Access', collapse: false, solr_params: @@MINCOUNT, query: { +# 'Online' => { :label => 'Online', :fq => "{!join from=cluster_id to=cluster_id v=access_f:Online}"}, +# 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} +# } + config.add_facet_field 'cluster', label: 'Prioritize your institution', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { 'Brown' => { :label => 'Brown', :fq => generate_cluster_fq(0, 6)}, 'Columbia' => { :label => 'Columbia', :fq => generate_cluster_fq(1, 6)}, 'Cornell' => { :label => 'Cornell', :fq => generate_cluster_fq(2, 6)}, @@ -341,17 +337,27 @@ def self.generate_cluster_fq(index, limit, elvl = false) 'Princeton' => { :label => 'Princeton', :fq => generate_cluster_fq(5, 6)}, 'Stanford' => { :label => 'Stanford', :fq => generate_cluster_fq(6, 6)}, 'HathiTrust' => { :label => 'HathiTrust', :fq => generate_cluster_fq(7, 6)}, - 'Brown-e' => { :label => 'Brown-e', :fq => generate_cluster_fq(0, 6, true)}, - 'Columbia-e' => { :label => 'Columbia-e', :fq => generate_cluster_fq(1, 6, true)}, - 'Cornell-e' => { :label => 'Cornell-e', :fq => generate_cluster_fq(2, 6, true)}, - 'Duke-e' => { :label => 'Duke-e', :fq => generate_cluster_fq(3, 6, true)}, - 'Penn-e' => { :label => 'Penn-e', :fq => generate_cluster_fq(4, 6, true)}, - 'Princeton-e' => { :label => 'Princeton-e', :fq => generate_cluster_fq(5, 6, true)}, - 'Stanford-e' => { :label => 'Stanford-e', :fq => generate_cluster_fq(6, 6, true)}, - 'HathiTrust-e' => { :label => 'HathiTrust-e', :fq => generate_cluster_fq(7, 6, true)}, +# 'Brown-e' => { :label => 'Brown-e', :fq => generate_cluster_fq(0, 6, true)}, +# 'Columbia-e' => { :label => 'Columbia-e', :fq => generate_cluster_fq(1, 6, true)}, +# 'Cornell-e' => { :label => 'Cornell-e', :fq => generate_cluster_fq(2, 6, true)}, +# 'Duke-e' => { :label => 'Duke-e', :fq => generate_cluster_fq(3, 6, true)}, +# 'Penn-e' => { :label => 'Penn-e', :fq => generate_cluster_fq(4, 6, true)}, +# 'Princeton-e' => { :label => 'Princeton-e', :fq => generate_cluster_fq(5, 6, true)}, +# 'Stanford-e' => { :label => 'Stanford-e', :fq => generate_cluster_fq(6, 6, true)}, +# 'HathiTrust-e' => { :label => 'HathiTrust-e', :fq => generate_cluster_fq(7, 6, true)}, 'Dynamic' => { :label => 'Dynamic', :fq => '*:*'} } - config.add_facet_field 'record_source_f', label: 'Record Source', collapse: false, solr_params: @@MINCOUNT, query: { + config.add_facet_field 'record_source_exclusive', label: 'See exclusively records from', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { + 'Brown' => { :label => 'Brown', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Brown}'}, + 'Columbia' => { :label => 'Columbia', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Columbia}'}, + 'Cornell' => { :label => 'Cornell', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Cornell}'}, + 'Duke' => { :label => 'Duke', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Duke}'}, + 'Penn' => { :label => 'Penn', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Penn}'}, + 'Princeton' => { :label => 'Princeton', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Princeton}'}, + 'Stanford' => { :label => 'Stanford', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Stanford}'}, + 'HathiTrust' => { :label => 'HathiTrust', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=HathiTrust}'}, + } + config.add_facet_field 'record_source_f', label: 'See any records clustered with holdings from', collapse: false, solr_params: @@MINCOUNT, query: { 'Brown' => { :label => 'Brown', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Brown}"}, 'Columbia' => { :label => 'Columbia', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Columbia}"}, 'Cornell' => { :label => 'Cornell', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Cornell}"}, @@ -361,15 +367,9 @@ def self.generate_cluster_fq(index, limit, elvl = false) 'Stanford' => { :label => 'Stanford', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Stanford}"}, 'HathiTrust' => { :label => 'HathiTrust', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:HathiTrust}"} } - config.add_facet_field 'record_source_exclusive', label: 'Record Source Exclusive', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { - 'Brown' => { :label => 'Brown', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Brown}'}, - 'Columbia' => { :label => 'Columbia', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Columbia}'}, - 'Cornell' => { :label => 'Cornell', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Cornell}'}, - 'Duke' => { :label => 'Duke', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Duke}'}, - 'Penn' => { :label => 'Penn', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Penn}'}, - 'Princeton' => { :label => 'Princeton', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Princeton}'}, - 'Stanford' => { :label => 'Stanford', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Stanford}'}, - 'HathiTrust' => { :label => 'HathiTrust', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=HathiTrust}'}, + config.add_facet_field 'elvl', label: 'Encoding Level', collapse: false, solr_params: @@MINCOUNT, query: { + 'Full' => { :label => 'Full', :fq => "elvl_rank_isort:0"}, + 'Other' => { :label => 'Other', :fq => '{!bool must_not=elvl_rank_isort:0}'} } config.add_facet_field 'format_f', label: 'Format', limit: 5, collapse: false, solr_params: @@MINCOUNT, query: { 'Book' => { :label => 'Book', :fq => "{!term f=format_f v='Book'}"}, @@ -397,17 +397,17 @@ def self.generate_cluster_fq(index, limit, elvl = false) #config.add_facet_field 'subject_taxonomy', label: 'Subject Taxonomy', collapse: false, :partial => 'blacklight/hierarchy/facet_hierarchy', :json_facet => @@SUBJECT_TAXONOMY, :top_level_field => 'toplevel_subject_f', :helper_method => :render_subcategories config.add_facet_field 'subject_f', label: 'Subject', limit: 5, index_range: 'A'..'Z', collapse: false, solr_params: @@MINCOUNT config.add_facet_field 'language_f', label: 'Language', limit: 5, collapse: false, solr_params: @@MINCOUNT - config.add_facet_field 'library_f', label: 'Library', limit: 5, collapse: false, solr_params: @@MINCOUNT - config.add_facet_field 'specific_location_f', label: 'Specific location', limit: 5, solr_params: @@MINCOUNT +# config.add_facet_field 'library_f', label: 'Library', limit: 5, collapse: false, solr_params: @@MINCOUNT +# config.add_facet_field 'specific_location_f', label: 'Specific location', limit: 5, solr_params: @@MINCOUNT config.add_facet_field 'publication_date_f', label: 'Publication date', limit: 5, collapse: false, solr_params: @@MINCOUNT - config.add_facet_field 'classification_f', label: 'Classification', limit: 5, collapse: false, solr_params: @@MINCOUNT +# config.add_facet_field 'classification_f', label: 'Classification', limit: 5, collapse: false, solr_params: @@MINCOUNT config.add_facet_field 'genre_f', label: 'Form/Genre', limit: 5, solr_params: @@MINCOUNT - config.add_facet_field 'recently_added_f', label: 'Recently added', solr_params: @@MINCOUNT, :query => { - :within_90_days => { label: 'Within 90 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (90 * SECONDS_PER_DAY) } TO *]" }, - :within_60_days => { label: 'Within 60 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (60 * SECONDS_PER_DAY) } TO *]" }, - :within_30_days => { label: 'Within 30 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (30 * SECONDS_PER_DAY) } TO *]" }, - :within_15_days => { label: 'Within 15 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (15 * SECONDS_PER_DAY) } TO *]" }, - } +# config.add_facet_field 'recently_added_f', label: 'Recently added', solr_params: @@MINCOUNT, :query => { +# :within_90_days => { label: 'Within 90 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (90 * SECONDS_PER_DAY) } TO *]" }, +# :within_60_days => { label: 'Within 60 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (60 * SECONDS_PER_DAY) } TO *]" }, +# :within_30_days => { label: 'Within 30 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (30 * SECONDS_PER_DAY) } TO *]" }, +# :within_15_days => { label: 'Within 15 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (15 * SECONDS_PER_DAY) } TO *]" }, +# } #config.add_facet_field 'example_pivot_field', label: 'Pivot Field', :pivot => ['format_f', 'language_f'] # config.add_facet_field 'example_query_facet_field', label: 'Publish Date', :query => { From ab64f6f888da88766f1103ec6870bdb5692ea2ae Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Fri, 20 Dec 2019 13:49:47 -0500 Subject: [PATCH 18/77] change catalog landing page Franklin text for POD explanation --- app/views/catalog/_home_text.html.erb | 36 ++------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/app/views/catalog/_home_text.html.erb b/app/views/catalog/_home_text.html.erb index bb2e685a..1871ec15 100644 --- a/app/views/catalog/_home_text.html.erb +++ b/app/views/catalog/_home_text.html.erb @@ -1,40 +1,8 @@
-

Franklin Catalog  lets you:

+

POD (Platform for Open Discovery)  proof of concept test instance

    -
  • Find print books, ebooks, journal and newspaper titles, sound recordings, videos, conference proceedings, music scores, manuscripts, and other resources owned or subscribed to by the Penn Libraries.
    • -
  • See location and status information for print works, and links to online materials.
    • -
  • Place requests (when logged in).
    • -
  • Find and access public-domain books and journals from HathiTrust.
    • -
- -

Other Penn Resources

- - - - -

Search and Request from Other Libraries

- -
    - -
  • BorrowDirect/E-ZBorrow (Search multiple library collections simultaneously).
    • - -
  • WorldCat (Union catalog of the holdings of libraries worldwide).
    • - +
  • This proof-of-concept test interface is intended only for testing by librarians and showcasing shared indexing issues. It does not represent an end-user experience. We used Penn’s blacklight-test instance to load the data, so it does have some of the end-user UI of the Franklin catalog, but that was just for speed and convenience of getting a demo running.
From b433349b9976cd3c59190ca7429ca515d913557e Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Fri, 20 Dec 2019 16:09:23 -0500 Subject: [PATCH 19/77] add Harvard --- app/controllers/catalog_controller.rb | 19 +++--- app/models/harvard_indexer.rb | 83 +++++++++++++++++++++++++++ app/models/search_builder.rb | 10 ++-- index_solr_harvard.sh | 3 + 4 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 app/models/harvard_indexer.rb create mode 100755 index_solr_harvard.sh diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index f5fe1f44..a0d533c4 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -329,14 +329,15 @@ def self.generate_cluster_fq(index, limit, elvl = false) # 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} # } config.add_facet_field 'cluster', label: 'Prioritize your institution', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { - 'Brown' => { :label => 'Brown', :fq => generate_cluster_fq(0, 6)}, - 'Columbia' => { :label => 'Columbia', :fq => generate_cluster_fq(1, 6)}, - 'Cornell' => { :label => 'Cornell', :fq => generate_cluster_fq(2, 6)}, - 'Duke' => { :label => 'Duke', :fq => generate_cluster_fq(3, 6)}, - 'Penn' => { :label => 'Penn', :fq => generate_cluster_fq(4, 6)}, - 'Princeton' => { :label => 'Princeton', :fq => generate_cluster_fq(5, 6)}, - 'Stanford' => { :label => 'Stanford', :fq => generate_cluster_fq(6, 6)}, - 'HathiTrust' => { :label => 'HathiTrust', :fq => generate_cluster_fq(7, 6)}, + 'Brown' => { :label => 'Brown', :fq => generate_cluster_fq(0, 7)}, + 'Columbia' => { :label => 'Columbia', :fq => generate_cluster_fq(1, 7)}, + 'Cornell' => { :label => 'Cornell', :fq => generate_cluster_fq(2, 7)}, + 'Duke' => { :label => 'Duke', :fq => generate_cluster_fq(3, 7)}, + 'Harvard' => { :label => 'Harvard', :fq => generate_cluster_fq(4, 7)}, + 'Penn' => { :label => 'Penn', :fq => generate_cluster_fq(5, 7)}, + 'Princeton' => { :label => 'Princeton', :fq => generate_cluster_fq(6, 7)}, + 'Stanford' => { :label => 'Stanford', :fq => generate_cluster_fq(7, 7)}, + 'HathiTrust' => { :label => 'HathiTrust', :fq => generate_cluster_fq(8, 7)}, # 'Brown-e' => { :label => 'Brown-e', :fq => generate_cluster_fq(0, 6, true)}, # 'Columbia-e' => { :label => 'Columbia-e', :fq => generate_cluster_fq(1, 6, true)}, # 'Cornell-e' => { :label => 'Cornell-e', :fq => generate_cluster_fq(2, 6, true)}, @@ -352,6 +353,7 @@ def self.generate_cluster_fq(index, limit, elvl = false) 'Columbia' => { :label => 'Columbia', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Columbia}'}, 'Cornell' => { :label => 'Cornell', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Cornell}'}, 'Duke' => { :label => 'Duke', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Duke}'}, + 'Harvard' => { :label => 'Harvard', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Harvard}'}, 'Penn' => { :label => 'Penn', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Penn}'}, 'Princeton' => { :label => 'Princeton', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Princeton}'}, 'Stanford' => { :label => 'Stanford', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Stanford}'}, @@ -362,6 +364,7 @@ def self.generate_cluster_fq(index, limit, elvl = false) 'Columbia' => { :label => 'Columbia', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Columbia}"}, 'Cornell' => { :label => 'Cornell', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Cornell}"}, 'Duke' => { :label => 'Duke', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Duke}"}, + 'Harvard' => { :label => 'Harvard', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Harvard}"}, 'Penn' => { :label => 'Penn', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Penn}"}, 'Princeton' => { :label => 'Princeton', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Princeton}"}, 'Stanford' => { :label => 'Stanford', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Stanford}"}, diff --git a/app/models/harvard_indexer.rb b/app/models/harvard_indexer.rb new file mode 100644 index 00000000..b72e4d52 --- /dev/null +++ b/app/models/harvard_indexer.rb @@ -0,0 +1,83 @@ + +class HarvardIndexer < FranklinIndexer + + def define_record_source_id + to_field 'record_source_id' do |rec, acc| + acc << RecordSource::HARVARD + end + end + + def define_record_source_facet + to_field 'record_source_f' do |rec, acc| + acc << 'Harvard' + end + end + + def get_namespaced_id(rec) + id = get_001_id(rec) + id.blank? ? nil : "HARVARD_#{id}" + end + + def link_to_source_context(rec) + system_id = get_001_id(rec) + "http://id.lib.harvard.edu/alma/#{system_id}/catalog" + end + + def define_mms_id + # no-op + end + + def define_id + to_field 'id' do |rec, acc, context| + id = get_namespaced_id(rec) + if id.nil? + context.skip!('Skipping institutional record with no 001') + end + acc.replace([id]) + end + end + + def get_001_id(rec) + id = rec.fields('001').first&.value&.strip + id.blank? ? nil : id + end + + def define_grouped_id + to_field 'grouped_id' do |rec, acc| + oclc_ids = pennlibmarc.get_oclc_id_values(rec) + if oclc_ids.size > 1 + puts 'Warning: Multiple OCLC IDs found, using the first one' + end + oclc_id = oclc_ids.first + id = get_namespaced_id(rec) + + prefix = oclc_id.present? ? "#{oclc_id}!" : '' + acc << "#{prefix}#{id}" + end + end + + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| + + links = [] + + links << { + linktext: 'View record in Harvard\'s catalog', + linkurl: link_to_source_context(rec) + } + + acc << links.to_json + + end + end + + def get_cluster_id(rec) + pennlibmarc.get_oclc_id_values(rec).first || begin + id = get_namespaced_id(rec) + digest = Digest::MD5.hexdigest(id) + # first 8 hex digits = first 4 bytes. construct an int out of that hex str. + digest[0,8].hex + end + end + +end diff --git a/app/models/search_builder.rb b/app/models/search_builder.rb index 3014f730..b1b9835d 100644 --- a/app/models/search_builder.rb +++ b/app/models/search_builder.rb @@ -68,7 +68,7 @@ def with(blacklight_params = {}) super(params_copy) end - @@record_sources = ['Brown', 'Columbia', 'Cornell', 'Duke', 'Penn', 'Princeton', 'Stanford', 'HathiTrust'] + @@record_sources = ['Brown', 'Columbia', 'Cornell', 'Duke', 'Harvard', 'Penn', 'Princeton', 'Stanford', 'HathiTrust'] def add_cluster_params(solr_parameters) if 'Dynamic' == blacklight_params.dig(:f, :cluster, 0) @@ -81,12 +81,12 @@ def add_cluster_params(solr_parameters) clause = 0 loop do solr_parameters["x#{clause}_#{source_idx}"] = "{!bool filter=$j#{clause}_#{source_idx} filter=$o#{clause}_#{source_idx}}" - solr_parameters["y#{clause}_#{source_idx}"] = "{!bool filter=$k#{clause}_#{source_idx} filter=$o#{clause}_#{source_idx}}" - solr_parameters["z#{clause}_#{source_idx}"] = "{!bool filter=$j#{clause}_#{source_idx} filter=$p#{clause}_#{source_idx}}" +# solr_parameters["y#{clause}_#{source_idx}"] = "{!bool filter=$k#{clause}_#{source_idx} filter=$o#{clause}_#{source_idx}}" +# solr_parameters["z#{clause}_#{source_idx}"] = "{!bool filter=$j#{clause}_#{source_idx} filter=$p#{clause}_#{source_idx}}" solr_parameters["j#{clause}_#{source_idx}"] = "{!join from=cluster_id to=cluster_id v=record_source_f:#{cluster}}" - solr_parameters["k#{clause}_#{source_idx}"] = "{!join from=cluster_id to=cluster_id v='{!bool filter=record_source_f:#{cluster} filter=elvl_rank_isort:0}'}" +# solr_parameters["k#{clause}_#{source_idx}"] = "{!join from=cluster_id to=cluster_id v='{!bool filter=record_source_f:#{cluster} filter=elvl_rank_isort:0}'}" solr_parameters["o#{clause}_#{source_idx}"] = "record_source_f:(#{other_sources.join(' OR ')})" - solr_parameters["p#{clause}_#{source_idx}"] = "{!bool filter='record_source_f:(#{other_sources.join(' OR ')})' must_not=elvl_rank_isort:0}" +# solr_parameters["p#{clause}_#{source_idx}"] = "{!bool filter='record_source_f:(#{other_sources.join(' OR ')})' must_not=elvl_rank_isort:0}" break if other_sources.length == 1 cluster = other_sources.shift clause += 1 diff --git a/index_solr_harvard.sh b/index_solr_harvard.sh new file mode 100755 index 00000000..ade0ba44 --- /dev/null +++ b/index_solr_harvard.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export MARC_SOURCE=HARVARD +./index_solr.sh "$@" From 2779d915a9a7afe21d41b55ad6dda8428e200d75 Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Wed, 19 Feb 2020 11:07:13 -0500 Subject: [PATCH 20/77] Add Ivies+ Request button to items without an alma_mms_id --- app/views/catalog/_index_default.html.erb | 2 ++ app/views/catalog/_ivies_request.html.erb | 1 + app/views/catalog/_show_default.html.erb | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 app/views/catalog/_ivies_request.html.erb diff --git a/app/views/catalog/_index_default.html.erb b/app/views/catalog/_index_default.html.erb index c551492f..23e13436 100644 --- a/app/views/catalog/_index_default.html.erb +++ b/app/views/catalog/_index_default.html.erb @@ -15,6 +15,8 @@ <%# render partial: 'alma_availability_button', locals: { document: document } %> <%= render partial: 'alma_availability', locals: { document: document, collapsed: true } %> <%# render partial: 'request_button', locals: { document: document } %> + <% else %> + <%= render partial: 'ivies_request', locals: { document: document } %> <% end %> diff --git a/app/views/catalog/_ivies_request.html.erb b/app/views/catalog/_ivies_request.html.erb new file mode 100644 index 00000000..0db60661 --- /dev/null +++ b/app/views/catalog/_ivies_request.html.erb @@ -0,0 +1 @@ + diff --git a/app/views/catalog/_show_default.html.erb b/app/views/catalog/_show_default.html.erb index 017150ee..1c8e440f 100644 --- a/app/views/catalog/_show_default.html.erb +++ b/app/views/catalog/_show_default.html.erb @@ -16,6 +16,8 @@ <% if document.alma_mms_id.present? %> <%= render partial: 'alma_availability', locals: { document: document, collapsed: false } %> +<% else %> + <%= render partial: 'ivies_request', locals: { document: document } %> <% end %> <%#= render partial: 'alma_availability_iframe', locals: { document: document } %> From 4ed48d28a6d01dd61b258a7fb0c857f583ad6a9d Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 12 Feb 2020 14:12:47 -0500 Subject: [PATCH 21/77] add Chicago --- app/controllers/catalog_controller.rb | 21 ++++--- app/models/base_indexer.rb | 1 + app/models/chicago_indexer.rb | 83 +++++++++++++++++++++++++++ app/models/search_builder.rb | 2 +- index_solr_chicago.sh | 3 + lib/tasks/penn_lib.rake | 4 ++ 6 files changed, 104 insertions(+), 10 deletions(-) create mode 100644 app/models/chicago_indexer.rb create mode 100755 index_solr_chicago.sh diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index a0d533c4..d1bc5c2b 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -329,15 +329,16 @@ def self.generate_cluster_fq(index, limit, elvl = false) # 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} # } config.add_facet_field 'cluster', label: 'Prioritize your institution', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { - 'Brown' => { :label => 'Brown', :fq => generate_cluster_fq(0, 7)}, - 'Columbia' => { :label => 'Columbia', :fq => generate_cluster_fq(1, 7)}, - 'Cornell' => { :label => 'Cornell', :fq => generate_cluster_fq(2, 7)}, - 'Duke' => { :label => 'Duke', :fq => generate_cluster_fq(3, 7)}, - 'Harvard' => { :label => 'Harvard', :fq => generate_cluster_fq(4, 7)}, - 'Penn' => { :label => 'Penn', :fq => generate_cluster_fq(5, 7)}, - 'Princeton' => { :label => 'Princeton', :fq => generate_cluster_fq(6, 7)}, - 'Stanford' => { :label => 'Stanford', :fq => generate_cluster_fq(7, 7)}, - 'HathiTrust' => { :label => 'HathiTrust', :fq => generate_cluster_fq(8, 7)}, + 'Brown' => { :label => 'Brown', :fq => generate_cluster_fq(0, 8)}, + 'Chicago' => { :label => 'Chicago', :fq => generate_cluster_fq(1, 8)}, + 'Columbia' => { :label => 'Columbia', :fq => generate_cluster_fq(2, 8)}, + 'Cornell' => { :label => 'Cornell', :fq => generate_cluster_fq(3, 8)}, + 'Duke' => { :label => 'Duke', :fq => generate_cluster_fq(4, 8)}, + 'Harvard' => { :label => 'Harvard', :fq => generate_cluster_fq(5, 8)}, + 'Penn' => { :label => 'Penn', :fq => generate_cluster_fq(6, 8)}, + 'Princeton' => { :label => 'Princeton', :fq => generate_cluster_fq(7, 8)}, + 'Stanford' => { :label => 'Stanford', :fq => generate_cluster_fq(8, 8)}, + 'HathiTrust' => { :label => 'HathiTrust', :fq => generate_cluster_fq(9, 8)}, # 'Brown-e' => { :label => 'Brown-e', :fq => generate_cluster_fq(0, 6, true)}, # 'Columbia-e' => { :label => 'Columbia-e', :fq => generate_cluster_fq(1, 6, true)}, # 'Cornell-e' => { :label => 'Cornell-e', :fq => generate_cluster_fq(2, 6, true)}, @@ -350,6 +351,7 @@ def self.generate_cluster_fq(index, limit, elvl = false) } config.add_facet_field 'record_source_exclusive', label: 'See exclusively records from', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { 'Brown' => { :label => 'Brown', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Brown}'}, + 'Chicago' => { :label => 'Chicago', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Chicago}'}, 'Columbia' => { :label => 'Columbia', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Columbia}'}, 'Cornell' => { :label => 'Cornell', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Cornell}'}, 'Duke' => { :label => 'Duke', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Duke}'}, @@ -361,6 +363,7 @@ def self.generate_cluster_fq(index, limit, elvl = false) } config.add_facet_field 'record_source_f', label: 'See any records clustered with holdings from', collapse: false, solr_params: @@MINCOUNT, query: { 'Brown' => { :label => 'Brown', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Brown}"}, + 'Chicago' => { :label => 'Chicago', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Chicago}"}, 'Columbia' => { :label => 'Columbia', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Columbia}"}, 'Cornell' => { :label => 'Cornell', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Cornell}"}, 'Duke' => { :label => 'Duke', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Duke}"}, diff --git a/app/models/base_indexer.rb b/app/models/base_indexer.rb index 2c115718..ebcbc80a 100644 --- a/app/models/base_indexer.rb +++ b/app/models/base_indexer.rb @@ -12,6 +12,7 @@ module RecordSource HARVARD = 8 STANFORD = 9 PRINCETON = 10 + CHICAGO = 11 end end diff --git a/app/models/chicago_indexer.rb b/app/models/chicago_indexer.rb new file mode 100644 index 00000000..92c448be --- /dev/null +++ b/app/models/chicago_indexer.rb @@ -0,0 +1,83 @@ + +class ChicagoIndexer < FranklinIndexer + + def define_record_source_id + to_field 'record_source_id' do |rec, acc| + acc << RecordSource::CHICAGO + end + end + + def define_record_source_facet + to_field 'record_source_f' do |rec, acc| + acc << 'Chicago' + end + end + + def get_namespaced_id(rec) + id = get_001_id(rec) + id.blank? ? nil : "CHICAGO_#{id}" + end + + def link_to_source_context(rec) + system_id = get_001_id(rec) + "https://catalog.lib.uchicago.edu/vufind/Record/#{system_id}" + end + + def define_mms_id + # no-op + end + + def define_id + to_field 'id' do |rec, acc, context| + id = get_namespaced_id(rec) + if id.nil? + context.skip!('Skipping institutional record with no 001') + end + acc.replace([id]) + end + end + + def get_001_id(rec) + id = rec.fields('001').first&.value&.strip + id.blank? ? nil : id + end + + def define_grouped_id + to_field 'grouped_id' do |rec, acc| + oclc_ids = pennlibmarc.get_oclc_id_values(rec) + if oclc_ids.size > 1 + puts 'Warning: Multiple OCLC IDs found, using the first one' + end + oclc_id = oclc_ids.first + id = get_namespaced_id(rec) + + prefix = oclc_id.present? ? "#{oclc_id}!" : '' + acc << "#{prefix}#{id}" + end + end + + def define_full_text_link_a + to_field 'full_text_link_a' do |rec, acc| + + links = [] + + links << { + linktext: 'View record in Chicago\'s catalog', + linkurl: link_to_source_context(rec) + } + + acc << links.to_json + + end + end + + def get_cluster_id(rec) + pennlibmarc.get_oclc_id_values(rec).first || begin + id = get_namespaced_id(rec) + digest = Digest::MD5.hexdigest(id) + # first 8 hex digits = first 4 bytes. construct an int out of that hex str. + digest[0,8].hex + end + end + +end diff --git a/app/models/search_builder.rb b/app/models/search_builder.rb index b1b9835d..0cb07254 100644 --- a/app/models/search_builder.rb +++ b/app/models/search_builder.rb @@ -68,7 +68,7 @@ def with(blacklight_params = {}) super(params_copy) end - @@record_sources = ['Brown', 'Columbia', 'Cornell', 'Duke', 'Harvard', 'Penn', 'Princeton', 'Stanford', 'HathiTrust'] + @@record_sources = ['Brown', 'Chicago', 'Columbia', 'Cornell', 'Duke', 'Harvard', 'Penn', 'Princeton', 'Stanford', 'HathiTrust'] def add_cluster_params(solr_parameters) if 'Dynamic' == blacklight_params.dig(:f, :cluster, 0) diff --git a/index_solr_chicago.sh b/index_solr_chicago.sh new file mode 100755 index 00000000..e494238c --- /dev/null +++ b/index_solr_chicago.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export MARC_SOURCE=CHICAGO +./index_solr.sh "$@" diff --git a/lib/tasks/penn_lib.rake b/lib/tasks/penn_lib.rake index 1cc65792..53df8e58 100644 --- a/lib/tasks/penn_lib.rake +++ b/lib/tasks/penn_lib.rake @@ -41,6 +41,8 @@ namespace :pennlib do StanfordIndexer.new when 'PRINCETON' PrincetonIndexer.new + when 'CHICAGO' + ChicagoIndexer.new else FranklinIndexer.new end @@ -72,6 +74,8 @@ namespace :pennlib do StanfordIndexer.new when 'PRINCETON' PrincetonIndexer.new + when 'CHICAGO' + ChicagoIndexer.new else FranklinIndexer.new end From f47162289568c9c1322efce45a8ded6801c134b4 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 12 Feb 2020 14:13:57 -0500 Subject: [PATCH 22/77] prevent bloated requests by more carefully deduping dedupe params --- app/controllers/catalog_controller.rb | 7 ++++++- app/models/search_builder.rb | 22 +++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index d1bc5c2b..22764e0c 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -100,7 +100,12 @@ def self.generate_cluster_fq(index, limit, elvl = false) else i = 0 loop do - ret += " must_not=$x#{i}_#{index}" + if index < i + exclude_filter_label = "x#{i}" + else + exclude_filter_label = "x#{i}_#{index}" + end + ret += " must_not=$#{exclude_filter_label}" break if i >= limit i += 1 end diff --git a/app/models/search_builder.rb b/app/models/search_builder.rb index 0cb07254..e3b51b3e 100644 --- a/app/models/search_builder.rb +++ b/app/models/search_builder.rb @@ -76,22 +76,30 @@ def add_cluster_params(solr_parameters) end source_idx = 0 loop do + solr_parameters["j#{source_idx}"] = "{!join from=cluster_id to=cluster_id v=record_source_f:#{@@record_sources[source_idx]}}" other_sources = @@record_sources.dup cluster = other_sources.delete_at(source_idx) clause = 0 loop do - solr_parameters["x#{clause}_#{source_idx}"] = "{!bool filter=$j#{clause}_#{source_idx} filter=$o#{clause}_#{source_idx}}" -# solr_parameters["y#{clause}_#{source_idx}"] = "{!bool filter=$k#{clause}_#{source_idx} filter=$o#{clause}_#{source_idx}}" -# solr_parameters["z#{clause}_#{source_idx}"] = "{!bool filter=$j#{clause}_#{source_idx} filter=$p#{clause}_#{source_idx}}" - solr_parameters["j#{clause}_#{source_idx}"] = "{!join from=cluster_id to=cluster_id v=record_source_f:#{cluster}}" -# solr_parameters["k#{clause}_#{source_idx}"] = "{!join from=cluster_id to=cluster_id v='{!bool filter=record_source_f:#{cluster} filter=elvl_rank_isort:0}'}" - solr_parameters["o#{clause}_#{source_idx}"] = "record_source_f:(#{other_sources.join(' OR ')})" -# solr_parameters["p#{clause}_#{source_idx}"] = "{!bool filter='record_source_f:(#{other_sources.join(' OR ')})' must_not=elvl_rank_isort:0}" + if source_idx >= clause + join_filter_idx = clause == 0 ? clause : clause - 1 + if source_idx == clause + other_filter_label = "o#{clause}" + else + other_filter_label = "o#{clause}_#{source_idx}" + end + solr_parameters[other_filter_label] = "record_source_f:(#{other_sources.join(' OR ')})" + solr_parameters["x#{clause}_#{source_idx}"] = "{!bool filter=$j#{join_filter_idx} filter=$#{other_filter_label}}" + end break if other_sources.length == 1 cluster = other_sources.shift clause += 1 end + last_source_idx = source_idx break unless (source_idx += 1) < @@record_sources.length + if source_idx != 0 + solr_parameters["x#{last_source_idx}"] = "{!bool filter=$j#{last_source_idx} filter=$o#{last_source_idx}}" + end end end From 96e832c5186c15107503b7d0b694272ef321e04b Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 19 Feb 2020 12:00:46 -0500 Subject: [PATCH 23/77] modified facet options for local deployment --- app/controllers/catalog_controller.rb | 78 ++++++++------------------- 1 file changed, 21 insertions(+), 57 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 22764e0c..36d563d7 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -228,6 +228,10 @@ def self.generate_cluster_fq(index, limit, elvl = false) a.params.dig(:f, :format_f)&.include?('Database & Article Index') } + local_only = lambda { |a, b, c| + a.params.dig(:f, :cluster)&.include?('Penn') + } + config.induce_sort = lambda { |blacklight_params| return 'title_nssort asc' if blacklight_params.dig(:f, :format_f)&.include?('Database & Article Index') } @@ -329,60 +333,20 @@ def self.generate_cluster_fq(index, limit, elvl = false) 'Z' => { :label => 'Z', :fq => "{!prefix tag=azlist ex=azlist f=title_xfacet v='z'}"}, 'Other' => { :label => 'Other', :fq => "{!tag=azlist ex=azlist}title_xfacet:/[ -`{-~].*/"} } -# config.add_facet_field 'access_f', label: 'Access', collapse: false, solr_params: @@MINCOUNT, query: { -# 'Online' => { :label => 'Online', :fq => "{!join from=cluster_id to=cluster_id v=access_f:Online}"}, -# 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} -# } - config.add_facet_field 'cluster', label: 'Prioritize your institution', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { - 'Brown' => { :label => 'Brown', :fq => generate_cluster_fq(0, 8)}, - 'Chicago' => { :label => 'Chicago', :fq => generate_cluster_fq(1, 8)}, - 'Columbia' => { :label => 'Columbia', :fq => generate_cluster_fq(2, 8)}, - 'Cornell' => { :label => 'Cornell', :fq => generate_cluster_fq(3, 8)}, - 'Duke' => { :label => 'Duke', :fq => generate_cluster_fq(4, 8)}, - 'Harvard' => { :label => 'Harvard', :fq => generate_cluster_fq(5, 8)}, - 'Penn' => { :label => 'Penn', :fq => generate_cluster_fq(6, 8)}, - 'Princeton' => { :label => 'Princeton', :fq => generate_cluster_fq(7, 8)}, - 'Stanford' => { :label => 'Stanford', :fq => generate_cluster_fq(8, 8)}, - 'HathiTrust' => { :label => 'HathiTrust', :fq => generate_cluster_fq(9, 8)}, -# 'Brown-e' => { :label => 'Brown-e', :fq => generate_cluster_fq(0, 6, true)}, -# 'Columbia-e' => { :label => 'Columbia-e', :fq => generate_cluster_fq(1, 6, true)}, -# 'Cornell-e' => { :label => 'Cornell-e', :fq => generate_cluster_fq(2, 6, true)}, -# 'Duke-e' => { :label => 'Duke-e', :fq => generate_cluster_fq(3, 6, true)}, -# 'Penn-e' => { :label => 'Penn-e', :fq => generate_cluster_fq(4, 6, true)}, -# 'Princeton-e' => { :label => 'Princeton-e', :fq => generate_cluster_fq(5, 6, true)}, -# 'Stanford-e' => { :label => 'Stanford-e', :fq => generate_cluster_fq(6, 6, true)}, -# 'HathiTrust-e' => { :label => 'HathiTrust-e', :fq => generate_cluster_fq(7, 6, true)}, - 'Dynamic' => { :label => 'Dynamic', :fq => '*:*'} + config.add_facet_field 'access_f', label: 'Access', collapse: false, solr_params: @@MINCOUNT, :if => local_only, query: { + 'Online' => { :label => 'Online', :fq => "{!join from=cluster_id to=cluster_id v=access_f:Online}"}, + 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} } - config.add_facet_field 'record_source_exclusive', label: 'See exclusively records from', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { - 'Brown' => { :label => 'Brown', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Brown}'}, - 'Chicago' => { :label => 'Chicago', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Chicago}'}, - 'Columbia' => { :label => 'Columbia', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Columbia}'}, - 'Cornell' => { :label => 'Cornell', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Cornell}'}, - 'Duke' => { :label => 'Duke', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Duke}'}, - 'Harvard' => { :label => 'Harvard', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Harvard}'}, - 'Penn' => { :label => 'Penn', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Penn}'}, - 'Princeton' => { :label => 'Princeton', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Princeton}'}, - 'Stanford' => { :label => 'Stanford', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=Stanford}'}, - 'HathiTrust' => { :label => 'HathiTrust', :fq => '{!term tag=rsx ex=rsx f=record_source_f v=HathiTrust}'}, - } - config.add_facet_field 'record_source_f', label: 'See any records clustered with holdings from', collapse: false, solr_params: @@MINCOUNT, query: { - 'Brown' => { :label => 'Brown', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Brown}"}, - 'Chicago' => { :label => 'Chicago', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Chicago}"}, - 'Columbia' => { :label => 'Columbia', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Columbia}"}, - 'Cornell' => { :label => 'Cornell', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Cornell}"}, - 'Duke' => { :label => 'Duke', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Duke}"}, - 'Harvard' => { :label => 'Harvard', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Harvard}"}, - 'Penn' => { :label => 'Penn', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Penn}"}, - 'Princeton' => { :label => 'Princeton', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Princeton}"}, - 'Stanford' => { :label => 'Stanford', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:Stanford}"}, - 'HathiTrust' => { :label => 'HathiTrust', :fq => "{!join from=cluster_id to=cluster_id v=record_source_f:HathiTrust}"} + config.add_facet_field 'cluster', label: 'Search domain', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { + 'Penn' => { :label => 'Penn', :fq => '{!term tag=cluster ex=cluster f=record_source_f v=Penn}'}, + 'All' => { :label => 'All (preclustered)', :fq => generate_cluster_fq(6, 8)}, + 'Dynamic' => { :label => 'All (dynamic)', :fq => '{!tag=cluster ex=cluster v=\'*:*\'}'} } config.add_facet_field 'elvl', label: 'Encoding Level', collapse: false, solr_params: @@MINCOUNT, query: { 'Full' => { :label => 'Full', :fq => "elvl_rank_isort:0"}, 'Other' => { :label => 'Other', :fq => '{!bool must_not=elvl_rank_isort:0}'} } - config.add_facet_field 'format_f', label: 'Format', limit: 5, collapse: false, solr_params: @@MINCOUNT, query: { + config.add_facet_field 'format_f', label: 'Format', limit: 5, collapse: false, solr_params: @@MINCOUNT, :if => local_only, query: { 'Book' => { :label => 'Book', :fq => "{!term f=format_f v='Book'}"}, 'Government document' => { :label => 'Government document', :fq => "{!term f=format_f v='Government document'}"}, 'Journal/Periodical' => { :label => 'Journal/Periodical', :fq => "{!term f=format_f v='Journal/Periodical'}"}, @@ -408,17 +372,17 @@ def self.generate_cluster_fq(index, limit, elvl = false) #config.add_facet_field 'subject_taxonomy', label: 'Subject Taxonomy', collapse: false, :partial => 'blacklight/hierarchy/facet_hierarchy', :json_facet => @@SUBJECT_TAXONOMY, :top_level_field => 'toplevel_subject_f', :helper_method => :render_subcategories config.add_facet_field 'subject_f', label: 'Subject', limit: 5, index_range: 'A'..'Z', collapse: false, solr_params: @@MINCOUNT config.add_facet_field 'language_f', label: 'Language', limit: 5, collapse: false, solr_params: @@MINCOUNT -# config.add_facet_field 'library_f', label: 'Library', limit: 5, collapse: false, solr_params: @@MINCOUNT -# config.add_facet_field 'specific_location_f', label: 'Specific location', limit: 5, solr_params: @@MINCOUNT + config.add_facet_field 'library_f', label: 'Library', limit: 5, collapse: false, :if => local_only, solr_params: @@MINCOUNT + config.add_facet_field 'specific_location_f', label: 'Specific location', limit: 5, :if => local_only, solr_params: @@MINCOUNT config.add_facet_field 'publication_date_f', label: 'Publication date', limit: 5, collapse: false, solr_params: @@MINCOUNT -# config.add_facet_field 'classification_f', label: 'Classification', limit: 5, collapse: false, solr_params: @@MINCOUNT + config.add_facet_field 'classification_f', label: 'Classification', limit: 5, collapse: false, :if => local_only, solr_params: @@MINCOUNT config.add_facet_field 'genre_f', label: 'Form/Genre', limit: 5, solr_params: @@MINCOUNT -# config.add_facet_field 'recently_added_f', label: 'Recently added', solr_params: @@MINCOUNT, :query => { -# :within_90_days => { label: 'Within 90 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (90 * SECONDS_PER_DAY) } TO *]" }, -# :within_60_days => { label: 'Within 60 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (60 * SECONDS_PER_DAY) } TO *]" }, -# :within_30_days => { label: 'Within 30 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (30 * SECONDS_PER_DAY) } TO *]" }, -# :within_15_days => { label: 'Within 15 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (15 * SECONDS_PER_DAY) } TO *]" }, -# } + config.add_facet_field 'recently_added_f', label: 'Recently added', solr_params: @@MINCOUNT, :if => local_only, :query => { + :within_90_days => { label: 'Within 90 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (90 * SECONDS_PER_DAY) } TO *]" }, + :within_60_days => { label: 'Within 60 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (60 * SECONDS_PER_DAY) } TO *]" }, + :within_30_days => { label: 'Within 30 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (30 * SECONDS_PER_DAY) } TO *]" }, + :within_15_days => { label: 'Within 15 days', fq: "recently_added_isort:[#{PennLib::Util.today_midnight - (15 * SECONDS_PER_DAY) } TO *]" }, + } #config.add_facet_field 'example_pivot_field', label: 'Pivot Field', :pivot => ['format_f', 'language_f'] # config.add_facet_field 'example_query_facet_field', label: 'Publish Date', :query => { From 7cf6ed03efa88cafeebe5b9d4212054c765dc737 Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Wed, 19 Feb 2020 12:32:34 -0500 Subject: [PATCH 24/77] Updated BD+ to Ivies+ Request for demo --- app/assets/javascripts/availability.js.erb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/assets/javascripts/availability.js.erb b/app/assets/javascripts/availability.js.erb index ca108d2d..9cf4daf2 100644 --- a/app/assets/javascripts/availability.js.erb +++ b/app/assets/javascripts/availability.js.erb @@ -223,6 +223,16 @@ $(document).ready(function() { } } + // Replace BD+ w/Ivies+ Request for demo purposes; correct when fully implemented + if(globaldata[mmsid]["inventory_type"] == "physical") { + for(i = 0; i < data.length; i++) { + if(data[i]['option_name'] == 'BorrowDirect+') { + data[i]['option_name'] = 'Ivies+ Request'; + break; + } + } + } + data.forEach(function(element) { var containingdiv = $('
'); var highlightable = element['highlightable'] ? ' highlightable' : '' From 706aac90888c8b9df023850c259e8fd193a664c5 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 24 Feb 2020 12:05:21 -0500 Subject: [PATCH 25/77] adjustments to facets and link display, per egmowens --- app/controllers/catalog_controller.rb | 15 +++++---------- app/helpers/document_render_helper.rb | 10 ++++++---- app/models/search_builder.rb | 7 +++++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 36d563d7..e285b094 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -229,7 +229,7 @@ def self.generate_cluster_fq(index, limit, elvl = false) } local_only = lambda { |a, b, c| - a.params.dig(:f, :cluster)&.include?('Penn') + 'Include Partner Libraries' != a.params.dig(:f, :cluster, 0) } config.induce_sort = lambda { |blacklight_params| @@ -296,6 +296,10 @@ def self.generate_cluster_fq(index, limit, elvl = false) @@MINCOUNT = { 'facet.mincount' => 1 } + config.add_facet_field 'cluster', label: 'Search domain', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { + #'Penn' => { :label => 'Penn', :fq => '{!term tag=cluster ex=cluster f=record_source_f v=Penn}'}, + 'Include Partner Libraries' => { :label => 'Include Partner Libraries', :fq => generate_cluster_fq(6, 8)} + } config.add_facet_field 'db_subcategory_f', label: 'Database Subject', if: lambda { |a,b,c| false } config.add_facet_field 'db_category_f', label: 'Database Subject', collapse: false, :partial => 'blacklight/hierarchy/facet_hierarchy', :json_facet => @@DATABASE_CATEGORY_TAXONOMY, :top_level_field => 'db_category_f', :facet_type => :database, @@ -337,15 +341,6 @@ def self.generate_cluster_fq(index, limit, elvl = false) 'Online' => { :label => 'Online', :fq => "{!join from=cluster_id to=cluster_id v=access_f:Online}"}, 'At the library' => { :label => 'At the library', :fq => "{!join from=cluster_id to=cluster_id v='{!term f=access_f v=\\'At the library\\'}'}"} } - config.add_facet_field 'cluster', label: 'Search domain', collapse: false, single: :manual, solr_params: @@MINCOUNT, query: { - 'Penn' => { :label => 'Penn', :fq => '{!term tag=cluster ex=cluster f=record_source_f v=Penn}'}, - 'All' => { :label => 'All (preclustered)', :fq => generate_cluster_fq(6, 8)}, - 'Dynamic' => { :label => 'All (dynamic)', :fq => '{!tag=cluster ex=cluster v=\'*:*\'}'} - } - config.add_facet_field 'elvl', label: 'Encoding Level', collapse: false, solr_params: @@MINCOUNT, query: { - 'Full' => { :label => 'Full', :fq => "elvl_rank_isort:0"}, - 'Other' => { :label => 'Other', :fq => '{!bool must_not=elvl_rank_isort:0}'} - } config.add_facet_field 'format_f', label: 'Format', limit: 5, collapse: false, solr_params: @@MINCOUNT, :if => local_only, query: { 'Book' => { :label => 'Book', :fq => "{!term f=format_f v='Book'}"}, 'Government document' => { :label => 'Government document', :fq => "{!term f=format_f v='Government document'}"}, diff --git a/app/helpers/document_render_helper.rb b/app/helpers/document_render_helper.rb index 48ef5657..6d68bd4a 100644 --- a/app/helpers/document_render_helper.rb +++ b/app/helpers/document_render_helper.rb @@ -53,13 +53,15 @@ def render_electronic_holdings_links(electronic_holdings_str) def render_online_resource_display_for_index_view(options) values = options[:value] - values.map do |value| + suppress_remote_links = 'Include Partner Libraries' != params.dig('f', 'cluster', 0) + ret = values.map do |value| JSON.parse(value).map do |link_struct| url = link_struct['linkurl'] text = link_struct['linktext'] - %Q{#{text}} - end.join('
') - end.join('
').html_safe + (suppress_remote_links && text =~ /View record in .*\'s catalog/) ? nil : %Q{#{text}} + end.compact.join('
') + end.reject { |item| item.blank? }.join('
') + ret.blank? ? 'Has partner library holdings' : ret.html_safe end def render_online_display_for_show_view(options) diff --git a/app/models/search_builder.rb b/app/models/search_builder.rb index e3b51b3e..0a9b7c06 100644 --- a/app/models/search_builder.rb +++ b/app/models/search_builder.rb @@ -71,8 +71,11 @@ def with(blacklight_params = {}) @@record_sources = ['Brown', 'Chicago', 'Columbia', 'Cornell', 'Duke', 'Harvard', 'Penn', 'Princeton', 'Stanford', 'HathiTrust'] def add_cluster_params(solr_parameters) - if 'Dynamic' == blacklight_params.dig(:f, :cluster, 0) - solr_parameters[:fq] << '{!collapse tag=cluster ex=cluster field=cluster_id nullPolicy=expand size=3000000}' +# if 'Dynamic' == blacklight_params.dig(:f, :cluster, 0) +# solr_parameters[:fq] << '{!collapse tag=cluster ex=cluster field=cluster_id nullPolicy=expand size=3000000}' +# end + if 'Include Partner Libraries' != blacklight_params.dig(:f, :cluster, 0) + solr_parameters[:fq] << '{!term tag=cluster ex=cluster f=record_source_f v=Penn}' end source_idx = 0 loop do From a11e1f3d93e9c677c4ef5dd59ed62be960ca87b6 Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Fri, 6 Mar 2020 13:51:32 -0500 Subject: [PATCH 26/77] Added methods to interact with BD API --- lib/requesting/relais.rb | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 lib/requesting/relais.rb diff --git a/lib/requesting/relais.rb b/lib/requesting/relais.rb new file mode 100644 index 00000000..e2690a00 --- /dev/null +++ b/lib/requesting/relais.rb @@ -0,0 +1,53 @@ +require 'httparty' + +module Relais + + def self.authenticate(patron_id) + url = URI.join(ENV['RELAIS_API_HOST'], 'portal-service/user/authentication') + + body = { :ApiKey => "#{ENV['RELAIS_API_KEY']}", + :UserGroup => "PATRON", + :LibrarySymbol => "PENN", + :PatronId => patron_id + } + + response = HTTParty.post(url, + :body => body.to_json, + :headers => { 'Content-Type' => 'application/json' } + ) + + return response['AuthorizationId'] + end + + def self.addRequest(authz_id, patron_id, title) + url = URI.join(ENV['RELAIS_API_HOST'], "portal-service/request/add?aid=#{authz_id}") + + body = { "BibliographicInfo": { + "Title": title + }, + "RequestFor": { + "PortalGroup": "PATRON", + "LibrarySymbol": "PENN", + "PatronId": patron_id + } + } + + response = HTTParty.post(url, + :body => body.to_json, + :headers => { 'Content-Type' => 'application/json' } + ) + + return response + end + + def self.getRequests(authz_id) + url = URI.join(ENV['RELAIS_API_HOST'], "portal-service/request/query/my?aid=#{authz_id}&type=open") + + response = HTTParty.get(url, + :headers => { 'Content-Type' => 'application/json' } + ) + + return response + end + +end From f6604fb608d2561d0c9e7b7bab9c0850c5abeb5b Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Fri, 6 Mar 2020 13:52:38 -0500 Subject: [PATCH 27/77] Added controller, views, and routes for BD requesting --- app/controllers/ivies_plus_controller.rb | 31 ++++++++++++++++++++ app/views/ivies_plus/new_request.html.erb | 22 ++++++++++++++ app/views/ivies_plus/request_placed.html.erb | 4 +++ app/views/ivies_plus/requests.html.erb | 27 +++++++++++++++++ config/routes.rb | 4 +++ 5 files changed, 88 insertions(+) create mode 100644 app/controllers/ivies_plus_controller.rb create mode 100644 app/views/ivies_plus/new_request.html.erb create mode 100644 app/views/ivies_plus/request_placed.html.erb create mode 100644 app/views/ivies_plus/requests.html.erb diff --git a/app/controllers/ivies_plus_controller.rb b/app/controllers/ivies_plus_controller.rb new file mode 100644 index 00000000..476a8314 --- /dev/null +++ b/app/controllers/ivies_plus_controller.rb @@ -0,0 +1,31 @@ +require 'requesting/relais' + +class IviesPlusController < ApplicationController + + def index + render :requests, locals: {:requests => open_requests} + end + + def new_request + # TODO: Add check and handling for empty title + title = SolrDocument.find(params['id']).fetch(:title, '') + + render :new_request, locals: {:title => title} + end + + def place_request + user_id = session['id'].presence + title = params[:title] + aid = Relais.authenticate(user_id) + response = Relais.addRequest(aid, user_id, title) + + render :request_placed, locals: {:message => response["ConfirmMessage"] || response["Problem"]["Message"]} + end + + def open_requests + user_id = session['id'].presence + aid = Relais.authenticate(user_id) + Relais.getRequests(aid)['MyRequestRecords'] || [] + end + +end diff --git a/app/views/ivies_plus/new_request.html.erb b/app/views/ivies_plus/new_request.html.erb new file mode 100644 index 00000000..cf9fbe25 --- /dev/null +++ b/app/views/ivies_plus/new_request.html.erb @@ -0,0 +1,22 @@ +<% @page_title = "Request Item" %> + +<%= form_tag('/new_request', method: 'post') do |f| %> + + <%= hidden_field_tag("title", title) %> +
+
+

Request Item

+
+
+
+ <%= label_tag(:title, 'Title', :class => 'control-label col-sm-6 col-md-2') %> +
+

<%= title %>

+
+
+ + <%= submit_tag('Submit Request', :class => 'btn btn-default') %> +
+
+ +<% end %> diff --git a/app/views/ivies_plus/request_placed.html.erb b/app/views/ivies_plus/request_placed.html.erb new file mode 100644 index 00000000..0ce225e8 --- /dev/null +++ b/app/views/ivies_plus/request_placed.html.erb @@ -0,0 +1,4 @@ +<%= message %> +
+
+You can view your open requests in Franklin via the "View Ivies+ Requests" link in the header. diff --git a/app/views/ivies_plus/requests.html.erb b/app/views/ivies_plus/requests.html.erb new file mode 100644 index 00000000..2cde4ba9 --- /dev/null +++ b/app/views/ivies_plus/requests.html.erb @@ -0,0 +1,27 @@ +<% @page_title = "Ivies+ Requests" %> + +
+
+

Ivies+ Requests

+
+
+ + + + + + + + + + <% requests.each do |req| %> + + + + + + <% end %> + +
Request #TitleDate Submitted
<%= req['RequestNumber'] %><%= req['Title'] %><%= req['DateSubmitted'] %>
+
+
diff --git a/config/routes.rb b/config/routes.rb index 72364788..661d9580 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -65,6 +65,10 @@ get 'alma/request' => 'franklin_alma#load_request' post 'alma/request' => 'franklin_alma#create_request' + get 'requests' => 'ivies_plus#index' + get 'new_request' => 'ivies_plus#new_request' + post 'new_request' => 'ivies_plus#place_request' + devise_scope :user do get 'alma/social_login_callback' => 'sessions#social_login_callback' get 'accounts/login' => 'sessions#sso_login_callback' From 86de85b3437ef4942fd490657591c04b8bcecc2f Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Fri, 6 Mar 2020 13:53:28 -0500 Subject: [PATCH 28/77] Set URL for Ivies+ requesting --- app/assets/javascripts/availability.js.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/assets/javascripts/availability.js.erb b/app/assets/javascripts/availability.js.erb index 9cf4daf2..1fe2d9ae 100644 --- a/app/assets/javascripts/availability.js.erb +++ b/app/assets/javascripts/availability.js.erb @@ -228,6 +228,7 @@ $(document).ready(function() { for(i = 0; i < data.length; i++) { if(data[i]['option_name'] == 'BorrowDirect+') { data[i]['option_name'] = 'Ivies+ Request'; + data[i]['option_url'] = '/new_request?id=FRANKLIN_' + mmsid; break; } } From b977b9d1774f23564c11372fc446c6becf565c53 Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Fri, 6 Mar 2020 13:54:15 -0500 Subject: [PATCH 29/77] Added link to Ivies+ Requests page and changed button handling for unauth'd users --- app/views/_user_util_links.html.erb | 3 +++ app/views/catalog/_ivies_request.html.erb | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/app/views/_user_util_links.html.erb b/app/views/_user_util_links.html.erb index bbd70b55..291ec4b0 100644 --- a/app/views/_user_util_links.html.erb +++ b/app/views/_user_util_links.html.erb @@ -52,6 +52,9 @@ Library Home <% if current_user %> +
  • + <%= link_to "Ivies+ Requests", {:controller => "ivies_plus", :action => "index"} %> +
  • <% if session['first_name'].present? %><%= session['first_name'] %> - <% end %>PennKey Library Account
    • diff --git a/app/views/catalog/_ivies_request.html.erb b/app/views/catalog/_ivies_request.html.erb index 0db60661..5d801ee5 100644 --- a/app/views/catalog/_ivies_request.html.erb +++ b/app/views/catalog/_ivies_request.html.erb @@ -1 +1,17 @@ - +<% if current_user %> + +<% else %> +
    +
    +
    + + +
    +
    +<% end %> From 71c9a887c6311fa86336084f10e414d84d28d2a5 Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Fri, 6 Mar 2020 13:59:10 -0500 Subject: [PATCH 30/77] Added additional env vars to nginx conf --- nginx/webapp-env.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nginx/webapp-env.conf b/nginx/webapp-env.conf index 38636814..3eb941d4 100644 --- a/nginx/webapp-env.conf +++ b/nginx/webapp-env.conf @@ -12,6 +12,9 @@ env DEVISE_SECRET_KEY; env ENABLE_DEBUG_URLS; env RAILS_ENV; env RAILS_LOG_LEVEL; +env RELAIS_API_KEY; +env RELAIS_PARTNERSHIP_ID; +env RELAIS_API_HOST; env SECRET_KEY_BASE; env SHOW_KNOWN_ISSUES; env SOLR_URL; From 606606ad53797a273804b5f2551c0b0c6b91828a Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Mon, 9 Mar 2020 14:42:49 -0400 Subject: [PATCH 31/77] Temp fix for broken auth --- app/controllers/catalog_controller.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index e285b094..7058c41a 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -57,7 +57,9 @@ def search_results(user_params) # manually expire the session if user has exceeded 'hard expiration' or if # shib session has become inactive def expire_session - invalid_shib = has_shib_session? && !shib_session_valid? + #invalid_shib = has_shib_session? && !shib_session_valid? + # TODO: the above line is broken when deployed to the test site; Investigate and fix + invalid_shib = false if (session[:hard_expiration] && session[:hard_expiration] < Time.now.to_i) || invalid_shib reset_session url = invalid_shib ? "/Shibboleth.sso/Logout?return=#{URI.encode(expire_shib_session_return_url)}" : expire_shib_session_return_url From ac25dd27c36bac6e92e6dadd1496968dc3a9bc21 Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Mon, 9 Mar 2020 14:48:45 -0400 Subject: [PATCH 32/77] Replaced references to Ivies+ with BorrowDirect+ --- app/assets/javascripts/availability.js.erb | 4 ++-- app/views/_user_util_links.html.erb | 2 +- app/views/catalog/_ivies_request.html.erb | 2 +- app/views/ivies_plus/request_placed.html.erb | 2 +- app/views/ivies_plus/requests.html.erb | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/assets/javascripts/availability.js.erb b/app/assets/javascripts/availability.js.erb index 1fe2d9ae..24aed695 100644 --- a/app/assets/javascripts/availability.js.erb +++ b/app/assets/javascripts/availability.js.erb @@ -223,11 +223,11 @@ $(document).ready(function() { } } - // Replace BD+ w/Ivies+ Request for demo purposes; correct when fully implemented + // Replace BD+ w/BD+ Request for demo purposes; correct when fully implemented if(globaldata[mmsid]["inventory_type"] == "physical") { for(i = 0; i < data.length; i++) { if(data[i]['option_name'] == 'BorrowDirect+') { - data[i]['option_name'] = 'Ivies+ Request'; + data[i]['option_name'] = 'BorrowDirect+ Request'; data[i]['option_url'] = '/new_request?id=FRANKLIN_' + mmsid; break; } diff --git a/app/views/_user_util_links.html.erb b/app/views/_user_util_links.html.erb index 291ec4b0..77910d48 100644 --- a/app/views/_user_util_links.html.erb +++ b/app/views/_user_util_links.html.erb @@ -53,7 +53,7 @@ <% if current_user %>
  • - <%= link_to "Ivies+ Requests", {:controller => "ivies_plus", :action => "index"} %> + <%= link_to "BorrowDirect+ Requests", {:controller => "ivies_plus", :action => "index"} %>
  • <% if session['first_name'].present? %><%= session['first_name'] %> - <% end %>PennKey Library Account diff --git a/app/views/catalog/_ivies_request.html.erb b/app/views/catalog/_ivies_request.html.erb index 5d801ee5..91fddb55 100644 --- a/app/views/catalog/_ivies_request.html.erb +++ b/app/views/catalog/_ivies_request.html.erb @@ -1,5 +1,5 @@ <% if current_user %> - + <% else %>
    diff --git a/app/views/ivies_plus/request_placed.html.erb b/app/views/ivies_plus/request_placed.html.erb index 0ce225e8..a60c89c2 100644 --- a/app/views/ivies_plus/request_placed.html.erb +++ b/app/views/ivies_plus/request_placed.html.erb @@ -1,4 +1,4 @@ <%= message %>

    -You can view your open requests in Franklin via the "View Ivies+ Requests" link in the header.     +You can view your open requests in Franklin via the "View BorrowDirect+ Requests" link in the header. diff --git a/app/views/ivies_plus/requests.html.erb b/app/views/ivies_plus/requests.html.erb index 2cde4ba9..e3da3ae5 100644 --- a/app/views/ivies_plus/requests.html.erb +++ b/app/views/ivies_plus/requests.html.erb @@ -1,8 +1,8 @@ -<% @page_title = "Ivies+ Requests" %> +<% @page_title = "BorrowDirect+ Requests" %>
    -

    Ivies+ Requests

    +

    BorrowDirect+ Requests

    From 219653d2e31bf498900a9421ca8ad16cefd85d3b Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Fri, 11 Oct 2019 12:52:36 -0400 Subject: [PATCH 33/77] Bibs with a single holding and item descriptions automatically shows the items table --- app/assets/javascripts/availability.js.erb | 30 ++++++++++++--------- app/controllers/franklin_alma_controller.rb | 16 ++++++++--- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/app/assets/javascripts/availability.js.erb b/app/assets/javascripts/availability.js.erb index 24aed695..4d9421f6 100644 --- a/app/assets/javascripts/availability.js.erb +++ b/app/assets/javascripts/availability.js.erb @@ -143,17 +143,23 @@ $(document).ready(function() { } var dataLen = json.data.length; - if(dataLen == 0) { - //$('#table-wrapper-' + mmsid).hide(); - } - // Reference: https://stackoverflow.com/a/15458987 - // Check if the location of the first holding is an HTML element, - // indicating this is for a serial - else if(/<[a-z][\s\S]*>/i.test(json.data[0][2])) { - $('#requestOptionsAvailability-' + mmsid).text('Holdings'); + if(dataLen > 0) { + // Reference: https://stackoverflow.com/a/15458987 + // Check if the location of the first holding is an HTML element, + // indicating this is for a serial + if(/<[a-z][\s\S]*>/i.test(json.data[0][2])) { + $('#requestOptionsAvailability-' + mmsid).text('Holdings'); + } + else { + $('#requestOptionsAvailability-' + mmsid).text('Availability'); + } } - else { - $('#requestOptionsAvailability-' + mmsid).text('Availability'); + else if(dataLen === undefined && !Array.isArray(json.data)) { + // Single holding case + var show_location = true; + loadItems(json.data.mmsid, json.data.holding_id, json.data.location_code, show_location); + $('.backbutton-top-' + mmsid).hide(); + $('.backbutton-' + mmsid).hide(); } $('.load-holding-details').each(function(idx, element) { @@ -352,11 +358,11 @@ $(document).ready(function() { ba.loadRequestOptions(); }); -function loadItems(mms_id, holding_id, location_code) { +function loadItems(mms_id, holding_id, location_code, show_location = false) { renderdata[mms_id] = []; var holdingItemsTable = $('#holdingItems-' + mms_id).DataTable(); holdingItemsTable.clear().draw(); - holdingItemsTable.ajax.url('/alma/holding_items.json?mms_id=' + mms_id + "&holding_id=" + holding_id + "¤t_location=" + location_code).load(); + holdingItemsTable.ajax.url('/alma/holding_items.json?mms_id=' + mms_id + "&holding_id=" + holding_id + "¤t_location=" + location_code + "&show_location=" + show_location).load(); swapDataTables(mms_id); } diff --git a/app/controllers/franklin_alma_controller.rb b/app/controllers/franklin_alma_controller.rb index a915198b..440417e2 100644 --- a/app/controllers/franklin_alma_controller.rb +++ b/app/controllers/franklin_alma_controller.rb @@ -264,6 +264,11 @@ def single_availability end end + if bib_data['availability'][mmsid]['holdings'].length() == 1 && has_holding_info + holding = bib_data['availability'][mmsid]['holdings'].first + table_data = {"mmsid": mmsid, "holding_id": holding['holding_id'], "location_code": holding['location_code']} + end + metadata[mmsid][:inventory_type] = inventory_type render :json => {"metadata": metadata, "data": table_data} end @@ -297,6 +302,7 @@ def check_requestable(has_holding_info = false) def holding_items userid = session['id'].presence || nil due_date_policy = 'Please log in for loan and request information' if userid.nil? + show_location = params.delete("show_location") == "true" api_instance = BlacklightAlma::BibsApi.instance api = api_instance.ezwadl_api[0] options = {:expand => 'due_date_policy', :offset => 0, :limit => 100, :user_id => userid, :order_by => 'description'} @@ -309,14 +315,18 @@ def holding_items policies = {} pids_to_check = [] - table_data = response_data['item'].map { |item| data = item['item_data'] + + location = show_location ? + "#{item['item_data']['location']['desc']}
    #{item['holding_data']['call_number']}
    " : + "" + unless(policies.has_key?(data['policy']['value']) || data['base_status']['desc'] != "Item in place" || userid.nil?) policies[data['policy']['value']] = nil pids_to_check << [data['pid'], data['policy']['value']] end - [data['policy']['value'], data['pid'], data['description'], data['base_status']['desc'], data['barcode'], due_date_policy || data['due_date_policy'], [], params['mms_id'], params['holding_id']] + [data['policy']['value'], data['pid'], location + data['description'], data['base_status']['desc'], data['barcode'], due_date_policy || data['due_date_policy'], [], params['mms_id'], params['holding_id']] } while options[:offset] + options[:limit] < response_data['total_record_count'] @@ -329,7 +339,7 @@ def holding_items policies[data['policy']['value']] = nil pids_to_check << [data['pid'], data['policy']['value']] end - [data['policy']['value'], data['pid'], data['description'], due_date_policy || data['due_date_policy'], data['base_status']['desc'], data['barcode'], [], params['mms_id'], params['holding_id']] + [data['policy']['value'], data['pid'], location + data['description'], due_date_policy || data['due_date_policy'], data['base_status']['desc'], data['barcode'], [], params['mms_id'], params['holding_id']] } end From 9f5343609060df9e92777aeeb1690965ae6f6bca Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Fri, 11 Oct 2019 13:52:37 -0400 Subject: [PATCH 34/77] Fixed HTML --- app/controllers/franklin_alma_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/franklin_alma_controller.rb b/app/controllers/franklin_alma_controller.rb index 440417e2..0b193a93 100644 --- a/app/controllers/franklin_alma_controller.rb +++ b/app/controllers/franklin_alma_controller.rb @@ -319,7 +319,7 @@ def holding_items data = item['item_data'] location = show_location ? - "#{item['item_data']['location']['desc']}
    #{item['holding_data']['call_number']}
    " : + "#{item['item_data']['location']['desc']}
    #{item['holding_data']['call_number']}
    " : "" unless(policies.has_key?(data['policy']['value']) || data['base_status']['desc'] != "Item in place" || userid.nil?) From a218fb168c751adda0f87ee12687519843093435 Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Fri, 11 Oct 2019 14:22:12 -0400 Subject: [PATCH 35/77] Fix to account for asset precompilation --- app/assets/javascripts/availability.js.erb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/availability.js.erb b/app/assets/javascripts/availability.js.erb index 4d9421f6..c5778b42 100644 --- a/app/assets/javascripts/availability.js.erb +++ b/app/assets/javascripts/availability.js.erb @@ -358,8 +358,13 @@ $(document).ready(function() { ba.loadRequestOptions(); }); -function loadItems(mms_id, holding_id, location_code, show_location = false) { +function loadItems(mms_id, holding_id, location_code, show_location) { renderdata[mms_id] = []; + + // Reference: https://stackoverflow.com/a/35464024 + // This is needed so that asset precompilation doesn't fail + show_location = typeof show_location !== 'undefined' ? show_location : false; + var holdingItemsTable = $('#holdingItems-' + mms_id).DataTable(); holdingItemsTable.clear().draw(); holdingItemsTable.ajax.url('/alma/holding_items.json?mms_id=' + mms_id + "&holding_id=" + holding_id + "¤t_location=" + location_code + "&show_location=" + show_location).load(); From ab2d681d378bbfdb533c7a3dc172b32b23056d5a Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Wed, 30 Oct 2019 14:19:20 -0400 Subject: [PATCH 36/77] Revert "Merge pull request #18 from upenn-libraries/single_holding_fix" This reverts commit d8655d51a37daea8ad8153ff1b3776564d54acb7, reversing changes made to e0d7298825b241ce691239be0291341269d576f2. --- app/assets/javascripts/availability.js.erb | 35 +++++++-------------- app/controllers/franklin_alma_controller.rb | 16 ++-------- 2 files changed, 15 insertions(+), 36 deletions(-) diff --git a/app/assets/javascripts/availability.js.erb b/app/assets/javascripts/availability.js.erb index c5778b42..24aed695 100644 --- a/app/assets/javascripts/availability.js.erb +++ b/app/assets/javascripts/availability.js.erb @@ -143,23 +143,17 @@ $(document).ready(function() { } var dataLen = json.data.length; - if(dataLen > 0) { - // Reference: https://stackoverflow.com/a/15458987 - // Check if the location of the first holding is an HTML element, - // indicating this is for a serial - if(/<[a-z][\s\S]*>/i.test(json.data[0][2])) { - $('#requestOptionsAvailability-' + mmsid).text('Holdings'); - } - else { - $('#requestOptionsAvailability-' + mmsid).text('Availability'); - } + if(dataLen == 0) { + //$('#table-wrapper-' + mmsid).hide(); + } + // Reference: https://stackoverflow.com/a/15458987 + // Check if the location of the first holding is an HTML element, + // indicating this is for a serial + else if(/<[a-z][\s\S]*>/i.test(json.data[0][2])) { + $('#requestOptionsAvailability-' + mmsid).text('Holdings'); } - else if(dataLen === undefined && !Array.isArray(json.data)) { - // Single holding case - var show_location = true; - loadItems(json.data.mmsid, json.data.holding_id, json.data.location_code, show_location); - $('.backbutton-top-' + mmsid).hide(); - $('.backbutton-' + mmsid).hide(); + else { + $('#requestOptionsAvailability-' + mmsid).text('Availability'); } $('.load-holding-details').each(function(idx, element) { @@ -358,16 +352,11 @@ $(document).ready(function() { ba.loadRequestOptions(); }); -function loadItems(mms_id, holding_id, location_code, show_location) { +function loadItems(mms_id, holding_id, location_code) { renderdata[mms_id] = []; - - // Reference: https://stackoverflow.com/a/35464024 - // This is needed so that asset precompilation doesn't fail - show_location = typeof show_location !== 'undefined' ? show_location : false; - var holdingItemsTable = $('#holdingItems-' + mms_id).DataTable(); holdingItemsTable.clear().draw(); - holdingItemsTable.ajax.url('/alma/holding_items.json?mms_id=' + mms_id + "&holding_id=" + holding_id + "¤t_location=" + location_code + "&show_location=" + show_location).load(); + holdingItemsTable.ajax.url('/alma/holding_items.json?mms_id=' + mms_id + "&holding_id=" + holding_id + "¤t_location=" + location_code).load(); swapDataTables(mms_id); } diff --git a/app/controllers/franklin_alma_controller.rb b/app/controllers/franklin_alma_controller.rb index 0b193a93..a915198b 100644 --- a/app/controllers/franklin_alma_controller.rb +++ b/app/controllers/franklin_alma_controller.rb @@ -264,11 +264,6 @@ def single_availability end end - if bib_data['availability'][mmsid]['holdings'].length() == 1 && has_holding_info - holding = bib_data['availability'][mmsid]['holdings'].first - table_data = {"mmsid": mmsid, "holding_id": holding['holding_id'], "location_code": holding['location_code']} - end - metadata[mmsid][:inventory_type] = inventory_type render :json => {"metadata": metadata, "data": table_data} end @@ -302,7 +297,6 @@ def check_requestable(has_holding_info = false) def holding_items userid = session['id'].presence || nil due_date_policy = 'Please log in for loan and request information' if userid.nil? - show_location = params.delete("show_location") == "true" api_instance = BlacklightAlma::BibsApi.instance api = api_instance.ezwadl_api[0] options = {:expand => 'due_date_policy', :offset => 0, :limit => 100, :user_id => userid, :order_by => 'description'} @@ -315,18 +309,14 @@ def holding_items policies = {} pids_to_check = [] + table_data = response_data['item'].map { |item| data = item['item_data'] - - location = show_location ? - "#{item['item_data']['location']['desc']}
    #{item['holding_data']['call_number']}
    " : - "" - unless(policies.has_key?(data['policy']['value']) || data['base_status']['desc'] != "Item in place" || userid.nil?) policies[data['policy']['value']] = nil pids_to_check << [data['pid'], data['policy']['value']] end - [data['policy']['value'], data['pid'], location + data['description'], data['base_status']['desc'], data['barcode'], due_date_policy || data['due_date_policy'], [], params['mms_id'], params['holding_id']] + [data['policy']['value'], data['pid'], data['description'], data['base_status']['desc'], data['barcode'], due_date_policy || data['due_date_policy'], [], params['mms_id'], params['holding_id']] } while options[:offset] + options[:limit] < response_data['total_record_count'] @@ -339,7 +329,7 @@ def holding_items policies[data['policy']['value']] = nil pids_to_check << [data['pid'], data['policy']['value']] end - [data['policy']['value'], data['pid'], location + data['description'], due_date_policy || data['due_date_policy'], data['base_status']['desc'], data['barcode'], [], params['mms_id'], params['holding_id']] + [data['policy']['value'], data['pid'], data['description'], due_date_policy || data['due_date_policy'], data['base_status']['desc'], data['barcode'], [], params['mms_id'], params['holding_id']] } end From 4c74f7eb15d81bab80b3f87541ac16637a5ba2ab Mon Sep 17 00:00:00 2001 From: Katherine Schultz Date: Wed, 30 Oct 2019 15:14:57 -0400 Subject: [PATCH 37/77] edit FEATURED loc to add Moelis Lounge --- config/translation_maps/locations.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/translation_maps/locations.xml b/config/translation_maps/locations.xml index f19a5d7b..1cc79e7c 100644 --- a/config/translation_maps/locations.xml +++ b/config/translation_maps/locations.xml @@ -699,7 +699,7 @@ Van Pelt - Featured Books--First Floor Van Pelt-Dietrich Library Center - Van Pelt - Featured Books--First Floor + Van Pelt - Featured Books--First Floor, see also table near Moelis Lounge Fisher Fine Arts Library From 2a527df44411a6f858e2a92cbdf22c55e527fb1a Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Mon, 11 Nov 2019 16:29:48 -0500 Subject: [PATCH 38/77] Fixed data offset issue that was hiding 'Request' link for the 101st item onward --- app/controllers/franklin_alma_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/franklin_alma_controller.rb b/app/controllers/franklin_alma_controller.rb index a915198b..1e36f7ea 100644 --- a/app/controllers/franklin_alma_controller.rb +++ b/app/controllers/franklin_alma_controller.rb @@ -329,7 +329,7 @@ def holding_items policies[data['policy']['value']] = nil pids_to_check << [data['pid'], data['policy']['value']] end - [data['policy']['value'], data['pid'], data['description'], due_date_policy || data['due_date_policy'], data['base_status']['desc'], data['barcode'], [], params['mms_id'], params['holding_id']] + [data['policy']['value'], data['pid'], data['description'], data['base_status']['desc'], data['barcode'], due_date_policy || data['due_date_policy'], [], params['mms_id'], params['holding_id']] } end From 3e5129ad2f6e0aaf0911c341a8c5d591ada0d8fc Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Thu, 5 Dec 2019 10:42:36 -0500 Subject: [PATCH 39/77] update collection news blog feed, per vallhonr --- app/controllers/collection_news_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/collection_news_controller.rb b/app/controllers/collection_news_controller.rb index 1e335094..a928c8f6 100644 --- a/app/controllers/collection_news_controller.rb +++ b/app/controllers/collection_news_controller.rb @@ -7,7 +7,7 @@ class CollectionNewsController < ApplicationController include RssProxy def index - rss_proxy('https://pennlibnews.wordpress.com/category/collection-news/feed/') + rss_proxy('http://www.library.upenn.edu/blogs/libraries-news/category/Collections/rss.xml') end end From 58997f73c385eeb9278e44b49f8d5e2525a2ddfd Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 9 Dec 2019 12:02:43 -0500 Subject: [PATCH 40/77] fix "More collection news" link; point to new blog --- app/views/catalog/_landing_bento.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/catalog/_landing_bento.html.erb b/app/views/catalog/_landing_bento.html.erb index 8f9f5047..73b0effb 100644 --- a/app/views/catalog/_landing_bento.html.erb +++ b/app/views/catalog/_landing_bento.html.erb @@ -49,7 +49,7 @@
    -
    More collection news...
    +
    More collection news...
    From a6ddd9e2d00245e6c73486aeb769f5f396374e71 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 23 Dec 2019 10:22:14 -0500 Subject: [PATCH 41/77] bookplates link to pantheon, not nebraska (per vallhonr) --- lib/penn_lib/marc.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/penn_lib/marc.rb b/lib/penn_lib/marc.rb index 097ca5fa..7f364afa 100644 --- a/lib/penn_lib/marc.rb +++ b/lib/penn_lib/marc.rb @@ -2374,7 +2374,7 @@ def get_web_link_display(rec) .split(/\W+/) .select { |word| !words_to_remove_from_web_link.member?(word.downcase) } .join('') - imagesource = "//www.library.upenn.edu/images/alum/bookplates/#{imagename}.gif" + imagesource = "//www.library.upenn.edu/sites/default/files/images/bookplates/#{imagename}.gif" links << { img_src: imagesource, img_alt: "#{linktext} Bookplate", From 3833f1bf629abd391bd1a0c60d2bfc390d4358fa Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Fri, 24 Jan 2020 11:11:22 -0500 Subject: [PATCH 42/77] Added test_access flag to correct eresource linking --- app/assets/javascripts/availability.js.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/availability.js.erb b/app/assets/javascripts/availability.js.erb index 24aed695..e6639e5c 100644 --- a/app/assets/javascripts/availability.js.erb +++ b/app/assets/javascripts/availability.js.erb @@ -43,7 +43,7 @@ $(document).ready(function() { if (holding['activation_status'] == 'Available') { var url = null; if (holding['portfolio_pid']) { - url = "https://<%= ENV['ALMA_DELIVERY_DOMAIN'] %>/view/uresolver/<%= ENV['ALMA_INSTITUTION_CODE'] %>/openurl?Force_direct=true&portfolio_pid=" + + url = "https://<%= ENV['ALMA_DELIVERY_DOMAIN'] %>/view/uresolver/<%= ENV['ALMA_INSTITUTION_CODE'] %>/openurl?Force_direct=true&test_access=true&&portfolio_pid=" + holding['portfolio_pid'] + "&rfr_id=info%3Asid%2Fprimo.exlibrisgroup.com&u.ignore_date_coverage=true" } From e45ab4c8b3953fb45b0ab400472a494b9651fb56 Mon Sep 17 00:00:00 2001 From: Katherine Schultz Date: Tue, 7 Jan 2020 15:45:26 -0500 Subject: [PATCH 43/77] create new loc Featured Books Display in MPA library --- config/translation_maps/locations.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/translation_maps/locations.xml b/config/translation_maps/locations.xml index 1cc79e7c..ec72be8a 100644 --- a/config/translation_maps/locations.xml +++ b/config/translation_maps/locations.xml @@ -877,6 +877,11 @@ Math/Physics/Astronomy Library Math/Physics/Astronomy - Circulation Desk     + + Math/Physics/Astronomy Library - Featured Books Display + Math/Physics/Astronomy Library + Math/Physics/Astronomy Library - Featured Books Display + Math/Physics/Astronomy Library - New Book Shelf Math/Physics/Astronomy Library From 18edd9834827f185ee14a3e50e021917c8fa2f8e Mon Sep 17 00:00:00 2001 From: Nikitas Tampakis Date: Thu, 11 Jun 2020 16:02:22 -0400 Subject: [PATCH 44/77] update database a-z list url --- app/views/catalog/_home_text.html.erb | 36 ++++++++++++++++++++ app/views/catalog/_landing_articles.html.erb | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/app/views/catalog/_home_text.html.erb b/app/views/catalog/_home_text.html.erb index 1871ec15..f042e3cd 100644 --- a/app/views/catalog/_home_text.html.erb +++ b/app/views/catalog/_home_text.html.erb @@ -2,7 +2,43 @@

    POD (Platform for Open Discovery)  proof of concept test instance

      +<<<<<<< HEAD
    • This proof-of-concept test interface is intended only for testing by librarians and showcasing shared indexing issues. It does not represent an end-user experience. We used Penn’s blacklight-test instance to load the data, so it does have some of the end-user UI of the Franklin catalog, but that was just for speed and convenience of getting a demo running.
      • +======= +
    • Find print books, ebooks, journal and newspaper titles, sound recordings, videos, conference proceedings, music scores, manuscripts, and other resources owned or subscribed to by the Penn Libraries.
      • +
    • See location and status information for print works, and links to online materials.
      • +
    • Place requests (when logged in).
      • +
    • Find and access public-domain books and journals from HathiTrust.
      • +
    + +

    Other Penn Resources

    + + + + +

    Search and Request from Other Libraries

    + +
      + +
    • BorrowDirect/E-ZBorrow (Search multiple library collections simultaneously).
      • + +
    • WorldCat (Union catalog of the holdings of libraries worldwide).
      • + +>>>>>>> 4e1331c... update database a-z list url
    diff --git a/app/views/catalog/_landing_articles.html.erb b/app/views/catalog/_landing_articles.html.erb index 2af70fa4..437d4581 100644 --- a/app/views/catalog/_landing_articles.html.erb +++ b/app/views/catalog/_landing_articles.html.erb @@ -14,7 +14,7 @@ From d956f4cc62f30b97d12cb1118189a79975209b02 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Fri, 31 Jan 2020 13:48:37 -0500 Subject: [PATCH 45/77] add last-10-years publication date facet bucket --- app/models/franklin_indexer.rb | 2 +- lib/penn_lib/marc.rb | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/app/models/franklin_indexer.rb b/app/models/franklin_indexer.rb index aaab1399..50eb7a9c 100644 --- a/app/models/franklin_indexer.rb +++ b/app/models/franklin_indexer.rb @@ -354,7 +354,7 @@ def define_all_fields to_field 'publication_date_f' do |rec, acc, ctx| val = ctx.clipboard.dig(:dates, :pub_date_decade) - acc << val if val + acc.concat(val.nil? ? [] : val) end to_field 'publication_dr' do |rec, acc, ctx| diff --git a/lib/penn_lib/marc.rb b/lib/penn_lib/marc.rb index 7f364afa..56fb07e0 100644 --- a/lib/penn_lib/marc.rb +++ b/lib/penn_lib/marc.rb @@ -952,9 +952,17 @@ def build_dates_hash(raw_pub_date_start, raw_pub_date_end = nil, content_date = content_date_end = pub_date_end end end + if current_year + 15 > pub_date_start.to_i + pub_date_decade = [ pub_date_start[0,3] + '0s' ] + if current_year - pub_date_start.to_i < 10 || current_year - pub_date_end.to_i < 10 + pub_date_decade << 'Last 10 years' + end + else + pub_date_decade = nil + end { :pub_date_sort => pub_date_start, - :pub_date_decade => current_year + 15 > pub_date_start.to_i ? pub_date_start[0,3] + '0s' : nil, + :pub_date_decade => pub_date_decade, :pub_date_range => "[#{pub_date_start} TO #{pub_date_end}]", :content_date_range => "[#{content_date_start} TO #{content_date_end}]", :pub_date_minsort => "#{pub_date_start}-01-01T00:00:00Z", From d17bda32093b0f996c55fc20b6f0e344e9d9406d Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Fri, 31 Jan 2020 13:49:13 -0500 Subject: [PATCH 46/77] filter out non-whitelisted "source specified" subject vocabs --- lib/penn_lib/marc.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/penn_lib/marc.rb b/lib/penn_lib/marc.rb index 56fb07e0..9c3656bd 100644 --- a/lib/penn_lib/marc.rb +++ b/lib/penn_lib/marc.rb @@ -341,7 +341,10 @@ def subject_codes_to_xfacet_prefixes def is_subject_field(field) # 10/2018 kms: add 2nd Ind 7 - subject_codes.member?(field.tag) && %w(0 2 4 7).member?(field.indicator2) + subject_codes.member?(field.tag) && (%w(0 2 4).member?(field.indicator2) || + (field.indicator2 == '7' && field.any? do |sf| + sf.code == '2' && %w(aat cct fast jlabsh lcsh lcstt lctgm local/osu mesh ndlsh nlksh).member?(sf.value) + end)) end def reject_pro_chr(sf) From bea2f9af5a4a8b6a1b66c184ec4ecd84f46344ea Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Fri, 31 Jan 2020 16:12:41 -0500 Subject: [PATCH 47/77] move "Penn Library Web" access facet to "Online library" location facet --- lib/penn_lib/marc.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/penn_lib/marc.rb b/lib/penn_lib/marc.rb index 9c3656bd..b8a011c1 100644 --- a/lib/penn_lib/marc.rb +++ b/lib/penn_lib/marc.rb @@ -712,12 +712,12 @@ def get_prt_count(rec) end def get_access_values(rec) - acc = rec.flat_map do |f| + acc = rec.map do |f| case f.tag when EnrichedMarc::TAG_HOLDING 'At the library' when EnrichedMarc::TAG_ELECTRONIC_INVENTORY - ['Online', 'Penn Library Web'] + 'Online' end end.compact acc += rec.fields('856') @@ -879,7 +879,7 @@ def holdings_location_mappings(rec, display_fieldname) # we don't facet for 'web' which is the 'Penn Library Web' location used in Voyager. # this location should eventually go away completely with data cleanup in Alma. - rec.fields(tag).flat_map do |field| + acc = rec.fields(tag).flat_map do |field| results = field.find_all { |sf| sf.code == subfield_code } .select { |sf| sf.value != 'web' } .map { |sf| @@ -893,6 +893,10 @@ def holdings_location_mappings(rec, display_fieldname) # flatten multiple 'library' values results.select(&:present?).flatten end.uniq + if rec.fields(EnrichedMarc::TAG_ELECTRONIC_INVENTORY).any? + acc << 'Online library' + end + return acc end def get_library_values(rec) From cba919492807fa3e2a8c140ceadc87ff5c23fe64 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 24 Feb 2020 13:42:37 -0500 Subject: [PATCH 48/77] Revert "add last-10-years publication date facet bucket" This reverts commit 659d581258d3ab2f24823364d5405d04cb570ba7. will be handled instead by specific "recently published" facet based on pub_max date field --- app/models/franklin_indexer.rb | 2 +- lib/penn_lib/marc.rb | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/app/models/franklin_indexer.rb b/app/models/franklin_indexer.rb index 50eb7a9c..aaab1399 100644 --- a/app/models/franklin_indexer.rb +++ b/app/models/franklin_indexer.rb @@ -354,7 +354,7 @@ def define_all_fields to_field 'publication_date_f' do |rec, acc, ctx| val = ctx.clipboard.dig(:dates, :pub_date_decade) - acc.concat(val.nil? ? [] : val) + acc << val if val end to_field 'publication_dr' do |rec, acc, ctx| diff --git a/lib/penn_lib/marc.rb b/lib/penn_lib/marc.rb index b8a011c1..9555b41f 100644 --- a/lib/penn_lib/marc.rb +++ b/lib/penn_lib/marc.rb @@ -959,17 +959,9 @@ def build_dates_hash(raw_pub_date_start, raw_pub_date_end = nil, content_date = content_date_end = pub_date_end end end - if current_year + 15 > pub_date_start.to_i - pub_date_decade = [ pub_date_start[0,3] + '0s' ] - if current_year - pub_date_start.to_i < 10 || current_year - pub_date_end.to_i < 10 - pub_date_decade << 'Last 10 years' - end - else - pub_date_decade = nil - end { :pub_date_sort => pub_date_start, - :pub_date_decade => pub_date_decade, + :pub_date_decade => current_year + 15 > pub_date_start.to_i ? pub_date_start[0,3] + '0s' : nil, :pub_date_range => "[#{pub_date_start} TO #{pub_date_end}]", :content_date_range => "[#{content_date_start} TO #{content_date_end}]", :pub_date_minsort => "#{pub_date_start}-01-01T00:00:00Z", From 4ac8abb7af508876fb3c3241295ec2f4de3cc109 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 24 Feb 2020 13:44:18 -0500 Subject: [PATCH 49/77] also whitelist subject display terms where source thesaurus specified --- lib/penn_lib/marc.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/penn_lib/marc.rb b/lib/penn_lib/marc.rb index 9555b41f..699debdb 100644 --- a/lib/penn_lib/marc.rb +++ b/lib/penn_lib/marc.rb @@ -506,7 +506,9 @@ def get_subjects_from_600s_and_800(rec, indicator2) acc += rec.fields .select { |f| subject_600s.member?(f.tag) || (f.tag == '880' && has_subfield6_value(f, /^(#{subject_600s.join('|')})/)) } - .select { |f| f.indicator2 == indicator2 || (f.indicator2 == '7' && indicator2 == '0') } + .select { |f| f.indicator2 == indicator2 || (f.indicator2 == '7' && indicator2 == '0' && f.any? do |sf| + sf.code == '2' && %w(aat cct fast jlabsh lcsh lcstt lctgm local/osu mesh ndlsh nlksh).member?(sf.value) + end)} .map do |field| #added 2017/04/10: filter out 0 (authority record numbers) added by Alma value_for_link = join_subfields(field, &subfield_not_in(%w{0 6 8 2 e w})) From 2923087a6ef2b21b51051af4f1805ef44a8eced1 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 24 Feb 2020 14:08:05 -0500 Subject: [PATCH 50/77] prevent duplicate subjects from being displayed no differentiation in source/type/ontology for now; they're just strings, linked/searched as strings, so there is no point whatsoever in duplicating them for display. --- lib/penn_lib/marc.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/penn_lib/marc.rb b/lib/penn_lib/marc.rb index 699debdb..a767eb8c 100644 --- a/lib/penn_lib/marc.rb +++ b/lib/penn_lib/marc.rb @@ -499,6 +499,7 @@ def subject_69X # 11/2018: add 69x as local subj, add 650 _7 as subj def get_subjects_from_600s_and_800(rec, indicator2) + track_dups = Set.new acc = [] if %w{0 1 2}.member?(indicator2) #Subjects, Childrens subjects, and Medical Subjects all share this code @@ -527,7 +528,7 @@ def get_subjects_from_600s_and_800(rec, indicator2) link_type: 'subject_xfacet2' } end - end.compact + end.compact.select { |val| track_dups.add?(val) } elsif indicator2 == '4' # Local subjects # either a tag in subject_600s list with ind2==4, or a tag in subject_69X list with any ind2. @@ -557,7 +558,7 @@ def get_subjects_from_600s_and_800(rec, indicator2) link_type: 'subject_search' } end - end.compact + end.compact.select { |val| track_dups.add?(val) } end acc end From b71b0fd6d5c01be490800463c8f5f8588c449200 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 24 Feb 2020 14:15:43 -0500 Subject: [PATCH 51/77] add top-level catch-all "Veterinary" library alias --- config/translation_maps/locations.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/config/translation_maps/locations.xml b/config/translation_maps/locations.xml index ec72be8a..f57d9321 100644 --- a/config/translation_maps/locations.xml +++ b/config/translation_maps/locations.xml @@ -1031,18 +1031,21 @@ Veterinary: du Pont Library (New Bolton) Health Sciences Libraries + Veterinary Veterinary: du Pont Library (New Bolton) Veterinary: du Pont Library (New Bolton) Veterinary: du Pont Library (New Bolton) - Circulation Desk Health Sciences Libraries + Veterinary Veterinary: du Pont Library (New Bolton) Veterinary: du Pont Library (New Bolton) - Circulation Desk Veterinary: du Pont Library (New Bolton) - Office Health Sciences Libraries + Veterinary Veterinary: du Pont Library (New Bolton) Veterinary: du Pont Library (New Bolton) - Office @@ -1056,12 +1059,14 @@ Veterinary: du Pont Library (New Bolton) - Reference Health Sciences Libraries + Veterinary Veterinary: du Pont Library (New Bolton) Veterinary: du Pont Library (New Bolton) - Reference Veterinary: du Pont Library (New Bolton) - Reserve Health Sciences Libraries + Veterinary Veterinary: du Pont Library (New Bolton) Reserve Veterinary: du Pont Library (New Bolton) - Reserve @@ -1131,6 +1136,7 @@ On Order Health Sciences Libraries + Veterinary Veterinary: du Pont Library (New Bolton) Ordered for New Bolton Library @@ -1152,6 +1158,7 @@ On Order Health Sciences Libraries + Veterinary Ordered for Veterinary: Atwood Library (Campus) @@ -1745,42 +1752,49 @@ Veterinary: Atwood Library (Campus) Health Sciences Libraries + Veterinary Veterinary: Atwood Library (Campus) Veterinary: Atwood Library (Campus) Veterinary: Atwood Library (Campus) - Display Health Sciences Libraries + Veterinary Veterinary: Atwood Library (Campus) Veterinary: Atwood Library (Campus) - Display Veterinary: Atwood Library (Campus) - Librarian's Office Health Sciences Libraries + Veterinary Veterinary: Atwood Library (Campus) Veterinary: Atwood Library (Campus) - Librarian's Office Veterinary: Atwood Library (Campus) - Oversize Health Sciences Libraries + Veterinary Veterinary: Atwood Library (Campus) Veterinary: Atwood Library (Campus) - Oversize Veterinary: Atwood Library (Campus) - Reference Health Sciences Libraries + Veterinary Veterinary: Atwood Library (Campus) Veterinary: Atwood Library (Campus) - Reference Veterinary: Atwood Library (Campus) - Reference Oversize Health Sciences Libraries + Veterinary Veterinary: Atwood Library (Campus) Veterinary: Atwood Library (Campus) - Reference Oversize Veterinary: Atwood Library (Campus) - Reserve Health Sciences Libraries + Veterinary Veterinary: Atwood Library (Campus) Reserve Veterinary: Atwood Library (Campus) - Reserve @@ -1788,6 +1802,7 @@ Veterinary: Atwood Library (Campus) - Reserve Oversize Health Sciences Libraries + Veterinary Veterinary: Atwood Library (Campus) Reserve Veterinary: Atwood Library (Campus) - Reserve Oversize From 262eef87a96c64c1c5abb07cb766223bbd85ef35 Mon Sep 17 00:00:00 2001 From: Chris Clement Date: Mon, 16 Mar 2020 11:25:51 -0400 Subject: [PATCH 52/77] Removed linking to physical item services --- app/assets/javascripts/availability.js.erb | 11 ++++-- app/controllers/franklin_alma_controller.rb | 40 +++++++++++---------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/app/assets/javascripts/availability.js.erb b/app/assets/javascripts/availability.js.erb index e6639e5c..90028c85 100644 --- a/app/assets/javascripts/availability.js.erb +++ b/app/assets/javascripts/availability.js.erb @@ -14,6 +14,9 @@ $(document).ready(function() { } function constructAeonLink(mmsid, holding) { + // TODO: remove this line once the libraries are open again + return null; + if(holding['link_to_aeon']) { return 'Request to view'; } @@ -113,7 +116,9 @@ $(document).ready(function() { var requests = globaldata[data[i][7]]; if(validRequestTypes.indexOf(requests[0]) != -1) { renderdata[mmsid].push(data[i][7]); - data[i][5][0] = "Request"; +// TODO: Uncomment when libraries reopen + //data[i][5][0] = "Request"; + data[i][5][0] = ""; table.row(i).invalidate().draw(); } } @@ -305,7 +310,9 @@ $(document).ready(function() { var requests = globaldata[data[i][0]]; if(validRequestTypes.indexOf(requests[0]) != -1) { renderdata[mmsid].push(data[i][0]); - data[i][5].push("Request"); +// TODO: Uncomment when libraries reopen + //data[i][5].push("Request"); + data[i][5].push(""); table.row(i).invalidate().draw(); } } diff --git a/app/controllers/franklin_alma_controller.rb b/app/controllers/franklin_alma_controller.rb index 1e36f7ea..78c670ec 100644 --- a/app/controllers/franklin_alma_controller.rb +++ b/app/controllers/franklin_alma_controller.rb @@ -218,7 +218,8 @@ def single_availability else bib_data['availability'][mmsid]['holdings'].each do |holding| links = [] - links << "Request to view in reading room" if holding['link_to_aeon'] +# TODO: Uncomment when libraries reopen + #links << "Request to view in reading room" if holding['link_to_aeon'] holding['availability'] = availability_status[holding['availability']] || 'Requestable' if has_holding_info @@ -345,11 +346,12 @@ def holding_items end policies[policy] = "/alma/request/?mms_id=%{mms_id}&holding_id=%{holding_id}&item_pid=%{item_pid}" unless not_requestable } - table_data.each { |item| policy = item.shift() request_url = (policies[policy] || '') % params.merge({:item_pid => item[0]}) - item[5] << "Request" unless (request_url.empty? || item[2] != 'Item in place') +# TODO: Uncomment when libraries reopen + #item[5] << "Request" unless (request_url.empty? || item[2] != 'Item in place') + item[5] << "" unless (request_url.empty? || item[2] != 'Item in place') } render :json => {"data": table_data} @@ -371,15 +373,16 @@ def request_options nil else case option['type']['value'] - when 'HOLD' - { - :option_name => 'Request', - #:option_url => option['request_url'], - :option_url => "/alma/request?mms_id=#{params['mms_id']}", - :avail_for_physical => true, - :avail_for_electronic => true, - :highlightable => true - } +# TODO: Uncomment when libraries reopen + #when 'HOLD' + #{ + #:option_name => 'Request', + ##:option_url => option['request_url'], + #:option_url => "/alma/request?mms_id=#{params['mms_id']}", + #:avail_for_physical => true, + #:avail_for_electronic => true, + #:highlightable => true + #} when 'GES' option_url = option['request_url'] if option_url.index('?') @@ -434,12 +437,13 @@ def request_options end } - results.append({ - :option_name => "Books By Mail", - :option_url => "https://franklin.library.upenn.edu/redir/booksbymail?bibid=#{params['mms_id']}", - :avail_for_physical => true, - :avail_for_electronic => false - }) if ['Athenaeum Member','Faculty','Faculty Express','Grad Student','Library Staff'].member?(session['user_group']) +# TODO: Uncomment when libraries reopen + #results.append({ + #:option_name => "Books By Mail", + #:option_url => "https://franklin.library.upenn.edu/redir/booksbymail?bibid=#{params['mms_id']}", + #:avail_for_physical => true, + #:avail_for_electronic => false + #}) if ['Athenaeum Member','Faculty','Faculty Express','Grad Student','Library Staff'].member?(session['user_group']) render :json => results end From d450b67d53ccf8a6dc5e6b77c711a7971e120d9e Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 24 Mar 2020 12:42:42 -0400 Subject: [PATCH 53/77] removed call to deprecated franklin_alerts, per ivangold --- app/views/shared/_footer.html.erb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/views/shared/_footer.html.erb b/app/views/shared/_footer.html.erb index a8c212b6..cdc932e5 100644 --- a/app/views/shared/_footer.html.erb +++ b/app/views/shared/_footer.html.erb @@ -91,5 +91,3 @@ - - From 55d7890349d6e22e953ba4b869e72c07c70ebb66 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 24 Mar 2020 12:46:29 -0400 Subject: [PATCH 54/77] hotfix: only proxy "View and filter" link when user has proxy-equivalent auth --- app/assets/javascripts/login-ezproxy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/login-ezproxy.js b/app/assets/javascripts/login-ezproxy.js index 140ac3eb..53a36ca0 100644 --- a/app/assets/javascripts/login-ezproxy.js +++ b/app/assets/javascripts/login-ezproxy.js @@ -45,7 +45,7 @@ $(document).ready(function() { var proxyUrl = PROXY_PREPEND + 'http://127.0.0.1:8082/?redirect=' + encodeURIComponent(currentUrl); $(div).find(".ezproxy-login-link").attr("href", proxyUrl); - if (auth !== null && auth !== undefined) { + if (auth !== null && auth !== undefined && auth.loggedIn) { $(div).find(".view-and-filter").find("a").each(function (idx, element) { // ezproxy handles url param specifically so we don't need to escape it var viewAndFilterUrl = $(element).attr("href"); From d45f0a988f63ddfb3326f0292896f122bc348398 Mon Sep 17 00:00:00 2001 From: Nikitas Tampakis Date: Thu, 11 Jun 2020 16:04:42 -0400 Subject: [PATCH 55/77] temporarily add links to hathi records corresponding to local holdings --- app/controllers/catalog_controller.rb | 4 ++++ app/helpers/document_render_helper.rb | 11 ++++++++++- app/views/catalog/_show_expanded.html.erb | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 7058c41a..f99d7b91 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -168,7 +168,11 @@ def self.generate_cluster_fq(index, limit, elvl = false) 'facet.mincount': 0, # fq: '{!tag=cluster}{!collapse field=cluster_id nullPolicy=expand size=5000000 min=record_source_id}', # this approach needs expand.field=cluster_id +<<<<<<< HEAD # MOVE TO FACET: fq: '{!bool tag=cluster must_not=$x1 must_not=$x2 must_not=$x3 must_not=$x4 must_not=$x5 must_not=$x6 must_not=$x7}', +======= + fq: %q~{!tag=cluster}NOT ({!join from=cluster_id to=cluster_id v='record_source_f:"Penn"'} AND record_source_f:"HathiTrust") NOT record_source_id:3~, +>>>>>>> 4618572... temporarily add links to hathi records corresponding to local holdings expand: 'true', 'expand.field': 'cluster_id', 'expand.q': '*:*', diff --git a/app/helpers/document_render_helper.rb b/app/helpers/document_render_helper.rb index 6d68bd4a..8f881783 100644 --- a/app/helpers/document_render_helper.rb +++ b/app/helpers/document_render_helper.rb @@ -51,6 +51,10 @@ def render_electronic_holdings_links(electronic_holdings_str) end end + @@HATHI_TMP_TEXT = 'HathiTrust Digital Library Login for full text' + @@HATHI_REPLACEMENT_TEXT = 'COVID-19 Special Access from HathiTrust — Login for full text' + @@HATHI_LOGIN_PREFIX = 'https://babel.hathitrust.org/Shibboleth.sso/Login?entityID=https://idp.pennkey.upenn.edu/idp/shibboleth&target=https%3A%2F%2Fbabel.hathitrust.org%2Fcgi%2Fping%2Fpong%3Ftarget%3D' + def render_online_resource_display_for_index_view(options) values = options[:value] suppress_remote_links = 'Include Partner Libraries' != params.dig('f', 'cluster', 0) @@ -71,11 +75,16 @@ def render_online_display_for_show_view(options) JSON.parse(value).map do |link_struct| url = link_struct['linkurl'] text = link_struct['linktext'] + orig_url = url + if text == @@HATHI_TMP_TEXT + text = @@HATHI_REPLACEMENT_TEXT + url = @@HATHI_LOGIN_PREFIX + URI.encode_www_form_component(url) + end html = %Q{
    #{text}} html += '
    '.html_safe if !text.start_with?('http') - html += + url + html += + orig_url end if link_struct['volumes'] diff --git a/app/views/catalog/_show_expanded.html.erb b/app/views/catalog/_show_expanded.html.erb index ded9f117..413325c0 100644 --- a/app/views/catalog/_show_expanded.html.erb +++ b/app/views/catalog/_show_expanded.html.erb @@ -1,4 +1,4 @@ -<% if document.expanded_docs.size > 0 %> +<% if document.expanded_docs.select { |doc| !doc.id.start_with?('HATHIALL_') }.size > 0 %>
    Other records:
    @@ -7,7 +7,7 @@
    From 75bf95b444a4b1e42d92b69d62a2d46993b8a6f6 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Mon, 13 Apr 2020 09:51:03 -0400 Subject: [PATCH 56/77] remove VIVO from "Other resources", per baowei --- app/views/catalog/_landing_bento.html.erb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/views/catalog/_landing_bento.html.erb b/app/views/catalog/_landing_bento.html.erb index 73b0effb..c9eee0fc 100644 --- a/app/views/catalog/_landing_bento.html.erb +++ b/app/views/catalog/_landing_bento.html.erb @@ -17,7 +17,6 @@
  • PennText Article Finder
  • Research guides
  • ScholarlyCommons
    • -
  • VIVO
  • WorldCat
  • From 2bb318112101e2afa4fde778acd9447d01c8a15d Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 21 Apr 2020 15:46:29 -0400 Subject: [PATCH 57/77] clarify patron classes authorized for Hathi COVID-19 special access --- app/helpers/document_render_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/helpers/document_render_helper.rb b/app/helpers/document_render_helper.rb index 8f881783..933aa9de 100644 --- a/app/helpers/document_render_helper.rb +++ b/app/helpers/document_render_helper.rb @@ -52,7 +52,7 @@ def render_electronic_holdings_links(electronic_holdings_str) end @@HATHI_TMP_TEXT = 'HathiTrust Digital Library Login for full text' - @@HATHI_REPLACEMENT_TEXT = 'COVID-19 Special Access from HathiTrust — Login for full text' + @@HATHI_REPLACEMENT_TEXT = 'COVID-19 Special Access from HathiTrust — Full text access only for students, active faculty, and permanent staff' @@HATHI_LOGIN_PREFIX = 'https://babel.hathitrust.org/Shibboleth.sso/Login?entityID=https://idp.pennkey.upenn.edu/idp/shibboleth&target=https%3A%2F%2Fbabel.hathitrust.org%2Fcgi%2Fping%2Fpong%3Ftarget%3D' def render_online_resource_display_for_index_view(options) From 3263afbec94a7681239ab136a7eecf3f6bb6baba Mon Sep 17 00:00:00 2001 From: Nikitas Tampakis Date: Thu, 11 Jun 2020 16:05:07 -0400 Subject: [PATCH 58/77] link to details regarding Hathi ETAS access authorization --- app/helpers/document_render_helper.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/helpers/document_render_helper.rb b/app/helpers/document_render_helper.rb index 933aa9de..19fde933 100644 --- a/app/helpers/document_render_helper.rb +++ b/app/helpers/document_render_helper.rb @@ -52,7 +52,8 @@ def render_electronic_holdings_links(electronic_holdings_str) end @@HATHI_TMP_TEXT = 'HathiTrust Digital Library Login for full text' - @@HATHI_REPLACEMENT_TEXT = 'COVID-19 Special Access from HathiTrust — Full text access only for students, active faculty, and permanent staff' + @@HATHI_REPLACEMENT_TEXT = 'COVID-19 Special Access from HathiTrust' + @@HATHI_INFO = ' — Full text access only for students, active faculty, and permanent staff' @@HATHI_LOGIN_PREFIX = 'https://babel.hathitrust.org/Shibboleth.sso/Login?entityID=https://idp.pennkey.upenn.edu/idp/shibboleth&target=https%3A%2F%2Fbabel.hathitrust.org%2Fcgi%2Fping%2Fpong%3Ftarget%3D' def render_online_resource_display_for_index_view(options) @@ -75,12 +76,14 @@ def render_online_display_for_show_view(options) JSON.parse(value).map do |link_struct| url = link_struct['linkurl'] text = link_struct['linktext'] + append = '' orig_url = url if text == @@HATHI_TMP_TEXT text = @@HATHI_REPLACEMENT_TEXT url = @@HATHI_LOGIN_PREFIX + URI.encode_www_form_component(url) + append = @@HATHI_INFO end - html = %Q{
    #{text}} + html = %Q{
    #{text}#{append}} html += '
    '.html_safe if !text.start_with?('http') From 69aab77f30417014ef2581c0bbbb3a3e214e8619 Mon Sep 17 00:00:00 2001 From: Nikitas Tampakis Date: Wed, 20 May 2020 12:06:26 -0400 Subject: [PATCH 59/77] Adds initial colenda bento integration Co-authored-by: Iva Goldsmith --- .gitignore | 3 + .../bento_search/colenda_engine.rb | 71 +++++++++++++++++++ app/views/catalog/bento.html.erb | 13 +++- .../_colenda_ajax_results_wrapper.html.erb | 14 ++++ .../_hide_colenda_zero_results.html.erb | 6 ++ config/initializers/bento_search.rb | 9 +++ 6 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 app/search_engines/bento_search/colenda_engine.rb create mode 100644 app/views/layouts/_colenda_ajax_results_wrapper.html.erb create mode 100644 app/views/layouts/_hide_colenda_zero_results.html.erb diff --git a/.gitignore b/.gitignore index 57b25409..57869270 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ local_dev_env .idea public/assets + +.byebug_history +.ruby-version diff --git a/app/search_engines/bento_search/colenda_engine.rb b/app/search_engines/bento_search/colenda_engine.rb new file mode 100644 index 00000000..6bba15a0 --- /dev/null +++ b/app/search_engines/bento_search/colenda_engine.rb @@ -0,0 +1,71 @@ +require 'httpclient' +require 'oga' + +class BentoSearch::ColendaEngine + + include BentoSearch::SearchEngine + + def search_implementation(args) + return if args[:query].nil? + terms = { :query_term => args[:query], + :field_term => 'keyword', + :f_term => args[:f_term].present? ? args[:f_term] : '', + :f_value => args[:f_value].present? ? args[:f_value] : '' + + } + + url = colenda_url(terms) + + http_client = HTTPClient.new + + results = BentoSearch::Results.new + + hash, response, exception = nil + + begin + response = http_client.get(url, nil, nil) + hash = JSON.parse(response.body) + + rescue BentoSearch::RubyTimeoutClass, HTTPClient::ConfigurationError, HTTPClient::BadResponseError, MultiJson::DecodeError, Nokogiri::SyntaxError => e + exception = e + end + if (response.nil? || hash.nil? || exception || + (! HTTP::Status.successful? response.status)) + results.error ||= {} + results.error[:exception] = e + results.error[:status] = response.status if response.present? + + return results + end + + results.total_items = hash['response']['pages']['total_count'].to_i + return results unless results.total_items > 0 + + entries = results.total_items > 1 ? hash['response']['docs'] : [hash['response']['docs']] + + entries.each do |entry| + online_resource = {} + item = BentoSearch::ResultItem.new + item.title = entry['title_ssim'].first + item.link = colenda_item_url(entry['id']) + item.authors = [entry['personal_name_sim'], entry['creator_sim'], entry['corporate_name_sim']].compact + item.format = entry['format_ssim'] + + results << item + + end + + return results + + end + + def colenda_item_url(id) + "https://colenda.library.upenn.edu/catalog/#{id}" + end + + def colenda_url(args) + "https://colenda.library.upenn.edu/catalog.json?per_page=5&q=#{CGI.escape(args[:query_term])}" + end + + +end diff --git a/app/views/catalog/bento.html.erb b/app/views/catalog/bento.html.erb index ace2a967..224b6a49 100644 --- a/app/views/catalog/bento.html.erb +++ b/app/views/catalog/bento.html.erb @@ -11,7 +11,7 @@
    <%= link_to "View and filter #{number_with_delimiter(@response.total)} #{@response.total != 1 ? 'results' : 'result'}", search_catalog_path(q: params[:q] || params[:query], search_field: 'keyword') %>
    - <%= bento_search 'catalog', :query => params[:q] || '', :per_page => 5, :load => :ajax_auto %> + <%= bento_search 'catalog', :query => params[:q] || '', :load => :ajax_auto %>
    + +
    + +
    +
    +

    Colenda

    + <%= bento_search 'colenda', :query => params[:q] || '', :load => :ajax_auto %>
    <%= render_expert_help(@response.aggregations["subject_specialists"]) %> @@ -62,4 +70,3 @@
    - diff --git a/app/views/layouts/_colenda_ajax_results_wrapper.html.erb b/app/views/layouts/_colenda_ajax_results_wrapper.html.erb new file mode 100644 index 00000000..b0166f50 --- /dev/null +++ b/app/views/layouts/_colenda_ajax_results_wrapper.html.erb @@ -0,0 +1,14 @@ +
    + <%= link_to "View and filter #{number_with_delimiter(results.total_items)} #{results.total_items != 1 ? 'results' : 'result'}", databases_results_url(params[:q] || params[:query]) %> +
    + +<%= yield %> + +<% if results.total_items > 0 %> +
    + <%= link_to(("View all results #{image_tag('view_all_arrow.png', :alt => '')}").html_safe, databases_results_url(params[:q] || params[:query])) %> +
    +<% end %> + +
    +
    diff --git a/app/views/layouts/_hide_colenda_zero_results.html.erb b/app/views/layouts/_hide_colenda_zero_results.html.erb new file mode 100644 index 00000000..d87d1eec --- /dev/null +++ b/app/views/layouts/_hide_colenda_zero_results.html.erb @@ -0,0 +1,6 @@ + diff --git a/config/initializers/bento_search.rb b/config/initializers/bento_search.rb index c368088d..b4cb122e 100644 --- a/config/initializers/bento_search.rb +++ b/config/initializers/bento_search.rb @@ -56,6 +56,15 @@ end end +BentoSearch.register_engine('colenda') do |conf| + conf.engine = 'BentoSearch::ColendaEngine' + conf.allow_routable_results = true + conf.for_display do |display| + display[:ajax] = { 'wrapper_template' => 'layouts/colenda_ajax_results_wrapper' } + display[:no_results_partial] = 'layouts/hide_colenda_zero_results' + end +end + BentoSearch.register_engine('catalog') do |conf| conf.engine = 'BentoSearch::CatalogEngine' conf.allow_routable_results = true From 60921b61927b3e6df642478614347bdc4c5a51fc Mon Sep 17 00:00:00 2001 From: Iva Goldsmith Date: Tue, 26 May 2020 15:46:19 -0400 Subject: [PATCH 60/77] Add colenda tab --- app/helpers/application_helper.rb | 8 ++++++++ app/views/catalog/_franklin_search_bar.html.erb | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 4e7a0ebd..081a95a3 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -20,6 +20,10 @@ def google_site_search_results_url(query) return "https://www.library.upenn.edu/search/web-pages?q=#{query}" end + def colenda_search_results_url(query) + return "https://colenda.library.upenn.edu/catalog?q=#{query}" + end + def catalog_results_url(query) return search_catalog_path(q: query, search_field: 'keyword') end @@ -62,6 +66,10 @@ def render_tab_link(tab_id, tab_label, anchor, url, data_target) attrs = { 'href': "http://www.library.upenn.edu/search/web-pages" } + elsif tab_id == 'colenda' && action_name == 'landing' + attrs = { + 'href': "https://colenda.library.upenn.edu" + } else attrs = { 'href': anchor, diff --git a/app/views/catalog/_franklin_search_bar.html.erb b/app/views/catalog/_franklin_search_bar.html.erb index 41878389..65737bff 100644 --- a/app/views/catalog/_franklin_search_bar.html.erb +++ b/app/views/catalog/_franklin_search_bar.html.erb @@ -46,6 +46,11 @@ <%= render_tab_link('databases', "Databases", 'catalog?utf8=âś“&search_field=keyword&f%5Bformat_f%5D%5B%5D=Database+%26+Article+Index&q=', databases_results_url(params[:q]), '#databases-main, #databases-search-navbar') %>
    +