diff --git a/.gitignore b/.gitignore index 89741e56..f626c9a8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,8 @@ # or operating system, you probably want to add a global ignore instead: # git config --global core.excludesfile '~/.gitignore_global' -# ignore nginx server config -/nginx/nginx.conf +# ignore nginx configuration +/nginx/ # Ignore bundler config. /.bundle @@ -38,6 +38,9 @@ .ruby-version VERSION +# don't track .env files +.env +.lando.env .env* *.key *.crt @@ -46,3 +49,4 @@ rdf_data *.tgz sdbm_data *.bak +test.ttl* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56cdd135..b43bc271 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -240,4 +240,4 @@ remove_nginx_image: rules: - when: always tags: - - build \ No newline at end of file + - build diff --git a/Gemfile.lock b/Gemfile.lock index f58f49cf..0e3dcdc3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -309,7 +309,7 @@ GEM rspec-mocks (~> 3.1.0) rspec-support (~> 3.1.0) rspec-support (3.1.2) - rubyzip (1.2.2) + rubyzip (1.3.0) sanitize (5.0.0) crass (~> 1.0.2) nokogiri (>= 1.8.0) diff --git a/README-development.md b/README-development.md new file mode 100644 index 00000000..14b8004c --- /dev/null +++ b/README-development.md @@ -0,0 +1,179 @@ +Installation +===== + +**1. Clone the repository** + +DEPRECATED -- USING ANSIBLE + + git clone https://github.com/upenn-libraries/sdbmss.git + +**2. Create file docker environment file `.env` in the root folder of the cloned repository. Define the following environment variables:** + +This is the password for the public access point for the Jena server, both for making updates and for downloading RDF data file + + ADMIN_PASSWORD= + +Login for private staging server deployment, if used + + RAILS_USER= + RAILS_PASS= + +MYSQL database setup. MYSQL_HOST is the name of the docker service defined in docker-compose.yml - everything else is up to you + + MYSQL_HOST=db + MYSQL_DATABASE= + MYSQL_ROOT_PASSWORD= + MYSQL_USER= + MYSQL_PASSWORD= + +RabbitMQ user and password; used for RDF live-update messaging system. Define here, then set the correct values when instantiating the RabitMQ service **(done later)** + + RABBIT_USER= + RABBIT_PASSWORD= + +Rails email setup. Depends on where the application is running and what mailing service is available + + SDBMSS_APP_HOST= + SDBMSS_SMTP_HOST= + SDBMSS_EMAIL_FROM= + SDBMSS_NOTIFY_EMAIL= + SDBMSS_NOTIFY_EMAIL_PASSWORD= + # email address + SDBMSS_EMAIL_EXCEPTIONS_TO= + +Generate separate keys (for each) by running rails task secret. + + SDBMSS_BLACKLIGHT_SECRET_KEY= + SDBMSS_DEVISE_SECRET_KEY= + SDBMSS_SECRET_KEY_BASE= + SDBMSS_SECRET_TOKEN= + +URL for SOLR server, using relative location of docker service (using name 'solr' from docker-compose.yml) + + SOLR_URL=http://solr:8982/solr/development + +**3. Build and Run (First Time)** + +Start everything up: + + docker-compose up --build + +**4. RabbitMQ First Time Setup** + +First time we need to create user and grant permissions. Use the same values for USER/PASS as set in your .docker-environment file + + docker-compose -f docker-compose-dev.yml rabbitmq /bin/bash + rabbitmqctl add_user + rabbitmqctl set_user_tags adminstrator + rabbitmqctl set_permissions -p / ".*" ".*" ".*" + +Then restart dependent containers: + + docker-compose -f docker-compose-dev.yml restart interface + docker-compose -f docker-compose-dev.yml restart rails + + +**5. First Time Setup: Rails and SOLR** + +Setup database - perform setup: + + docker exec $(docker ps -q -f name=sdbmss_rails) bundle exec rake db:setup + +(Optional: Load data from .sql dump) + +```bash +docker cp sdbm.sql.gz $(docker ps -q -f name=sdbmss_db):/tmp/sdbm.sql.gz +docker exec -it $(docker ps -q -f name=sdbmss_db) bash +cd /tmp +gunzip sdbm.sql.gz +mysql -u -p < sdbm.sql +rm sdbm.sql # remove the sql file (it's very big) +exit # exit the MySQL container +docker exec $(docker ps -q -f name=sdbmss_rails) bundle exec rake db:migrate +``` + +**NOTE**: If you are importing from a data file that includes **Page** objects, the database records will be copied, but not the page files. You will need to move these manually to the appropriate place in the public/static folder (uploads/, tooltips/ or docs/) + +``` +docker cp docs $(docker ps -q -f name=sdbmss_rails):/usr/src/app/public/static/ +docker cp tooltips $(docker ps -q -f name=sdbmss_rails):/usr/src/app/public/static/ +docker cp uploads $(docker ps -q -f name=sdbmss_rails):/usr/src/app/public/static/ +``` + +Index in Solr: + + docker exec $(docker ps -q -f name=sdbmss_rails) bundle exec rake sunspot:reindex + +**6. Jena First Time Setup** + +``` +docker exec -t $(docker ps -q -f name=sdbmss_rails) bundle exec rake sparql:test +``` +File should be in `.`; gzip it. + +``` +gzip test.ttl +``` + +Copy file to Jena container and gunzip it + +``` +docker cp test.ttl.gz $(docker ps -q -f name=sdbmss_jena):/tmp/ +docker exec -t $(docker ps -q -f name=sdbmss_jena) gunzip /tmp/test.ttl.gz +``` + +Load the data into Jena + +``` +docker exec -t $(docker ps -q -f name=sdbmss_jena) sh -c 'cd /jena-fuseki && ./tdbloader --loc=/fuseki/databases/sdbm /tmp/test.ttl' +``` + +Clean up the files. + +``` +$ docker exec -t $(docker ps -q -f name=sdbmss_jena) rm /tmp/test.ttl +rm ~/deployments/sdbmss/test.ttl.gz +``` + +Create the datset in Jena Fuseki. + +Go here and create the sdbm dataset: + +- Click 'add new data set' +- Enter 'sdbm' +- Select 'Persistent – dataset will persist across Fuseki restarts' +- Click 'create dataset' + +Scale the services: + +``` +docker-compose -f docker-compose-dev.yml restart jena +docker-compose -f docker-compose-dev.yml restart rabbitmq +docker-compose -f docker-compose-dev.yml restart rails +``` + +Run the Jena verify task to confirm that it works. Be sure to hide the debugging output. + +``` +docker-compose -f docker-compose-dev.yml exec rails bundle exec rake jena:verify | grep -v DEBUG +``` + +NB: You may need to run the command more than once. + +``` +sdbm01[~]$ docker-compose -f docker-compose-dev.yml exec rails bundle exec rake jena:verify | grep -v DEBUG +Starting Queue Listening +No more messages in queue. +Remaining responses: 764 +$ docker-compose -f docker-compose-dev.yml exec rails bundle exec rake jena:verify | grep -v DEBUG +Starting Queue Listening +Parsed contents: {"id"=>300122, "code"=>"200", "message"=>"OK"} +Jena Update was Successful! +Parsed contents: {"id"=>300211, "code"=>"200", "message"=>"OK"} +Jena Update was Successful! +Parsed contents: {"id"=>300212, "code"=>"200", "message"=>"OK"} +Jena Update was Successful! +Parsed contents: {"id"=>300213, "code"=>"200", "message"=>"OK"} +Jena Update was Successful! +# ... etc. +``` \ No newline at end of file diff --git a/README.md b/README.md index 5dfdea55..1e56e568 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ DEPRECATED -- USING ANSIBLE URL for SOLR server, using relative location of docker service (using name 'solr' from docker-compose.yml) - SOLR_URL=http://solr:8983/solr/development + SOLR_URL=http://solr:8982/solr/development **3. (Optional) Create a file VERSION in the root folder of the cloned repository.** @@ -103,7 +103,7 @@ DEPRECATED -- USING ANSIBLE **5. First Time Setup: Rails and SOLR** -Before you begin, have the folliwing: +Before you begin, have the following: - Database backup from previous version sdbm.sql.gz diff --git a/app/models/concerns/tell_bunny.rb b/app/models/concerns/tell_bunny.rb index 5495530c..74c596ad 100644 --- a/app/models/concerns/tell_bunny.rb +++ b/app/models/concerns/tell_bunny.rb @@ -13,15 +13,73 @@ module TellBunny has_many :jena_responses, as: :record end + SINGLE_QUOTE_REGEXP = Regexp.new "'" + + ## + # Prepare the string for ingestion in to Jena: + # + # - Remove control characters Jena can't process + # - Escape leading and trailing +'+ as +\'+ + # + # + # @param value [String] a string value + # @return [String] + def rdf_string_prep value + return unless value.present? + # the next two steps have been moved here from `export_rdf.rake` + value.gsub!("\r\n", '') # remove CrLf + value.gsub!("\\", '') # remove "\" to avoid illegal control characters + return value unless value =~ SINGLE_QUOTE_REGEXP + value.strip! + # replace initial or final "'" with "\'" + value.gsub(%r{^'|'$}, %q{\\\'}) + end + + ## + # If +value+ is present, return a formatted rdf object string based on +data_type+; + # otherwise +nil+ is returned. + # + # format_triple_object 3, :integer # => "'3'^^xsd:integer" + # url_base = 'https://sdbm.library.upenn.edu/names/' + # format_triple_object 22104, :uri, url_base # => "" + # + # Note that when the data type is +uri+, +url_base+ is required and +value+ is appended to +url_base+. + # + # @param [Object] value the value of the property object + # @param [Symbol] data_type one of +:integer+, +:decimal+, +:boolean', + # +:string+, or +:uri+ + # @param [String] url_base a string like +https://sdbm.library.upenn.edu/names/+; + # required if +data_type+ is +:uri+ + # @return [String, nil] returns a formatted RDF object string or +nil+ if +value+ is blank + # @raise [RuntimeError] if +data_type+ is +:uri+ and +url_base+ is blank + def format_triple_object value, data_type, url_base=nil + return unless value.present? + return if value =~ %r{\A[^[:alnum:]]*\z} + case data_type + when :integer + "'#{value}'^^xsd:integer" + when :decimal + "'#{value}'^^xsd:decimal" + when :boolean + "'#{value}'^^xsd:boolean" + when :string + "'''#{rdf_string_prep value.to_s}'''" + when :uri + raise "No `url_base` supplied for #{value}" unless url_base.present? + "<#{url_base}#{value}>" + else + raise "Unknown triple object data_type: '#{data_type}'; expected one of :string, :decimal, :boolean, :string, :uri" + end + end + # inherited and overriden by relevent models. # NOTE: use triple single quotes to enclose string literals, to avoid confusion with quotes in the strings themsleves - def to_rdf %Q( - # sdbm:names/#{id} sdbm:names_id #{id} + # sdbm:names/#{id} sdbm:names_id #{id} ) end - + #private def update_bunny(jena_response_id = nil) @@ -54,7 +112,7 @@ def update_bunny(jena_response_id = nil) rescue Bunny::TCPConnectionFailed => e #puts "(Update) - Connection to RabbitMQ server failed" - self.jena_responses.destroy_all + self.jena_responses.destroy_all JenaResponse.create!(record: self, status: 0, message: "404: Failed to connect from Rails to RabbitMQ: #{e}") #rescue StandardError => e # self.jena_responses.destroy_all diff --git a/app/models/dericci_link.rb b/app/models/dericci_link.rb index c37a0252..88b05044 100644 --- a/app/models/dericci_link.rb +++ b/app/models/dericci_link.rb @@ -19,9 +19,9 @@ def to_rdf fields: {} } - map[:fields][:dericci_record_id] = "" if dericci_record_id.present? - map[:fields][:name_id] = "" if name_id.present? - map[:fields][:other_info] = "'''#{other_info}'''" if other_info.present? + map[:fields][:dericci_record_id] = format_triple_object dericci_record_id, :string + map[:fields][:name_id] = format_triple_object name_id, :uri, 'https://sdbm.library.upenn.edu/names/' + map[:fields][:other_info] = format_triple_object other_info, :string map end diff --git a/app/models/dericci_record.rb b/app/models/dericci_record.rb index dd9d88b7..8c2449b1 100644 --- a/app/models/dericci_record.rb +++ b/app/models/dericci_record.rb @@ -8,8 +8,8 @@ class DericciRecord < ActiveRecord::Base has_many :bookmarks, as: :document, dependent: :destroy has_many :comments, as: :commentable - - include Watchable + + include Watchable include UserFields include TellBunny @@ -48,15 +48,15 @@ def to_rdf fields: {} } - map[:fields][:dates] = "'''#{dates}'''" if dates.present? - map[:fields][:name] = "'''#{name}'''" if name.present? - map[:fields][:place] = "'''#{place}'''" if place.present? - map[:fields][:url] = "'''#{url}'''" if url.present? - map[:fields][:cards] = "'#{cards}'^^xsd:integer" if cards.present? - map[:fields][:size] = "'''#{size}'''" if size.present? - map[:fields][:other_info] = "'''#{other_info}'''" if other_info.present? - map[:fields][:senate_house] = "'''#{senate_house}'''" if senate_house.present? - map[:fields][:out_of_scope] = "'#{out_of_scope}'^^xsd:boolean" unless out_of_scope.nil? + map[:fields][:dates] = format_triple_object dates, :string + map[:fields][:name] = format_triple_object name, :string + map[:fields][:place] = format_triple_object place, :string + map[:fields][:url] = format_triple_object url, :string + map[:fields][:cards] = format_triple_object cards, :integer + map[:fields][:size] = format_triple_object size, :string + map[:fields][:other_info] = format_triple_object other_info, :string + map[:fields][:senate_house] = format_triple_object senate_house, :string + map[:fields][:out_of_scope] = format_triple_object out_of_scope, :boolean map end diff --git a/app/models/entry.rb b/app/models/entry.rb index 9edcfabe..ee8e52a0 100644 --- a/app/models/entry.rb +++ b/app/models/entry.rb @@ -70,7 +70,7 @@ class Entry < ActiveRecord::Base has_many :entry_places, -> { order(:order => :asc) }, inverse_of: :entry, dependent: :destroy has_many :places, through: :entry_places has_many :entry_uses, -> { order(:order => :asc) }, inverse_of: :entry, dependent: :destroy - + # has_many :entry_comments, dependent: :destroy has_many :comments, as: :commentable has_many :sales, inverse_of: :entry, dependent: :destroy @@ -92,8 +92,8 @@ class Entry < ActiveRecord::Base # list of args to pass to Entry.includes in various places, for fetching a 'complete' entry @@includes = [ - :created_by, - :updated_by, + :created_by, + :updated_by, :institution, :entry_titles, :entry_dates, @@ -103,7 +103,7 @@ class Entry < ActiveRecord::Base :supercedes, :groups, :manuscripts, - {:group_records => [:group]}, + {:group_records => [:group]}, {:sales => [:sale_agents => [:agent]]}, {:entry_authors => [:author]}, {:entry_artists => [:artist]}, @@ -215,7 +215,7 @@ def decrement_counters Language.decrement_counter(:entries_count, language.id) objects.push(language) end - + if sale sale.sale_agents.map(&:agent).uniq.each do |sale_agent| Name.decrement_counter(:sale_agents_count, sale_agent.id) @@ -250,7 +250,7 @@ def get_entries_for_manuscript def get_sale_agents_names(role) t = get_sale - + if !t return "" end @@ -301,9 +301,9 @@ def get_sale_price end def missing_authority_names - entry_authors.select{ |ea| ea.author_id.blank? && ea.observed_name.present? }.length + - entry_artists.select{ |ea| ea.artist_id.blank? && ea.observed_name.present? }.length + - entry_scribes.select{ |ea| ea.scribe_id.blank? && ea.observed_name.present? }.length + + entry_authors.select{ |ea| ea.author_id.blank? && ea.observed_name.present? }.length + + entry_artists.select{ |ea| ea.artist_id.blank? && ea.observed_name.present? }.length + + entry_scribes.select{ |ea| ea.scribe_id.blank? && ea.observed_name.present? }.length + provenance.select{ |ea| ea.provenance_agent_id.blank? && ea.observed_name.present? }.length end @@ -364,7 +364,7 @@ def cumulative_updated_at # details for public display, slimmer than 'as_flat_hash' and without things like user groups included def bookmark_details - results = { + results = { manuscript: manuscript ? manuscript.id : nil, source_date: SDBMSS::Util.format_fuzzy_date(source.date), source_title: source.title, @@ -390,7 +390,7 @@ def bookmark_details # export. Obviously, decisions have to be made here about how to # represent the nested associations for display and there has to be # some information tweaking/loss. - + def as_flat_hash(options: {}) @@ -492,7 +492,7 @@ def to_citation # # field_type should be a symbol that matches a DSL field, like # :text or :integer - + def define_field(field_type, *args, &block) # last argument should be hash of options, so add to it @@ -521,7 +521,7 @@ def define_field(field_type, *args, &block) (sale ? sale.sale_agents.map(&:display_value) : []) + # details groups.map(&:name) + - manuscripts.map(&:public_id) + + manuscripts.map(&:public_id) + entry_titles.map(&:display_value) + entry_authors.map(&:display_value) + entry_dates.map(&:display_value) + @@ -529,7 +529,7 @@ def define_field(field_type, *args, &block) entry_scribes.map(&:display_value) + entry_languages.map(&:display_value) + entry_materials.map(&:display_value) + - entry_places.map(&:display_value) + + entry_places.map(&:display_value) + entry_uses.map(&:use) + [ folios, @@ -585,7 +585,7 @@ def define_field(field_type, *args, &block) define_field(:text, :source_date_search, :stored => true) do source.date end - + define_field(:string, :source, :stored => true) do source.public_id @@ -751,7 +751,7 @@ def define_field(field_type, *args, &block) define_field(:string, :artist, :stored => true, :multiple => true) do entry_artists.select(&:facet_value).map(&:facet_value) end - + define_field(:string, :artist_flat, :stored => true) do entry_artists.map(&:display_value).join("; ") end @@ -818,15 +818,15 @@ def define_field(field_type, *args, &block) end define_field(:integer, :missing_authority_names, :stored => true) do - entry_authors.where(author_id: nil).where.not(observed_name: nil).count + - entry_artists.where(artist_id: nil).where.not(observed_name: nil).count + - entry_scribes.where(scribe_id: nil).where.not(observed_name: nil).count + + entry_authors.where(author_id: nil).where.not(observed_name: nil).count + + entry_artists.where(artist_id: nil).where.not(observed_name: nil).count + + entry_scribes.where(scribe_id: nil).where.not(observed_name: nil).count + provenance.where(provenance_agent_id: nil).where.not(observed_name: nil).count end define_field(:integer, :folios, :stored => true) { folios } #define_field(:text, :folios_search, :stored => true, :more_like_this => true) { folios.to_s } - + define_field(:integer, :num_columns, :stored => true) { num_columns } define_field(:integer, :num_lines, :stored => true) { num_lines } @@ -955,18 +955,18 @@ def to_i def dispute_reasons ["Malicious/misleading data", "I disagree with some of the data", "Other"] - end + end def self.do_csv_dump params = ActionController::Parameters.new - + filename = "#{self.model_name.to_s.pluralize.underscore}.csv" path = "public/static/docs/#{filename}" offset = 0 File.delete("#{path}.zip") if File.exist?("#{path}.zip") - + objects = [] headers = nil loop do @@ -982,7 +982,7 @@ def self.do_csv_dump csv << headers end objects.each do |r| - csv << r.values + csv << r.values end end end @@ -997,9 +997,9 @@ def self.do_csv_dump def self.do_csv_search(params, download) - + offset = 0 - + objects = [] filename = download.filename user = download.user @@ -1018,7 +1018,7 @@ def self.do_csv_search(params, download) csv << headers end objects.each do |r| - csv << r.values + csv << r.values end end end @@ -1031,34 +1031,34 @@ def self.do_csv_search(params, download) download.update({status: 1, filename: "#{filename}.zip"}) #download.created_by.notify("Your download '#{download.filename}' is ready.") - end + end # I don't love having to duplicate all the fields AGAIN here, but inheriting it all from blacklight doesn't seem to work - # + # def self.filters [ - ["Entry Id", "entry_id"], - ["Manuscript ID", "manuscript_id"], - ["Source ID (Full)", "source"], + ["Entry Id", "entry_id"], + ["Manuscript ID", "manuscript_id"], + ["Source ID (Full)", "source"], #["Source Date", "source_date"], ["Sale Sold", "sale_sold"], ["Sale Date", "sale_date"], - ["Price", "sale_price"], + ["Price", "sale_price"], ["Missing Authority Names", "missing_authority_names"], - ["Folios", "folios"], - ["Columns", "num_columns"], - ["Lines", "num_lines"], - ["Height", "height"], - ["Width", "width"], - ["Alternate Size", "alt_size"], - ["Fullpage Miniatures", "miniatures_fullpage"], - ["Large Miniatures", "miniatures_large"], - ["Small Miniatures", "miniatures_small"], - ["Unspecified Miniatures", "miniatures_unspec_size"], + ["Folios", "folios"], + ["Columns", "num_columns"], + ["Lines", "num_lines"], + ["Height", "height"], + ["Width", "width"], + ["Alternate Size", "alt_size"], + ["Fullpage Miniatures", "miniatures_fullpage"], + ["Large Miniatures", "miniatures_large"], + ["Small Miniatures", "miniatures_small"], + ["Unspecified Miniatures", "miniatures_unspec_size"], ["Historiated Initals", "initials_historiated"], ["Decorated Initials", "initials_decorated"], - ["Updated By", "updated_by"], - ["Created By", "created_by"], + ["Updated By", "updated_by"], + ["Created By", "created_by"], ["Approved", "approved"], ["Deprecated", "deprecated"], ["Unverified Legacy Record", "unverified_legacy_record"], @@ -1068,26 +1068,26 @@ def self.filters def self.fields [ - ["All Fields", "complete_entry"], - ["Source", "source_search"], - ["Source Title", "source_title"], + ["All Fields", "complete_entry"], + ["Source", "source_search"], + ["Source Title", "source_title"], ["Source Date", "source_date_search"], ["Catalog or Lot #", "catalog_or_lot_number_search"], - ["Institution", "institution_search"], - ["Selling Agent", "sale_selling_agent_search"], - ["Seller", "sale_seller_search"], - ["Buyer", "sale_buyer_search"], + ["Institution", "institution_search"], + ["Selling Agent", "sale_selling_agent_search"], + ["Seller", "sale_seller_search"], + ["Buyer", "sale_buyer_search"], ["Title", "title_search"], - ["Author", "author_search"], + ["Author", "author_search"], ["Date", "manuscript_date_search"], - ["Artist", "artist_search"], - ["Scribe", "scribe_search"], - ["Binding", "binding_search"], - ["Link", "manuscript_link_search"], - ["Other Info", "other_info_search"], - ["Language", "language_search"], - ["Material", "material_search"], - ["Place", "place_search"], + ["Artist", "artist_search"], + ["Scribe", "scribe_search"], + ["Binding", "binding_search"], + ["Link", "manuscript_link_search"], + ["Other Info", "other_info_search"], + ["Language", "language_search"], + ["Material", "material_search"], + ["Place", "place_search"], ["Use", "use_search"], ["Provenance", "provenance_search"], ] @@ -1095,7 +1095,7 @@ def self.fields def self.dates [ - ["Added On", "created_at"], + ["Added On", "created_at"], ["Updated On", "updated_at"] ] end @@ -1121,28 +1121,28 @@ def to_rdf fields: {} } - map[:fields][:catalog_or_lot_number] = "'''#{catalog_or_lot_number}'''" if catalog_or_lot_number.present? - map[:fields][:folios] = "'#{folios}'^^xsd:integer" if folios.present? - map[:fields][:num_columns] = "'#{num_columns}'^^xsd:integer" if num_columns.present? - map[:fields][:num_lines] = "'#{num_lines}'^^xsd:integer" if num_lines.present? - map[:fields][:height] = "'#{height}'^^xsd:integer" if height.present? - map[:fields][:width] = "'#{width}'^^xsd:integer" if width.present? - map[:fields][:alt_size] = "'''#{alt_size}'''" if alt_size.present? - map[:fields][:manuscript_binding] = "'''#{manuscript_binding.to_s.gsub("'", "")}'''" if manuscript_binding.present? - map[:fields][:other_info] = "'''#{other_info.to_s.gsub("'", "")}'''" if other_info.present? - map[:fields][:manuscript_link] = "'''#{manuscript_link}'''" if manuscript_link.present? - map[:fields][:miniatures_fullpage] = "'#{miniatures_fullpage}'^^xsd:integer" if miniatures_fullpage.present? - map[:fields][:miniatures_large] = "'#{miniatures_large}'^^xsd:integer" if miniatures_large.present? - map[:fields][:miniatures_small] = "'#{miniatures_small}'^^xsd:integer" if miniatures_small.present? - map[:fields][:miniatures_unspec_size] = "'#{miniatures_unspec_size}'^^xsd:integer" if miniatures_unspec_size.present? - map[:fields][:initials_historiated] = "'#{initials_historiated}'^^xsd:integer" if initials_historiated.present? - map[:fields][:initials_decorated] = "'#{initials_decorated}'^^xsd:integer" if initials_decorated.present? - map[:fields][:transaction_type] = "'''#{transaction_type}'''" if transaction_type.present? - map[:fields][:deprecated] = "'#{deprecated}'^^xsd:boolean" unless deprecated.nil? - map[:fields][:unverified_legacy_record] = "'#{unverified_legacy_record}'^^xsd:boolean" unless unverified_legacy_record.nil? - map[:fields][:institution_id] = "" if institution_id.present? - map[:fields][:superceded_by_id] = "" if superceded_by_id.present? - map[:fields][:source_id] = "" if source_id.present? + map[:fields][:catalog_or_lot_number] = format_triple_object catalog_or_lot_number, :string + map[:fields][:folios] = format_triple_object folios, :integer + map[:fields][:num_columns] = format_triple_object num_columns, :integer + map[:fields][:num_lines] = format_triple_object num_lines, :integer + map[:fields][:height] = format_triple_object height, :integer + map[:fields][:width] = format_triple_object width, :integer + map[:fields][:alt_size] = format_triple_object alt_size, :string + map[:fields][:manuscript_binding] = format_triple_object manuscript_binding, :string + map[:fields][:other_info] = format_triple_object other_info, :string + map[:fields][:manuscript_link] = format_triple_object manuscript_link, :string + map[:fields][:miniatures_fullpage] = format_triple_object miniatures_fullpage, :integer + map[:fields][:miniatures_large] = format_triple_object miniatures_large, :integer + map[:fields][:miniatures_small] = format_triple_object miniatures_small, :integer + map[:fields][:miniatures_unspec_size] = format_triple_object miniatures_unspec_size, :integer + map[:fields][:initials_historiated] = format_triple_object initials_historiated, :integer + map[:fields][:initials_decorated] = format_triple_object initials_decorated, :integer + map[:fields][:transaction_type] = format_triple_object transaction_type, :string + map[:fields][:deprecated] = format_triple_object deprecated, :boolean + map[:fields][:unverified_legacy_record] = format_triple_object unverified_legacy_record, :boolean + map[:fields][:institution_id] = format_triple_object institution_id, :uri, 'https://sdbm.library.upenn.edu/names/' + map[:fields][:superceded_by_id] = format_triple_object superceded_by_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:source_id] = format_triple_object source_id, :uri, 'https://sdbm.library.upenn.edu/sources/' map end @@ -1178,7 +1178,7 @@ def update_counters sale.sale_agents.group_by(&:agent_id).keep_if{ |k, v| v.length >= 1}.each do |k, sale_agent| agent = sale_agent.first.agent Name.update_counters(agent.id, :sale_agents_count => agent.sale_entries.where(deprecated: false, draft: false).count - agent.sale_agents_count) unless agent.nil? - end + end end entry_places.group_by(&:place_id).keep_if{ |k, v| v.length >= 1}.each do |k, entry_place| diff --git a/app/models/entry_artist.rb b/app/models/entry_artist.rb index 15e57fcb..76028b92 100644 --- a/app/models/entry_artist.rb +++ b/app/models/entry_artist.rb @@ -7,25 +7,25 @@ class EntryArtist < ActiveRecord::Base include TellBunny TYPES_ROLES = [ - ["Work", "Workshop"], - ["Styl", "Style"], - ["Circ", "Circle"], - ["Scho", "School"], - ["Asso", "Associate"], - ["Assos", "Associates"], - ["Atel", "Atelier"], - ["Arti", "Artist"], - ["Grou", "Group"], - ["Styls", "Styles"], - ["Foll", "Follower"], - ["Desc", "Descendant"], - ["Folls", "Followers"], - ["Cont", "Contemporary"], - ["Son", "Son"], - ["Mann", "Manner"], - ["Mini", "Miniatures"], - ["Prov", "Provincial"], - ["Pupi", "Pupil"], + ["Work", "Workshop"], + ["Styl", "Style"], + ["Circ", "Circle"], + ["Scho", "School"], + ["Asso", "Associate"], + ["Assos", "Associates"], + ["Atel", "Atelier"], + ["Arti", "Artist"], + ["Grou", "Group"], + ["Styls", "Styles"], + ["Foll", "Follower"], + ["Desc", "Descendant"], + ["Folls", "Followers"], + ["Cont", "Contemporary"], + ["Son", "Son"], + ["Mann", "Manner"], + ["Mini", "Miniatures"], + ["Prov", "Provincial"], + ["Pupi", "Pupil"], ["Coll", "Collaborators"], ["Mast", "Master"], ["Assi", "Assistant"], @@ -80,13 +80,13 @@ def to_rdf fields: {} } - map[:fields][:observed_name] = "'''#{observed_name.to_s.gsub("'", "")}'''" if observed_name.present? - map[:fields][:artist_id] = "" if artist_id.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:role] = "'''#{role}'''" if role.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? + map[:fields][:observed_name] = format_triple_object observed_name, :string + map[:fields][:artist_id] = format_triple_object artist_id, :uri, 'https://sdbm.library.upenn.edu/names/' + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:role] = format_triple_object role, :string + map[:fields][:order] = format_triple_object order, :integer + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean map end diff --git a/app/models/entry_author.rb b/app/models/entry_author.rb index 5c59ab3f..477c0052 100644 --- a/app/models/entry_author.rb +++ b/app/models/entry_author.rb @@ -44,7 +44,7 @@ def display_value def facet_value author ? author.name : nil - end + end def to_rdf map = { @@ -53,13 +53,13 @@ def to_rdf fields: {} } - map[:fields][:observed_name] = "'''#{observed_name.to_s.gsub("'", "")}'''" if observed_name.present? - map[:fields][:author_id] = "" if author_id.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:role] = "'''#{role}'''" if role.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? + map[:fields][:observed_name] = format_triple_object observed_name, :string + map[:fields][:author_id] = format_triple_object author_id, :uri, 'https://sdbm.library.upenn.edu/names/' + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:role] = format_triple_object role, :string + map[:fields][:order] = format_triple_object order, :integer + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean map end diff --git a/app/models/entry_date.rb b/app/models/entry_date.rb index 0f3a8e26..9904a123 100644 --- a/app/models/entry_date.rb +++ b/app/models/entry_date.rb @@ -114,13 +114,13 @@ def to_rdf fields: {} } - map[:fields][:observed_date] = "'''#{observed_date}'''" if observed_date.present? - map[:fields][:date_normalized_start] = "'''#{date_normalized_start}'''" if date_normalized_start.present? - map[:fields][:date_normalized_end] = "'''#{date_normalized_end}'''" if date_normalized_end.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? + map[:fields][:observed_date] = format_triple_object observed_date, :string + map[:fields][:date_normalized_start] = format_triple_object date_normalized_start, :string + map[:fields][:date_normalized_end] = format_triple_object date_normalized_end, :string + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:order] = format_triple_object order, :integer + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean map end diff --git a/app/models/entry_language.rb b/app/models/entry_language.rb index 526246f3..ea358964 100644 --- a/app/models/entry_language.rb +++ b/app/models/entry_language.rb @@ -35,12 +35,12 @@ def to_rdf fields: {} } - map[:fields][:observed_name] = "'''#{observed_name}'''" if observed_name.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:language_id] = "" if language_id.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? + map[:fields][:observed_name] = format_triple_object observed_name, :string + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:language_id] = format_triple_object language_id, :uri, 'https://sdbm.library.upenn.edu/languages/' + map[:fields][:order] = format_triple_object order, :integer + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean map end diff --git a/app/models/entry_manuscript.rb b/app/models/entry_manuscript.rb index 5e9a2196..a20f662d 100644 --- a/app/models/entry_manuscript.rb +++ b/app/models/entry_manuscript.rb @@ -93,9 +93,9 @@ def to_rdf fields: {} } - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:manuscript_id] = "" if manuscript_id.present? - map[:fields][:relation_type] = "'''#{relation_type}'''" if relation_type.present? + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:manuscript_id] = format_triple_object manuscript_id, :uri, 'https://sdbm.library.upenn.edu/manuscripts/' + map[:fields][:relation_type] = format_triple_object relation_type, :string map end diff --git a/app/models/entry_material.rb b/app/models/entry_material.rb index 3f39d52c..8c9d6ff7 100644 --- a/app/models/entry_material.rb +++ b/app/models/entry_material.rb @@ -48,12 +48,12 @@ def to_rdf fields: {} } - map[:fields][:material] = "'''#{material}'''" if material.present? - map[:fields][:observed_name] = "'''#{observed_name}'''" if observed_name.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? + map[:fields][:material] = format_triple_object material, :string + map[:fields][:observed_name] = format_triple_object observed_name, :string + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:order] = format_triple_object order, :integer + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean map end diff --git a/app/models/entry_place.rb b/app/models/entry_place.rb index 9072ad9b..36a654ee 100644 --- a/app/models/entry_place.rb +++ b/app/models/entry_place.rb @@ -31,12 +31,12 @@ def to_rdf fields: {} } - map[:fields][:observed_name] = "'''#{observed_name}'''" if observed_name.present? - map[:fields][:place_id] = "" if place_id.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? + map[:fields][:observed_name] = format_triple_object observed_name, :string + map[:fields][:place_id] = format_triple_object place_id, :uri, 'https://sdbm.library.upenn.edu/places/' + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:order] = format_triple_object order, :integer + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean map end diff --git a/app/models/entry_scribe.rb b/app/models/entry_scribe.rb index e39e086f..b47f54fa 100644 --- a/app/models/entry_scribe.rb +++ b/app/models/entry_scribe.rb @@ -45,12 +45,12 @@ def to_rdf fields: {} } - map[:fields][:observed_name] = "'''#{observed_name.to_s.gsub("'", "")}'''" if observed_name.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:scribe_id] = "" if scribe_id.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? + map[:fields][:observed_name] = format_triple_object observed_name, :string + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:scribe_id] = format_triple_object scribe_id, :uri, 'https://sdbm.library.upenn.edu/names/' + map[:fields][:order] = format_triple_object order, :integer + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean map end diff --git a/app/models/entry_title.rb b/app/models/entry_title.rb index 4d15c641..07e0011e 100644 --- a/app/models/entry_title.rb +++ b/app/models/entry_title.rb @@ -29,12 +29,12 @@ def to_rdf fields: {} } - map[:fields][:title] = "'''#{title.to_s.gsub("'", "")}'''" if title.present? - map[:fields][:common_title] = "'''#{common_title.to_s.gsub("'", "")}'''" if common_title.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? + map[:fields][:title] = format_triple_object title, :string + map[:fields][:common_title] = format_triple_object common_title, :string + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:order] = format_triple_object order, :integer + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean map end diff --git a/app/models/entry_use.rb b/app/models/entry_use.rb index 1f889324..7bcb24d6 100644 --- a/app/models/entry_use.rb +++ b/app/models/entry_use.rb @@ -15,9 +15,9 @@ def to_rdf fields: {} } - map[:fields][:use] = "'''#{use}'''" if use.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? + map[:fields][:use] = format_triple_object use, :string + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:order] = format_triple_object order, :integer map end diff --git a/app/models/language.rb b/app/models/language.rb index 2a2cbaf2..9a3e3193 100644 --- a/app/models/language.rb +++ b/app/models/language.rb @@ -88,7 +88,7 @@ def to_rdf model_class: "languages", id: id, fields: { - name: "'''#{name}'''" + name: "'''#{rdf_string_prep name}'''" } } end diff --git a/app/models/manuscript.rb b/app/models/manuscript.rb index a49adc58..ae052f97 100644 --- a/app/models/manuscript.rb +++ b/app/models/manuscript.rb @@ -17,7 +17,7 @@ class Manuscript < ActiveRecord::Base #include IndexAfterUpdate #include HasPaperTrail # do this manually, since we want to create versions when record is touched - has_paper_trail + has_paper_trail include CreatesActivity extend SolrSearchable @@ -25,7 +25,7 @@ class Manuscript < ActiveRecord::Base include TellBunny # searchable! - + searchable do string :created_by do created_by ? created_by.username : "" @@ -122,7 +122,7 @@ def to_s end def bookmark_details - results = { + results = { titles: all_titles.to_a.join(", "), location: location, url: url, @@ -157,9 +157,9 @@ def to_rdf fields: {} } - map[:fields][:name] = "'''#{name}'''" if name.present? - map[:fields][:location] = "'''#{location}'''" if location.present? - map[:fields][:url] = "'''#{url}'''" if url.present? + map[:fields][:name] = format_triple_object name, :string + map[:fields][:location] = format_triple_object location, :string + map[:fields][:url] = format_triple_object url, :string map end diff --git a/app/models/name.rb b/app/models/name.rb index 1f311200..8283ac6e 100644 --- a/app/models/name.rb +++ b/app/models/name.rb @@ -27,7 +27,7 @@ class Name < ActiveRecord::Base include Ratable include TellBunny - + extend SolrSearchable default_scope { where(deleted: false) } @@ -41,7 +41,7 @@ class Name < ActiveRecord::Base has_many :name_places, :dependent => :destroy has_many :places, -> {distinct}, through: :name_places - has_many :bookmarks, as: :document, dependent: :destroy + has_many :bookmarks, as: :document, dependent: :destroy has_many :entry_artists, foreign_key: "artist_id" has_many :artist_entries, -> {distinct}, through: :entry_artists, source: :entry @@ -96,12 +96,12 @@ class Name < ActiveRecord::Base name_obj.viaf_id = name_obj.viaf_id.strip unless name_obj.viaf_id.nil? end end - end + end before_validation :normalize def normalize - self.name = self.name.mb_chars.normalize + self.name = self.name.mb_chars.normalize end searchable :unless => :deleted do @@ -133,10 +133,10 @@ def normalize string :viaf_id string :subtype string :places do - places.map(&:name).join("; ") + places.map(&:name).join("; ") end text :places do - places.map(&:name).join("; ") + places.map(&:name).join("; ") end integer :created_by_id integer :artists_count @@ -164,18 +164,18 @@ def normalize integer :disputes do ratings.where(qualifier: "dispute").count - end + end end def self.filters super + [ - ["VIAF", "viaf_id"], - ["Author Count", "authors_count"], - ["Artist Count", "artists_count"], - ["Scribe Count", "scribes_count"], - ["Provenance Count", "provenance_count"], + ["VIAF", "viaf_id"], + ["Author Count", "authors_count"], + ["Artist Count", "artists_count"], + ["Scribe Count", "scribes_count"], + ["Provenance Count", "provenance_count"], ["Source Agent Count", "source_agents_count"], - ["# of Confirms", "confirms"], + ["# of Confirms", "confirms"], ["# of Disputes", "disputes"], ["Problem", "problem"], ["Type", "subtype"] @@ -222,7 +222,7 @@ def search_result_format end ### - # + # # This is all used only for creating reference data # ### @@ -318,7 +318,7 @@ def self.suggestions(name, check_if_name_already_exists: true, debug: false) # include both People (personalNames) and Organizations (corporateNames) cql = "(local.personalNames all \"#{name}\" or local.corporateNames all \"#{name}\")" - + response = VIAF.sru_search(cql) if debug @@ -456,11 +456,11 @@ def merge_into(target) ids = EntryArtist.where(artist_id: self.id).pluck(:id) EntryArtist.where(artist_id: self.id).update_all({ artist_id: target_id }) EntryArtist.where( id: ids ).each(&:update_bunny) - + ids = EntryAuthor.where(author_id: self.id).pluck(:id) EntryAuthor.where(author_id: self.id).update_all({ author_id: target_id }) EntryAuthor.where( id: ids ).each(&:update_bunny) - + ids = EntryScribe.where(scribe_id: self.id).pluck(:id) EntryScribe.where(scribe_id: self.id).update_all({ scribe_id: target_id }) EntryScribe.where( id: ids ).each(&:update_bunny) @@ -468,19 +468,19 @@ def merge_into(target) ids = SaleAgent.where(agent_id: self.id).pluck(:id) SaleAgent.where(agent_id: self.id).update_all({ agent_id: target_id }) SaleAgent.where( id: ids ).each(&:update_bunny) - + ids = SourceAgent.where(agent_id: self.id).pluck(:id) SourceAgent.where(agent_id: self.id).update_all({ agent_id: target_id }) - SourceAgent.where( id: ids ).each(&:update_bunny) - + SourceAgent.where( id: ids ).each(&:update_bunny) + ids = Provenance.where(provenance_agent_id: self.id).pluck(:id) Provenance.where(provenance_agent_id: self.id).update_all({ provenance_agent_id: target_id }) Provenance.where( id: ids ).each(&:update_bunny) - + ids = DericciLink.where(name_id: self.id).pluck(:id) DericciLink.where(name_id: self.id).update_all({ name_id: target_id }) DericciLink.where( id: ids ).each(&:update_bunny) - + ids = DericciRecord.where(verified_id: self.id).pluck(:id) DericciRecord.where(verified_id: self.id).update_all({verified_id: target_id}) DericciRecord.where( id: ids ).each(&:update_bunny) @@ -492,19 +492,19 @@ def merge_into(target) target.is_provenance_agent ||= self.is_provenance_agent target.save - + # but ... CAN't SAVE when name is BLANK (nil) # self.name = nil self.viaf_id = nil self.deleted = true self.save! - # slice into managable chunks to avoid running out of space in mysql + # slice into managable chunks to avoid running out of space in mysql entry_ids.each_slice(200) do |slice| SDBMSS::IndexJob.perform_later(Entry.to_s, slice) end - Name.update_counters(target.id, + Name.update_counters(target.id, :authors_count => target.author_entries.where(deprecated: false, draft: false).count - target.authors_count, :artists_count => target.artist_entries.where(deprecated: false, draft: false).count - target.artists_count, :scribes_count => target.scribe_entries.where(deprecated: false, draft: false).count - target.scribes_count, @@ -525,13 +525,13 @@ def to_rdf fields: {} } - map[:fields][:name] = "'''#{name.to_s.gsub("'", "")}'''" if name.present? - map[:fields][:viaf_id] = "'''#{viaf_id}'''" if viaf_id.present? - map[:fields][:subtype] = "'''#{subtype}'''" if subtype.present? - map[:fields][:startdate] = "'''#{startdate}'''" if startdate.present? - map[:fields][:enddate] = "'''#{enddate}'''" if enddate.present? - map[:fields][:other_info] = "'''#{other_info.to_s.gsub("'", "")}'''" if other_info.present? - map[:fields][:deleted] = "'#{deleted}'^^xsd:boolean" unless deleted.nil? + map[:fields][:name] = format_triple_object name, :string + map[:fields][:viaf_id] = format_triple_object viaf_id, :string + map[:fields][:subtype] = format_triple_object subtype, :string + map[:fields][:startdate] = format_triple_object startdate, :string + map[:fields][:enddate] = format_triple_object enddate, :string + map[:fields][:other_info] = format_triple_object other_info, :string + map[:fields][:deleted] = format_triple_object deleted, :boolean map end diff --git a/app/models/name_place.rb b/app/models/name_place.rb index 7d6b7cf3..e21ffd40 100644 --- a/app/models/name_place.rb +++ b/app/models/name_place.rb @@ -31,10 +31,10 @@ def to_rdf fields: {} } - map[:fields][:place_id] = "" if place_id.present? - map[:fields][:name_id] = "" if name_id.present? - map[:fields][:notbefore] = "'''#{notbefore}'''" if notbefore.present? - map[:fields][:notafter] = "'''#{notafter}'''" if notafter.present? + map[:fields][:place_id] = format_triple_object place_id, :uri, 'https://sdbm.library.upenn.edu/places/' + map[:fields][:name_id] = format_triple_object name_id, :uri, 'https://sdbm.library.upenn.edu/names/' + map[:fields][:notbefore] = format_triple_object notbefore, :string + map[:fields][:notafter] = format_triple_object notafter, :string map end diff --git a/app/models/place.rb b/app/models/place.rb index be871626..c51392a2 100644 --- a/app/models/place.rb +++ b/app/models/place.rb @@ -89,7 +89,7 @@ def entries_to_index_on_update def self.filters super + [ - ["Authority Id", "authority_id"], + ["Authority Id", "authority_id"], ["Authority Source", "authority_source"], ["Problem", "problem"], ["Parent", "parent"], @@ -148,13 +148,13 @@ def to_rdf fields: {} } - map[:fields][:name] = "'''#{name.to_s.gsub("'", "")}'''" if name.present? - map[:fields][:authority_id] = "'''#{authority_id}'''" if authority_id.present? - map[:fields][:authority_source] = "'''#{authority_source}'''" if authority_source.present? - map[:fields][:parent_id] = "" if parent_id.present? - map[:fields][:latitude] = "'#{latitude}'^^xsd:decimal" if latitude.present? - map[:fields][:longitude] = "'#{longitude}'^^xsd:decimal" if longitude.present? - map[:fields][:deleted] = "'#{deleted}'^^xsd:boolean" unless deleted.nil? + map[:fields][:name] = format_triple_object name, :string + map[:fields][:authority_id] = format_triple_object authority_id, :string + map[:fields][:authority_source] = format_triple_object authority_source, :string + map[:fields][:parent_id] = format_triple_object parent_id, :uri, 'https://sdbm.library.upenn.edu/places/' + map[:fields][:latitude] = format_triple_object latitude, :decimal + map[:fields][:longitude] = format_triple_object longitude, :decimal + map[:fields][:deleted] = format_triple_object deleted, :boolean map end diff --git a/app/models/provenance.rb b/app/models/provenance.rb index 485c52d6..50a0bde6 100644 --- a/app/models/provenance.rb +++ b/app/models/provenance.rb @@ -74,7 +74,7 @@ class Provenance < ActiveRecord::Base # Returns a 2-item Array with start_date and end_date in the format # YYYY or YYYY-MM-DD, depending on how much information is in the # approximate date string. - + after_save do |agent| if agent.provenance_agent && !agent.provenance_agent.is_provenance_agent agent.provenance_agent.is_provenance_agent = true @@ -137,24 +137,24 @@ def to_rdf fields: {} } - map[:fields][:observed_name] = "'''#{observed_name.to_s.gsub("'", "")}'''" if observed_name.present? - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:provenance_agent_id] = "" if provenance_agent_id.present? - map[:fields][:order] = "'#{order}'^^xsd:integer" if order.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? - map[:fields][:associated_date] = "'''#{associated_date}'''" if associated_date.present? - map[:fields][:associated_date_normalized_start] = "'''#{associated_date_normalized_start}'''" if associated_date_normalized_start.present? - map[:fields][:associated_date_normalized_end] = "'''#{associated_date_normalized_end}'''" if associated_date_normalized_end.present? - map[:fields][:start_date] = "'''#{start_date}'''" if start_date.present? - map[:fields][:start_date_normalized_start] = "'''#{start_date_normalized_start}'''" if start_date_normalized_start.present? - map[:fields][:start_date_normalized_end] = "'''#{start_date_normalized_end}'''" if start_date_normalized_end.present? - map[:fields][:end_date] = "'''#{end_date}'''" if end_date.present? - map[:fields][:end_date_normalized_start] = "'''#{end_date_normalized_start}'''" if end_date_normalized_start.present? - map[:fields][:end_date_normalized_end] = "'''#{end_date_normalized_end}'''" if end_date_normalized_end.present? - map[:fields][:comment] = "'''#{comment.to_s.gsub("'", "")}'''" if comment.present? - map[:fields][:direct_transfer] = "'#{direct_transfer}'^^xsd:boolean" unless direct_transfer.nil? - map[:fields][:acquisition_method] = "'''#{acquisition_method}'''" if acquisition_method.present? + map[:fields][:observed_name] = format_triple_object observed_name, :string + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:provenance_agent_id] = format_triple_object provenance_agent_id, :uri, 'https://sdbm.library.upenn.edu/names/' + map[:fields][:order] = format_triple_object order, :integer + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean + map[:fields][:associated_date] = format_triple_object associated_date, :string + map[:fields][:associated_date_normalized_start] = format_triple_object associated_date_normalized_start, :string + map[:fields][:associated_date_normalized_end] = format_triple_object associated_date_normalized_end, :string + map[:fields][:start_date] = format_triple_object start_date, :string + map[:fields][:start_date_normalized_start] = format_triple_object start_date_normalized_start, :string + map[:fields][:start_date_normalized_end] = format_triple_object start_date_normalized_end, :string + map[:fields][:end_date] = format_triple_object end_date, :string + map[:fields][:end_date_normalized_start] = format_triple_object end_date_normalized_start, :string + map[:fields][:end_date_normalized_end] = format_triple_object end_date_normalized_end, :string + map[:fields][:comment] = format_triple_object comment, :string + map[:fields][:direct_transfer] = format_triple_object direct_transfer, :boolean + map[:fields][:acquisition_method] = format_triple_object acquisition_method, :string map end diff --git a/app/models/sale.rb b/app/models/sale.rb index f46b1ff9..ffa75b20 100644 --- a/app/models/sale.rb +++ b/app/models/sale.rb @@ -138,12 +138,12 @@ def to_rdf fields: {} } - map[:fields][:entry_id] = "" if entry_id.present? - map[:fields][:date] = "'''#{date}'''" if date.present? - map[:fields][:price] = "'#{price}'^^xsd:decimal" if price.present? - map[:fields][:currency] = "'''#{currency}'''" if currency.present? - map[:fields][:other_currency] = "'''#{other_currency}'''" if other_currency.present? - map[:fields][:sold] = "'''#{sold}'''" if sold.present? + map[:fields][:entry_id] = format_triple_object entry_id, :uri, 'https://sdbm.library.upenn.edu/entries/' + map[:fields][:date] = format_triple_object date, :string + map[:fields][:price] = format_triple_object price, :decimal + map[:fields][:currency] = format_triple_object currency, :string + map[:fields][:other_currency] = format_triple_object other_currency, :string + map[:fields][:sold] = format_triple_object sold, :string map end diff --git a/app/models/sale_agent.rb b/app/models/sale_agent.rb index e8c0f0c2..53020914 100644 --- a/app/models/sale_agent.rb +++ b/app/models/sale_agent.rb @@ -50,12 +50,12 @@ def to_rdf fields: {} } - map[:fields][:sale_id] = "" if sale_id.present? - map[:fields][:observed_name] = "'''#{observed_name}'''" if observed_name.present? - map[:fields][:agent_id] = "" if agent_id.present? - map[:fields][:role] = "'''#{role}'''" if role.present? - map[:fields][:supplied_by_data_entry] = "'#{supplied_by_data_entry}'^^xsd:boolean" unless supplied_by_data_entry.nil? - map[:fields][:uncertain_in_source] = "'#{uncertain_in_source}'^^xsd:boolean" unless uncertain_in_source.nil? + map[:fields][:sale_id] = format_triple_object sale_id, :uri, 'https://sdbm.library.upenn.edu/sales/' + map[:fields][:observed_name] = format_triple_object observed_name, :string + map[:fields][:agent_id] = format_triple_object agent_id, :uri, 'https://sdbm.library.upenn.edu/names/' + map[:fields][:role] = format_triple_object role, :string + map[:fields][:supplied_by_data_entry] = format_triple_object supplied_by_data_entry, :boolean + map[:fields][:uncertain_in_source] = format_triple_object uncertain_in_source, :boolean map end diff --git a/app/models/source.rb b/app/models/source.rb index 1564d959..a0154cc6 100644 --- a/app/models/source.rb +++ b/app/models/source.rb @@ -67,7 +67,7 @@ class Source < ActiveRecord::Base has_many :source_agents, inverse_of: :source has_many :bookmarks, as: :document, dependent: :destroy - + # validates_inclusion_of :whether_mss, in: HAS_MANUSCRIPT_TYPES.map(&:first), message: 'whether_mss is invalid', allow_blank: true validates_inclusion_of :medium, in: MEDIUM_TYPES.map(&:first), message: 'medium is invalid', allow_blank: true @@ -224,7 +224,7 @@ def to_i end def bookmark_details - results = { + results = { type: source_type.display_name, date: SDBMSS::Util.format_fuzzy_date(date), author: author, @@ -325,7 +325,7 @@ def merge_into (target) self.deleted = true self.save! - # slice into managable chunks to avoid running out of space in mysql + # slice into managable chunks to avoid running out of space in mysql entry_ids.each_slice(200) do |slice| SDBMSS::IndexJob.perform_later(Entry.to_s, slice) end @@ -333,11 +333,11 @@ def merge_into (target) def self.fields [ - ["Source Type", "source_type"], - ["Date", "date"], - ["Title", "title"], - ["Author", "author"], - ["Source Agent/Institution", "agent_name"], + ["Source Type", "source_type"], + ["Date", "date"], + ["Title", "title"], + ["Author", "author"], + ["Source Agent/Institution", "agent_name"], ["Location", "location"], ["Location/Institution", "location_institution"], ["Date Accessed", "date_accessed"], @@ -387,19 +387,19 @@ def to_rdf fields: {} } - map[:fields][:source_type_id] = "" if source_type_id.present? - map[:fields][:legacy] = "'#{legacy}'^^xsd:boolean" unless legacy.nil? - map[:fields][:date_accessed] = "'''#{date_accessed}'''" if date_accessed.present? - map[:fields][:medium] = "'''#{medium}'''" if medium.present? - map[:fields][:location] = "'''#{location}'''" if location.present? - map[:fields][:location_institution] = "'''#{location_institution}'''" if location_institution.present? - map[:fields][:status] = "'''#{status}'''" if status.present? - map[:fields][:other_info] = "'''#{other_info.to_s.gsub("'", "")}'''" if other_info.present? - map[:fields][:deleted] = "'#{deleted}'^^xsd:boolean" unless deleted.nil? - map[:fields][:author] = "'''#{author}'''" if author.present? - map[:fields][:title] = "'''#{title.to_s.gsub("'", "")}'''" if title.present? - map[:fields][:date] = "'''#{date}'''" if date.present? - map[:fields][:link] = "'''#{link}'''" if link.present? + map[:fields][:source_type_id] = format_triple_object source_type_id, :string + map[:fields][:legacy] = format_triple_object legacy, :boolean + map[:fields][:date_accessed] = format_triple_object date_accessed, :string + map[:fields][:medium] = format_triple_object medium, :string + map[:fields][:location] = format_triple_object location, :string + map[:fields][:location_institution] = format_triple_object location_institution, :string + map[:fields][:status] = format_triple_object status, :string + map[:fields][:other_info] = format_triple_object other_info, :string + map[:fields][:deleted] = format_triple_object deleted, :boolean + map[:fields][:author] = format_triple_object author, :string + map[:fields][:title] = format_triple_object title, :string + map[:fields][:date] = format_triple_object date, :string + map[:fields][:link] = format_triple_object link, :string map end diff --git a/app/models/source_agent.rb b/app/models/source_agent.rb index 2a4d3a8d..607452a0 100644 --- a/app/models/source_agent.rb +++ b/app/models/source_agent.rb @@ -3,7 +3,7 @@ class SourceAgent < ActiveRecord::Base belongs_to :agent, class_name: 'Name', counter_cache: :source_agents_count validate :validate_role - validate :observed_or_authority + validate :observed_or_authority include HasPaperTrail include TellBunny @@ -50,7 +50,7 @@ def valid_roles_for_source_type def facet_value agent ? agent.name : nil - end + end def to_rdf map = { @@ -59,10 +59,10 @@ def to_rdf fields: {} } - map[:fields][:observed_name] = "'''#{observed_name}'''" if observed_name.present? - map[:fields][:agent_id] = "" if agent_id.present? - map[:fields][:role] = "'''#{role}'''" if role.present? - map[:fields][:source_id] = "" if source_id.present? + map[:fields][:observed_name] = format_triple_object observed_name, :string + map[:fields][:agent_id] = format_triple_object agent_id, :uri, 'https://sdbm.library.upenn.edu/names/' + map[:fields][:role] = format_triple_object role, :string + map[:fields][:source_id] = format_triple_object source_id, :uri, 'https://sdbm.library.upenn.edu/sources/' map end diff --git a/app/models/source_type.rb b/app/models/source_type.rb index 2c5a5fc8..31310cb8 100644 --- a/app/models/source_type.rb +++ b/app/models/source_type.rb @@ -17,7 +17,7 @@ class SourceType < ActiveRecord::Base # ex: spreadsheet of Duke Univ. collection, Benjy's spreadsheet, and pretty much everything else. include TellBunny - + default_scope { order("id = 4 desc") } AUCTION_CATALOG = 'auction_catalog' @@ -64,10 +64,10 @@ def to_rdf fields: {} } - map[:fields][:name] = "'''#{name}'''" if name.present? - map[:fields][:display_name] = "'''#{display_name}'''" if display_name.present? - map[:fields][:entries_transaction_field] = "'#{entries_transaction_field}'^^xsd:boolean" unless entries_transaction_field.nil? - map[:fields][:entries_have_institution_field] = "'#{entries_have_institution_field}'^^xsd:boolean" unless entries_have_institution_field.nil? + map[:fields][:name] = format_triple_object name, :string + map[:fields][:display_name] = format_triple_object display_name, :string + map[:fields][:entries_transaction_field] = format_triple_object entries_transaction_field, :boolean + map[:fields][:entries_have_institution_field] = format_triple_object entries_have_institution_field, :boolean map end diff --git a/app/views/shared/_control_panel.html.erb b/app/views/shared/_control_panel.html.erb index 9278d74e..b6052e70 100644 --- a/app/views/shared/_control_panel.html.erb +++ b/app/views/shared/_control_panel.html.erb @@ -173,6 +173,12 @@ A Legacy Inscribed +
  • + + + Mapping Manuscript Migrations + +
  • diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 00000000..e7a9b43d --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,256 @@ +version: '3.7' + +services: + rabbitmq: + image: rabbitmq:3.7 + restart: always + hostname: 'rabbitmq' + ports: + - '5672:5672' + environment: + RABBITMQ_ERLANG_COOKIE: 'secret cookie here' # needed for rabbitmqctl to work (??) + RABBIT_PASSWORD: + RABBIT_USER: + ADMIN_PASSWORD: + volumes: + - rabbitmq_data:/var/lib/rabbitmq + + db: + image: mysql:5.7 + restart: always + env_file: .docker-environment + # logging: + # driver: "journald" + ports: + - "3307:3307" + volumes: + - sdbm_data:/var/lib/mysql + # We can define ENV vars using secret files because the MySQL image will + # recognize value is a path and read it from the file. + # + # THIS WILL LIKELY NOT WORK WITH OTHER IMAGES + environment: + MYSQL_DATABASE: + MYSQL_ROOT_PASSWORD: + MYSQL_USER: + MYSQL_PASSWORD: + solr: + env_file: .docker-environment + restart: always + image: sdbmss + build: + context: . + dockerfile: Dockerfile + ports: + - "8982:8982" + # logging: + # driver: "journald" + links: + - db + volumes: + - sdbm_solr:/usr/src/app/solr + command: bundle exec rake sunspot:solr:run + environment: + ADMIN_PASSWORD: + RAILS_USER: + RAILS_PASS: + MYSQL_HOST: + MYSQL_DATABASE: + MYSQL_ROOT_PASSWORD: + MYSQL_USER: + MYSQL_PASSWORD: + RABBIT_PASSWORD: + RABBIT_USER: + SDBMSS_APP_HOST: + SDBMSS_SMTP_HOST: + SDBMSS_EMAIL_FROM: + SDBMSS_NOTIFY_EMAIL: + SDBMSS_NOTIFY_EMAIL_PASSWORD: + SDBMSS_EMAIL_EXCEPTIONS_TO: + SDBMSS_BLACKLIGHT_SECRET_KEY: + SDBMSS_DEVISE_SECRET_KEY: + SDBMSS_SECRET_KEY_BASE: + SDBMSS_SECRET_TOKEN: + SOLR_URL: + rails: + env_file: .docker-environment + image: sdbmss + build: + context: . + dockerfile: Dockerfile + restart: always + ports: + - "3000:3000" + environment: + RAILS_ENV: + MYSQL_HOST: + ADMIN_PASSWORD: + RAILS_USER: + RAILS_PASS: + MYSQL_DATABASE: + MYSQL_ROOT_PASSWORD: + MYSQL_USER: + MYSQL_PASSWORD: + RABBIT_PASSWORD: + RABBIT_USER: + SDBMSS_IMAGE_TAG: + SDBMSS_APP_HOST: + SDBMSS_SMTP_HOST: + SDBMSS_EMAIL_FROM: + SDBMSS_NOTIFY_EMAIL: + SDBMSS_NOTIFY_EMAIL_PASSWORD: + SDBMSS_EMAIL_EXCEPTIONS_TO: + SDBMSS_BLACKLIGHT_SECRET_KEY: + SDBMSS_DEVISE_SECRET_KEY: + SDBMSS_SECRET_KEY_BASE: + SDBMSS_SECRET_TOKEN: + SOLR_URL: + links: + - db + - solr + - rabbitmq + depends_on: + - "rabbitmq" + volumes: + - .:/usr/src/app + - sdbm_assets:/usr/src/app/public/assets + - sdbm_docs:/usr/src/app/public/static + delayedjob: + env_file: .docker-environment + image: sdbmss + build: + context: . + dockerfile: Dockerfile + restart: always + depends_on: + - "db" + - "rabbitmq" + links: + - db + - solr + volumes: + - .:/usr/src/app + environment: + ADMIN_PASSWORD: + RAILS_ENV: + RAILS_USER: + RAILS_PASS: + MYSQL_HOST: + MYSQL_DATABASE: + MYSQL_ROOT_PASSWORD: + MYSQL_USER: + MYSQL_PASSWORD: + RABBIT_PASSWORD: + RABBIT_USER: + SDBMSS_IMAGE_TAG: + SDBMSS_APP_HOST: + SDBMSS_SMTP_HOST: + SDBMSS_EMAIL_FROM: + SDBMSS_NOTIFY_EMAIL: + SDBMSS_NOTIFY_EMAIL_PASSWORD: + SDBMSS_EMAIL_EXCEPTIONS_TO: + SDBMSS_BLACKLIGHT_SECRET_KEY: + SDBMSS_DEVISE_SECRET_KEY: + SDBMSS_SECRET_KEY_BASE: + SDBMSS_SECRET_TOKEN: + SOLR_URL: + command: bundle exec rake jobs:work + # logging: + # driver: "journald" + jena: + env_file: .docker-environment + image: sdbmss_jena + build: + context: sdbmss_jena + dockerfile: Dockerfile + restart: always + # logging: + # driver: "journald" + ports: + - "3030:3030" + volumes: + - rdf_data:/fuseki + environment: + ADMIN_PASSWORD: + RAILS_USER: + RAILS_PASS: + MYSQL_HOST: + MYSQL_DATABASE: + MYSQL_ROOT_PASSWORD: + MYSQL_USER: + MYSQL_PASSWORD: + RABBIT_PASSWORD: + RABBIT_USER: + SDBMSS_APP_HOST: + SDBMSS_SMTP_HOST: + SDBMSS_EMAIL_FROM: + SDBMSS_NOTIFY_EMAIL: + SDBMSS_NOTIFY_EMAIL_PASSWORD: + SDBMSS_EMAIL_EXCEPTIONS_TO: + SDBMSS_BLACKLIGHT_SECRET_KEY: + SDBMSS_DEVISE_SECRET_KEY: + SDBMSS_SECRET_KEY_BASE: + SDBMSS_SECRET_TOKEN: + SOLR_URL: + interface: + env_file: .docker-environment + image: sdbmss_interface + build: + context: sdbmss_interface + dockerfile: Dockerfile + restart: always + command: ruby interface.rb + # logging: + # driver: "journald" + links: + - jena + - rabbitmq + depends_on: + - "rabbitmq" + environment: + ADMIN_PASSWORD: + RAILS_ENV: + RAILS_USER: + RAILS_PASS: + MYSQL_HOST: + MYSQL_DATABASE: + MYSQL_ROOT_PASSWORD: + MYSQL_USER: + MYSQL_PASSWORD: + RABBIT_PASSWORD: + RABBIT_USER: + SDBMSS_APP_HOST: + SDBMSS_SMTP_HOST: + SDBMSS_EMAIL_FROM: + SDBMSS_NOTIFY_EMAIL: + SDBMSS_NOTIFY_EMAIL_PASSWORD: + SDBMSS_EMAIL_EXCEPTIONS_TO: + SDBMSS_BLACKLIGHT_SECRET_KEY: + SDBMSS_DEVISE_SECRET_KEY: + SDBMSS_SECRET_KEY_BASE: + SDBMSS_SECRET_TOKEN: + SOLR_URL: + nginx: + image: sdbmss_nginx + build: + context: sdbmss_nginx + dockerfile: Dockerfile + restart: always + volumes: + - ./nginx/ssl:/etc/httpd/ssl + # logging: + # driver: "journald" + links: + - rails + - jena + ports: + - "80:80" + - "443:443" +volumes: + sdbm_docs: + sdbm_data: + rdf_data: + sdbm_assets: + sdbm_solr: + rabbitmq_data: + diff --git a/docker-environment-sample b/docker-environment-sample new file mode 100644 index 00000000..058dac88 --- /dev/null +++ b/docker-environment-sample @@ -0,0 +1,22 @@ +MYSQL_HOST=db +RAILS_ENV=development +SDBMSS_APP_HOST=localhost +SDBMSS_SMTP_HOST=mail.somewhere.com +ADMIN_PASSWORD=x +RAILS_USER=x +RAILS_PASS=x +MYSQL_DATABASE=x +MYSQL_USER=x +MYSQL_PASSWORD=x +MYSQL_ROOT_PASSWORD=x +RABBIT_PASSWORD=x +RABBIT_USER=x +SDBMSS_EMAIL_FROM=user@example.com +SDBMSS_NOTIFY_EMAIL=user@example.com +SDBMSS_NOTIFY_EMAIL_PASSWORD=x +SDBMSS_EMAIL_EXCEPTIONS_TO=user@example.com +SDBMSS_BLACKLIGHT_SECRET_KEY=x +SDBMSS_DEVISE_SECRET_KEY=x +SDBMSS_SECRET_KEY_BASE=x +SDBMSS_SECRET_TOKEN=x +SOLR_URL=x diff --git a/lib/tasks/export_rdf.rake b/lib/tasks/export_rdf.rake index f6721e2a..0022a538 100644 --- a/lib/tasks/export_rdf.rake +++ b/lib/tasks/export_rdf.rake @@ -4,7 +4,7 @@ namespace :sparql do task :test => :environment do entities = [ Name, Place, Language, Source, DericciLink, DericciRecord, Entry, EntryArtist, EntryAuthor, EntryDate, EntryLanguage, EntryManuscript, EntryMaterial, EntryPlace, EntryScribe, EntryTitle, EntryUse, Manuscript, NamePlace, Provenance, Sale, SaleAgent, SourceAgent ] - + total = 0 possible = 0 origin = Time.now @@ -29,7 +29,8 @@ namespace :sparql do f.puts " sdbm:#{class_name}_id #{instance.id} ;" f.puts "rdf:type sdbm:#{class_name} ;" instance.to_rdf[:fields].each do | key, value | - f.puts "sdbm:#{class_name}_#{key} #{value.gsub("\r\n", '').gsub("\\", '')} ;" + next if value.nil? + f.puts "sdbm:#{class_name}_#{key} #{value} ;" end f.puts "." count += 1 diff --git a/sdbmss_interface/Dockerfile b/sdbmss_interface/Dockerfile index 011f40a2..6b5e3d02 100644 --- a/sdbmss_interface/Dockerfile +++ b/sdbmss_interface/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby +FROM ruby:2.7 COPY Gemfile ./ COPY interface.rb ./ diff --git a/sdbmss_interface/Gemfile.lock b/sdbmss_interface/Gemfile.lock new file mode 100644 index 00000000..aa1ceda3 --- /dev/null +++ b/sdbmss_interface/Gemfile.lock @@ -0,0 +1,15 @@ +GEM + remote: https://rubygems.org/ + specs: + amq-protocol (2.3.0) + bunny (2.14.1) + amq-protocol (~> 2.3, >= 2.3.0) + +PLATFORMS + ruby + +DEPENDENCIES + bunny + +BUNDLED WITH + 1.17.2 diff --git a/sdbmss_interface/interface.rb b/sdbmss_interface/interface.rb index 2d2cebf6..1520612e 100644 --- a/sdbmss_interface/interface.rb +++ b/sdbmss_interface/interface.rb @@ -72,8 +72,8 @@ request.set_form_data({"update" => query}) request.basic_auth("admin", ENV["ADMIN_PASSWORD"]) response = http.request(request) - if response.code != 200 - puts "PROBLEM: #{response} #{query}" + if response.code.to_i != 200 + puts "PROBLEM: #{response}: code #{response.code.inspect} \n #{query}" end status_queue = channel.queue("sdbm_status") status_queue.publish({id: message['response_id'], code: response.code, message: response.message}.to_json) @@ -89,12 +89,21 @@ message['fields'].each do |field, new_value| predicate = "sdbm:#{message['model_class']}_#{field}" -# When the record is updated, the process is the same, but as well as the old triple being deleted (for each triple in the + # When the record is updated, the process is the same, but as well as the old triple being deleted (for each triple in the # record), the new triple is added with the new value. query += %Q( DELETE { ?subject #{predicate} ?object } + ) + + # Don't insert triples with empty objects + unless new_value.to_s.empty? + query += %Q( INSERT { ?subject #{predicate} #{new_value} } + ) + end + + query += %Q( WHERE { BIND ( as ?subject) . OPTIONAL { ?subject #{predicate} ?object } @@ -110,16 +119,17 @@ }; ) -# The message response handling is the same in both cases, however. + # The message response handling is the same in both cases, however. begin request = Net::HTTP::Post.new(uri.request_uri) request.set_form_data({"update" => query}) request.basic_auth("admin", ENV["ADMIN_PASSWORD"]) response = http.request(request) - if response.code != 200 - puts "PROBLEM: #{response} #{query}" + if response.code.to_i != 200 + puts "PROBLEM: #{response}: code #{response.code.inspect} \n #{query}" end + status_queue = channel.queue("sdbm_status") status_queue.publish({id: message['response_id'], code: response.code, message: response.message}.to_json) rescue Exception => err diff --git a/sdbmss_nginx/nginx.conf b/sdbmss_nginx/nginx.conf index 51bbd751..0376505b 100644 --- a/sdbmss_nginx/nginx.conf +++ b/sdbmss_nginx/nginx.conf @@ -18,7 +18,7 @@ http { upstream solr { least_conn; - server solr:8983 weight=10 max_fails=3 fail_timeout=30s; + server solr:8982 weight=10 max_fails=3 fail_timeout=30s; } server { diff --git a/solr/conf/scripts.conf b/solr/conf/scripts.conf index f58b262a..16a2f07c 100644 --- a/solr/conf/scripts.conf +++ b/solr/conf/scripts.conf @@ -15,7 +15,7 @@ user= solr_hostname=localhost -solr_port=8983 +solr_port=8982 rsyncd_port=18983 data_dir= webapp_name=solr