Merge pull request #197 from unepwcmc/release-1.3.0

Release 1.3.0

.gitignore

@@ -30,4 +30,5 @@ yarn-debug.log*
 .env
 
 previous_failed_specs.txt
-coverage
+coverage
+/TAGS

CHANGELOG.md

@@ -27,4 +27,19 @@
 system specs. Also included is a FactoryBot linter for linting factories. A couple of
 redundant methods have also been removed from the models.
 * Merged in a number of PRs fixing vulnerabilities on the frontend and backend that were 
-opened by Dependabot.
+opened by Dependabot.
+
+## 1.3.0
+
+* chore: add `/TAGS` to .gitignore
+* chore: upgrade puma from 3.7 ~> 4.3
+* fix: add missing "`WHERE geo_entities.iso3 IS NOT NULL`" to *countries* scope in `geo_entity.rb` and "`WHERE geo_entities.iso3 <> 'GBL'`" to *country_stats* scope in `geo_entity_stat.rb`
+* fix: change global protection stats to be static within habitats table, as there was a problem with the automatic generation of global stats not including non-member state data yet.
+* refactor: partially automate country last-updated by using the geo_entity's `updated_at` property
+* refactor: set Red List last updated date (`@red_list_last_updated`) in site & countries controller
+* feat: add rake invoke task to deploy.rb so that you can run any specified rake task before the deploy is published e.g. `bundle exec cap staging deploy TASK=import:refresh`
+* feat: add `BRANCH=` option `cap staging deploy` so you can specify the branch to deploy to staging (default to develop)
+* refactor: raise more informative exception in Esri class if the response is not what was expected
+* feat: add utilities/files.rb with `latest_file_by_glob` method to help select the latest filename-timestamped CSV that is used in the _habitat coverage protection_ imports
+* refactor: add more informative error when a CSV is missing an expected header
+* fix: wrap _import:refresh_ within a DB transaction so that it doesn't commit anything on failure

app/controllers/countries_controller.rb

@@ -6,7 +6,7 @@ def show
 
     return redirect_to(action: 'not_found', controller: 'errors') unless @country
 
-    # TODO - work out how to integrate i18n with this for country names - we may want to
+    # TODO: - work out how to integrate i18n with this for country names - we may want to
     # have a list of country names in each language which is then dynamically fetched from
     # a CSV/yml depending on the language selected
     @iso3 = @country&.iso3
@@ -26,8 +26,10 @@ def show
     @example_species_common = Serializers::SpeciesImagesSerializer.new(@country.all_species).to_json
     @example_species_threatened = Serializers::SpeciesImagesSerializer.new(@country.all_species, true).to_json
 
-    @example_species_select = habitats.reject { |habitat| habitat['data'].nil? }.sort { |h1, h2| h2['data'].last[1] <=> h1['data'].last[1] }.
-    map { |habitat| { id: habitat[:id], name: habitat[:title] }}
+    @example_species_select = habitats
+      .reject { |habitat| habitat['data'].nil? }
+      .sort { |h1, h2| h2['data'].last[1] <=> h1['data'].last[1] }
+      .map { |habitat| { id: habitat[:id], name: habitat[:title] } }
 
     @habitat_change = Serializers::HabitatCountryChangeSerializer.new(@country, habitats_present_status).serialize
 
@@ -38,15 +40,17 @@ def show
     @next_country_url = country_link_path(@next_country.actual_name)
     @next_country_name = @next_country.actual_name
     @next_country_flag = helpers.if_country_get_flag(@next_country.iso3)
+
+    @red_list_last_updated = Date.parse('2021-08-01').strftime('%b, %Y') # TODO: automate based on date imported
   end
 
   private
 
   def next_country
-    following_countries = GeoEntity.permitted_countries.select { |geo_entity| geo_entity.name > @country.name } 
-    
+    following_countries = GeoEntity.permitted_countries.select { |geo_entity| geo_entity.name > @country.name }
+
     return GeoEntity.permitted_countries.first if following_countries.blank?
-    
-    following_countries.min_by { |geo_entity| geo_entity.name }
+
+    following_countries.min_by(&:name)
   end
 end

app/controllers/site_controller.rb

@@ -12,27 +12,27 @@ def index
     red_list_data = Species.count_species
 
     @red_list_data = @habitats.each { |habitat| habitat['data'] = red_list_data[habitat[:id]] }
-    
+
     @habitat_cover = Serializers::HabitatCoverSerializer.new.serialize
 
     doughnut_chart = I18n.t('home.sdg.doughnut_chart_data')
     @doughnut_chart = []
 
     doughnut_chart.each do |item|
       @doughnut_chart.push({
-        'title': item[:title],
-        'colour': item[:colour],
-        'icon': ActionController::Base.helpers.image_url(item[:icon]),
-        'description': item[:description],
-        'url': item[:url],
-        'source': item[:source]
-      })
+                             'title': item[:title],
+                             'colour': item[:colour],
+                             'icon': ActionController::Base.helpers.image_url(item[:icon]),
+                             'description': item[:description],
+                             'url': item[:url],
+                             'source': item[:source]
+                           })
     end
-  end
 
-  def about
+    @red_list_last_updated = Date.parse('2021-08-01').strftime('%b, %Y') # TODO: automate based on date imported
   end
 
-  def legal
-  end
+  def about; end
+
+  def legal; end
 end

app/models/geo_entity.rb

@@ -15,7 +15,7 @@ class GeoEntity < ApplicationRecord
 
   has_many :country_citations, foreign_key: 'country_id'
 
-  scope :countries, -> { where.not(iso3: nil || 'GBL') }
+  scope :countries, -> { where.not(iso3: nil).where.not(iso3: 'GBL') }
   scope :regions, -> { where(iso3: nil) }
 
   # Only allowing actual countries to be considered for the 'Next country' button

app/models/geo_entity_stat.rb

@@ -4,16 +4,19 @@ class GeoEntityStat < ApplicationRecord
   has_many :sources, through: :geo_entity_stats_sources
   has_many :geo_entity_stats_sources, class_name: 'GeoEntityStatsSources', dependent: :destroy
 
-  scope :country_stats, -> { joins(:geo_entity).where('geo_entities.iso3 IS NOT NULL') }
+  scope :country_stats, lambda {
+    joins(:geo_entity).where.not('geo_entities.iso3': nil)
+      .where.not('geo_entities.iso3': 'GBL')
+  }
 
-  # TODO - consider adding :present-but-unknown to this list which will require some thinking
-  enum occurrence: [:absent, :unknown, :present]
+  # TODO: - consider adding :present-but-unknown to this list which will require some thinking
+  enum occurrence: %i[absent unknown present]
 
   BASE_OCCURRENCES = {
     'coralreefs' => 'unknown',
     'saltmarshes' => 'unknown',
     'mangroves' => 'unknown',
     'seagrasses' => 'unknown',
     'coldcorals' => 'unknown'
-  }
+  }.freeze
 end

app/models/habitat.rb

@@ -13,39 +13,75 @@ def protected_title
   end
 
   def calculate_country_cover_change(country_name)
-    country_cover_change = { change_km: 0, change_percentage: 0 }
+    country_cover_change = {
+      change_km: 0,
+      change_percentage: 0
+    }
+
+    # We only have mangroves data at the moment
+    return country_cover_change unless name == 'mangroves'
 
-    # We only got mangroves data at the moment
-    return country_cover_change unless name == "mangroves"
     geo_entity_id = GeoEntity.find_by(name: country_name).id
-    habitat_base_year = ChangeStat.find_by(habitat_id: id, geo_entity_id: geo_entity_id)&.send("total_value_#{baseline_year}".to_sym)
-    habitat_last_year = ChangeStat.find_by(habitat_id: id, geo_entity_id: geo_entity_id)&.send(latest_year)
-    return country_cover_change if (habitat_base_year.nil? || habitat_last_year.nil?)
+    habitat_base_year = ChangeStat.find_by(
+      habitat_id: id,
+      geo_entity_id: geo_entity_id
+    )
+      &.send("total_value_#{baseline_year}".to_sym)
+
+    habitat_last_year = ChangeStat.find_by(
+      habitat_id: id,
+      geo_entity_id: geo_entity_id
+    )
+      &.send(latest_year)
+
+    return country_cover_change if habitat_base_year.nil? || habitat_last_year.nil?
+
     change_km = habitat_last_year - habitat_base_year
-    change_percentage = (change_km/habitat_base_year) * 100
+    change_percentage = (change_km / habitat_base_year) * 100
 
-    country_cover_change.merge!({change_km: ActiveSupport::NumberHelper.number_to_delimited(change_km.round(2)), change_percentage: change_percentage.round(2)})
+    country_cover_change.merge!(
+      {
+        change_km: ActiveSupport::NumberHelper.number_to_delimited(change_km.round(2)),
+        change_percentage: change_percentage.round(2)
+      }
+    )
   end
 
   def calculate_global_cover_change
-    global_cover_change = { change_km: 0, change_percentage: 0, baseline_year: baseline_year, original_total: 0 }
-    return global_cover_change unless name == "mangroves"
-    habitat_base_year = ChangeStat.includes(:geo_entity).where.not(geo_entities: { iso3: nil }).where(habitat_id: id).pluck("total_value_#{baseline_year}".to_sym).inject(0) { |sum, x| sum + x }
-    habitat_last_year = ChangeStat.includes(:geo_entity).where.not(geo_entities: { iso3: nil }).where(habitat_id: id).pluck(latest_year).inject(0) { |sum, x| sum + x }
+    global_cover_change = {
+      change_km: 0,
+      change_percentage: 0,
+      baseline_year: baseline_year,
+      original_total: 0
+    }
+
+    return global_cover_change unless name == 'mangroves'
+
+    habitat_base_year = ChangeStat.includes(:geo_entity)
+      .where
+      .not(geo_entities: { iso3: nil })
+      .where(habitat_id: id)
+      .pluck("total_value_#{baseline_year}".to_sym)
+      .inject(0) { |sum, x| sum + x }
+
+    habitat_last_year = ChangeStat.includes(:geo_entity)
+      .where
+      .not(geo_entities: { iso3: nil })
+      .where(habitat_id: id)
+      .pluck(latest_year)
+      .inject(0) { |sum, x| sum + x }
+
     total_value_change = habitat_last_year - habitat_base_year
     total_value_change_percentage = (total_value_change / habitat_base_year) * 100
 
-    global_cover_change.merge!({
-      change_km: total_value_change.round(2), change_percentage: total_value_change_percentage.round(2),
-      baseline_year: baseline_year, original_total: habitat_base_year.round(2)
-    })
-  end
-
-  def global_stats 
-    {
-      total_habitat_cover: geo_entity_stats.country_stats.pluck(:total_value).compact.reduce(&:+),
-      protected_habitat_cover: geo_entity_stats.country_stats.pluck(:protected_value).compact.reduce(&:+)
-    }
+    global_cover_change.merge!(
+      {
+        change_km: total_value_change.round(2),
+        change_percentage: total_value_change_percentage.round(2),
+        baseline_year: baseline_year,
+        original_total: habitat_base_year.round(2)
+      }
+    )
   end
 
   def calculate_global_protection
@@ -70,7 +106,7 @@ def total_value_by_country
     c = Carto.new(name)
     total_value_by_country = 0
 
-    if name == "coldcorals"
+    if name == 'coldcorals'
       total_value_by_country = c.total_points_by_country
       total_value_by_country = sort_country_count(total_value_by_country)
     else
@@ -81,17 +117,32 @@ def total_value_by_country
   end
 
   def type
-    name == "coldcorals" ? "points" : "area"
+    name == 'coldcorals' ? 'points' : 'area'
+  end
+
+  def global_stats
+    stats = geo_entity_stats.country_stats.to_a # reduce hits to database
+    {
+      total_habitat_cover: stats.pluck(:total_value).compact.reduce(&:+),
+      protected_habitat_cover: stats.pluck(:protected_value).compact.reduce(&:+)
+    }
   end
 
   def global_protection
-    stats = { 'name' => name, 'total_value' => 0, 'protected_value' => 0 }
+    stats = {
+      'name' => name,
+      'total_value' => 0,
+      'protected_value' => 0
+    }
+
+    global_stats_data = global_stats # reduce hits to database
 
-    stats['total_value'] = global_stats[:total_habitat_cover]
-    stats['protected_value'] = global_stats[:protected_habitat_cover]
+    stats['total_value'] = global_stats_data[:total_habitat_cover]
+    stats['protected_value'] = global_stats_data[:protected_habitat_cover]
 
-    protected_value = stats['protected_value'] > 0 ? stats['protected_value'] : 1
-    stats.merge({'protected_percentage' => protected_value / stats['total_value'] * 100})
+    protected_value = stats['protected_value'].positive? ? stats['protected_value'] : 1
+
+    stats.merge({ 'protected_percentage' => protected_value / stats['total_value'] * 100 })
   end
 
   def self.global_protection
@@ -100,30 +151,90 @@ def self.global_protection
 
   def self.global_protection_by_id
     hash = {}
-    self.global_protection.each do |habitat_stats|
+    global_protection.each do |habitat_stats|
+      hash[habitat_stats['name']] = habitat_stats.except('name')
+    end
+    hash
+  end
+
+  # experimental alternative to self.global_protection_by_id: not currently used
+  def self.global_protection_by_id_v2
+    hash = {}
+    global_protection_v2.each do |habitat_stats|
       hash[habitat_stats['name']] = habitat_stats.except('name')
     end
     hash
   end
 
+  # experimental alternative to self.global_protection: not currently used
+  # TODO: be brave, use this method instead of the original, it's cool and its
+  # 2x faster than the original
+  # TODO: check codebase for how other related methods are used e.g.
+  # global_stats and global_protection methods. e.g. they are in habitat_spec.rb
+  def self.global_protection_v2
+    geo_entities = GeoEntity.arel_table
+    geo_entity_stats = GeoEntityStat.arel_table
+    habitats = Habitat.arel_table
+
+    # get all the data we need in a single query
+    # @see scuttle.io for sql->arel help
+    query = GeoEntityStat.select(
+      [
+        habitats[:name],
+        geo_entity_stats[:total_value].sum.as('total_value'),
+        geo_entity_stats[:protected_value].sum.as('protected_value')
+      ]
+    )
+      .where(
+        geo_entities[:iso3].not_eq(nil).and(geo_entities[:iso3].not_eq('GBL'))
+      )
+      .joins(
+        geo_entity_stats.join(geo_entities).on(
+          geo_entities[:id].eq(geo_entity_stats[:geo_entity_id])
+        ).join_sources
+      )
+      .joins(
+        geo_entity_stats.join(habitats).on(
+          habitats[:id].eq(geo_entity_stats[:habitat_id])
+        ).join_sources
+      )
+      .group(habitats[:name])
+
+    results = ActiveRecord::Base.connection.execute(query.to_sql)
+
+    results.map do |row|
+      total_value = row['total_value'].to_f
+      protected_value = row['protected_value'].to_f
+      protected_percentage = protected_value / total_value * 100
+      {
+        name: row['name'],
+        total_value: total_value.round(2),
+        protected_value: protected_value.round(2),
+        protected_percentage: protected_percentage.round(2)
+      }.stringify_keys
+    end
+  end
+
   private
 
   def sum_country_areas(total_area_by_country)
     country_total_area = {}
     total_area_by_country.flatten.each do |country_data|
-      next if country_data["iso3"].include? "/" #remove areas which have multiple iso
-      next if country_data["iso3"].include? "ABNJ" #remove ABNJ
-      country_total_area[country_data["iso3"]] ||= 0
-      country_total_area[country_data["iso3"]] += country_data["sum"]
+      next if country_data['iso3'].include? '/' # remove areas which have multiple iso
+      next if country_data['iso3'].include? 'ABNJ' # remove ABNJ
+
+      country_total_area[country_data['iso3']] ||= 0
+      country_total_area[country_data['iso3']] += country_data['sum']
     end
     country_total_area
   end
 
   def sort_country_count(total_value_by_country)
     country_total_points = {}
     total_value_by_country.each do |total_value|
-      next if total_value["iso3"].include? "ABNJ"
-      country_total_points[total_value["iso3"]] = total_value["count"]
+      next if total_value['iso3'].include? 'ABNJ'
+
+      country_total_points[total_value['iso3']] = total_value['count']
     end
     country_total_points
   end