diff --git a/.env.example b/.env.example index 95976f05..5b6a03b6 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,3 @@ MAPBOX_TOKEN=XXXXXXXXXX -CARTO_USERNAME=XXXXXXXXXX WDPA_POLY_TABLE=XXXXXXXXXX WDPA_POINT_TABLE=XXXXXXXXXX -CARTO_API_KEY=XXXXXXXXXX diff --git a/README.md b/README.md index fcdc878e..c5f404e9 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,6 @@ * If you simply want to regenerate your statistics, you don't have to drop your database. Just run `rake import:refresh` instead. ## Data - -We are still trying to see if it is feasible to generate map tiles and data dynamically using Carto and in the meantime we are using static data. - -Data is fetched from CartoDB using the `Carto` module defined in `lib/modules`. -Some details about the layers in Carto are in `config/habitats.yml`. - At the moment static data is provided related to the protected areas coverage. Ideally, the protected areas coverage data should always be dynamically generated, as this depends on the WDPA release which is updated every month. diff --git a/app/models/habitat.rb b/app/models/habitat.rb index 3142e28d..44465b73 100644 --- a/app/models/habitat.rb +++ b/app/models/habitat.rb @@ -122,24 +122,6 @@ def latest_year :total_value_2016 end - def total_value_by_country - c = Carto.new(name) - total_value_by_country = 0 - - if name == 'coldcorals' - total_value_by_country = c.total_points_by_country - total_value_by_country = sort_country_count(total_value_by_country) - else - total_value_by_country = c.total_area_by_country - total_value_by_country = sum_country_areas(total_value_by_country) - end - total_value_by_country - end - - def type - name == 'coldcorals' ? 'points' : 'area' - end - def global_stats { total_habitat_cover: total_area, @@ -198,62 +180,62 @@ def self.global_protection_by_id end # experimental alternative to self.global_protection_by_id: not currently used - def self.global_protection_by_id_v2 - hash = {} - global_protection_v2.each do |habitat_stats| - hash[habitat_stats['name']] = habitat_stats.except('name') - end - hash - end + # def self.global_protection_by_id_v2 + # hash = {} + # global_protection_v2.each do |habitat_stats| + # hash[habitat_stats['name']] = habitat_stats.except('name') + # end + # hash + # end # experimental alternative to self.global_protection: not currently used # TODO: be brave, use this method instead of the original, it's cool and its # 2x faster than the original # TODO: check codebase for how other related methods are used e.g. # global_stats and global_protection methods. e.g. they are in habitat_spec.rb - def self.global_protection_v2 - geo_entities = GeoEntity.arel_table - geo_entity_stats = GeoEntityStat.arel_table - habitats = Habitat.arel_table - - # get all the data we need in a single query - # @see scuttle.io for sql->arel help - query = GeoEntityStat.select( - [ - habitats[:name], - geo_entity_stats[:total_value].sum.as('total_value'), - geo_entity_stats[:protected_value].sum.as('protected_value') - ] - ) - .where( - geo_entities[:iso3].not_eq(nil).and(geo_entities[:iso3].not_eq('GBL')) - ) - .joins( - geo_entity_stats.join(geo_entities).on( - geo_entities[:id].eq(geo_entity_stats[:geo_entity_id]) - ).join_sources - ) - .joins( - geo_entity_stats.join(habitats).on( - habitats[:id].eq(geo_entity_stats[:habitat_id]) - ).join_sources - ) - .group(habitats[:name]) - - results = ActiveRecord::Base.connection.execute(query.to_sql) - - results.map do |row| - total_value = row['total_value'].to_f - protected_value = row['protected_value'].to_f - protected_percentage = protected_value / total_value * 100 - { - name: row['name'], - total_value: total_value.round(2), - protected_value: protected_value.round(2), - protected_percentage: protected_percentage.round(2) - }.stringify_keys - end - end + # def self.global_protection_v2 + # geo_entities = GeoEntity.arel_table + # geo_entity_stats = GeoEntityStat.arel_table + # habitats = Habitat.arel_table + + # # get all the data we need in a single query + # # @see scuttle.io for sql->arel help + # query = GeoEntityStat.select( + # [ + # habitats[:name], + # geo_entity_stats[:total_value].sum.as('total_value'), + # geo_entity_stats[:protected_value].sum.as('protected_value') + # ] + # ) + # .where( + # geo_entities[:iso3].not_eq(nil).and(geo_entities[:iso3].not_eq('GBL')) + # ) + # .joins( + # geo_entity_stats.join(geo_entities).on( + # geo_entities[:id].eq(geo_entity_stats[:geo_entity_id]) + # ).join_sources + # ) + # .joins( + # geo_entity_stats.join(habitats).on( + # habitats[:id].eq(geo_entity_stats[:habitat_id]) + # ).join_sources + # ) + # .group(habitats[:name]) + + # results = ActiveRecord::Base.connection.execute(query.to_sql) + + # results.map do |row| + # total_value = row['total_value'].to_f + # protected_value = row['protected_value'].to_f + # protected_percentage = protected_value / total_value * 100 + # { + # name: row['name'], + # total_value: total_value.round(2), + # protected_value: protected_value.round(2), + # protected_percentage: protected_percentage.round(2) + # }.stringify_keys + # end + # end private @@ -268,14 +250,4 @@ def sum_country_areas(total_area_by_country) end country_total_area end - - def sort_country_count(total_value_by_country) - country_total_points = {} - total_value_by_country.each do |total_value| - next if total_value['iso3'].include? 'ABNJ' - - country_total_points[total_value['iso3']] = total_value['count'] - end - country_total_points - end end diff --git a/config/habitats.yml b/config/habitats.yml index 1691b961..2a5bd05d 100644 --- a/config/habitats.yml +++ b/config/habitats.yml @@ -3,8 +3,6 @@ habitats: name: 'coralreefs' title: 'Warm-water corals' theme: 'orange' - poly_table: 'wcmc008_coralreef2010_py_v4' - point_table: 'wcmc008_coralreef2010_pt_v4' wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_008_CoralReefs_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' total_area: 149886.974126351 protected_area: 67498.1537285504 @@ -12,8 +10,6 @@ habitats: name: 'saltmarshes' title: 'Saltmarshes' theme: 'green' - poly_table: 'wcmc027_saltmarshes_py_v6' - point_table: 'wcmc027_saltmarshes_pt_v6' wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_027_Saltmarsh_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' total_area: 224435.075089817 protected_area: 111454.965592792 @@ -21,8 +17,6 @@ habitats: name: 'mangroves' title: 'Mangroves' theme: 'yellow' - poly_table: 'wcmc011_atlasmangrove2010_py_v3' - point_table: wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_011_WorldAtlasMangroves_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' total_area: 135869.552270024 protected_area: 58847.6773406482 @@ -30,8 +24,6 @@ habitats: name: 'seagrasses' title: 'Seagrasses' theme: 'blue' - poly_table: 'wcmc_013_014_seagrassespy_v6' - point_table: 'wcmc_013_014_seagrassespt_v6' wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_013_014_Seagrass_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' total_area: 314001.940552758 protected_area: 83689.7720939699 @@ -39,8 +31,6 @@ habitats: name: 'coldcorals' title: 'Cold-water corals' theme: 'pink' - poly_table: 'wcmc001_coldcorals2017_py_v5' - point_table: 'wcmc001_coldcorals2017_pt_v5' wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_001_ColdCorals2017_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' total_area: 15336.9752787556 protected_area: 4511.04992438828 diff --git a/db/migrate/20221115093841_remove_carto_tables_from_habitats.rb b/db/migrate/20221115093841_remove_carto_tables_from_habitats.rb new file mode 100644 index 00000000..e9ef93b4 --- /dev/null +++ b/db/migrate/20221115093841_remove_carto_tables_from_habitats.rb @@ -0,0 +1,6 @@ +class RemoveCartoTablesFromHabitats < ActiveRecord::Migration[5.1] + def change + remove_column :habitats, :point_table, :string + remove_column :habitats, :poly_table, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 5518d7ca..01b0a60c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20210812135252) do +ActiveRecord::Schema.define(version: 20221115093841) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -118,8 +118,6 @@ t.string "name", default: "", null: false t.string "title", default: "", null: false t.string "theme", default: "", null: false - t.string "poly_table" - t.string "point_table" t.integer "global_coverage", default: 0, null: false t.integer "protected_percentage", default: 0, null: false t.datetime "created_at", null: false diff --git a/lib/modules/carto.rb b/lib/modules/carto.rb deleted file mode 100644 index 1b10aa35..00000000 --- a/lib/modules/carto.rb +++ /dev/null @@ -1,62 +0,0 @@ -class Carto - include HTTParty - - attr_reader :response - - def initialize(habitat, constraints=nil) - @habitat = Habitat.find_by_name(habitat) - @constraints = constraints - carto_username = Rails.application.secrets.dig(:carto_username) - self.class.base_uri "https://#{carto_username}.carto.com/api/v2" - end - - def total_area - carto_api_key = Rails.application.secrets.dig(:carto_api_key) - @options = { query: { q: total_area_query, api_key: carto_api_key } } - @response = self.class.get("/sql", @options) - @response.parsed_response['rows'] - end - - def total_area_by_country - carto_api_key = Rails.application.secrets.dig(:carto_api_key) - @options = { query: { q: total_area_by_country_query, api_key: carto_api_key } } - @response = self.class.get("/sql", @options) - @response.parsed_response['rows'] - end - - def total_points_by_country - carto_api_key = Rails.application.secrets.dig(:carto_api_key) - @options = { query: { q: total_points_by_country_query, api_key: carto_api_key } } - @response = self.class.get("/sql", @options) - @response.parsed_response['rows'] - end - - def intersect - @response = self.class.get("/sql", @options) - @response.parsed_response['rows'] - end - - private - - def total_area_query - query = "SELECT SUM(gis_area_k) FROM #{from}" - query << " WHERE #{@constraints}" if @constraints.present? - query - end - - def total_area_by_country_query - "SELECT SUM(gis_area_k),iso3 FROM #{from} GROUP BY iso3 ORDER by SUM(gis_area_k) desc" - end - - def total_points_by_country_query - "SELECT COUNT(*),iso3 FROM #{from} GROUP BY iso3 ORDER by COUNT(*) desc" - end - - def from - @habitat.poly_table - end - - def intersection - PostgisUtils.intersection(@geom) - end -end diff --git a/lib/tasks/import_habitats.rake b/lib/tasks/import_habitats.rake index 6e8617b1..4254c5ef 100644 --- a/lib/tasks/import_habitats.rake +++ b/lib/tasks/import_habitats.rake @@ -9,8 +9,6 @@ namespace :import do attributes = { title: data['title'], theme: data['theme'], - poly_table: data['poly_table'], - point_table: data['point_table'], wms_url: data['wms_url'], total_area: data['total_area'].to_f, # see NOTE(1) protected_area: data['protected_area'].to_f # see NOTE(1) diff --git a/spec/models/habitat_spec.rb b/spec/models/habitat_spec.rb index a172fa61..10f7c8ac 100644 --- a/spec/models/habitat_spec.rb +++ b/spec/models/habitat_spec.rb @@ -73,33 +73,5 @@ it 'retrieves the occurrence of the habitat' do expect(habitat.occurrence(country.id)).to eq('present') end - - describe '#total_value_by_country' do - let(:carto) { instance_double(Carto) } - - before { allow(Carto).to receive(:new).and_return(carto) } - - context 'when habitat is coldcorals' do - let(:habitat) { FactoryBot.create(:coldcorals) } - - it 'returns the total number of points' do - allow(carto).to receive(:total_points_by_country).and_return( - [{ "iso3": country.iso3, "count": 30 }.with_indifferent_access] - ) - - expect(habitat.total_value_by_country).to eq({ "#{country.iso3}" => 30 }) - end - end - - context 'when habitat is not coldcorals' do - it 'returns the total area' do - allow(carto).to receive(:total_area_by_country).and_return( - [[{ "iso3": country.iso3, "sum": 30 }.with_indifferent_access]] - ) - - expect(habitat.total_value_by_country).to eq({ "#{country.iso3}" => 30 }) - end - end - end end end