From 6ce9c5ab75807ddc79c8719a6eea9ad0bc9d127e Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Tue, 13 Feb 2018 13:59:36 +0100 Subject: [PATCH 01/19] Fix tests to use proper organization --- spec/models/carto/api_key_spec.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/spec/models/carto/api_key_spec.rb b/spec/models/carto/api_key_spec.rb index ca1101a7ba93..add7288ecab6 100644 --- a/spec/models/carto/api_key_spec.rb +++ b/spec/models/carto/api_key_spec.rb @@ -244,7 +244,7 @@ def with_connection_from_api_key(api_key) api_key_permissions(api_key, @table1.database_schema, @table1.name).permissions.should include(permission) end - sql = "drop table #{@user1.database_schema}.#{@table1.name}" + sql = "drop table \"#{@user1.database_schema}\".\"#{@table1.name}\"" @user1.in_database(as: :superuser).run(sql) api_key_permissions(api_key, @table1.database_schema, @table1.name).should be_nil @@ -318,14 +318,11 @@ def with_connection_from_api_key(api_key) end describe 'with organization users' do - include_context 'organization with users helper' - before(:all) do @auth_api_feature_flag = FactoryGirl.create(:feature_flag, name: 'auth_api', restricted: false) - @auth_organization = test_organization - @auth_organization.save - @user1 = create_auth_api_user(@auth_organization) - @carto_user1 = Carto::User.where(id: @user1.id).first + @auth_organization = FactoryGirl.create(:organization, quota_in_bytes: 1.gigabytes) + @user1 = TestUserFactory.new.create_owner(@auth_organization) + @carto_user1 = Carto::User.find(@user1.id) end after(:all) do @@ -337,13 +334,16 @@ def with_connection_from_api_key(api_key) it_behaves_like 'api key' it 'fails to grant to a non-owned table' do - table = create_table(user_id: @carto_org_user_2.id) + other_user = TestUserFactory.new.create_test_user(unique_name('user'), @auth_organization) + + table = create_table(user_id: other_user.id) grants = [database_grant(table.database_schema, table.name), apis_grant] expect { @carto_user1.api_keys.create_regular_key!(name: 'full', grants: grants) }.to raise_exception ActiveRecord::RecordInvalid table.destroy + other_user.destroy end end end From 39f4b618be0eb6d8578cc88a9b2f8ac11d87dbc2 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Wed, 14 Feb 2018 11:33:33 +0100 Subject: [PATCH 02/19] Grant access only to quota check function to api keys --- app/models/carto/api_key.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/models/carto/api_key.rb b/app/models/carto/api_key.rb index 0484a9940129..4ef02b34a6f9 100644 --- a/app/models/carto/api_key.rb +++ b/app/models/carto/api_key.rb @@ -228,7 +228,7 @@ def create_role db_run("ALTER ROLE \"#{db_role}\" SET search_path TO #{user.db_service.build_search_path}") if user.organization_user? - db_run("GRANT \"#{user.service.organization_member_group_role_member_name}\" TO \"#{db_role}\"") + db_run("GRANT ALL ON FUNCTION \"#{user.database_schema}\"._CDB_UserQuotaInBytes() TO \"#{db_role}\"") end end @@ -280,6 +280,10 @@ def revoke_privileges db_run("REVOKE USAGE ON SCHEMA \"#{schema}\" FROM \"#{db_role}\"") db_run("REVOKE USAGE, SELECT ON ALL SEQUENCES IN SCHEMA \"#{schema}\" FROM \"#{db_role}\"") end + + if user.organization_user? + db_run("REVOKE ALL ON FUNCTION \"#{user.database_schema}\"._CDB_UserQuotaInBytes() FROM \"#{db_role}\"") + end end def grant_aux_write_privileges_for_schema(s) From 1948b55d0e009cff9e8f6ba0d9ef69ab822cef84 Mon Sep 17 00:00:00 2001 From: Alberto Romeu Date: Wed, 14 Feb 2018 12:59:10 +0100 Subject: [PATCH 03/19] add test for gpkg with no srs --- db/fake_data/no_coordinate_system_dataset.gpkg | Bin 0 -> 12288 bytes spec/models/data_import_spec.rb | 13 +++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 db/fake_data/no_coordinate_system_dataset.gpkg diff --git a/db/fake_data/no_coordinate_system_dataset.gpkg b/db/fake_data/no_coordinate_system_dataset.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..a1e0d3db307cf015448aecb51c5c676ae03d8b49 GIT binary patch literal 12288 zcmeHNO^@0}7#`!@B_A8LQnf-oM42KXK=G#7z_M%=sdRTjXcpLjQA<`U%fSO{1sh@; zC7WA5)Jna#hu-=J`Y(FRDYxX1R6XvGF_2 zo4dM$VRv8|sspoN32+?v8bScT>iA0%_gW#pk#U|ToS!f6i5WjXldgjGzrI+e!u&t9 z>719$NUFWK=dkD<1QI3Q^Q@|#2W13 zLC;c$hk6G#8_in1hQP>goMeJhox#A;bW@j@n>i60!yY1zn18?|wc)^o&LM_lH0|Lbwg$RJ*FVSq3do0D3;~9~{{{h#JC81+3s~J;GYJ1u`EM?A#mytkhmyXyU1}1@A%|3ia}nU zRFT|h6A!)n{|mssICbtagBb!0ffEr(ETOfx?gZ@l|3v&)14H1{Bk)eo|4ZOG0MEHs z{7?KJr`}`Vfk!+Sjpg&)WQsy(U^>`zJoo(qf81}s z38vh>b6gI&M&Cv&&e3od7F{EQ>0KRLp0OkvcC&?u6Jb-~4!Txq?aDBj6#A;|D4j#q z>|sa(dq^hXwke*zV9Iu^uA>_`l}xnP6UKT%lM~ms64k9l_f}HaZZ#UHCM&daQ!Z8a z(rLlaP31n^skg}F3XqnL{F0&G4>pm~q$xbH!gjsZlp7^7&#-$(>2(xK8B1KNH&CTq zqciQXMAy==fvzE9$Tn)urIX7hB9|`mE3s;l13HP!-nZ}e$*~+o9XSKHthh|{XG9T0 z#iioXyLXjBU6<+q|7B1Hpd9%&@^j=*{&#-?y;u4KEJtI70{7#kDbMIX4eVG)gv|Ys zVS0R z^C6lpl7uPuh{Hg;9o)ME_W(#v9Mmf{qOeM~CxM=8FrfdB910{}AKn-RIJwuP_s*`w zE??%_t}o*~!p3(;^nAulaW!P|Ok;|w6iCms_x|rS;9tMjdS}860fxY-L|~O8=W*9Q hyLYHM Date: Wed, 14 Feb 2018 13:00:43 +0100 Subject: [PATCH 04/19] add support for gpkg without srs --- services/importer/lib/importer/loader.rb | 8 +++++++- services/importer/lib/importer/ogr2ogr.rb | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/services/importer/lib/importer/loader.rb b/services/importer/lib/importer/loader.rb index 012f4aedea0f..800cc4ef8c45 100644 --- a/services/importer/lib/importer/loader.rb +++ b/services/importer/lib/importer/loader.rb @@ -254,7 +254,7 @@ def run_ogr2ogr(append_mode=false) ogr2ogr.run(append_mode) #In case there are not an specific error we try to fix it - if ogr2ogr.generic_error? && ogr2ogr.exit_code == 0 + if ogr2ogr.generic_error? && ogr2ogr.exit_code == 0 || ogr2ogr.missing_srs? try_fallback(append_mode) end @@ -291,6 +291,12 @@ def try_fallback(append_mode) ogr2ogr.overwrite = true ogr2ogr.encoding = "ISO-8859-1" ogr2ogr.run(append_mode) + elsif ogr2ogr.missing_srs? + job.log "Fallback: Source dataset has no coordinate system, forcing -s_srs 4326" + @job.fallback_executed = "srs 4326" + ogr2ogr.overwrite = true + ogr2ogr.shape_coordinate_system = '4326' + ogr2ogr.run(append_mode) end ogr2ogr.set_default_properties end diff --git a/services/importer/lib/importer/ogr2ogr.rb b/services/importer/lib/importer/ogr2ogr.rb index 861e98f91db2..0f8d56dd2bd9 100644 --- a/services/importer/lib/importer/ogr2ogr.rb +++ b/services/importer/lib/importer/ogr2ogr.rb @@ -114,6 +114,10 @@ def too_many_columns? command_output =~ /tables can have at most 1600 columns/i end + def missing_srs? + exit_code == 256 && command_output =~ /Use -s_srs to set one/i + end + def unsupported_format? exit_code == 256 && command_output =~ /Unable to open(.*)with the following drivers/i end From c50a4d8d5acf28d11727c399450ff83cc246f189 Mon Sep 17 00:00:00 2001 From: Alberto Romeu Date: Wed, 14 Feb 2018 13:02:03 +0100 Subject: [PATCH 05/19] update NEWS.md --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index b5520ed05c5d..5d4d92bd2a1b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -138,6 +138,7 @@ ion for time-series (#12670) * Add titles (and description) to embeds in mobile viewports (#13517) * User feed renders google maps properly when user has it enabled * Prevent destroying modals with `keepOpenOnRouteChange` property enabled on Builder when route changes. ([Support#1293](https://github.com/CartoDB/support/issues/1293)) +* Import gpkg without coordinate system. ([Support#1303](https://github.com/CartoDB/support/issues/1303)) * Improved bundling aliases * Remove Tangram's vector rendering support in Builder embeds ([#13461](https://github.com/CartoDB/cartodb/issues/13461)) * Remove Tangram references (#13461) From 485e8c478fc4bb4cd2db63e2a9bb3b78b9b74a75 Mon Sep 17 00:00:00 2001 From: Alberto Romeu Date: Wed, 14 Feb 2018 13:08:53 +0100 Subject: [PATCH 06/19] Hound --- services/importer/lib/importer/loader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/importer/lib/importer/loader.rb b/services/importer/lib/importer/loader.rb index 800cc4ef8c45..7e3224f90968 100644 --- a/services/importer/lib/importer/loader.rb +++ b/services/importer/lib/importer/loader.rb @@ -254,7 +254,7 @@ def run_ogr2ogr(append_mode=false) ogr2ogr.run(append_mode) #In case there are not an specific error we try to fix it - if ogr2ogr.generic_error? && ogr2ogr.exit_code == 0 || ogr2ogr.missing_srs? + if ogr2ogr.generic_error? && ogr2ogr.exit_code.zero? || ogr2ogr.missing_srs? try_fallback(append_mode) end From 9a0714152de90b937a59b33b1328988b6621af92 Mon Sep 17 00:00:00 2001 From: Alberto Romeu Date: Wed, 14 Feb 2018 13:34:07 +0100 Subject: [PATCH 07/19] fix tests --- services/importer/spec/doubles/ogr2ogr.rb | 1 + services/importer/spec/unit/loader_spec.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/services/importer/spec/doubles/ogr2ogr.rb b/services/importer/spec/doubles/ogr2ogr.rb index 4c8361f36645..47cf9898de35 100644 --- a/services/importer/spec/doubles/ogr2ogr.rb +++ b/services/importer/spec/doubles/ogr2ogr.rb @@ -27,6 +27,7 @@ def file_too_big?; return; end def statement_timeout?; return; end def segfault_error?; return; end def kml_style_missing?; return; end + def missing_srs?; return; end end end end diff --git a/services/importer/spec/unit/loader_spec.rb b/services/importer/spec/unit/loader_spec.rb index 6c30c4954b31..9c8ae824bc16 100644 --- a/services/importer/spec/unit/loader_spec.rb +++ b/services/importer/spec/unit/loader_spec.rb @@ -49,6 +49,7 @@ ogr2ogr_mock.stubs(:duplicate_column?).returns(false) ogr2ogr_mock.stubs(:segfault_error?).returns(false) ogr2ogr_mock.stubs(:kml_style_missing?).returns(false) + ogr2ogr_mock.stubs(:missing_srs?).returns(false) ogr2ogr_mock.stubs(:exit_code).returns(0) ogr2ogr_mock.stubs(:run).returns(Object.new).at_least_once From c042015b9f076bc7faf99e9ce83cff0f45729ec4 Mon Sep 17 00:00:00 2001 From: Alberto Romeu Date: Wed, 14 Feb 2018 13:38:12 +0100 Subject: [PATCH 08/19] Hound --- services/importer/spec/doubles/ogr2ogr.rb | 40 ++++++++++++++--------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/services/importer/spec/doubles/ogr2ogr.rb b/services/importer/spec/doubles/ogr2ogr.rb index 47cf9898de35..d019203721b8 100644 --- a/services/importer/spec/doubles/ogr2ogr.rb +++ b/services/importer/spec/doubles/ogr2ogr.rb @@ -14,22 +14,32 @@ def run(append_mode=false) Object.new end - def set_default_properties; return; end - - def generic_error?; return; end - def encoding_error?; return; end - def invalid_dates?; return; end - def duplicate_column?; return; end - def invalid_geojson?; return; end - def too_many_columns?; return; end - def unsupported_format?; return; end - def file_too_big?; return; end - def statement_timeout?; return; end - def segfault_error?; return; end - def kml_style_missing?; return; end - def missing_srs?; return; end + def set_default_properties; end + + def generic_error?; end + + def encoding_error?; end + + def invalid_dates?; end + + def duplicate_column?; end + + def invalid_geojson?; end + + def too_many_columns?; end + + def unsupported_format?; end + + def file_too_big?; end + + def statement_timeout?; end + + def segfault_error?; end + + def kml_style_missing?; end + + def missing_srs?; end end end end end - From b82926f1daee02dfc3ebd6d9c2c231aa5dd8e32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Mart=C3=ADn?= Date: Wed, 14 Feb 2018 15:38:40 +0100 Subject: [PATCH 09/19] Show empty permission list for master and default_public keys --- .../carto/api/api_key_presenter.rb | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/app/controllers/carto/api/api_key_presenter.rb b/app/controllers/carto/api/api_key_presenter.rb index 5939b6b616a8..fe6c6e328608 100644 --- a/app/controllers/carto/api/api_key_presenter.rb +++ b/app/controllers/carto/api/api_key_presenter.rb @@ -21,19 +21,27 @@ def to_poro }, { type: 'database', - tables: @api_key.table_permissions_from_db.map do |p| - { - schema: p.schema, - name: p.name, - permissions: p.permissions - } - end + tables: table_permissions_for_api_key } ], created_at: @api_key.created_at.to_s, updated_at: @api_key.updated_at.to_s } end + + private + + def table_permissions_for_api_key + return [] if @api_key.master? || @api_key.default_public? + + @api_key.table_permissions_from_db.map do |p| + { + schema: p.schema, + name: p.name, + permissions: p.permissions + } + end + end end end end From 54dd5ccb4c73bbde3f6cc075d30c48224fd5f7e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Mart=C3=ADn?= Date: Wed, 14 Feb 2018 15:38:50 +0100 Subject: [PATCH 10/19] Skip inherited permissions --- app/models/carto/api_key.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/models/carto/api_key.rb b/app/models/carto/api_key.rb index f684fe9a0392..1edf2471b6ed 100644 --- a/app/models/carto/api_key.rb +++ b/app/models/carto/api_key.rb @@ -130,10 +130,8 @@ def table_permissions_from_db string_agg(DISTINCT lower(privilege_type),',') privilege_types FROM information_schema.table_privileges tp - LEFT JOIN - information_schema.applicable_roles ar ON tp.grantee = ar.role_name WHERE - ar.grantee = '#{db_role}' OR tp.grantee = '#{db_role}' + tp.grantee = '#{db_role}' GROUP BY table_schema, table_name; From 82242815a82d5fd4ca56f1a64ca35c938d4d6844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Mart=C3=ADn?= Date: Wed, 14 Feb 2018 16:51:05 +0100 Subject: [PATCH 11/19] Remove old behaviour test --- spec/models/carto/api_key_spec.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/spec/models/carto/api_key_spec.rb b/spec/models/carto/api_key_spec.rb index 671ec0fbfb12..89aec1058493 100644 --- a/spec/models/carto/api_key_spec.rb +++ b/spec/models/carto/api_key_spec.rb @@ -251,12 +251,6 @@ def with_connection_from_api_key(api_key) api_key.destroy end - - it 'shows public tables' do - api_key = @carto_user1.api_keys.default_public.first - - api_key_permissions(api_key, @public_table.database_schema, @public_table.name).permissions.should eq ['select'] - end end describe 'master api key' do From 29117d32b0ff7acecb09ff3bf00e19d614f5119b Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Wed, 14 Feb 2018 16:54:53 +0100 Subject: [PATCH 12/19] Comment explaining the quota trigger GRANT --- app/models/carto/api_key.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models/carto/api_key.rb b/app/models/carto/api_key.rb index 869b09a224b2..e764917ea0cb 100644 --- a/app/models/carto/api_key.rb +++ b/app/models/carto/api_key.rb @@ -197,6 +197,11 @@ def role_creation_queries "ALTER ROLE \"#{db_role}\" SET search_path TO #{user.db_service.build_search_path}" ] + # This is GRANTED to the organizational role for organization users, and the PUBLIC users for non-orgs + # We do not want to grant the organization role to the Api Keys, since that also opens access to the analysis + # catalog and tablemetadata. To be more consistent, we should probably GRANT this to the organization public + # user instead, but that has the downside of leaking quotas to the public. + # This works for now, but if you are adding new permissions, please reconsider this decision. if user.organization_user? queries << "GRANT ALL ON FUNCTION \"#{user.database_schema}\"._CDB_UserQuotaInBytes() TO \"#{db_role}\"" end From b8f617050bdaa1d670002c23268030b5a19e05aa Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Wed, 14 Feb 2018 16:55:16 +0100 Subject: [PATCH 13/19] NEWS --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 8f09d2ee3267..f411b5c330a5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -124,7 +124,7 @@ ion for time-series (#12670) * Header authentication (#13329) * Keep API Key permissions up to date when tables change (#13333) * Delete API keys on user deletion (#13470) - * Inherit from public user for API key permissions (#13464) + * Inherit from public user for API key permissions (#13464, #13550) * Sync master key with user model (#13540) * Do not allow empty api list in Auth API [#13291](https://github.com/CartoDB/cartodb/issues/13291) * Conventions (#13491) From bb9f5dee49e84dfce236c066c0a8ed7cdcb08dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Mart=C3=ADn?= Date: Thu, 15 Feb 2018 11:05:23 +0100 Subject: [PATCH 14/19] Fix test --- spec/models/carto/api_key_spec.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spec/models/carto/api_key_spec.rb b/spec/models/carto/api_key_spec.rb index 89aec1058493..8bfe37f9844a 100644 --- a/spec/models/carto/api_key_spec.rb +++ b/spec/models/carto/api_key_spec.rb @@ -251,6 +251,14 @@ def with_connection_from_api_key(api_key) api_key.destroy end + + it 'shows public tables' do + api_key = @carto_user1.api_keys.default_public.first + unless @carto_user1.has_organization? + api_key_permissions(api_key, @public_table.database_schema, @public_table.name) + .permissions.should eq ['select'] + end + end end describe 'master api key' do From 3e081a4ee6c054047e321b673d244274b4b9d316 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 15 Feb 2018 11:24:40 +0100 Subject: [PATCH 15/19] Test for failure to access system tables --- spec/models/carto/api_key_spec.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/spec/models/carto/api_key_spec.rb b/spec/models/carto/api_key_spec.rb index 671ec0fbfb12..1d48a511bcd0 100644 --- a/spec/models/carto/api_key_spec.rb +++ b/spec/models/carto/api_key_spec.rb @@ -115,6 +115,18 @@ def with_connection_from_api_key(api_key) }.to raise_exception ActiveRecord::RecordInvalid end + it 'fails to access system tables' do + api_key = @carto_user1.api_keys.create_regular_key!(name: 'full', grants: [apis_grant]) + + with_connection_from_api_key(api_key) do |connection| + ['cdb_tablemetadata', 'cdb_analysis_catalog'].each do |table| + expect { + connection.execute("select count(1) from cartodb.#{table}") + }.to raise_exception /permission denied/ + end + end + end + describe '#destroy' do it 'removes the role from DB' do grants = [database_grant(@table1.database_schema, @table1.name), apis_grant] From db5cf4af0c718d84ac77c8b2d8cb4c5397eef1fe Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 15 Feb 2018 13:53:45 +0100 Subject: [PATCH 16/19] Upgrade instructions: split general/version specific steps --- UPGRADE | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/UPGRADE b/UPGRADE index b7b0d65da570..85bc2158dac0 100644 --- a/UPGRADE +++ b/UPGRADE @@ -1,5 +1,32 @@ +Standard migration (default) +---------------------------- + +Unless stated otherwise, assume any new CARTO release requires calling these commands: + + $ bundle exec rake db:migrate + +NOTES: + - Redis server must be listening on the configured port (see config/app_config.yml) for some of the steps below to work + - Rails must be restarted after an upgrade + +Extension migration +------------------- + +Most CARTO releases need an updated the cartodb-postgresql extension. After installing it, it must be upgraded in all existing databases: + + $ bundle exec rake cartodb:db:load_functions + +Specific version notes +====================== + +3.1.0 onwards +------------- + +Upgrade notes are now kept in NEWS.md. Check the NOTICE section of the release you are upgrading to. + 3.0.0 -> 3.1.0 -------------- + * Run rake task: bundle exec rake cartodb:overlays:create_overlays @@ -11,30 +38,13 @@ 2.x -> 3.0.0 ------------ + * Install CartoDB extension version 0.3.x * Run rake tasks: - bundle exec rake cartodb:db:create_default_vis_permissions - bundle exec rake cartodb:db:populate_permission_entity_id -General notes -------------- - -NOTES: - - Redis server must be listening on the configured - port (see config/app_config.yml) for some of the - steps below to work - - Rails must be restarted after an upgrade - - -Standard migration (default) ----------------------------- - -Unless stated otherwise, assume any new CartoDB release requires calling these commands: - - $ bundle exec rake db:migrate - - Mandatory migration ------------------- From 42e693c9f426532c525cccfa0f0c0a6c6bc207f1 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 15 Feb 2018 14:02:07 +0100 Subject: [PATCH 17/19] Make upgrade_postgres_extension easier to use --- lib/tasks/db_maintenance.rake | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/tasks/db_maintenance.rake b/lib/tasks/db_maintenance.rake index c5586af98f9d..6b675ee43fbd 100644 --- a/lib/tasks/db_maintenance.rake +++ b/lib/tasks/db_maintenance.rake @@ -247,8 +247,6 @@ namespace :cartodb do desc 'Upgrade cartodb postgresql extension' task :upgrade_postgres_extension, [:database_host, :version, :sleep, :statement_timeout] => :environment do |task_name, args| - raise "Sample usage: rake cartodb:db:upgrade_postgres_extension['127.0.0.1','0.5.2']" if args[:database_host].blank? or args[:version].blank? - # Send this as string, not as number extension_version = args[:version] database_host = args[:database_host] @@ -256,14 +254,17 @@ namespace :cartodb do statement_timeout = args[:statement_timeout].blank? ? 180000 : args[:statement_timeout] # 3 min by default puts "Upgrading cartodb extension with following config:" - puts "extension_version: #{extension_version}" - puts "database_host: #{database_host}" + puts "extension_version: #{extension_version || 'LATEST'}" + puts "database_host: #{database_host || 'ALL'}" puts "sleep: #{sleep}" puts "statement_timeout: #{statement_timeout}" - count = ::User.where(database_host: database_host).count + query = User + query = query.where(database_host: database_host) if database_host + + count = query.count - ::User.where(database_host: database_host).order(Sequel.asc(:created_at)).each_with_index do |user, i| + query.order(Sequel.asc(:created_at)).each_with_index do |user, i| begin # We grant 2 x statement_timeout, by default 6 min Timeout::timeout(statement_timeout/1000 * 2) do From 8c82089d5164df22f98c59e43196589091bbd77c Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 15 Feb 2018 14:02:26 +0100 Subject: [PATCH 18/19] Suggest using upgrade_postgres_extension --- UPGRADE | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/UPGRADE b/UPGRADE index 85bc2158dac0..60d70b04585d 100644 --- a/UPGRADE +++ b/UPGRADE @@ -4,7 +4,7 @@ Standard migration (default) Unless stated otherwise, assume any new CARTO release requires calling these commands: $ bundle exec rake db:migrate - + NOTES: - Redis server must be listening on the configured port (see config/app_config.yml) for some of the steps below to work - Rails must be restarted after an upgrade @@ -13,8 +13,8 @@ Extension migration ------------------- Most CARTO releases need an updated the cartodb-postgresql extension. After installing it, it must be upgraded in all existing databases: - - $ bundle exec rake cartodb:db:load_functions + + $ bundle exec rake cartodb:db:upgrade_postgres_extension Specific version notes ====================== From 75e286f4371fbdc4e336a3aaf019ee1394e90184 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 15 Feb 2018 14:03:35 +0100 Subject: [PATCH 19/19] Typo --- UPGRADE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UPGRADE b/UPGRADE index 60d70b04585d..9aca938c7bbf 100644 --- a/UPGRADE +++ b/UPGRADE @@ -12,7 +12,7 @@ NOTES: Extension migration ------------------- -Most CARTO releases need an updated the cartodb-postgresql extension. After installing it, it must be upgraded in all existing databases: +Most CARTO releases need an updated cartodb-postgresql extension. After installing it, it must be upgraded in all existing databases: $ bundle exec rake cartodb:db:upgrade_postgres_extension