From f7186674706a5461ae5d09dbe56bd82ded8e1459 Mon Sep 17 00:00:00 2001 From: tkuzynow Date: Mon, 12 Feb 2024 15:59:59 +0100 Subject: [PATCH 1/6] fix: added pagination to keycloak service --- .../config/AgencyServiceJpaConfiguration.java | 1 - .../keycloak/KeycloakService.java | 68 ++++++++++++++----- ...ingleTenantAdminsAllowedMigrationTask.java | 6 +- .../userservice/UserServiceAddAdminTask.java | 2 +- 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/vi/migrationtool/agencyservice/config/AgencyServiceJpaConfiguration.java b/src/main/java/com/vi/migrationtool/agencyservice/config/AgencyServiceJpaConfiguration.java index ac8a65d..48c0f2e 100644 --- a/src/main/java/com/vi/migrationtool/agencyservice/config/AgencyServiceJpaConfiguration.java +++ b/src/main/java/com/vi/migrationtool/agencyservice/config/AgencyServiceJpaConfiguration.java @@ -1,7 +1,6 @@ package com.vi.migrationtool.agencyservice.config; import javax.sql.DataSource; - import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; import org.springframework.boot.context.properties.ConfigurationProperties; diff --git a/src/main/java/com/vi/migrationtool/keycloak/KeycloakService.java b/src/main/java/com/vi/migrationtool/keycloak/KeycloakService.java index 66c4b6a..6731dbb 100644 --- a/src/main/java/com/vi/migrationtool/keycloak/KeycloakService.java +++ b/src/main/java/com/vi/migrationtool/keycloak/KeycloakService.java @@ -6,7 +6,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.client.util.Lists; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -35,7 +37,7 @@ public class KeycloakService { private static final String SEARCH_PARAM = "search"; - private static final String MAX_USERS_TO_MIGRATE = "500"; + private static final short USER_PAGE_SIZE = 500; private static final String PROVIDED_ROLE_DOESNT_EXISTS_IN_KEYCLOAK_MSG = "The provided role {} doesn't exists in keycloak, please create it first"; private final KeycloakConfig keycloakConfig; @@ -78,7 +80,15 @@ public void addRolesToUsersWithRoleName( KeycloakLoginResponseDTO loginResponse = keycloakLoginService.loginAdminUser(); httpHeaders.setBearerAuth(loginResponse.getAccessToken()); - List keycloakUsers = getUsersWithRoleName(roleNameToSearchForUsers); + var pageNumber = 1; + var users = getUsersWithRoleName(roleNameToSearchForUsers, pageNumber); + List keycloakUsers = Lists.newArrayList(); + while (!users.isEmpty()) { + keycloakUsers.addAll(users); + pageNumber++; + users = getUsersWithRoleName(roleNameToSearchForUsers, pageNumber); + } + if (keycloakUsers.isEmpty()) { log.info( "No users found with the given role {}. Migration will not be applied", @@ -172,23 +182,25 @@ private List getUsersBy(final String searchTerm, final HttpHeaders return List.of(response.getBody()); } - public List getUsersWithRoleName(final String roleName) { + public List getUsersWithRoleName(final String roleName, int pageNumberedFromOne) { var httpHeaders = new HttpHeaders(); httpHeaders.setContentType(MediaType.APPLICATION_JSON); KeycloakLoginResponseDTO loginResponse = keycloakLoginService.loginAdminUser(); httpHeaders.setBearerAuth(loginResponse.getAccessToken()); - return getUsersWithRoleName(roleName, httpHeaders); + return getUsersWithRoleName(roleName, getFirstElementIndex(pageNumberedFromOne), httpHeaders); } private List getUsersWithRoleName( - final String roleName, final HttpHeaders httpHeaders) { + final String roleName, final int firstElementIndex, final HttpHeaders httpHeaders) { var url = keycloakConfig.getAuthServerUrl() + "/admin/realms/online-beratung/roles/" + roleName - + "/users?first=0&max=" - + MAX_USERS_TO_MIGRATE; + + "/users?first=" + + firstElementIndex + + "&max=" + + USER_PAGE_SIZE; var getUsersBySearchTermURL = getUrl(url); HttpEntity requestEntity = new HttpEntity<>(httpHeaders); @@ -202,6 +214,7 @@ private List getUsersWithRoleName( new Object[] {}); if (isNull(response.getBody())) { log.warn("No user found in keycloak using search param {}", getUsersBySearchTermURL); + return Lists.newArrayList(); } return List.of(response.getBody()); } @@ -292,20 +305,41 @@ private UsersWithRole addCustomAttributeToUsersWithRole( var restTemplate = new RestTemplate(); restTemplate.setErrorHandler(faultTolerantResponseErrorHandler()); - var users = getUsersWithRoleName(roleName, httpHeaders); - var updatedUsers = - users.stream() - .map( - user -> - addCustomAttributeToUserIfDoesNotExist( - customAttribute, value, httpHeaders, restTemplate, user)) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); + + Collection updatedUsers = Lists.newArrayList(); + var pageNumber = 1; + var users = getUsersWithRoleName(roleName, getFirstElementIndex(pageNumber)); + while (!users.isEmpty()) { + var migratedUsersPage = + addCustomAttributeToUsers(customAttribute, value, httpHeaders, restTemplate, users); + pageNumber++; + updatedUsers.addAll(migratedUsersPage); + users = getUsersWithRoleName(roleName, pageNumber); + } return new UsersWithRole(roleName, updatedUsers); } + private List addCustomAttributeToUsers( + String customAttribute, + Long value, + HttpHeaders httpHeaders, + RestTemplate restTemplate, + List users) { + return users.stream() + .map( + user -> + addCustomAttributeToUserIfDoesNotExist( + customAttribute, value, httpHeaders, restTemplate, user)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + private int getFirstElementIndex(int pageNumberedFromOne) { + return (pageNumberedFromOne - 1) * USER_PAGE_SIZE; + } + private Optional addCustomAttributeToUserIfDoesNotExist( String customAttribute, Long customAttributeValue, diff --git a/src/main/java/com/vi/migrationtool/mongodb/UpdateLegalContentChangesBySingleTenantAdminsAllowedMigrationTask.java b/src/main/java/com/vi/migrationtool/mongodb/UpdateLegalContentChangesBySingleTenantAdminsAllowedMigrationTask.java index 67ed2e0..fa0b7b0 100644 --- a/src/main/java/com/vi/migrationtool/mongodb/UpdateLegalContentChangesBySingleTenantAdminsAllowedMigrationTask.java +++ b/src/main/java/com/vi/migrationtool/mongodb/UpdateLegalContentChangesBySingleTenantAdminsAllowedMigrationTask.java @@ -11,7 +11,8 @@ @Data @Slf4j -public class UpdateLegalContentChangesBySingleTenantAdminsAllowedMigrationTask extends MigrationTasks { +public class UpdateLegalContentChangesBySingleTenantAdminsAllowedMigrationTask + extends MigrationTasks { private Boolean value; @@ -31,7 +32,8 @@ public void migrate(ApplicationSettingService applicationSettingService) { private void applyMigration( ApplicationSettingService applicationSettingService, ApplicationSettingsEntity applicationSettingsEntity) { - LegalContentChangesBySingleTenantAdminsAllowed toggle = new LegalContentChangesBySingleTenantAdminsAllowed(); + LegalContentChangesBySingleTenantAdminsAllowed toggle = + new LegalContentChangesBySingleTenantAdminsAllowed(); toggle.setValue(value); applicationSettingsEntity.setLegalContentChangesBySingleTenantAdminsAllowed(toggle); applicationSettingService.updateApplicationSettings(applicationSettingsEntity); diff --git a/src/main/java/com/vi/migrationtool/userservice/UserServiceAddAdminTask.java b/src/main/java/com/vi/migrationtool/userservice/UserServiceAddAdminTask.java index ef89527..10f64f2 100644 --- a/src/main/java/com/vi/migrationtool/userservice/UserServiceAddAdminTask.java +++ b/src/main/java/com/vi/migrationtool/userservice/UserServiceAddAdminTask.java @@ -27,7 +27,7 @@ public class UserServiceAddAdminTask extends MigrationTasks { public void execute(Database database) { KeycloakService keycloakService = BeanAwareSpringLiquibase.getBean(KeycloakService.class); - List adminUsersWithRoleName = keycloakService.getUsersWithRoleName(roleName); + List adminUsersWithRoleName = keycloakService.getUsersWithRoleName(roleName, 1); log.info("Found users with role name of size: {}", adminUsersWithRoleName.size()); JdbcTemplate jdbcTemplate = BeanAwareSpringLiquibase.getBean(JdbcTemplate.class); createTenantAdminUsersIfNotExist(adminUsersWithRoleName, jdbcTemplate); From d8293cee8dd9310e7aed9cdd75bba60b0ebcf4f2 Mon Sep 17 00:00:00 2001 From: tkuzynow Date: Fri, 23 Feb 2024 18:44:44 +0100 Subject: [PATCH 2/6] feat: add migration to enable tenant level toggle --- ...ctivateTenantLevelToggleMigrationTask.java | 33 +++++++++++++++++++ ..._enable_central_data_protection_toggle.xml | 11 +++++++ 2 files changed, 44 insertions(+) create mode 100644 src/main/java/com/vi/migrationtool/tenantservice/ActivateTenantLevelToggleMigrationTask.java create mode 100644 src/main/resources/migrations/changeset/0027_enable_central_data_protection_toggle/0027_enable_central_data_protection_toggle.xml diff --git a/src/main/java/com/vi/migrationtool/tenantservice/ActivateTenantLevelToggleMigrationTask.java b/src/main/java/com/vi/migrationtool/tenantservice/ActivateTenantLevelToggleMigrationTask.java new file mode 100644 index 0000000..d29ff9e --- /dev/null +++ b/src/main/java/com/vi/migrationtool/tenantservice/ActivateTenantLevelToggleMigrationTask.java @@ -0,0 +1,33 @@ +package com.vi.migrationtool.tenantservice; + +import com.vi.migrationtool.common.MigrationTasks; +import com.vi.migrationtool.config.BeanAwareSpringLiquibase; +import liquibase.database.Database; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; + +@Slf4j +public class ActivateTenantLevelToggleMigrationTask extends MigrationTasks { + + String featureToggleName = "featureCentralDataProtectionTemplateEnabled"; + + @Override + public void execute(Database database) { + var tenantServiceJdbcTemplate = + BeanAwareSpringLiquibase.getNamedBean("tenantServiceJdbcTemplate", JdbcTemplate.class); + log.info("Activating tenant level toggle" + featureToggleName + " for all tenants"); + int[] updatedTenants = activateFeatureToggle(tenantServiceJdbcTemplate); + log.info("Updated {} tenants", updatedTenants.length); + } + + private int[] activateFeatureToggle(JdbcTemplate tenantServiceJdbcTemplate) { + int[] updatedTenants = + tenantServiceJdbcTemplate.batchUpdate( + "update tenant set settings = REPLACE( settings ,'\"" + + featureToggleName + + "\":false','\"" + + featureToggleName + + "\":true')"); + return updatedTenants; + } +} diff --git a/src/main/resources/migrations/changeset/0027_enable_central_data_protection_toggle/0027_enable_central_data_protection_toggle.xml b/src/main/resources/migrations/changeset/0027_enable_central_data_protection_toggle/0027_enable_central_data_protection_toggle.xml new file mode 100644 index 0000000..4db0653 --- /dev/null +++ b/src/main/resources/migrations/changeset/0027_enable_central_data_protection_toggle/0027_enable_central_data_protection_toggle.xml @@ -0,0 +1,11 @@ + + + + + + + + + \ No newline at end of file From 9850cc2ecebc8f6ae4b36d3738eb854d229f03bb Mon Sep 17 00:00:00 2001 From: tkuzynow Date: Mon, 26 Feb 2024 13:57:52 +0100 Subject: [PATCH 3/6] feat: add migration to enable tenant level toggle --- ...ctivateTenantLevelToggleMigrationTask.java | 33 -------------- .../EnableTenantLevelToggleMigrationTask.java | 43 +++++++++++++++++++ ..._enable_central_data_protection_toggle.xml | 4 +- src/main/resources/migrations/master.xml | 1 + 4 files changed, 46 insertions(+), 35 deletions(-) delete mode 100644 src/main/java/com/vi/migrationtool/tenantservice/ActivateTenantLevelToggleMigrationTask.java create mode 100644 src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java diff --git a/src/main/java/com/vi/migrationtool/tenantservice/ActivateTenantLevelToggleMigrationTask.java b/src/main/java/com/vi/migrationtool/tenantservice/ActivateTenantLevelToggleMigrationTask.java deleted file mode 100644 index d29ff9e..0000000 --- a/src/main/java/com/vi/migrationtool/tenantservice/ActivateTenantLevelToggleMigrationTask.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.vi.migrationtool.tenantservice; - -import com.vi.migrationtool.common.MigrationTasks; -import com.vi.migrationtool.config.BeanAwareSpringLiquibase; -import liquibase.database.Database; -import lombok.extern.slf4j.Slf4j; -import org.springframework.jdbc.core.JdbcTemplate; - -@Slf4j -public class ActivateTenantLevelToggleMigrationTask extends MigrationTasks { - - String featureToggleName = "featureCentralDataProtectionTemplateEnabled"; - - @Override - public void execute(Database database) { - var tenantServiceJdbcTemplate = - BeanAwareSpringLiquibase.getNamedBean("tenantServiceJdbcTemplate", JdbcTemplate.class); - log.info("Activating tenant level toggle" + featureToggleName + " for all tenants"); - int[] updatedTenants = activateFeatureToggle(tenantServiceJdbcTemplate); - log.info("Updated {} tenants", updatedTenants.length); - } - - private int[] activateFeatureToggle(JdbcTemplate tenantServiceJdbcTemplate) { - int[] updatedTenants = - tenantServiceJdbcTemplate.batchUpdate( - "update tenant set settings = REPLACE( settings ,'\"" - + featureToggleName - + "\":false','\"" - + featureToggleName - + "\":true')"); - return updatedTenants; - } -} diff --git a/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java b/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java new file mode 100644 index 0000000..dc3dc80 --- /dev/null +++ b/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java @@ -0,0 +1,43 @@ +package com.vi.migrationtool.tenantservice; + +import com.vi.migrationtool.common.MigrationTasks; +import com.vi.migrationtool.config.BeanAwareSpringLiquibase; +import liquibase.database.Database; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.helper.Validate; +import org.springframework.jdbc.core.JdbcTemplate; + +@Slf4j +@Setter +public class EnableTenantLevelToggleMigrationTask extends MigrationTasks { + + String featureToggleName; + + @Override + public void execute(Database database) { + Validate.notNull(featureToggleName, "Feature toggle name must not be null"); + var tenantServiceJdbcTemplate = + BeanAwareSpringLiquibase.getNamedBean("tenantServiceJdbcTemplate", JdbcTemplate.class); + log.info("Activating tenant level toggle" + featureToggleName + " for all tenants"); + activateFeatureToggle(tenantServiceJdbcTemplate); + } + + private void activateFeatureToggle(JdbcTemplate tenantServiceJdbcTemplate) { + int[] updatedTenants = + tenantServiceJdbcTemplate.batchUpdate( + "update tenant set settings = REPLACE( settings ,'\"" + + featureToggleName + + "\":false','\"" + + featureToggleName + + "\":true') where settings like '%" + featureToggleName + ":false%';"); + + log.info("Updated toggle value for {} tenants", updatedTenants[0]); + + int[] tenantsWithAddedSettings = tenantServiceJdbcTemplate.batchUpdate( + "update tenant set settings = REPLACE(settings, '}',',\"" + featureToggleName + "\":true}')" + + " where settings not like '%" + featureToggleName + "%';"); + + log.info("Added feature toggle to {} tenants", tenantsWithAddedSettings[0]); + } +} diff --git a/src/main/resources/migrations/changeset/0027_enable_central_data_protection_toggle/0027_enable_central_data_protection_toggle.xml b/src/main/resources/migrations/changeset/0027_enable_central_data_protection_toggle/0027_enable_central_data_protection_toggle.xml index 4db0653..0773e52 100644 --- a/src/main/resources/migrations/changeset/0027_enable_central_data_protection_toggle/0027_enable_central_data_protection_toggle.xml +++ b/src/main/resources/migrations/changeset/0027_enable_central_data_protection_toggle/0027_enable_central_data_protection_toggle.xml @@ -3,8 +3,8 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.6.xsd"> - - + + diff --git a/src/main/resources/migrations/master.xml b/src/main/resources/migrations/master.xml index 71a5c2c..70e6b5a 100644 --- a/src/main/resources/migrations/master.xml +++ b/src/main/resources/migrations/master.xml @@ -32,4 +32,5 @@ + From a57035dbcc65ec04e779c890637c94e11ee3711a Mon Sep 17 00:00:00 2001 From: tkuzynow Date: Mon, 26 Feb 2024 13:58:36 +0100 Subject: [PATCH 4/6] feat: add migration to enable tenant level toggle --- .../EnableTenantLevelToggleMigrationTask.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java b/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java index dc3dc80..ee3232e 100644 --- a/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java +++ b/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java @@ -30,13 +30,20 @@ private void activateFeatureToggle(JdbcTemplate tenantServiceJdbcTemplate) { + featureToggleName + "\":false','\"" + featureToggleName - + "\":true') where settings like '%" + featureToggleName + ":false%';"); + + "\":true') where settings like '%" + + featureToggleName + + ":false%';"); log.info("Updated toggle value for {} tenants", updatedTenants[0]); - int[] tenantsWithAddedSettings = tenantServiceJdbcTemplate.batchUpdate( - "update tenant set settings = REPLACE(settings, '}',',\"" + featureToggleName + "\":true}')" - + " where settings not like '%" + featureToggleName + "%';"); + int[] tenantsWithAddedSettings = + tenantServiceJdbcTemplate.batchUpdate( + "update tenant set settings = REPLACE(settings, '}',',\"" + + featureToggleName + + "\":true}')" + + " where settings not like '%" + + featureToggleName + + "%';"); log.info("Added feature toggle to {} tenants", tenantsWithAddedSettings[0]); } From 675499e03ef92a0d86eb02f01b805be227f66ed2 Mon Sep 17 00:00:00 2001 From: tkuzynow Date: Mon, 26 Feb 2024 17:19:43 +0100 Subject: [PATCH 5/6] feat: add migration to enable tenant level toggle --- .../tenantservice/EnableTenantLevelToggleMigrationTask.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java b/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java index ee3232e..df0b596 100644 --- a/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java +++ b/src/main/java/com/vi/migrationtool/tenantservice/EnableTenantLevelToggleMigrationTask.java @@ -31,7 +31,9 @@ private void activateFeatureToggle(JdbcTemplate tenantServiceJdbcTemplate) { + "\":false','\"" + featureToggleName + "\":true') where settings like '%" + + "\"" + featureToggleName + + "\"" + ":false%';"); log.info("Updated toggle value for {} tenants", updatedTenants[0]); From e70981b90731cb057f22ffed977a53a0fe19c08a Mon Sep 17 00:00:00 2001 From: Patric Dosch Date: Fri, 15 Mar 2024 11:58:48 +0100 Subject: [PATCH 6/6] chore: fix diakonie migrations --- .../resources/migrations/master-diakonie.xml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/resources/migrations/master-diakonie.xml b/src/main/resources/migrations/master-diakonie.xml index eeb5506..91d2fad 100644 --- a/src/main/resources/migrations/master-diakonie.xml +++ b/src/main/resources/migrations/master-diakonie.xml @@ -4,11 +4,25 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.2.xsd"> - + + + + + + + + + + + + + + - + +