Skip to content

Commit

Permalink
Issue 30815 populate the unique fields table when the database valida…
Browse files Browse the repository at this point in the history 
…tion is enabled (#30862)

### Proposed Changes
* Create a Initializer to check if the Database validation was enabled 


https://github.com/dotCMS/core/pull/30862/files#diff-b1a4fb7917e1a3e9e98ded9a8d7f31ffd2b90a7aa873c73ba2a8f35b82282a12R17

* Create a Method to create the table 

https://github.com/dotCMS/core/pull/30862/files#diff-b0aed2eddcc36be2a2e2f46624345a83c8698db238fe4dc228c0fb13a11e344fR419

* Create a method to drop the table


https://github.com/dotCMS/core/pull/30862/files#diff-b0aed2eddcc36be2a2e2f46624345a83c8698db238fe4dc228c0fb13a11e344fR440

* Create a method to populate the table


https://github.com/dotCMS/core/pull/30862/files#diff-b0aed2eddcc36be2a2e2f46624345a83c8698db238fe4dc228c0fb13a11e344fR464

* Add WrapInTransaction and CloseDBIfOpened annotation to each method in
UniqueFieldDataBaseUtil


https://github.com/dotCMS/core/pull/30862/files#diff-b0aed2eddcc36be2a2e2f46624345a83c8698db238fe4dc228c0fb13a11e344fR138-R353

---------

Co-authored-by: fabrizzio-dotCMS <[email protected]>
  • Loading branch information
@freddyDOTCMS @fabrizzio-dotCMS
freddyDOTCMS and fabrizzio-dotCMS authored Dec 6, 2024
1 parent e32a4d3 commit 632b9af
Show file tree
Hide file tree
Showing 8 changed files with 1,063 additions and 634 deletions.
3 changes: 2 additions & 1 deletion dotCMS/src/main/java/com/dotcms/config/DotInitializationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import com.dotcms.business.SystemTableInitializer;
import com.dotcms.cdi.CDIUtils;
import com.dotcms.contenttype.business.ContentTypeInitializer;
import com.dotcms.contenttype.business.uniquefields.UniqueFieldValidationStrategyResolver;
import com.dotcms.contenttype.business.uniquefields.UniqueFieldsValidationInitializer;
import com.dotcms.contenttype.business.uniquefields.extratable.UniqueFieldsTableCleanerInitializer;
import com.dotcms.rendering.velocity.events.ExceptionHandlersInitializer;
import com.dotcms.system.event.local.business.LocalSystemEventSubscribersInitializer;
Expand Down Expand Up @@ -136,6 +136,7 @@ private Set<DotInitializer> getInternalInitializers() {
new SystemTableInitializer(),
new EmbeddingsInitializer(),
CDIUtils.getBeanThrows(UniqueFieldsTableCleanerInitializer.class),
CDIUtils.getBeanThrows(UniqueFieldsValidationInitializer.class),
new AnalyticsInitializer()
);
} // getInternalInitializers.
Expand Down
61 changes: 61 additions & 0 deletions .../java/com/dotcms/contenttype/business/uniquefields/UniqueFieldsValidationInitializer.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package com.dotcms.contenttype.business.uniquefields;

import com.dotcms.config.DotInitializer;
import com.dotcms.content.elasticsearch.business.ESContentletAPIImpl;
import com.dotcms.contenttype.business.uniquefields.extratable.UniqueFieldDataBaseUtil;
import com.dotmarketing.common.db.DotDatabaseMetaData;
import com.dotmarketing.db.DbConnectionFactory;
import com.dotmarketing.exception.DotDataException;
import com.dotmarketing.util.Logger;

import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.context.Dependent;
import javax.inject.Inject;
import java.sql.SQLException;

/**
* Initializer in charge of check when dotCMS start up if the Unique Fields Data Base validation was enabled
* to create and populate the unique_fields table.
*
* It check if the table already exists and:
* - If it exists and the Database validation is disabled then drop the table.
* - If it does not exist and the Database validation is enabled then it created and populate it.
* - If it exists and the Database validation is enabled do nothing.
* - If it does not exist and the Database validation is disabled do nothing.
*/
@Dependent
public class UniqueFieldsValidationInitializer implements DotInitializer {

private UniqueFieldDataBaseUtil uniqueFieldDataBaseUtil;
private DotDatabaseMetaData dotDatabaseMetaData;

@Inject
public UniqueFieldsValidationInitializer(final UniqueFieldDataBaseUtil uniqueFieldDataBaseUtil){
this.uniqueFieldDataBaseUtil = uniqueFieldDataBaseUtil;
this.dotDatabaseMetaData = new DotDatabaseMetaData();
}

@Override
public void init() {
final boolean featureFlagDbUniqueFieldValidation = ESContentletAPIImpl.getFeatureFlagDbUniqueFieldValidation();
boolean uniqueFieldsTableExists = uniqueFieldsTableExists();

try {
if (featureFlagDbUniqueFieldValidation && !uniqueFieldsTableExists) {
this.uniqueFieldDataBaseUtil.createTableAnsPopulate();
} else if (!featureFlagDbUniqueFieldValidation && uniqueFieldsTableExists) {
this.uniqueFieldDataBaseUtil.dropUniqueFieldsValidationTable();
}
} catch (DotDataException e) {
Logger.error(UniqueFieldsValidationInitializer.class, e);
}
}

private boolean uniqueFieldsTableExists(){
try {
return dotDatabaseMetaData.tableExists(DbConnectionFactory.getConnection(), "unique_fields");
} catch (SQLException e) {
return false;
}
}
}
166 changes: 165 additions & 1 deletion ...java/com/dotcms/contenttype/business/uniquefields/extratable/UniqueFieldDataBaseUtil.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
package com.dotcms.contenttype.business.uniquefields.extratable;


import com.dotcms.business.CloseDBIfOpened;
import com.dotcms.business.WrapInTransaction;

import com.dotcms.contenttype.model.field.Field;
import com.dotmarketing.common.db.DotConnect;
import com.dotmarketing.exception.DotDataException;
Expand All @@ -9,13 +13,17 @@
import com.liferay.util.StringPool;

import javax.enterprise.context.ApplicationScoped;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.Optional;


import static com.dotcms.content.elasticsearch.business.ESContentletAPIImpl.UNIQUE_PER_SITE_FIELD_VARIABLE_NAME;

import static com.dotcms.contenttype.business.uniquefields.extratable.UniqueFieldCriteria.*;
import static com.dotcms.util.CollectionsUtils.list;
import static org.apache.lucene.queries.function.valuesource.LiteralValueSource.hash;


/**
* Util class to handle QL statement related with the unique_fiedls table
Expand Down Expand Up @@ -81,6 +89,44 @@ public class UniqueFieldDataBaseUtil {
private final static String DELETE_UNIQUE_FIELDS_BY_FIELD = "DELETE FROM unique_fields " +
"WHERE supporting_values->>'" + FIELD_VARIABLE_NAME_ATTR + "' = ?";

private final static String POPULATE_UNIQUE_FIELDS_VALUES_QUERY = "INSERT INTO unique_fields (unique_key_val, supporting_values) " +
"SELECT encode(sha256(CONCAT(content_type_id, field_var_name, language_id, field_value, " +
" CASE WHEN uniquePerSite = 'true' THEN host_id ELSE '' END)::bytea), 'hex') as unique_key_val, " +
" json_build_object('" + CONTENT_TYPE_ID_ATTR + "', content_type_id, " +
"'" + FIELD_VARIABLE_NAME_ATTR + "', field_var_name, " +
"'" + LANGUAGE_ID_ATTR + "', language_id, " +
"'" + FIELD_VALUE_ATTR +"', field_value, " +
"'" + SITE_ID_ATTR + "', host_id, " +
"'" + VARIANT_ATTR + "', variant_id, " +
"'" + UNIQUE_PER_SITE_ATTR + "', " + "uniquePerSite, " +
"'" + LIVE_ATTR + "', live, " +
"'" + CONTENTLET_IDS_ATTR + "', contentlet_identifier) AS supporting_values " +
"FROM (" +
" SELECT structure.inode AS content_type_id," +
" field.velocity_var_name AS field_var_name," +
" contentlet.language_id AS language_id," +
" identifier.host_inode AS host_id," +
" jsonb_extract_path_text(contentlet_as_json -> 'fields', field.velocity_var_name)::jsonb ->>'value' AS field_value," +
" ARRAY_AGG(DISTINCT contentlet.identifier) AS contentlet_identifier," +
" (CASE WHEN COUNT(DISTINCT contentlet_version_info.variant_id) > 1 THEN 'DEFAULT' ELSE MAX(contentlet_version_info.variant_id) END) AS variant_id, " +
" ((CASE WHEN COUNT(*) > 1 AND COUNT(DISTINCT contentlet_version_info.live_inode = contentlet.inode) > 1 THEN 0 " +
" ELSE MAX((CASE WHEN contentlet_version_info.live_inode = contentlet.inode THEN 1 ELSE 0 END)::int) " +
" END) = 1) AS live," +
" (MAX(CASE WHEN field_variable.variable_value = 'true' THEN 1 ELSE 0 END)) = 1 AS uniquePerSite" +
" FROM contentlet" +
" INNER JOIN structure ON structure.inode = contentlet.structure_inode" +
" INNER JOIN field ON structure.inode = field.structure_inode" +
" INNER JOIN identifier ON contentlet.identifier = identifier.id" +
" INNER JOIN contentlet_version_info ON contentlet_version_info.live_inode = contentlet.inode OR" +
" contentlet_version_info.working_inode = contentlet.inode" +
" LEFT JOIN field_variable ON field_variable.field_id = field.inode AND field_variable.variable_key = '" + UNIQUE_PER_SITE_FIELD_VARIABLE_NAME + "'" +
" WHERE jsonb_extract_path_text(contentlet_as_json -> 'fields', field.velocity_var_name) IS NOT NULL" +
" AND field.unique_ = true" +
" GROUP BY structure.inode," +
" field.velocity_var_name," +
" contentlet.language_id," +
" identifier.host_inode," +
" jsonb_extract_path_text(contentlet_as_json -> 'fields', field.velocity_var_name)::jsonb ->>'value') as data_to_populate";

/**
* Insert a new register into the unique_fields table, if already exists another register with the same
Expand All @@ -89,10 +135,12 @@ public class UniqueFieldDataBaseUtil {
* @param key
* @param supportingValues
*/
@WrapInTransaction
public void insertWithHash(final String key, final Map<String, Object> supportingValues) throws DotDataException {
new DotConnect().setSQL(INSERT_SQL_WIT_HASH).addParam(key).addJSONParam(supportingValues).loadObjectResults();
}

@WrapInTransaction
public void insert(final String key, final Map<String, Object> supportingValues) throws DotDataException {
new DotConnect()
.setSQL(INSERT_SQL)
Expand All @@ -110,6 +158,7 @@ public void insert(final String key, final Map<String, Object> supportingValues)
* representing the unique field.
* @param contentletId The Contentlet ID to be added to the list.
*/
@WrapInTransaction
public void updateContentList(final UniqueFieldCriteria uniqueFieldCriteria, final String contentletId) throws DotDataException {
updateContentList(uniqueFieldCriteria.criteria(), list(contentletId));
}
Expand All @@ -123,6 +172,7 @@ public void updateContentList(final UniqueFieldCriteria uniqueFieldCriteria, fin
*
* @throws DotDataException An error occurred when interacting with the database.
*/
@WrapInTransaction
public void updateContentList(final String criteria, final List<String> contentletIds) throws DotDataException {
new DotConnect().setSQL(UPDATE_CONTENT_LIST)
.addJSONParam(contentletIds)
Expand All @@ -139,6 +189,7 @@ public void updateContentList(final String criteria, final List<String> contentl
*
* @throws DotDataException An error occurred when interacting with the database.
*/
@WrapInTransaction
public void updateContentListWithHash(final String hash, final List<String> contentletIds) throws DotDataException {
new DotConnect().setSQL(UPDATE_CONTENT_LIST_WITH_HASH)
.addJSONParam(contentletIds)
Expand All @@ -156,6 +207,7 @@ public void updateContentListWithHash(final String hash, final List<String> cont
*
* @throws DotDataException If an error occurs when interacting with the database.
*/
@CloseDBIfOpened
public Optional<Map<String, Object>> get(final Contentlet contentlet, final Field field) throws DotDataException {
try {
final List<Map<String, Object>> results = new DotConnect().setSQL(GET_UNIQUE_FIELDS_BY_CONTENTLET)
Expand All @@ -180,6 +232,7 @@ public Optional<Map<String, Object>> get(final Contentlet contentlet, final Fiel
*
* @throws DotDataException If an error occurs when interacting with the database.
*/
@WrapInTransaction
public void delete(final String hash, final String fieldVariable) throws DotDataException {
new DotConnect().setSQL(DELETE_UNIQUE_FIELD)
.addParam(hash)
Expand All @@ -198,6 +251,7 @@ public void delete(final String hash, final String fieldVariable) throws DotData
*
* @throws DotDataException If an error occurs when interacting with the database.
*/
@WrapInTransaction
public void recalculate(final String contentTypeId, final String fieldVarName, final boolean uniquePerSite) throws DotDataException {
new DotConnect().setSQL(getUniqueRecalculationQuery(uniquePerSite))
.addParam(contentTypeId)
Expand All @@ -220,6 +274,7 @@ private static String getUniqueRecalculationQuery(final boolean uniquePerSite) {
uniquePerSite);
}

@CloseDBIfOpened
public List<Map<String, Object>> get(final String contentId, final long languegeId) throws DotDataException {
return new DotConnect().setSQL(GET_UNIQUE_FIELDS_BY_CONTENTLET_AND_LANGUAGE)
.addParam("\"" + contentId + "\"")
Expand All @@ -235,6 +290,7 @@ public List<Map<String, Object>> get(final String contentId, final long languege
* @return
* @throws DotDataException
*/
@CloseDBIfOpened
public List<Map<String, Object>> get(final String contentId, final String variantId) throws DotDataException {
return new DotConnect().setSQL(GET_UNIQUE_FIELDS_BY_CONTENTLET_AND_VARIANT)
.addParam("\"" + contentId + "\"")
Expand All @@ -248,6 +304,7 @@ public List<Map<String, Object>> get(final String contentId, final String varian
* @param hash
* @throws DotDataException
*/
@WrapInTransaction
public void delete(final String hash) throws DotDataException {
new DotConnect().setSQL(DELETE_UNIQUE_FIELDS)
.addParam(hash)
Expand All @@ -260,6 +317,7 @@ public void delete(final String hash) throws DotDataException {
* @param field
* @throws DotDataException
*/
@WrapInTransaction
public void delete(final Field field) throws DotDataException {
new DotConnect().setSQL(DELETE_UNIQUE_FIELDS_BY_FIELD)
.addParam(field.variable())
Expand All @@ -273,6 +331,7 @@ public void delete(final Field field) throws DotDataException {
* @param liveValue
* @throws DotDataException
*/
@WrapInTransaction
public void setLive(Contentlet contentlet, final boolean liveValue) throws DotDataException {

new DotConnect().setSQL(SET_LIVE_BY_CONTENTLET)
Expand All @@ -291,6 +350,7 @@ public void setLive(Contentlet contentlet, final boolean liveValue) throws DotDa
*
* @throws DotDataException
*/
@WrapInTransaction
public void removeLive(Contentlet contentlet) throws DotDataException {

new DotConnect().setSQL(DELETE_UNIQUE_FIELDS_BY_CONTENTLET)
Expand All @@ -300,4 +360,108 @@ public void removeLive(Contentlet contentlet) throws DotDataException {
.addParam(true)
.loadObjectResults();
}

/**
* Create the {@code unique_fields} table for the new Unique Field Data base
* Validation mechanism. The new {@code unique_fields} table will be used to validate fields that must be
* unique, and what parameters were used to defined such a uniqueness feature.
*
* <h4>Table Definition:</h4>
* <pre>
* {@code
* CREATE TABLE unique_fields (
* unique_key_val VARCHAR(64) PRIMARY KEY,
* supporting_values JSONB
* );
* }
* </pre>
* <h4>Columns:</h4>
* The {@code unique_key_val} column will store a hash created from a combination of the following:
* <ul>
* <li>Content type ID.</li>
* <li>Field variable name.</li>
* <li>Field value.</li>
* <li>Language.</li>
* <li>Site ID (if the {@code uniquePerSite} option is enabled).</li>
* </ul>
* <p>
* The {@code supporting_values} column contains a JSON object with the following format:
* <pre>
* {@code
* {
* "contentTypeID": "",
* "fieldVariableName": "",
* "fieldValue": "",
* "languageId": "",
* "hostId": "",
* "uniquePerSite": true|false,
* "contentletsId": [...],
* "variant": "",
* "live": true|fsle
* }
* }
* </pre>
* <p>The {@code contentletsId} array holds the IDs of contentlets with the same field value that
* existed before the database was upgraded. After the upgrade, no more contentlets with
* duplicate values will be allowed.</p>
*
* <h4>Additional Details:</h4>
* <ul>
* <li>The Host ID is included in the hash calculation only if the {@code uniquePerSite}
* field variable is enabled.</li>
* <li>The {@code unique_key_val} field ensures that only truly unique values can be inserted
* moving forward.</li>
* <li>This upgrade task also populates the {@code unique_fields} table with the existing
* unique field values from the current database.</li>
* </ul>
*/
@WrapInTransaction
public void createUniqueFieldsValidationTable() throws DotDataException {
new DotConnect().setSQL("CREATE TABLE IF NOT EXISTS unique_fields (" +
"unique_key_val VARCHAR(64) PRIMARY KEY," +
"supporting_values JSONB" +
" )").loadObjectResults();
}

@WrapInTransaction
public void createTableAnsPopulate() throws DotDataException {
createUniqueFieldsValidationTable();
populateUniqueFieldsTable();
}

/**
* Drop the {@code unique_fields} table for the new Unique Field Data base validation mechanism.
*
* @see UniqueFieldDataBaseUtil#createUniqueFieldsValidationTable()
*
* @throws DotDataException
*/
@WrapInTransaction
public void dropUniqueFieldsValidationTable() throws DotDataException {
try {
new DotConnect().setSQL("DROP TABLE unique_fields").loadObjectResults();
} catch (DotDataException e) {
final Throwable cause = e.getCause();

if (!SQLException.class.isInstance(cause) ||
!"ERROR: table \"unique_fields\" does not exist".equals(cause.getMessage())) {
throw e;
}
}
}

/**
* Populates the {@code unique_fields} table with unique field values extracted from the {@code contentlet} table.
*
* The process involves:
* - Identifying all {@link com.dotcms.contenttype.model.type.ContentType} objects with unique fields.
* - Retrieving all {@link Contentlet} entries and their corresponding values for both LIVE and Working versions.
* - Storing these unique field values into the {@code unique_fields} table with all this data.
*
* @throws DotDataException
*/
@WrapInTransaction
public void populateUniqueFieldsTable() throws DotDataException {
new DotConnect().setSQL(POPULATE_UNIQUE_FIELDS_VALUES_QUERY).loadObjectResults();
}
}
Loading

0 comments on commit 632b9af

Please sign in to comment.