From 55a1d4fb6bf349fa89fcb7236012e81bcac0d980 Mon Sep 17 00:00:00 2001 From: freddyDOTCMS Date: Tue, 24 Dec 2024 10:43:43 -0600 Subject: [PATCH] #30802 Updating CubeJs Schema to match with the last attribute renaming to match with ES naming --- .../setup/config/dev/cube/schema/Events.js | 121 +++++++++--------- .../collectors/AsyncVanitiesCollector.java | 1 - .../collectors/BasicProfileCollector.java | 1 - .../analytics/track/collectors/Collector.java | 8 +- .../collectors/CustomerEventCollector.java | 4 + .../track/collectors/FilesCollector.java | 2 + .../track/collectors/PageDetailCollector.java | 1 + .../track/collectors/PagesCollector.java | 1 + .../AsyncVanitiesCollectorTest.java | 1 - .../WebEventsCollectorServiceImplTest.java | 1 - 10 files changed, 72 insertions(+), 69 deletions(-) diff --git a/docker/docker-compose-examples/analytics/setup/config/dev/cube/schema/Events.js b/docker/docker-compose-examples/analytics/setup/config/dev/cube/schema/Events.js index 5950342bc17d..1bbdd94756d4 100644 --- a/docker/docker-compose-examples/analytics/setup/config/dev/cube/schema/Events.js +++ b/docker/docker-compose-examples/analytics/setup/config/dev/cube/schema/Events.js @@ -157,70 +157,75 @@ cube('request', { MIN(utc_time) as createdAt, MAX(source_ip) as source_ip, MAX(language) as language, + MAX(languageid) as languageId, + MAX(object_live) as live, + MAX(object_working) as working, MAX(useragent) as user_agent, MAX(persona) as persona, - MAX(rendermode) as rendermode, MAX(referer) as referer, - MAX(host) as host, - MAX(CASE WHEN event_type = 'PAGE_REQUEST' THEN object_id ELSE NULL END) as page_id, - MAX(CASE WHEN event_type = 'PAGE_REQUEST' THEN object_title ELSE NULL END) as page_title, - MAX(CASE WHEN event_type = 'FILE_REQUEST' THEN object_id ELSE NULL END) as file_id, - MAX(CASE WHEN event_type = 'FILE_REQUEST' THEN object_title ELSE NULL END) as file_title, - MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_id ELSE NULL END) as vanity_id, + MAX(conhost) as conHost, + MAX(conhostname) as conHostName, + CASE + WHEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_identifier END) IS NOT NULL + THEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_identifier END) + WHEN MAX(CASE WHEN event_type = 'URL_MAP' THEN object_identifier END) IS NOT NULL + THEN MAX(CASE WHEN event_type = 'URL_MAP' THEN object_identifier END) + WHEN COUNT(DISTINCT object_identifier) == 1 THEN MAX(object_identifier) + ELSE MAX(object_identifier) + END AS identifier, + MAX(object_title) as title, + CASE + WHEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_basetype END) IS NOT NULL + THEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_basetype END) + WHEN MAX(CASE WHEN event_type = 'URL_MAP' THEN object_basetype END) IS NOT NULL + THEN MAX(CASE WHEN event_type = 'URL_MAP' THEN object_basetype END) + WHEN COUNT(DISTINCT object_basetype) == 1 THEN MAX(object_basetype) + ELSE MAX(object_basetype) + END AS baseType, + MAX(object_contenttype) as contentType, + MAX(object_contenttypename) as contentTypeName, + MAX(object_contenttypeid) as contentTypeId, MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_forward_to ELSE NULL END) as vanity_forward_to, - MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_response ELSE NULL END) as vanity_response, - (SUM(CASE WHEN event_type = 'VANITY_REQUEST' THEN 1 ELSE 0 END) > 0)::bool as was_vanity_url_hit, - MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN comefromvanityurl ELSE NULL END) as come_from_vanity_url, - (SUM(CASE WHEN event_type = 'URL_MAP' THEN 1 ELSE 0 END) > 0)::bool as url_map_match, - MAX(CASE WHEN event_type = 'URL_MAP' THEN object_id ELSE NULL END) as url_map_content_detail_id, - MAX(CASE WHEN event_type = 'URL_MAP' THEN object_title ELSE NULL END) as url_map_content_detail_title, - MAX(CASE WHEN event_type = 'URL_MAP' THEN object_content_type_id ELSE NULL END) as url_map_content_type_id, - MAX(CASE WHEN event_type = 'URL_MAP' THEN object_content_type_name ELSE NULL END) as url_map_content_type_name, - MAX(CASE WHEN event_type = 'URL_MAP' THEN object_content_type_var_name ELSE NULL END) as url_map_content_type_var_name, - MAX(object_detail_page_url) as url_map_detail_page_url, + MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_action ELSE NULL END) as vanity_action, + CASE + WHEN SUM(CASE WHEN event_type = 'URL_MAP' THEN 1 ELSE 0 END) > 0 THEN 'URL_MAP' + WHEN SUM(CASE WHEN event_type = 'VANITY_REQUEST' THEN 1 ELSE 0 END) > 0 THEN 'VANITY_URL' + ELSE 'NOTHING' + END mappingType, MAX(url) AS url, MAX(cluster_id) AS cluster_id, - MAX(customer_id) AS customer_id, - CASE - WHEN MAX(CASE WHEN event_type = 'FILE_REQUEST' THEN 1 ELSE 0 END) = 1 THEN 'FILE' - WHEN MAX(CASE WHEN event_type = 'PAGE_REQUEST' THEN 1 ELSE 0 END) = 1 THEN 'PAGE' - WHEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' AND object_response != '200' THEN 1 ELSE 0 END) = 1 THEN 'VANITY_REDIRECT' - ELSE 'OTHER' - END AS what_am_i + MAX(customer_id) AS customer_id FROM events GROUP BY request_id`, dimensions: { + mappingType: { sql: 'mappingType', type: `string` }, + conHost: { sql: 'conHost', type: `string` }, + conHostName: { sql: 'conHostName', type: `string` }, + contentTypeName: { sql: 'contentTypeName', type: `string` }, + contentTypeId: { sql: 'contentTypeId', type: `string` }, + contentType: { sql: 'contentType', type: `string` }, + live: { sql: 'live', type: `boolean` }, + working: { sql: 'working', type: `boolean` }, + baseType: { sql: 'baseType', type: `string` }, + identifier: { sql: 'identifier', type: `string` }, + title: { sql: 'title', type: `string` }, requestId: { sql: 'request_id', type: `string` }, clusterId: { sql: 'cluster_id', type: `string` }, customerId: { sql: 'customer_id', type: `string` }, sessionId: { sql: 'sessionid', type: `string` }, isSessionNew: { sql: 'isSessionNew', type: `boolean` }, createdAt: { sql: 'createdAt', type: `time`, }, - whatAmI: { sql: 'what_am_i', type: `string` }, sourceIp: { sql: 'source_ip', type: `string` }, language: { sql: 'language', type: `string` }, + languageId: { sql: 'languageid', type: `string` }, userAgent: { sql: 'user_agent', type: `string` }, referer: { sql: 'referer', type: `string` }, - renderMode: { sql: 'rendermode', type: `string` }, persona: { sql: 'persona', type: `string` }, host: { sql: 'host', type: `string` }, url: { sql: 'url', type: `string` }, - pageId: { sql: 'page_id', type: `string` }, - pageTitle: { sql: 'page_title', type: `string` }, - fileId: { sql: 'file_id', type: `string` }, - fileTitle: { sql: 'file_title', type: `string` }, - wasVanityHit: { sql: 'was_vanity_url_hit', type: `boolean` }, - vanityId: { sql: 'vanity_id', type: `string` }, - vanityForwardTo: { sql: 'vanity_forward_to', type: `string` }, - vanityResponse: { sql: 'vanity_response', type: `string` }, - comeFromVanityURL: { sql: 'come_from_vanity_url', type: `boolean` }, + forwardTo: { sql: 'vanity_forward_to', type: `string` }, + action: { sql: 'vanity_action', type: `string` }, urlMapWasHit: { sql: 'url_map_match', type: `boolean` }, - isDetailPage: { sql: "url_map_detail_page_url is not null and url_map_detail_page_url != ''", type: `boolean` }, - urlMapContentDetailId: { sql: 'url_map_content_detail_id', type: `string` }, - urlMapContentDetailTitle: { sql: 'url_map_content_detail_title', type: `string` }, - urlMapContentId: { sql: 'url_map_content_type_id', type: `string` }, - urlMapContentTypeName: { sql: 'url_map_content_type_name', type: `string` }, - urlMapContentTypeVarName: { sql: 'url_map_content_type_var_name', type: `string` } }, measures: { count: { @@ -236,36 +241,32 @@ cube('request', { type: 'countDistinct', title: 'Total Requests' }, - pageRequest: { - sql: 'page_id', + fileRequest: { + sql: `CASE WHEN ${CUBE}.baseType = 'FILEASSET' THEN 1 ELSE NULL END`, type: 'count', - title: 'Count of Page request' + title: 'Count of FileAsset Request' }, - uniquePageRequest: { - sql: 'page_id', - type: 'countDistinct', - title: 'Unique Page Ids by Session' - }, - pageRequestAverage: { + fileRequestAverage: { sql: `${fileRequest} / NULLIF(${totalRequest}, 0)`, type: 'number', title: 'FileRequest Average' }, - fileRequest: { - sql: 'file_id', + pageRequest: { + sql: `CASE WHEN ${CUBE}.baseType = 'HTMLPAGE' THEN 1 ELSE NULL END`, type: 'count', - title: 'Count of File Request' + title: 'Count of Page request' }, - uniqueFileRequest: { - sql: 'file_id', - type: 'countDistinct', - title: 'Unique Count of File Request' + pageRequestAverage: { + sql: `${pageRequest} / NULLIF(${totalRequest}, 0)`, + type: 'number', + title: 'Page Request Average' }, - fileRequestAverage: { - sql: `${fileRequest} / NULLIF(${totalRequest}, 0)`, + otherRequestAverage: { + sql: `(${totalRequest} - (${fileRequest} + ${pageRequest})) / NULLIF(${totalRequest}, 0)`, type: 'number', - title: 'FileRequest Average' + title: 'No FIle OR Page Average' } + } }); diff --git a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollector.java b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollector.java index 822b95d4da6d..ee2c49c093be 100644 --- a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollector.java +++ b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollector.java @@ -67,7 +67,6 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa match.get(whoIAM).collect(innerCollectorContextMap, collectorPayloadBean); } - collectorPayloadBean.put(COME_FROM_VANITY_URL, true); return collectorPayloadBean; } diff --git a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/BasicProfileCollector.java b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/BasicProfileCollector.java index 8d5d5c4e53a4..6057c93b9221 100644 --- a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/BasicProfileCollector.java +++ b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/BasicProfileCollector.java @@ -70,7 +70,6 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa WebAPILocator.getPersonalizationWebAPI().getContainerPersonalization(request)); // Include default value for other boolean fields in the Clickhouse table - collectorPayloadBean.put(COME_FROM_VANITY_URL, false); collectorPayloadBean.put(IS_EXPERIMENT_PAGE, false); collectorPayloadBean.put(IS_TARGET_PAGE, false); diff --git a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/Collector.java b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/Collector.java index f6437c0dee41..f77ed70d7aed 100644 --- a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/Collector.java +++ b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/Collector.java @@ -23,8 +23,8 @@ public interface Collector { String INODE = "inode"; // NOT USED YET BUT MAY BE DESIRED String BASE_TYPE = "baseType"; String PATH = "path"; // NOT USED YET BUT MAY BE DESIRED - String LIVE = "live"; // NOT USED YET BUT MAY BE DESIRED - String WORKING = "working"; // NOT USED YET BUT MAY BE DESIRED + String LIVE = "live"; + String WORKING = "working"; String CONTENT_TYPE_VAR_NAME = "contentType"; String CONTENT_TYPE_NAME = "contentTypeName"; String CONTENT_TYPE_ID = "contentTypeId"; @@ -48,12 +48,10 @@ public interface Collector { String UTC_TIME = "utc_time"; - String RENDER_MODE = "renderMode"; - String RESPONSE = "response"; + String RESPONSE = "action"; String RESPONSE_CODE = "response_code"; - String COME_FROM_VANITY_URL = "comeFromVanityURL"; String DETAIL_PAGE_URL = "detail_page_url"; String IS_EXPERIMENT_PAGE = "isexperimentpage"; String IS_TARGET_PAGE = "istargetpage"; diff --git a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CustomerEventCollector.java b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CustomerEventCollector.java index 64628e4d464f..aec8b8260c33 100644 --- a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CustomerEventCollector.java +++ b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CustomerEventCollector.java @@ -2,6 +2,7 @@ import com.dotcms.analytics.track.matchers.UserCustomDefinedRequestMatcher; import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; import com.liferay.util.StringPool; import java.util.Objects; @@ -26,9 +27,12 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa final String host = (String)collectorContextMap.get(CollectorContextMap.HOST); final Host site = (Host) collectorContextMap.get(CollectorContextMap.CURRENT_HOST); final String language = (String)collectorContextMap.get(CollectorContextMap.LANG); + final Long languageId = (Long)collectorContextMap.get(CollectorContextMap.LANG_ID); + collectorPayloadBean.put(URL, uri); collectorPayloadBean.put(SITE_NAME, Objects.nonNull(site)?site.getHostname():host); collectorPayloadBean.put(LANGUAGE, language); + collectorPayloadBean.put(LANGUAGE_ID, languageId); collectorPayloadBean.put(SITE_ID, null != site?site.getIdentifier():StringPool.UNKNOWN); final String eventType = collectorContextMap.get(CollectorContextMap.EVENT_TYPE) == null? EventType.CUSTOM_USER_EVENT.getType():(String)collectorContextMap.get(CollectorContextMap.EVENT_TYPE); diff --git a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/FilesCollector.java b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/FilesCollector.java index 571be4f7d7ef..e92872967ee0 100644 --- a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/FilesCollector.java +++ b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/FilesCollector.java @@ -50,6 +50,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa final Host site = (Host) collectorContextMap.get(CollectorContextMap.CURRENT_HOST); final Long languageId = (Long)collectorContextMap.get(CollectorContextMap.LANG_ID); final String language = (String)collectorContextMap.get(CollectorContextMap.LANG); + final HashMap fileObject = new HashMap<>(); if (Objects.nonNull(uri) && Objects.nonNull(site) && Objects.nonNull(languageId)) { @@ -71,6 +72,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa collectorPayloadBean.put(URL, uri); collectorPayloadBean.put(SITE_NAME, Objects.nonNull(site)?site.getHostname():host); collectorPayloadBean.put(LANGUAGE, language); + collectorPayloadBean.put(LANGUAGE_ID, languageId); collectorPayloadBean.put(SITE_ID, null != site?site.getIdentifier():StringPool.UNKNOWN); collectorPayloadBean.put(EVENT_TYPE, EventType.FILE_REQUEST.getType()); diff --git a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PageDetailCollector.java b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PageDetailCollector.java index 370aa5182821..f2da25166452 100644 --- a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PageDetailCollector.java +++ b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PageDetailCollector.java @@ -91,6 +91,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa collectorPayloadBean.put(EVENT_TYPE, EventType.PAGE_REQUEST.getType()); collectorPayloadBean.put(URL, uri); collectorPayloadBean.put(LANGUAGE, language); + collectorPayloadBean.put(LANGUAGE_ID, languageId); if (Objects.nonNull(site)) { collectorPayloadBean.put(SITE_NAME, site.getHostname()); diff --git a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PagesCollector.java b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PagesCollector.java index 1266da3e3805..1f8a1cd951b9 100644 --- a/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PagesCollector.java +++ b/dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PagesCollector.java @@ -100,6 +100,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa collectorPayloadBean.put(OBJECT, pageObject); collectorPayloadBean.put(URL, uri); collectorPayloadBean.put(LANGUAGE, language); + collectorPayloadBean.put(LANGUAGE_ID, languageId); if (Objects.nonNull(site)) { collectorPayloadBean.put(SITE_NAME, site.getHostname()); diff --git a/dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollectorTest.java b/dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollectorTest.java index 807448c2d578..0541e022d16b 100644 --- a/dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollectorTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollectorTest.java @@ -86,7 +86,6 @@ public void collectAsyncVanityData() throws DotDataException, IOException, final Map expectedDataMap = Map.of( Collector.EVENT_TYPE, EventType.PAGE_REQUEST.getType(), Collector.SITE_NAME, testSite.getHostname(), - Collector.COME_FROM_VANITY_URL, true, Collector.LANGUAGE, defaultLanguage.getIsoCode(), Collector.URL, TEST_PAGE_URL, Collector.OBJECT, Map.of( diff --git a/dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/WebEventsCollectorServiceImplTest.java b/dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/WebEventsCollectorServiceImplTest.java index a1e344046cb8..90ae66b4456a 100644 --- a/dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/WebEventsCollectorServiceImplTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/WebEventsCollectorServiceImplTest.java @@ -372,7 +372,6 @@ public void testAsyncVanitiesCollector() throws DotDataException, IOException, D final Map expectedDataMap = Map.of( Collector.EVENT_TYPE, EventType.PAGE_REQUEST.getType(), Collector.SITE_NAME, testSite.getHostname(), - Collector.COME_FROM_VANITY_URL, true, Collector.LANGUAGE, defaultLanguage.getIsoCode(), Collector.URL, TEST_PAGE_URL, Collector.OBJECT, Map.of(