#30802 Updating CubeJs Schema to match with the last attribute renami…

…ng to match with ES naming

docker/docker-compose-examples/analytics/setup/config/dev/cube/schema/Events.js

@@ -157,70 +157,75 @@ cube('request', {
                MIN(utc_time) as createdAt,
                MAX(source_ip) as source_ip,
                MAX(language) as language,
+               MAX(languageid) as languageId,
+               MAX(object_live) as live,
+               MAX(object_working) as working,
                MAX(useragent) as user_agent,
                MAX(persona) as persona,
-               MAX(rendermode) as rendermode,
                MAX(referer) as referer,
-               MAX(host) as host,
-               MAX(CASE WHEN event_type = 'PAGE_REQUEST' THEN object_id ELSE NULL END) as page_id,
-               MAX(CASE WHEN event_type = 'PAGE_REQUEST' THEN object_title ELSE NULL END) as page_title,
-               MAX(CASE WHEN event_type = 'FILE_REQUEST' THEN object_id ELSE NULL END) as file_id,
-               MAX(CASE WHEN event_type = 'FILE_REQUEST' THEN object_title ELSE NULL END) as file_title,
-               MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_id ELSE NULL END) as vanity_id,
+               MAX(conhost) as conHost,
+               MAX(conhostname) as conHostName,
+              CASE
+                WHEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_identifier END) IS NOT NULL 
+                    THEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_identifier END)
+                WHEN MAX(CASE WHEN event_type = 'URL_MAP' THEN object_identifier END) IS NOT NULL 
+                    THEN MAX(CASE WHEN event_type = 'URL_MAP' THEN object_identifier END)
+                WHEN COUNT(DISTINCT object_identifier) == 1 THEN MAX(object_identifier)
+                ELSE MAX(object_identifier)
+               END AS  identifier,
+               MAX(object_title) as title,
+               CASE
+                  WHEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_basetype END) IS NOT NULL 
+                      THEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_basetype END)
+                  WHEN MAX(CASE WHEN event_type = 'URL_MAP' THEN object_basetype END) IS NOT NULL 
+                      THEN MAX(CASE WHEN event_type = 'URL_MAP' THEN object_basetype END)
+                  WHEN COUNT(DISTINCT object_basetype) == 1 THEN MAX(object_basetype) 
+                  ELSE MAX(object_basetype)
+                END AS  baseType,
+               MAX(object_contenttype) as contentType,
+               MAX(object_contenttypename) as contentTypeName,
+               MAX(object_contenttypeid) as contentTypeId,
                MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_forward_to ELSE NULL END) as vanity_forward_to,
-               MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_response ELSE NULL END) as vanity_response,
-               (SUM(CASE WHEN event_type = 'VANITY_REQUEST' THEN 1 ELSE 0 END) > 0)::bool as was_vanity_url_hit,
-               MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN comefromvanityurl ELSE NULL END) as come_from_vanity_url,
-               (SUM(CASE WHEN event_type = 'URL_MAP' THEN 1 ELSE 0 END) > 0)::bool as url_map_match,
-               MAX(CASE WHEN event_type = 'URL_MAP' THEN object_id ELSE NULL END) as url_map_content_detail_id,
-               MAX(CASE WHEN event_type = 'URL_MAP' THEN object_title ELSE NULL END) as url_map_content_detail_title,
-               MAX(CASE WHEN event_type = 'URL_MAP' THEN object_content_type_id ELSE NULL END) as url_map_content_type_id,
-               MAX(CASE WHEN event_type = 'URL_MAP' THEN object_content_type_name ELSE NULL END) as url_map_content_type_name,
-               MAX(CASE WHEN event_type = 'URL_MAP' THEN object_content_type_var_name ELSE NULL END) as url_map_content_type_var_name,
-               MAX(object_detail_page_url) as url_map_detail_page_url,
+               MAX(CASE WHEN event_type = 'VANITY_REQUEST' THEN object_action ELSE NULL END) as vanity_action,
+               CASE 
+                    WHEN SUM(CASE WHEN event_type = 'URL_MAP' THEN 1 ELSE 0 END) > 0 THEN 'URL_MAP'
+                    WHEN SUM(CASE WHEN event_type = 'VANITY_REQUEST' THEN 1 ELSE 0 END) > 0 THEN 'VANITY_URL'
+                    ELSE 'NOTHING'
+               END mappingType,
                MAX(url) AS url,
                MAX(cluster_id) AS cluster_id,
-               MAX(customer_id) AS customer_id,
-               CASE
-                 WHEN MAX(CASE WHEN event_type = 'FILE_REQUEST' THEN 1 ELSE 0 END) = 1 THEN 'FILE'
-                 WHEN MAX(CASE WHEN event_type = 'PAGE_REQUEST' THEN 1 ELSE 0 END) = 1 THEN 'PAGE'
-                 WHEN MAX(CASE WHEN event_type = 'VANITY_REQUEST' AND object_response != '200' THEN 1 ELSE 0 END) = 1 THEN 'VANITY_REDIRECT'
-                 ELSE 'OTHER'
-               END  AS what_am_i
+               MAX(customer_id) AS customer_id
         FROM events
         GROUP BY request_id`,
   dimensions: {
+    mappingType: { sql: 'mappingType', type: `string` },
+    conHost: { sql: 'conHost', type: `string` },
+    conHostName: { sql: 'conHostName', type: `string` },
+    contentTypeName: { sql: 'contentTypeName', type: `string` },
+    contentTypeId: { sql: 'contentTypeId', type: `string` },
+    contentType: { sql: 'contentType', type: `string` },
+    live: { sql: 'live', type: `boolean` },
+    working: { sql: 'working', type: `boolean` },
+    baseType: { sql: 'baseType', type: `string` },
+    identifier: { sql: 'identifier', type: `string` },
+    title: { sql: 'title', type: `string` },
     requestId: { sql: 'request_id', type: `string` },
     clusterId: { sql: 'cluster_id', type: `string` },
     customerId: { sql: 'customer_id', type: `string` },
     sessionId: { sql: 'sessionid', type: `string` },
     isSessionNew: { sql: 'isSessionNew', type: `boolean` },
     createdAt: { sql: 'createdAt', type: `time`, },
-    whatAmI: { sql: 'what_am_i', type: `string` },
     sourceIp: { sql: 'source_ip', type: `string` },
     language: { sql: 'language', type: `string` },
+    languageId: { sql: 'languageid', type: `string` },
     userAgent: { sql: 'user_agent', type: `string` },
     referer: { sql: 'referer', type: `string` },
-    renderMode: { sql: 'rendermode', type: `string` },
     persona: { sql: 'persona', type: `string` },
     host: { sql: 'host', type: `string` },
     url: { sql: 'url', type: `string` },
-    pageId: { sql: 'page_id', type: `string` },
-    pageTitle: { sql: 'page_title', type: `string` },
-    fileId: { sql: 'file_id', type: `string` },
-    fileTitle: { sql: 'file_title', type: `string` },
-    wasVanityHit: { sql: 'was_vanity_url_hit', type: `boolean` },
-    vanityId: { sql: 'vanity_id', type: `string` },
-    vanityForwardTo: { sql: 'vanity_forward_to', type: `string` },
-    vanityResponse: { sql: 'vanity_response', type: `string` },
-    comeFromVanityURL: { sql: 'come_from_vanity_url', type: `boolean` },
+    forwardTo: { sql: 'vanity_forward_to', type: `string` },
+    action: { sql: 'vanity_action', type: `string` },
     urlMapWasHit: { sql: 'url_map_match', type: `boolean` },
-    isDetailPage: { sql: "url_map_detail_page_url is not null and url_map_detail_page_url != ''", type: `boolean` },
-    urlMapContentDetailId: { sql: 'url_map_content_detail_id', type: `string` },
-    urlMapContentDetailTitle: { sql: 'url_map_content_detail_title', type: `string` },
-    urlMapContentId: { sql: 'url_map_content_type_id', type: `string` },
-    urlMapContentTypeName: { sql: 'url_map_content_type_name', type: `string` },
-    urlMapContentTypeVarName: { sql: 'url_map_content_type_var_name', type: `string` }
   },
   measures: {
     count: {
@@ -236,36 +241,32 @@ cube('request', {
       type: 'countDistinct',
       title: 'Total Requests'
     },
-    pageRequest: {
-      sql: 'page_id',
+    fileRequest: {
+      sql: `CASE WHEN ${CUBE}.baseType = 'FILEASSET' THEN 1 ELSE NULL END`,
       type: 'count',
-      title: 'Count of Page request'
+      title: 'Count of FileAsset Request'
     },
-    uniquePageRequest: {
-      sql: 'page_id',
-      type: 'countDistinct',
-      title: 'Unique Page Ids by Session'
-    },
-    pageRequestAverage: {
+    fileRequestAverage: {
       sql: `${fileRequest} / NULLIF(${totalRequest}, 0)`,
       type: 'number',
       title: 'FileRequest Average'
     },
-    fileRequest: {
-      sql: 'file_id',
+    pageRequest: {
+      sql: `CASE WHEN ${CUBE}.baseType = 'HTMLPAGE' THEN 1 ELSE NULL END`,
       type: 'count',
-      title: 'Count of File Request'
+      title: 'Count of Page request'
     },
-    uniqueFileRequest: {
-      sql: 'file_id',
-      type: 'countDistinct',
-      title: 'Unique Count of File Request'
+    pageRequestAverage: {
+      sql: `${pageRequest} / NULLIF(${totalRequest}, 0)`,
+      type: 'number',
+      title: 'Page Request Average'
     },
-    fileRequestAverage: {
-      sql: `${fileRequest} / NULLIF(${totalRequest}, 0)`,
+    otherRequestAverage: {
+      sql: `(${totalRequest} - (${fileRequest} + ${pageRequest})) / NULLIF(${totalRequest}, 0)`,
       type: 'number',
-      title: 'FileRequest Average'
+      title: 'No FIle OR Page Average'
     }
+
   }
 });
 

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollector.java

@@ -67,7 +67,6 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
             match.get(whoIAM).collect(innerCollectorContextMap, collectorPayloadBean);
         }
 
-        collectorPayloadBean.put(COME_FROM_VANITY_URL, true);
         return collectorPayloadBean;
     }
 

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/BasicProfileCollector.java

@@ -70,7 +70,6 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
                 WebAPILocator.getPersonalizationWebAPI().getContainerPersonalization(request));
 
         // Include default value for other boolean fields in the Clickhouse table
-        collectorPayloadBean.put(COME_FROM_VANITY_URL, false);
         collectorPayloadBean.put(IS_EXPERIMENT_PAGE, false);
         collectorPayloadBean.put(IS_TARGET_PAGE, false);
 

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/Collector.java

@@ -23,8 +23,8 @@ public interface Collector {
     String INODE = "inode"; // NOT USED YET BUT MAY BE DESIRED
     String BASE_TYPE = "baseType";
     String PATH = "path"; // NOT USED YET BUT MAY BE DESIRED
-    String LIVE = "live"; // NOT USED YET BUT MAY BE DESIRED
-    String WORKING = "working"; // NOT USED YET BUT MAY BE DESIRED
+    String LIVE = "live";
+    String WORKING = "working";
     String CONTENT_TYPE_VAR_NAME = "contentType";
     String CONTENT_TYPE_NAME = "contentTypeName";
     String CONTENT_TYPE_ID = "contentTypeId";
@@ -48,12 +48,10 @@ public interface Collector {
     String UTC_TIME = "utc_time";
 
 
-    String RENDER_MODE = "renderMode";
-    String RESPONSE = "response";
+    String RESPONSE = "action";
     String RESPONSE_CODE = "response_code";
 
 
-    String COME_FROM_VANITY_URL = "comeFromVanityURL";
     String DETAIL_PAGE_URL = "detail_page_url";
     String IS_EXPERIMENT_PAGE = "isexperimentpage";
     String IS_TARGET_PAGE = "istargetpage";

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CustomerEventCollector.java

@@ -2,6 +2,7 @@
 
 import com.dotcms.analytics.track.matchers.UserCustomDefinedRequestMatcher;
 import com.dotmarketing.beans.Host;
+import com.dotmarketing.business.APILocator;
 import com.liferay.util.StringPool;
 
 import java.util.Objects;
@@ -26,9 +27,12 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
         final String host = (String)collectorContextMap.get(CollectorContextMap.HOST);
         final Host site = (Host) collectorContextMap.get(CollectorContextMap.CURRENT_HOST);
         final String language = (String)collectorContextMap.get(CollectorContextMap.LANG);
+        final Long languageId = (Long)collectorContextMap.get(CollectorContextMap.LANG_ID);
+
         collectorPayloadBean.put(URL, uri);
         collectorPayloadBean.put(SITE_NAME, Objects.nonNull(site)?site.getHostname():host);
         collectorPayloadBean.put(LANGUAGE, language);
+        collectorPayloadBean.put(LANGUAGE_ID, languageId);
         collectorPayloadBean.put(SITE_ID, null != site?site.getIdentifier():StringPool.UNKNOWN);
         final String eventType = collectorContextMap.get(CollectorContextMap.EVENT_TYPE) == null?
                 EventType.CUSTOM_USER_EVENT.getType():(String)collectorContextMap.get(CollectorContextMap.EVENT_TYPE);

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/FilesCollector.java

@@ -50,6 +50,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
         final Host site = (Host) collectorContextMap.get(CollectorContextMap.CURRENT_HOST);
         final Long languageId = (Long)collectorContextMap.get(CollectorContextMap.LANG_ID);
         final String language = (String)collectorContextMap.get(CollectorContextMap.LANG);
+
         final HashMap<String, String> fileObject = new HashMap<>();
 
         if (Objects.nonNull(uri) && Objects.nonNull(site) && Objects.nonNull(languageId)) {
@@ -71,6 +72,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
         collectorPayloadBean.put(URL, uri);
         collectorPayloadBean.put(SITE_NAME, Objects.nonNull(site)?site.getHostname():host);
         collectorPayloadBean.put(LANGUAGE, language);
+        collectorPayloadBean.put(LANGUAGE_ID, languageId);
         collectorPayloadBean.put(SITE_ID, null != site?site.getIdentifier():StringPool.UNKNOWN);
         collectorPayloadBean.put(EVENT_TYPE, EventType.FILE_REQUEST.getType());
 

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PageDetailCollector.java

@@ -91,6 +91,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
         collectorPayloadBean.put(EVENT_TYPE, EventType.PAGE_REQUEST.getType());
         collectorPayloadBean.put(URL, uri);
         collectorPayloadBean.put(LANGUAGE, language);
+        collectorPayloadBean.put(LANGUAGE_ID, languageId);
 
         if (Objects.nonNull(site)) {
             collectorPayloadBean.put(SITE_NAME, site.getHostname());

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PagesCollector.java

@@ -100,6 +100,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
         collectorPayloadBean.put(OBJECT,  pageObject);
         collectorPayloadBean.put(URL, uri);
         collectorPayloadBean.put(LANGUAGE, language);
+        collectorPayloadBean.put(LANGUAGE_ID, languageId);
 
         if (Objects.nonNull(site)) {
             collectorPayloadBean.put(SITE_NAME,  site.getHostname());

dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollectorTest.java

@@ -86,7 +86,6 @@ public void collectAsyncVanityData() throws DotDataException, IOException,
         final Map<String, Object> expectedDataMap = Map.of(
                 Collector.EVENT_TYPE, EventType.PAGE_REQUEST.getType(),
                 Collector.SITE_NAME, testSite.getHostname(),
-                Collector.COME_FROM_VANITY_URL, true,
                 Collector.LANGUAGE, defaultLanguage.getIsoCode(),
                 Collector.URL, TEST_PAGE_URL,
                 Collector.OBJECT, Map.of(

dotcms-integration/src/test/java/com/dotcms/analytics/track/collectors/WebEventsCollectorServiceImplTest.java

@@ -372,7 +372,6 @@ public void testAsyncVanitiesCollector() throws DotDataException, IOException, D
         final Map<String, Object> expectedDataMap = Map.of(
                 Collector.EVENT_TYPE, EventType.PAGE_REQUEST.getType(),
                 Collector.SITE_NAME, testSite.getHostname(),
-                Collector.COME_FROM_VANITY_URL, true,
                 Collector.LANGUAGE, defaultLanguage.getIsoCode(),
                 Collector.URL, TEST_PAGE_URL,
                 Collector.OBJECT, Map.of(