Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 30411 refactor collector payload key names #30784

Merged
merged 9 commits into from
Dec 6, 2024
Merged

Issue 30411 refactor collector payload key names #30784

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7e1ea55
#30411 first shot with some addition and name refactoring
jdotcms Nov 26, 2024
6ba6527
#30411 first shot with some addition and name refactoring
jdotcms Nov 26, 2024
1ff54d8
#30411 adding last renames
jdotcms Nov 27, 2024
783a7d8
#30411 adding fixes for it test
jdotcms Nov 29, 2024
58b029a
#30411 adding fixes
jdotcms Dec 2, 2024
5bd3182
#30411 adding fixes
jdotcms Dec 3, 2024
db8f96a
Merge branch 'main' of github.com:dotCMS/core into issue-30411-refact…
jdotcms Dec 3, 2024
e06b079
#30411 adding fixes
jdotcms Dec 3, 2024
0d9f38e
#30411 fixing junits
jdotcms Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dotCMS/src/main/java/com/dotcms/analytics/track/collectors/AsyncVanitiesCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
if (UtilMethods.isSet(whoIAM)) {

final CollectorContextMap innerCollectorContextMap = new WrapperCollectorContextMap(collectorContextMap,
Map.of("uri", cachedVanityUrl.forwardTo));
Map.of(URL, cachedVanityUrl.forwardTo));
match.get(whoIAM).collect(innerCollectorContextMap, collectorPayloadBean);
}

Expand Down
19 changes: 17 additions & 2 deletions dotCMS/src/main/java/com/dotcms/analytics/track/collectors/BasicProfileCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,16 @@
import com.dotmarketing.business.web.WebAPILocator;
import com.dotmarketing.util.PageMode;
import com.dotmarketing.util.UtilMethods;
import com.liferay.portal.model.User;
import com.liferay.util.StringPool;

import javax.servlet.http.HttpServletRequest;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

/**
Expand Down Expand Up @@ -66,8 +69,6 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
collectorPayloadBean.put(PERSONA,
WebAPILocator.getPersonalizationWebAPI().getContainerPersonalization(request));

collectorPayloadBean.put(RENDER_MODE, PageMode.get(request).toString().replace("_MODE", StringPool.BLANK));

// Include default value for other boolean fields in the Clickhouse table
collectorPayloadBean.put(COME_FROM_VANITY_URL, false);
collectorPayloadBean.put(IS_EXPERIMENT_PAGE, false);
Expand All @@ -78,9 +79,23 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
collectorPayloadBean.put(EVENT_SOURCE, EventSource.DOT_CMS.getName());
}

setUserInfo(request, collectorPayloadBean);

return collectorPayloadBean;
}

private void setUserInfo(final HttpServletRequest request, final CollectorPayloadBean collectorPayloadBean) {

final User user = WebAPILocator.getUserWebAPI().getUser(request);
if (Objects.nonNull(user)) {

final HashMap<String, String> userObject = new HashMap<>();
userObject.put(ID, user.getUserId().toString());
userObject.put(EMAIL, user.getEmailAddress());
collectorPayloadBean.put(USER_OBJECT, userObject);
}
}

@Override
public boolean isEventCreator(){
return false;
Expand Down
48 changes: 31 additions & 17 deletions dotCMS/src/main/java/com/dotcms/analytics/track/collectors/Collector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,37 +18,51 @@
*/
public interface Collector {

String CLUSTER = "cluster";
String COME_FROM_VANITY_URL = "comeFromVanityURL";
String CONTENT_TYPE_ID = "content_type_id";
String CONTENT_TYPE_NAME = "content_type_name";
String CONTENT_TYPE_VAR_NAME = "content_type_var_name";
String DETAIL_PAGE_URL = "detail_page_url";
String ID = "identifier";
String TITLE = "title";
String INODE = "inode"; // NOT USED YET BUT MAY BE DESIRED
String BASE_TYPE = "baseType";
String PATH = "path"; // NOT USED YET BUT MAY BE DESIRED
String LIVE = "live"; // NOT USED YET BUT MAY BE DESIRED
String WORKING = "working"; // NOT USED YET BUT MAY BE DESIRED
String CONTENT_TYPE_VAR_NAME = "contentType";
String CONTENT_TYPE_NAME = "contentTypeName";
String CONTENT_TYPE_ID = "contentTypeId";
String SITE_ID = "conHost";
String SITE_NAME = "conHostName";
String LANGUAGE_ID = "languageId";
String LANGUAGE = "language";

String EVENT_SOURCE = "event_source";
String EVENT_TYPE = "event_type";
String HOST = "host";
String ID = "id";
String IS_EXPERIMENT_PAGE = "isexperimentpage";
String IS_TARGET_PAGE = "istargetpage";
String LANGUAGE = "language";
String LANGUAGE_ID = "language_id";
String CLUSTER = "cluster";
String OBJECT = "object";
String PERSONA = "persona";
String REFERER = "referer";
String RENDER_MODE = "renderMode";
String REQUEST_ID = "request_id";
String RESPONSE = "response";
String RESPONSE_CODE = "response_code";
String SERVER = "server";
String SESSION_ID = "sessionId";
String SESSION_NEW = "sessionNew";
String SITE = "site";
String TITLE = "title";
String URL = "url";
String USER_AGENT = "userAgent";
String UTC_TIME = "utc_time";


String RENDER_MODE = "renderMode";
String RESPONSE = "response";
String RESPONSE_CODE = "response_code";


String COME_FROM_VANITY_URL = "comeFromVanityURL";
String DETAIL_PAGE_URL = "detail_page_url";
String IS_EXPERIMENT_PAGE = "isexperimentpage";
String IS_TARGET_PAGE = "istargetpage";
String VANITY_QUERY_STRING = "vanity_query_string";
String VANITY_URL_KEY = "vanity_url";
String FORWARD_TO = "forwardTo";

String EMAIL = "email";
String USER_OBJECT = "user";
/**
* Test if the collector should run
* @param collectorContextMap
Expand Down
1 change: 1 addition & 0 deletions dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CollectorContextMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
public interface CollectorContextMap {

String REQUEST_ID = "requestId";
String REQUEST = "request";
String TIME = "time";
String CLUSTER = "cluster";
String SERVER = "server";
Expand Down
4 changes: 2 additions & 2 deletions dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CustomerEventCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
final Host site = (Host) collectorContextMap.get(CollectorContextMap.CURRENT_HOST);
final String language = (String)collectorContextMap.get(CollectorContextMap.LANG);
collectorPayloadBean.put(URL, uri);
collectorPayloadBean.put(HOST, Objects.nonNull(site)?site.getHostname():host);
collectorPayloadBean.put(SITE_NAME, Objects.nonNull(site)?site.getHostname():host);
collectorPayloadBean.put(LANGUAGE, language);
collectorPayloadBean.put(SITE, null != site?site.getIdentifier():StringPool.UNKNOWN);
collectorPayloadBean.put(SITE_ID, null != site?site.getIdentifier():StringPool.UNKNOWN);
final String eventType = collectorContextMap.get(CollectorContextMap.EVENT_TYPE) == null?
EventType.CUSTOM_USER_EVENT.getType():(String)collectorContextMap.get(CollectorContextMap.EVENT_TYPE);
collectorPayloadBean.put(EVENT_TYPE, eventType);
Expand Down
11 changes: 9 additions & 2 deletions dotCMS/src/main/java/com/dotcms/analytics/track/collectors/FilesCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.dotmarketing.portlets.contentlet.model.Contentlet;
import com.dotmarketing.portlets.fileassets.business.FileAssetAPI;
import com.liferay.util.StringPool;
import io.vavr.control.Try;

import java.util.HashMap;
import java.util.Objects;
Expand Down Expand Up @@ -57,14 +58,20 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
fileObject.put(ID, fileAsset.getIdentifier());
fileObject.put(TITLE, fileAsset.getTitle());
fileObject.put(URL, uri);
fileObject.put(CONTENT_TYPE_ID, fileAsset.getContentType().id());
fileObject.put(CONTENT_TYPE_NAME, fileAsset.getContentType().name());
fileObject.put(CONTENT_TYPE_VAR_NAME, fileAsset.getContentType().variable());
fileObject.put(BASE_TYPE, fileAsset.getContentType().baseType().name());
fileObject.put(LIVE, String.valueOf(Try.of(()->fileAsset.isLive()).getOrElse(false)));
fileObject.put(WORKING, String.valueOf(Try.of(()->fileAsset.isWorking()).getOrElse(false)));
});
}

collectorPayloadBean.put(OBJECT, fileObject);
collectorPayloadBean.put(URL, uri);
collectorPayloadBean.put(HOST, Objects.nonNull(site)?site.getHostname():host);
collectorPayloadBean.put(SITE_NAME, Objects.nonNull(site)?site.getHostname():host);
collectorPayloadBean.put(LANGUAGE, language);
collectorPayloadBean.put(SITE, null != site?site.getIdentifier():StringPool.UNKNOWN);
collectorPayloadBean.put(SITE_ID, null != site?site.getIdentifier():StringPool.UNKNOWN);
collectorPayloadBean.put(EVENT_TYPE, EventType.FILE_REQUEST.getType());

return collectorPayloadBean;
Expand Down
8 changes: 7 additions & 1 deletion dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PageDetailCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
pageObject.put(ID, detailPageContent.getIdentifier());
pageObject.put(TITLE, detailPageContent.getTitle());
pageObject.put(URL, uri);
pageObject.put(CONTENT_TYPE_ID, urlMapContentlet.getContentType().id());
pageObject.put(CONTENT_TYPE_NAME, urlMapContentlet.getContentType().name());
pageObject.put(CONTENT_TYPE_VAR_NAME, urlMapContentlet.getContentType().variable());
pageObject.put(BASE_TYPE, urlMapContentlet.getContentType().baseType().name());
pageObject.put(LIVE, String.valueOf(Try.of(()->urlMapContentlet.isLive()).getOrElse(false)));
pageObject.put(WORKING, String.valueOf(Try.of(()->urlMapContentlet.isWorking()).getOrElse(false)));
pageObject.put(DETAIL_PAGE_URL, Try.of(detailPageContent::getURI).getOrElse(StringPool.BLANK));
collectorPayloadBean.put(OBJECT, pageObject);
}
Expand All @@ -87,7 +93,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
collectorPayloadBean.put(LANGUAGE, language);

if (Objects.nonNull(site)) {
collectorPayloadBean.put(HOST, site.getHostname());
collectorPayloadBean.put(SITE_NAME, site.getHostname());
}
return collectorPayloadBean;
}
Expand Down
7 changes: 4 additions & 3 deletions dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PagesCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
pageObject.put(CONTENT_TYPE_ID, urlMapContentType.id());
pageObject.put(CONTENT_TYPE_NAME, urlMapContentType.name());
pageObject.put(CONTENT_TYPE_VAR_NAME, urlMapContentType.variable());
pageObject.put(BASE_TYPE, urlMapContentType.baseType().name());
pageObject.put(LIVE, String.valueOf(Try.of(()->urlMapContentlet.isLive()).getOrElse(false)));
pageObject.put(WORKING, String.valueOf(Try.of(()->urlMapContentlet.isWorking()).getOrElse(false)));
collectorPayloadBean.put(EVENT_TYPE, EventType.URL_MAP.getType());
}
} else {
Expand All @@ -83,8 +86,6 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
pageObject.put(ID, page.getIdentifier());
pageObject.put(TITLE, page.getTitle());
collectorPayloadBean.put(EVENT_TYPE, EventType.PAGE_REQUEST.getType());
pageObject.put("id", page.getIdentifier());
pageObject.put("title", page.getTitle());
}
pageObject.put(URL, uri);
}
Expand All @@ -94,7 +95,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
collectorPayloadBean.put(LANGUAGE, language);

if (Objects.nonNull(site)) {
collectorPayloadBean.put(HOST, site.getHostname());
collectorPayloadBean.put(SITE_NAME, site.getHostname());
}

return collectorPayloadBean;
Expand Down
8 changes: 3 additions & 5 deletions dotCMS/src/main/java/com/dotcms/analytics/track/collectors/SyncVanitiesCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
*/
public class SyncVanitiesCollector implements Collector {

public static final String VANITY_URL_KEY = "vanity_url";
public static final String FORWARD_TO = "forward_to";

public SyncVanitiesCollector() {
}
Expand All @@ -32,9 +30,9 @@ public boolean test(CollectorContextMap collectorContextMap) {
@Override
public CollectorPayloadBean collect(final CollectorContextMap collectorContextMap,
final CollectorPayloadBean collectorPayloadBean) {
if (null != collectorContextMap.get("request")) {
if (null != collectorContextMap.get(CollectorContextMap.REQUEST)) {

final HttpServletRequest request = (HttpServletRequest)collectorContextMap.get("request");
final HttpServletRequest request = (HttpServletRequest)collectorContextMap.get(CollectorContextMap.REQUEST);
final String vanityUrl = (String)request.getAttribute(Constants.CMS_FILTER_URI_OVERRIDE);
final String vanityQueryString = (String)request.getAttribute(Constants.CMS_FILTER_QUERY_STRING_OVERRIDE);
if (request instanceof VanityUrlRequestWrapper) {
Expand Down Expand Up @@ -66,7 +64,7 @@ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMa
collectorPayloadBean.put(URL, uri);
collectorPayloadBean.put(LANGUAGE, language);
collectorPayloadBean.put(LANGUAGE_ID, languageId);
collectorPayloadBean.put(SITE, site.getIdentifier());
collectorPayloadBean.put(SITE_ID, site.getIdentifier());
collectorPayloadBean.put(EVENT_TYPE, EventType.VANITY_REQUEST.getType());

return collectorPayloadBean;
Expand Down
27 changes: 17 additions & 10 deletions ...egration/src/test/java/com/dotcms/analytics/track/collectors/PageDetailCollectorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import com.dotmarketing.beans.Host;
import com.dotmarketing.business.APILocator;
import com.dotmarketing.exception.DotDataException;
import com.dotmarketing.exception.DotSecurityException;
import com.dotmarketing.portlets.contentlet.model.Contentlet;
import com.dotmarketing.portlets.contentlet.model.IndexPolicy;
import com.dotmarketing.portlets.htmlpageasset.model.HTMLPageAsset;
Expand Down Expand Up @@ -70,7 +71,7 @@ public static void prepare() throws Exception {
* </ul>
*/
@Test
public void testPageDetailCollector() throws DotDataException, UnknownHostException {
public void testPageDetailCollector() throws DotDataException, UnknownHostException, DotSecurityException {
final HttpServletResponse response = mock(HttpServletResponse.class);
final String requestId = UUIDUtil.uuid();
final HttpServletRequest request = Util.mockHttpRequestObj(response,
Expand Down Expand Up @@ -100,15 +101,21 @@ public void testPageDetailCollector() throws DotDataException, UnknownHostExcept
ContentletDataGen.publish(newsTestContent);

final Map<String, Object> expectedDataMap = Map.of(
"event_type", EventType.PAGE_REQUEST.getType(),
"host", testSite.getHostname(),
"language", language.getIsoCode(),
"url", TEST_URL_MAP_DETAIL_PAGE_URL,
"object", Map.of(
"id", testDetailPage.getIdentifier(),
"title", testDetailPage.getTitle(),
"url", TEST_URL_MAP_DETAIL_PAGE_URL,
"detail_page_url", testDetailPage.getURI())
Collector.EVENT_TYPE, EventType.PAGE_REQUEST.getType(),
Collector.SITE_NAME, testSite.getHostname(),
Collector.LANGUAGE, language.getIsoCode(),
Collector.URL, TEST_URL_MAP_DETAIL_PAGE_URL,
Collector.OBJECT, Map.of(
Collector.ID, testDetailPage.getIdentifier(),
Collector.TITLE, testDetailPage.getTitle(),
Collector.URL, TEST_URL_MAP_DETAIL_PAGE_URL,
Collector.CONTENT_TYPE_ID, testDetailPage.getContentType().id(),
Collector.CONTENT_TYPE_NAME, testDetailPage.getContentType().name(),
Collector.CONTENT_TYPE_VAR_NAME, testDetailPage.getContentType().variable(),
Collector.BASE_TYPE, testDetailPage.getContentType().baseType().name(),
Collector.LIVE, testDetailPage.isLive(),
Collector.WORKING, testDetailPage.isWorking(),
Collector.DETAIL_PAGE_URL, testDetailPage.getURI())
);

final Collector collector = new PageDetailCollector();
Expand Down
54 changes: 33 additions & 21 deletions ...s-integration/src/test/java/com/dotcms/analytics/track/collectors/PagesCollectorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public static void prepare() throws Exception {
* </ul>
*/
@Test
public void collectPageData() throws DotDataException, UnknownHostException {
public void collectPageData() throws Exception {
final HttpServletResponse response = mock(HttpServletResponse.class);
final String requestId = UUIDUtil.uuid();
final HttpServletRequest request = Util.mockHttpRequestObj(response, TEST_PAGE_URL,
Expand All @@ -82,14 +82,21 @@ public void collectPageData() throws DotDataException, UnknownHostException {
final HTMLPageAsset testPage = Util.createTestHTMLPage(testSite, TEST_PAGE_NAME);

final Map<String, Object> expectedDataMap = Map.of(
"event_type", EventType.PAGE_REQUEST.getType(),
"host", testSite.getHostname(),
"language", APILocator.getLanguageAPI().getDefaultLanguage().getIsoCode(),
"url", TEST_PAGE_URL,
"object", Map.of(
"id", testPage.getIdentifier(),
"title", testPage.getTitle(),
"url", testPage.getURI())
Collector.EVENT_TYPE, EventType.PAGE_REQUEST.getType(),
Collector.SITE_NAME, testSite.getHostname(),
Collector.LANGUAGE, APILocator.getLanguageAPI().getDefaultLanguage().getIsoCode(),
Collector.URL, TEST_PAGE_URL,
Collector.OBJECT, Map.of(
Collector.ID, testPage.getIdentifier(),
Collector.TITLE, testPage.getTitle(),
Collector.URL, TEST_PAGE_URL,
Collector.CONTENT_TYPE_ID, testPage.getContentType().id(),
Collector.CONTENT_TYPE_NAME, testPage.getContentType().name(),
Collector.CONTENT_TYPE_VAR_NAME, testPage.getContentType().variable(),
Collector.BASE_TYPE, testPage.getContentType().baseType().name(),
Collector.LIVE, testPage.isLive(),
Collector.WORKING, testPage.isWorking(),
Collector.DETAIL_PAGE_URL, testPage.getURI())
);

final Collector collector = new PagesCollector();
Expand All @@ -116,7 +123,7 @@ public void collectPageData() throws DotDataException, UnknownHostException {
* </ul>
*/
@Test
public void collectUrlMapPageData() throws DotDataException, UnknownHostException {
public void collectUrlMapPageData() throws Exception {
final HttpServletResponse response = mock(HttpServletResponse.class);
final String requestId = UUIDUtil.uuid();
final HttpServletRequest request = Util.mockHttpRequestObj(response,
Expand Down Expand Up @@ -148,17 +155,22 @@ public void collectUrlMapPageData() throws DotDataException, UnknownHostExceptio
ContentletDataGen.publish(newsTestContent);

final Map<String, Object> expectedDataMap = Map.of(
"event_type", EventType.URL_MAP.getType(),
"host", testSite.getHostname(),
"language", APILocator.getLanguageAPI().getDefaultLanguage().getIsoCode(),
"url", TEST_URL_MAP_DETAIL_PAGE_URL,
"object", Map.of(
"content_type_var_name", urlMappedContentType.variable(),
"content_type_id", urlMappedContentType.id(),
"id", newsTestContent.getIdentifier(),
"content_type_name", urlMappedContentType.name(),
"title", urlTitle,
"url", TEST_URL_MAP_DETAIL_PAGE_URL)
Collector.EVENT_TYPE, EventType.URL_MAP.getType(),
Collector.SITE_NAME, testSite.getHostname(),
Collector.LANGUAGE, APILocator.getLanguageAPI().getDefaultLanguage().getIsoCode(),
Collector.URL, TEST_URL_MAP_DETAIL_PAGE_URL,
Collector.OBJECT, Map.of(
Collector.ID, newsTestContent.getIdentifier(),
Collector.TITLE, urlTitle,
Collector.URL, TEST_URL_MAP_DETAIL_PAGE_URL,
Collector.CONTENT_TYPE_ID, urlMappedContentType.id(),
Collector.CONTENT_TYPE_NAME, urlMappedContentType.name(),
Collector.CONTENT_TYPE_VAR_NAME, urlMappedContentType.variable(),
Collector.BASE_TYPE, testDetailPage.getContentType().baseType().name(),
Collector.LIVE, newsTestContent.isLive(),
Collector.WORKING, newsTestContent.isWorking(),
Collector.DETAIL_PAGE_URL, testDetailPage.getURI()
)
);

final Collector collector = new PagesCollector();
Expand Down
Loading
Loading