diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTask.java index 906eb7c37..e89e59313 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTask.java @@ -22,6 +22,7 @@ import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState; import java.util.List; import java.util.OptionalLong; +import java.util.Set; import java.util.function.Predicate; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; @@ -160,7 +161,7 @@ public TeradataAssessmentLogsJdbcTask( SharedState state, String logTable, String queryTable, - List conditions, + Set conditions, ZonedInterval interval, @CheckForNull String logDateColumn, OptionalLong maxSqlLength, diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java index 2a537c2a1..d01f320bb 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java @@ -21,6 +21,7 @@ import com.google.auto.service.AutoService; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Range; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; @@ -44,7 +45,6 @@ import com.google.edwmigration.dumper.plugin.lib.dumper.spi.TeradataLogsDumpFormat; import java.time.Duration; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.OptionalLong; @@ -189,13 +189,13 @@ public void addTasksTo(List> out, @Nonnull ConnectorArguments ar logTable = alternates.get(0); queryTable = alternates.get(1); } - List conditions = new ArrayList<>(); + ImmutableSet.Builder conditionsBuilder = ImmutableSet.builder(); // if the user specifies an earliest start time there will be extraneous empty dump files // because we always iterate over the full 7 trailing days; maybe it's worth // preventing that in the future. To do that, we should require getQueryLogEarliestTimestamp() // to parse and return an ISO instant, not a database-server-specific format. if (!StringUtils.isBlank(arguments.getQueryLogEarliestTimestamp())) { - conditions.add("L.StartTime >= " + arguments.getQueryLogEarliestTimestamp()); + conditionsBuilder.add("L.StartTime >= " + arguments.getQueryLogEarliestTimestamp()); } Duration rotationDuration = arguments.getQueryLogRotationFrequency(); @@ -211,6 +211,10 @@ public void addTasksTo(List> out, @Nonnull ConnectorArguments ar OptionalLong maxSqlLength = PropertyParser.parseNumber( arguments, TeradataLogsConnectorProperty.MAX_SQL_LENGTH, MAX_SQL_LENGTH_RANGE); + if (!isAssessment) { + conditionsBuilder.add("L.UserName <> 'DBC'"); + } + ImmutableSet conditions = conditionsBuilder.build(); for (ZonedInterval interval : intervals) { String file = createFilename(ZIP_ENTRY_PREFIX, interval); if (isAssessment) { @@ -235,7 +239,6 @@ public void addTasksTo(List> out, @Nonnull ConnectorArguments ar utilityLogsTable, interval)); } else { - conditions.add("L.UserName <> 'DBC'"); out.add( new TeradataLogsJdbcTask( file, queryLogsState, logTable, queryTable, conditions, interval) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java index 0a262ca85..5b6be1d99 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java @@ -29,6 +29,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.io.ByteSink; import com.google.common.primitives.Ints; import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval; @@ -43,9 +44,9 @@ import java.sql.SQLException; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.Collections; import java.util.List; import java.util.OptionalLong; +import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; import javax.annotation.CheckForNull; @@ -100,18 +101,18 @@ public class TeradataLogsJdbcTask extends AbstractJdbcTask { protected final SharedState state; protected final String logTable; protected final String queryTable; - protected final List conditions; + protected final ImmutableSet conditions; protected final ZonedInterval interval; @CheckForNull private final String logDateColumn; private final OptionalLong maxSqlLength; - protected final List orderBy; + protected final ImmutableList orderBy; public TeradataLogsJdbcTask( @Nonnull String targetPath, SharedState state, String logTable, String queryTable, - List conditions, + Set conditions, ZonedInterval interval) { this( targetPath, @@ -122,7 +123,7 @@ public TeradataLogsJdbcTask( interval, /* logDateColumn= */ null, /* maxSqlLength= */ OptionalLong.empty(), - Collections.emptyList()); + /* orderBy= */ ImmutableList.of()); } protected TeradataLogsJdbcTask( @@ -130,7 +131,7 @@ protected TeradataLogsJdbcTask( SharedState state, String logTable, String queryTable, - List conditions, + Set conditions, ZonedInterval interval, @CheckForNull String logDateColumn, OptionalLong maxSqlLength, @@ -139,11 +140,11 @@ protected TeradataLogsJdbcTask( this.state = Preconditions.checkNotNull(state, "SharedState was null."); this.logTable = logTable; this.queryTable = queryTable; - this.conditions = conditions; + this.conditions = ImmutableSet.copyOf(conditions); this.interval = interval; this.logDateColumn = logDateColumn; this.maxSqlLength = maxSqlLength; - this.orderBy = orderBy; + this.orderBy = ImmutableList.copyOf(orderBy); } private static boolean isQueryTable(@Nonnull String expression) { @@ -248,7 +249,7 @@ private String getSql(@Nonnull JdbcHandle handle) { buf.append(" ORDER BY "); Joiner.on(", ").appendTo(buf, orderBy); } - return buf.toString().replace('\n', ' '); + return formatQuery(buf.toString()); } private String createLogDateColumnConditionStr() { diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/AbstractConnectorTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/AbstractConnectorTest.java index be2a81b46..11e1fd487 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/AbstractConnectorTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/AbstractConnectorTest.java @@ -40,7 +40,7 @@ public abstract class AbstractConnectorTest { @SuppressWarnings("UnusedVariable") private static final Logger LOG = LoggerFactory.getLogger(AbstractConnectorTest.class); - protected static enum SpecialTaskType { + protected enum SpecialTaskType { Version, Arguments, Format diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTaskTest.java index 4902b48c8..5b442a523 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTaskTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTaskTest.java @@ -20,6 +20,7 @@ import static java.util.Collections.emptyList; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval; import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState; import java.time.ZoneId; @@ -52,7 +53,7 @@ public void getSql_success() { queryLogsState, "SampleQueryTable", "SampleSqlTable", - /* conditions= */ emptyList(), + /* conditions= */ ImmutableSet.of(), interval, /* logDateColumn= */ null, /* maxSqlLength= */ OptionalLong.empty(), @@ -79,7 +80,7 @@ public void getSql_maxSqlLength() { queryLogsState, "SampleQueryTable", "SampleSqlTable", - /* conditions= */ emptyList(), + /* conditions= */ ImmutableSet.of(), interval, /* logDateColumn= */ null, /* maxSqlLength= */ OptionalLong.of(20000), @@ -114,7 +115,7 @@ public void getSql_noSecondTable() { queryLogsState, "SampleQueryTable", "SampleSqlTable", - /* conditions= */ emptyList(), + /* conditions= */ ImmutableSet.of(), interval, /* logDateColumn= */ null, /* maxSqlLength= */ OptionalLong.empty(), @@ -141,7 +142,7 @@ public void getSql_noSecondTableWithLogDateColumn() { queryLogsState, "SampleQueryTable", "SampleSqlTable", - /* conditions= */ emptyList(), + /* conditions= */ ImmutableSet.of(), interval, "SampleLogDate", /* orderBy */ /* maxSqlLength= */ OptionalLong.empty(), @@ -169,7 +170,7 @@ public void getSql_noSecondTableWithCondition() { queryLogsState, "SampleQueryTable", "SampleSqlTable", - /* conditions= */ ImmutableList.of("L.QueryID=7"), + /* conditions= */ ImmutableSet.of("L.QueryID=7"), interval, /* logDateColumn= */ null, /* maxSqlLength= */ OptionalLong.empty(), @@ -196,7 +197,7 @@ public void getSql_noSecondTableWithOrderBy() { queryLogsState, "SampleQueryTable", "SampleSqlTable", - /* conditions= */ emptyList(), + /* conditions= */ ImmutableSet.of(), interval, /* logDateColumn= */ null, /* maxSqlLength= */ OptionalLong.empty(), @@ -224,7 +225,7 @@ public void getSql_withLogDateColumn() { queryLogsState, "SampleQueryTable", "SampleSqlTable", - /* conditions= */ emptyList(), + /* conditions= */ ImmutableSet.of(), interval, "SampleLogDate", /* orderBy */ /* maxSqlLength= */ OptionalLong.empty(), @@ -253,7 +254,7 @@ public void getSql_withLogDateColumnAndMaxSqlLength() { queryLogsState, "SampleQueryTable", "SampleSqlTable", - /* conditions= */ emptyList(), + /* conditions= */ ImmutableSet.of(), interval, "SampleLogDate", /* orderBy */ /* maxSqlLength= */ OptionalLong.of(20000), @@ -292,7 +293,7 @@ public void getSql_fullQuery() { queryLogsState, "SampleQueryTable", "SampleSqlTable", - ImmutableList.of("QueryID=7", "QueryText LIKE '%abc%'"), + ImmutableSet.of("QueryID=7", "QueryText LIKE '%abc%'"), interval, "SampleLogDate", /* maxSqlLength= */ OptionalLong.empty(), diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnectorTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnectorTest.java index 71319f23a..62df27c10 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnectorTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnectorTest.java @@ -16,10 +16,15 @@ */ package com.google.edwmigration.dumper.application.dumper.connector.teradata; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.edwmigration.dumper.application.dumper.task.AbstractJdbcTask.setParameterValues; +import static com.google.edwmigration.dumper.application.dumper.test.DumperTestUtils.assertQueryEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.Resources; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; @@ -29,6 +34,8 @@ import com.google.edwmigration.dumper.application.dumper.handle.JdbcHandle; import com.google.edwmigration.dumper.application.dumper.io.FileSystemOutputHandleFactory; import com.google.edwmigration.dumper.application.dumper.io.OutputHandleFactory; +import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask; +import com.google.edwmigration.dumper.application.dumper.task.FormatTask; import com.google.edwmigration.dumper.application.dumper.task.Task; import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext; import com.google.edwmigration.dumper.application.dumper.test.DummyTaskRunContext; @@ -133,14 +140,166 @@ public void testExecution() throws Exception { List> tasks = new ArrayList<>(); connector.addTasksTo( tasks, - new ConnectorArguments( - new String[] {"--connector", connector.getName(), "--query-log-days", "1"})); + new ConnectorArguments("--connector", connector.getName(), "--query-log-days", "1")); for (Task task : tasks) { task.run(runContext); } } } + @Test + public void addTasksTo_dumpMetadataTask() throws Exception { + List> tasks = new ArrayList<>(); + + // Act + connector.addTasksTo(tasks, new ConnectorArguments("--connector", connector.getName())); + + // Assert + assertEquals(1, tasks.stream().filter(task -> task instanceof DumpMetadataTask).count()); + } + + @Test + public void addTasksTo_formatTask() throws Exception { + List> tasks = new ArrayList<>(); + + // Act + connector.addTasksTo(tasks, new ConnectorArguments("--connector", connector.getName())); + + // Assert + assertEquals(1, tasks.stream().filter(task -> task instanceof FormatTask).count()); + } + + @Test + public void addTasksTo_noTeradataAssessmentLogsJdbcTaskWithoutAssessmentFlag() throws Exception { + List> tasks = new ArrayList<>(); + + // Act + connector.addTasksTo(tasks, new ConnectorArguments("--connector", connector.getName())); + + // Assert + assertTrue(tasks.stream().noneMatch(task -> task instanceof TeradataAssessmentLogsJdbcTask)); + } + + @Test + public void + addTasksTo_allTeradataLogsJdbcTasksAreTeradataAssessmentLogsJdbcTasksWithAssessmentFlag() + throws Exception { + List> tasks = new ArrayList<>(); + + // Act + connector.addTasksTo( + tasks, new ConnectorArguments("--connector", connector.getName(), "--assessment")); + + // Assert + assertTrue( + tasks.stream() + .filter(task -> task instanceof TeradataLogsJdbcTask) + .allMatch(task -> task instanceof TeradataAssessmentLogsJdbcTask)); + } + + @Test + public void addTasksTo_teradataLogsJdbcTaskForOneHour() throws Exception { + List> tasks = new ArrayList<>(); + + // Act + connector.addTasksTo( + tasks, + new ConnectorArguments( + "--connector", + connector.getName(), + "--query-log-start", + "2023-12-22 00:00:00", + "--query-log-end", + "2023-12-22 01:00:00")); + + // Assert + List queries = + tasks.stream() + .filter(task -> task instanceof TeradataLogsJdbcTask) + .map( + task -> + ((TeradataLogsJdbcTask) task) + .getSql(unused -> true, new String[] {"SampleColumn"})) + .collect(toImmutableList()); + assertEquals(1, queries.size()); + assertQueryEquals( + "SELECT SampleColumn FROM dbc.DBQLogTbl L WHERE L.ErrorCode=0 AND" + + " L.StartTime >= CAST('2023-12-22T00:00:00Z' AS TIMESTAMP) AND" + + " L.StartTime < CAST('2023-12-22T01:00:00Z' AS TIMESTAMP) AND L.UserName <> 'DBC'", + getOnlyElement(queries)); + } + + @Test + public void addTasksTo_teradataLogsJdbcTasksForTwoHours() throws Exception { + List> tasks = new ArrayList<>(); + + // Act + connector.addTasksTo( + tasks, + new ConnectorArguments( + "--connector", + connector.getName(), + "--query-log-start", + "2023-12-22 00:00:00", + "--query-log-end", + "2023-12-22 02:00:00")); + + // Assert + List queries = + tasks.stream() + .filter(task -> task instanceof TeradataLogsJdbcTask) + .map( + task -> + ((TeradataLogsJdbcTask) task) + .getSql(unused -> true, new String[] {"SampleColumn"})) + .collect(toImmutableList()); + assertEquals( + ImmutableList.of( + "SELECT SampleColumn FROM dbc.DBQLogTbl L WHERE L.ErrorCode=0 AND" + + " L.StartTime >= CAST('2023-12-22T00:00:00Z' AS TIMESTAMP) AND" + + " L.StartTime < CAST('2023-12-22T01:00:00Z' AS TIMESTAMP) AND L.UserName <> 'DBC'", + "SELECT SampleColumn FROM dbc.DBQLogTbl L WHERE L.ErrorCode=0 AND" + + " L.StartTime >= CAST('2023-12-22T01:00:00Z' AS TIMESTAMP) AND" + + " L.StartTime < CAST('2023-12-22T02:00:00Z' AS TIMESTAMP) AND L.UserName <> 'DBC'"), + queries); + } + + @Test + public void addTasksTo_teradataAssessmentLogsJdbcTaskWithAssessmentFlag() throws Exception { + List> tasks = new ArrayList<>(); + + // Act + connector.addTasksTo( + tasks, + new ConnectorArguments( + "--connector", + connector.getName(), + "--query-log-start", + "2023-12-22 00:00:00", + "--query-log-end", + "2023-12-22 01:00:00", + "--assessment")); + + // Assert + List queries = + tasks.stream() + .filter(task -> task instanceof TeradataAssessmentLogsJdbcTask) + .map( + task -> + ((TeradataAssessmentLogsJdbcTask) task) + .getSql(unused -> true, new String[] {"ST.QueryID"})) + .collect(toImmutableList()); + assertEquals(1, queries.size()); + assertQueryEquals( + "SELECT ST.QueryID" + + " FROM dbc.QryLogV L LEFT OUTER JOIN dbc.DBQLSQLTbl ST ON (L.QueryID=ST.QueryID)" + + " WHERE L.ErrorCode=0" + + " AND L.StartTime >= CAST('2023-12-22T00:00:00Z' AS TIMESTAMP)" + + " AND L.StartTime < CAST('2023-12-22T01:00:00Z' AS TIMESTAMP)" + + " ORDER BY ST.QueryID, ST.SQLRowNo", + getOnlyElement(queries)); + } + @Test public void testConnectorSynthetic() throws Exception { Assume.assumeTrue(isDumperTest()); diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTaskTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTaskTest.java new file mode 100644 index 000000000..7a7c1f201 --- /dev/null +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTaskTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2022-2023 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.connector.teradata; + +import static com.google.edwmigration.dumper.application.dumper.test.DumperTestUtils.assertQueryEquals; + +import com.google.common.collect.ImmutableSet; +import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval; +import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import org.junit.Test; + +public class TeradataLogsJdbcTaskTest { + + private SharedState queryLogsState = new SharedState(); + + @Test + public void getSql_success() { + ZonedInterval interval = + new ZonedInterval( + ZonedDateTime.of(2023, 3, 4, 16, 0, 0, 0, ZoneId.systemDefault()), + ZonedDateTime.of(2023, 3, 4, 17, 0, 0, 0, ZoneId.systemDefault())); + TeradataLogsJdbcTask jdbcTask = + new TeradataLogsJdbcTask( + "result.csv", + queryLogsState, + "SampleQueryTable", + "SampleSqlTable", + /* conditions= */ ImmutableSet.of(), + interval); + + // Act + String query = jdbcTask.getSql(s -> true, new String[] {"L.QueryID", "ST.QueryID"}); + + // Assert + assertQueryEquals( + "SELECT L.QueryID, ST.QueryID" + + " FROM SampleQueryTable L LEFT OUTER JOIN SampleSqlTable ST ON (L.QueryID=ST.QueryID)" + + " WHERE L.ErrorCode=0 AND" + + " L.StartTime >= CAST('2023-03-04T16:00:00Z' AS TIMESTAMP) AND" + + " L.StartTime < CAST('2023-03-04T17:00:00Z' AS TIMESTAMP)", + query); + } + + @Test + public void getSql_withCondition() { + ZonedInterval interval = + new ZonedInterval( + ZonedDateTime.of(2023, 3, 4, 16, 0, 0, 0, ZoneId.systemDefault()), + ZonedDateTime.of(2023, 3, 4, 17, 0, 0, 0, ZoneId.systemDefault())); + TeradataLogsJdbcTask jdbcTask = + new TeradataLogsJdbcTask( + "result.csv", + queryLogsState, + "SampleQueryTable", + "SampleSqlTable", + /* conditions= */ ImmutableSet.of("L.UserName <> 'DBC'"), + interval); + + // Act + String query = jdbcTask.getSql(s -> true, new String[] {"L.QueryID", "ST.QueryID"}); + + // Assert + assertQueryEquals( + "SELECT L.QueryID, ST.QueryID" + + " FROM SampleQueryTable L LEFT OUTER JOIN SampleSqlTable ST ON (L.QueryID=ST.QueryID)" + + " WHERE L.ErrorCode=0 AND" + + " L.StartTime >= CAST('2023-03-04T16:00:00Z' AS TIMESTAMP) AND" + + " L.StartTime < CAST('2023-03-04T17:00:00Z' AS TIMESTAMP) AND L.UserName <> 'DBC'", + query); + } +}