Skip to content

Commit

Permalink
cache database column types to improve serialization speed
Browse files Browse the repository at this point in the history
  • Loading branch information
AFine-gs committed Oct 17, 2023
1 parent c3d7cfa commit d16b95e
Show file tree
Hide file tree
Showing 3 changed files with 129 additions and 14 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -113,6 +114,7 @@ public class RelationalResult extends StreamingResult implements IRelationalResu
public MutableList<SetImplTransformers> setTransformers = Lists.mutable.empty();

public Builder builder;
private Calendar calendar;

public RelationalResult(MutableList<ExecutionActivity> activities, RelationalExecutionNode node, List<SQLResultColumn> sqlResultColumns, String databaseType, String databaseTimeZone, Connection connection, MutableList<CommonProfile> profiles, List<String> temporaryTables, Span topSpan)
{
Expand Down Expand Up @@ -444,16 +446,7 @@ public Object getValue(int columnIndex) throws SQLException
if (resultDBColumnsMetaData.isTimestampColumn(columnIndex))
{
Timestamp ts;
if (getRelationalDatabaseTimeZone() != null)
{
ts = resultSet.getTimestamp(columnIndex, new GregorianCalendar(TimeZone.getTimeZone(getRelationalDatabaseTimeZone())));
}
else
{
//TODO, throw exception, TZ should always be specified
//Till then, default to PURE default which is "GMT"
ts = resultSet.getTimestamp(columnIndex, new GregorianCalendar(TimeZone.getTimeZone("GMT")));
}
ts = resultSet.getTimestamp(columnIndex, getCalendar());
result = ts;
}
else if (resultDBColumnsMetaData.isDateColumn(columnIndex))
Expand Down Expand Up @@ -651,6 +644,25 @@ public boolean tryAdvance(Consumer<? super ObjectNode> action)
return StreamSupport.stream(spliterator, false).onClose(this::close);
}


private Calendar getCalendar()
{
String timeZoneId = getRelationalDatabaseTimeZone();
TimeZone timeZone = (timeZoneId != null) ? TimeZone.getTimeZone(timeZoneId) : TimeZone.getTimeZone("GMT");
if (calendar == null)
{
//TODO, throw exception, TZ should always be specified
//Till then, default to PURE default which is "GMT"
calendar = new GregorianCalendar(timeZone);
}
else
{
calendar.clear();
calendar.setTimeZone(timeZone);
}
return calendar;
}

@Override
public void cancel()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,25 +26,45 @@ public class SQLResultDBColumnsMetaData
{
private final List<SQLResultColumn> sqlResultColumns;
private final List<Integer> dbMetaDataType;
private final boolean[] timeStampColumns;
private final boolean[] dateColumns;

SQLResultDBColumnsMetaData(List<SQLResultColumn> resultColumns, ResultSetMetaData rsMetaData) throws SQLException
{
int size = resultColumns.size();
this.sqlResultColumns = resultColumns;
this.dbMetaDataType = Lists.multiReader.ofInitialCapacity(resultColumns.size());
for (int i = 1; i <= resultColumns.size(); i++)
this.dbMetaDataType = Lists.multiReader.ofInitialCapacity(size);
this.timeStampColumns = new boolean[size];
this.dateColumns = new boolean[size];


for (int i = 1; i <= size; i++)
{

this.dbMetaDataType.add(rsMetaData.getColumnType(i));
if (columnIsOfType(i, Types.TIMESTAMP, "TIMESTAMP"))
{
timeStampColumns[i - 1] = true;

}
else if (columnIsOfType(i, Types.DATE, "DATE"))
{
dateColumns[i - 1] = true;

}

}
}

boolean isTimestampColumn(int index)
{
return columnIsOfType(index, Types.TIMESTAMP, "TIMESTAMP");
return timeStampColumns[index - 1];
}


boolean isDateColumn(int index)
{
return columnIsOfType(index, Types.DATE, "DATE");
return dateColumns[index - 1];
}

private boolean columnIsOfType(int index, int dbColumnType, String alloyColumnType)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright 2023 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.finos.legend.engine.plan.execution.stores.relational.result;

import org.eclipse.collections.api.factory.Lists;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.result.SQLResultColumn;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Types;
import java.util.List;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;

public class TestSqlResultDBColumnMetaData
{

private SQLResultDBColumnsMetaData metaData;
private ResultSetMetaData resultSetMetaData;


@Before
public void setUp() throws SQLException
{
resultSetMetaData = Mockito.mock(ResultSetMetaData.class);
try
{
when(resultSetMetaData.getColumnType(1)).thenReturn(Types.TIMESTAMP);
when(resultSetMetaData.getColumnType(2)).thenReturn(Types.DATE);
when(resultSetMetaData.getColumnType(3)).thenReturn(Types.VARCHAR);

}
catch (SQLException e)
{
e.printStackTrace();
}

List<SQLResultColumn> resultColumns = Lists.mutable.of(
new SQLResultColumn("Column1", "TIMESTAMP"),
new SQLResultColumn("Column2", "DATE"),
new SQLResultColumn("Column3", "STRING")

);

metaData = new SQLResultDBColumnsMetaData(resultColumns, resultSetMetaData);
}

@Test
public void testIsTimestampColumn()
{
assertTrue(metaData.isTimestampColumn(1));
assertFalse(metaData.isTimestampColumn(2));
assertFalse(metaData.isTimestampColumn(3));

}


@Test
public void testIsDateColumn()
{
assertFalse(metaData.isDateColumn(1));
assertTrue(metaData.isDateColumn(2));
assertFalse(metaData.isDateColumn(3));

}

}

0 comments on commit d16b95e

Please sign in to comment.