Skip to content

Commit

Permalink
enabled google and excel sources
Browse files Browse the repository at this point in the history
  • Loading branch information
@datomo
datomo committed Sep 27, 2023
1 parent 7bbae4f commit 6968e7d
Show file tree
Hide file tree
Showing 20 changed files with 367 additions and 495 deletions.
6 changes: 3 additions & 3 deletions plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.polypheny.db.algebra.type.AlgDataTypeSystem;
import org.polypheny.db.catalog.entity.physical.PhysicalColumn;
import org.polypheny.db.catalog.entity.physical.PhysicalTable;
import org.polypheny.db.catalog.exceptions.GenericRuntimeException;
import org.polypheny.db.plan.Convention;
import org.polypheny.db.schema.Namespace.Schema;
import org.polypheny.db.schema.impl.AbstractNamespace;
Expand Down Expand Up @@ -101,12 +102,11 @@ public PhysicalTable createCsvTable( long id, PhysicalTable table, CsvSource csv
try {
source = Sources.of( new URL( directoryUrl, csvFileName ) );
} catch ( MalformedURLException e ) {
throw new RuntimeException( e );
throw new GenericRuntimeException( e );
}
int[] fields = fieldIds.stream().mapToInt( i -> i ).toArray();
CsvTable csvTable = createTable( id, source, table, fieldTypes, fields, csvSource );

return csvTable;
return createTable( id, source, table, fieldTypes, fields, csvSource );
}


Expand Down
3 changes: 2 additions & 1 deletion plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.polypheny.db.catalog.entity.allocation.AllocationTableWrapper;
import org.polypheny.db.catalog.entity.logical.LogicalTableWrapper;
import org.polypheny.db.catalog.entity.physical.PhysicalTable;
import org.polypheny.db.catalog.exceptions.GenericRuntimeException;
import org.polypheny.db.information.InformationGroup;
import org.polypheny.db.information.InformationTable;
import org.polypheny.db.prepare.Context;
Expand Down Expand Up @@ -262,7 +263,7 @@ public Map<String, List<ExportedColumn>> getExportedColumns() {
position++;
}
} catch ( IOException e ) {
throw new RuntimeException( e );
throw new GenericRuntimeException( e );
}

exportedColumnCache.put( physicalTableName, list );
Expand Down
147 changes: 59 additions & 88 deletions plugins/excel-adapter/src/main/java/org/polypheny/db/adapter/excel/ExcelEnumerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.calcite.avatica.util.DateTimeUtils;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellType;
Expand All @@ -39,16 +40,28 @@
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.polypheny.db.adapter.java.JavaTypeFactory;
import org.polypheny.db.algebra.type.AlgDataType;
import org.polypheny.db.catalog.exceptions.GenericRuntimeException;
import org.polypheny.db.type.PolyType;
import org.polypheny.db.type.entity.PolyBoolean;
import org.polypheny.db.type.entity.PolyDate;
import org.polypheny.db.type.entity.PolyDouble;
import org.polypheny.db.type.entity.PolyFloat;
import org.polypheny.db.type.entity.PolyInteger;
import org.polypheny.db.type.entity.PolyLong;
import org.polypheny.db.type.entity.PolyNull;
import org.polypheny.db.type.entity.PolyString;
import org.polypheny.db.type.entity.PolyTime;
import org.polypheny.db.type.entity.PolyTimeStamp;
import org.polypheny.db.type.entity.PolyValue;
import org.polypheny.db.util.Pair;
import org.polypheny.db.util.Source;

class ExcelEnumerator<E> implements Enumerator<E> {
class ExcelEnumerator implements Enumerator<PolyValue[]> {

Iterator<Row> reader;
private final AtomicBoolean cancelFlag;
private final RowConverter<E> rowConverter;
private E current;
private final RowConverter rowConverter;
private PolyValue[] current;

private static final FastDateFormat TIME_FORMAT_DATE;
private static final FastDateFormat TIME_FORMAT_TIME;
Expand All @@ -75,12 +88,11 @@ class ExcelEnumerator<E> implements Enumerator<E> {


ExcelEnumerator( Source source, AtomicBoolean cancelFlag, List<ExcelFieldType> fieldTypes, int[] fields, String sheet ) {
//noinspection unchecked
this( source, cancelFlag, false, null, (RowConverter<E>) converter( fieldTypes, fields ), sheet );
this( source, cancelFlag, false, null, converter( fieldTypes, fields ), sheet );
}


ExcelEnumerator( Source source, AtomicBoolean cancelFlag, boolean stream, String[] filterValues, RowConverter<E> rowConverter, String sheet ) {
ExcelEnumerator( Source source, AtomicBoolean cancelFlag, boolean stream, String[] filterValues, RowConverter rowConverter, String sheet ) {
this.cancelFlag = cancelFlag;
this.rowConverter = rowConverter;
try {
Expand All @@ -96,13 +108,8 @@ class ExcelEnumerator<E> implements Enumerator<E> {
}


private static RowConverter<?> converter( List<ExcelFieldType> fieldTypes, int[] fields ) {
if ( fields.length == 1 ) {
final int field = fields[0];
return new SingleColumnRowConverter( fieldTypes.get( field ), field );
} else {
return new ArrayRowConverter( fieldTypes, fields );
}
private static RowConverter converter( List<ExcelFieldType> fieldTypes, int[] fields ) {
return new ArrayRowConverter( fieldTypes, fields );
}


Expand Down Expand Up @@ -235,7 +242,7 @@ public static Iterator<Row> openExcel( Source source, String sheetname ) throws

Workbook workbook = WorkbookFactory.create( fileIn );
workbook.getNumberOfSheets();
if ( sheetname.equals( "" ) ) {
if ( sheetname.isEmpty() ) {
sheet = workbook.getSheetAt( 0 );
} else {
sheet = workbook.getSheet( sheetname );
Expand All @@ -252,7 +259,7 @@ public static void setSheet( String sheetName ) {


@Override
public E current() {
public PolyValue[] current() {
return current;
}

Expand Down Expand Up @@ -328,116 +335,83 @@ static int[] identityList( int n ) {

/**
* Row converter.
*
* @param <E> element type
*/
abstract static class RowConverter<E> {
abstract static class RowConverter {

abstract E convertRow( Row rows );
abstract PolyValue[] convertRow( Row rows );


protected Object convert( ExcelFieldType fieldType, Cell cell ) {
protected PolyValue convert( ExcelFieldType fieldType, Cell cell ) {
if ( fieldType == null ) {
return cell;
throw new NotImplementedException();
//return cell;
}
if ( cell == null ) {
return PolyNull.NULL;
}
try {

switch ( fieldType ) {
case BOOLEAN:
if ( cell == null ) {
return null;
}
return cell.getBooleanCellValue();
return PolyBoolean.of( cell.getBooleanCellValue() );
case BYTE:
if ( cell == null ) {
return null;
}
return Byte.parseByte( cell.getStringCellValue() );
return PolyInteger.of( Byte.parseByte( cell.getStringCellValue() ) );
case SHORT:
if ( cell == null ) {
return null;
}
return Short.parseShort( cell.getStringCellValue() );
return PolyInteger.of( Short.parseShort( cell.getStringCellValue() ) );
case INT:
if ( cell == null ) {
return null;
}
return (Double.valueOf( cell.getNumericCellValue() ).intValue());
return PolyInteger.of( cell.getNumericCellValue() );
case LONG:
if ( cell == null ) {
return null;
}

if ( cell.getCellType() == CellType.STRING ) {
return Long.parseLong( cell.getStringCellValue() );
} else if ( cell.getCellType() == CellType.NUMERIC ) {
return Long.toString( (long) cell.getNumericCellValue() );
return PolyLong.of( Long.parseLong( cell.getStringCellValue() ) );
}
return Long.parseLong( String.valueOf( cell.getNumericCellValue() ) );
return PolyLong.of( cell.getNumericCellValue() );
case FLOAT:
if ( cell == null ) {
return null;
}
if ( cell.getCellType() == CellType.STRING ) {
return Float.parseFloat( cell.getStringCellValue() );
} else if ( cell.getCellType() == CellType.NUMERIC ) {
return Float.parseFloat( String.valueOf( cell.getNumericCellValue() ) );
return PolyFloat.of( Float.parseFloat( cell.getStringCellValue() ) );
}
return Float.parseFloat( String.valueOf( cell.getNumericCellValue() ) );
return PolyFloat.of( cell.getNumericCellValue() );
case DOUBLE:
if ( cell == null ) {
return null;
}
return cell.getNumericCellValue();
return PolyDouble.of( cell.getNumericCellValue() );
case DATE:
if ( cell == null ) {
return null;
}
try {
//convert date from string to date
if ( cell.getCellType() == CellType.STRING ) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern( "dd.MM.yyyy", Locale.ENGLISH );
LocalDate date2 = LocalDate.parse( cell.getStringCellValue(), formatter );
return (int) (TimeUnit.DAYS.toMillis( date2.toEpochDay() ) / DateTimeUtils.MILLIS_PER_DAY);
} else {
Date date = cell.getDateCellValue();
return (int) (date.getTime() / DateTimeUtils.MILLIS_PER_DAY);
return PolyDate.of( (TimeUnit.DAYS.toMillis( date2.toEpochDay() ) / DateTimeUtils.MILLIS_PER_DAY) );
}
Date date = cell.getDateCellValue();
return PolyDate.of( (int) (date.getTime() / DateTimeUtils.MILLIS_PER_DAY) );


} catch ( Exception e ) {
return null;
throw new GenericRuntimeException( "Could not read the date field from the document." );
}
case TIME:
if ( cell == null ) {
return null;
}

try {
Date date = TIME_FORMAT_TIME.parse( cell
.getStringCellValue() );
return (int) date.getTime();
return PolyTime.of( date.getTime() );


} catch ( Exception e ) {
return null;
throw new GenericRuntimeException( "Could not read the time field from the document." );
}
case TIMESTAMP:
if ( cell == null ) {
return null;
}
try {
Date date = TIME_FORMAT_TIMESTAMP.parse( cell
.getStringCellValue() );
return date.getTime();
return PolyTimeStamp.of( date.getTime() );
} catch ( Exception e ) {
return null;
throw new GenericRuntimeException( "Could not read the timestamp field from the document." );
}
case STRING:
default:
return cell.getStringCellValue();
return PolyString.of( cell.getStringCellValue() );
}
} catch ( Exception e ) {
return cell.getStringCellValue();
throw new GenericRuntimeException( "Could not read %s from the document.", cell );
}
}

Expand All @@ -447,7 +421,7 @@ protected Object convert( ExcelFieldType fieldType, Cell cell ) {
/**
* Array row converter.
*/
static class ArrayRowConverter extends RowConverter<Object[]> {
static class ArrayRowConverter extends RowConverter {

private final ExcelFieldType[] fieldTypes;
private final int[] fields;
Expand All @@ -470,7 +444,7 @@ static class ArrayRowConverter extends RowConverter<Object[]> {


@Override
public Object[] convertRow( Row row ) {
public PolyValue[] convertRow( Row row ) {
if ( stream ) {
return convertStreamRow( row );
} else {
Expand All @@ -479,9 +453,9 @@ public Object[] convertRow( Row row ) {
}


public Object[] convertNormalRow( Row row ) {
public PolyValue[] convertNormalRow( Row row ) {
Iterator<Cell> cells = row.cellIterator();
final Object[] objects = new Object[fields.length];
final PolyValue[] objects = new PolyValue[fields.length];
while ( cells.hasNext() ) {
Cell cell = cells.next();
int field = fields[cell.getColumnIndex()] - 1;
Expand All @@ -491,12 +465,9 @@ public Object[] convertNormalRow( Row row ) {
}


public Object[] convertStreamRow( Row row ) {
final Object[] objects = new Object[fields.length + 1];
objects[0] = System.currentTimeMillis();
for ( int i = 0; i < fields.length; i++ ) {
int field = fields[i];
}
public PolyValue[] convertStreamRow( Row row ) {
final PolyValue[] objects = new PolyValue[fields.length + 1];
objects[0] = PolyLong.of( System.currentTimeMillis() );
return objects;
}

Expand All @@ -519,8 +490,8 @@ private SingleColumnRowConverter( ExcelFieldType fieldType, int fieldIndex ) {


@Override
public Object convertRow( Row row ) {
return convert( fieldType, row.getCell( fieldIndex ) );
public PolyValue[] convertRow( Row row ) {
return new PolyValue[]{ convert( fieldType, row.getCell( fieldIndex ) ) };
}

}
Expand Down
Loading

0 comments on commit 6968e7d

Please sign in to comment.