Skip to content

Commit

Permalink
Critical bug fixes
Browse files Browse the repository at this point in the history 
There is data corruption issues in the parser. Fixed that along with
handling null values
  • Loading branch information
@datasetutil
datasetutil committed Mar 6, 2015
1 parent 5c6720e commit 5b90a8b
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 34 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<description>Analytics Cloud Datatset Utils</description>
<artifactId>datasetutils</artifactId>
<packaging>jar</packaging>
<version>32.0.18-SNAPSHOT</version>
<version>32.0.19-SNAPSHOT</version>
<url>https://github.com/forcedotcom/Analytics-Cloud-Dataset-Utils</url>
<organization>
<name>salesforce.com</name>
Expand Down
16 changes: 13 additions & 3 deletions src/main/java/com/sforce/dataset/loader/EbinFormatWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,12 @@ public void addrow(String[] values) throws IOException,NumberFormatException, P

while(key_value_count<_dataTypes.size())
{
// System.err.println("Count: " + count + " ; key_value_count: " + key_value_count );
// System.err.println(_dataTypes.get(key_value_count).getName() + ":" + _dataTypes.get(key_value_count).getType() + " = " + values[count]);
// System.err.println();

if(_dataTypes.get(key_value_count).isComputedField)

if(_dataTypes.get(key_value_count).isSkipped)
{
key_value_count++;
Expand Down Expand Up @@ -268,7 +274,7 @@ public void addrow(String[] values) throws IOException,NumberFormatException, P
{
//The date is null we don't add null dims
// dim_values.add("");
dim_keys.add(_dataTypes.get(key_value_count).getName());
// dim_keys.add(_dataTypes.get(key_value_count).getName());
curr.put(_dataTypes.get(key_value_count).getName(), null);
key_value_count++;

Expand Down Expand Up @@ -392,6 +398,7 @@ public void addrow(String[] values) throws IOException,NumberFormatException, P
long sec_epoch = dt.getTime()/(1000);
long day_epoch = dt.getTime()/(1000*60*60*24);


measure_values.add(sec_epoch);
curr.put(_dataTypes.get(key_value_count).getName(), sec_epoch);
key_value_count++;
Expand All @@ -414,6 +421,7 @@ public void addrow(String[] values) throws IOException,NumberFormatException, P
dim_values.add(Integer.toString(year));
dim_keys.add(_dataTypes.get(key_value_count).getName());
curr.put(_dataTypes.get(key_value_count).getName(), year);
// System.out.println(_dataTypes.get(key_value_count).getName() + ": "+columnValue + " - "+ year);
key_value_count++;

dim_values.add(Integer.toString(quarter));
Expand Down Expand Up @@ -509,9 +517,9 @@ public void addrow(String[] values) throws IOException,NumberFormatException, P
dim_values.add(columnValue.toString());
dim_keys.add(_dataTypes.get(key_value_count).getName());
}
curr.put(_dataTypes.get(key_value_count).getName(), columnValue.toString());
key_value_count++;
}
curr.put(_dataTypes.get(key_value_count).getName(), columnValue);
key_value_count++;
}
count++;
}
Expand Down Expand Up @@ -566,6 +574,8 @@ protected void vInt(long i) throws IOException

protected void dict(LinkedList<String> dim_keys, LinkedList<String> dim_values) throws IOException
{
// System.out.println(dim_keys);
// System.out.println(dim_values);
int dlen = dim_values.size();
vInt(dlen);
for (int i = 0; i < dlen; i++) {
Expand Down
117 changes: 101 additions & 16 deletions src/main/java/com/sforce/dataset/loader/file/schema/FieldType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,24 +26,109 @@
package com.sforce.dataset.loader.file.schema;



public class FieldType {
public String name = null; //Required
public String fullyQualifiedName = null; //Required
public String label = null; //Required
public String description = null; //Optional
public String type = null; //Required - Text, Numeric, Date
public int precision = 0; //Required if type is Numeric, the number 256.99 has a precision of 5
public int scale = 0; //Required if type is Numeric, the number 256.99 has a scale of 2
public String decimalSeparator = ".";
public String defaultValue = null; //required for numeric types
public String format = null; //Required if type is Date
protected String name = null; //Required
protected String fullyQualifiedName = null; //Required
protected String label = null; //Required
protected String description = null; //Optional
protected String type = null; //Required - Text, Numeric, Date
protected int precision = 0; //Required if type is Numeric, the number 256.99 has a precision of 5
protected int scale = 0; //Required if type is Numeric, the number 256.99 has a scale of 2
protected String decimalSeparator = ".";
protected String defaultValue = null; //required for numeric types
protected String format = null; //Required if type is Date
public boolean isSystemField = false; //Optional
public boolean isUniqueId = false; //Optional
public boolean isMultiValue = false; //Optional
public String multiValueSeparator = null; //Optional - only used if IsMultiValue = true separator
public int fiscalMonthOffset = 0;
public int firstDayOfWeek = -1; //1=SUNDAY, 2=MONDAY etc.. -1 the week starts on 1st day of year and is always 7 days long
public boolean isYearEndFiscalYear = true; //Optional
public boolean canTruncateValue = true; //Optional
public boolean isSkipped = false; //Optional
protected String multiValueSeparator = null; //Optional - only used if IsMultiValue = true separator
protected int fiscalMonthOffset = 0; //The month in which the fiscal quarter starts
protected int firstDayOfWeek = -1; //1=SUNDAY, 2=MONDAY etc.. -1 the week starts on 1st day of year and is always 7 days long
public boolean isYearEndFiscalYear = true; //Optional
public boolean canTruncateValue = true; //Optional
public boolean isSkipped = false; //Optional

public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getFullyQualifiedName() {
return fullyQualifiedName;
}
public void setFullyQualifiedName(String fullyQualifiedName) {
this.fullyQualifiedName = fullyQualifiedName;
}
public String getLabel() {
return label;
}
public void setLabel(String label) {
this.label = label;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public int getPrecision() {
return precision;
}
public void setPrecision(int precision) {
this.precision = precision;
}
public int getScale() {
return scale;
}
public void setScale(int scale) {
this.scale = scale;
}
public String getDecimalSeparator() {
return decimalSeparator;
}
public void setDecimalSeparator(String decimalSeparator) {
this.decimalSeparator = decimalSeparator;
}
public String getDefaultValue() {
return defaultValue;
}
public void setDefaultValue(String defaultValue) {
if(defaultValue!=null && !defaultValue.isEmpty())
{
this.defaultValue = defaultValue;
}
}
public String getFormat() {
return format;
}
public void setFormat(String format) {
this.format = format;
}
public String getMultiValueSeparator() {
return multiValueSeparator;
}
public void setMultiValueSeparator(String multiValueSeparator) {
this.multiValueSeparator = multiValueSeparator;
}
public int getFiscalMonthOffset() {
return fiscalMonthOffset;
}
public void setFiscalMonthOffset(int fiscalMonthOffset) {
this.fiscalMonthOffset = fiscalMonthOffset;
}
public int getFirstDayOfWeek() {
return firstDayOfWeek;
}
public void setFirstDayOfWeek(int firstDayOfWeek) {
this.firstDayOfWeek = firstDayOfWeek;
}


}
23 changes: 13 additions & 10 deletions src/main/java/com/sforce/dataset/loader/file/schema/ext/FieldType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,22 +350,25 @@ public void setMultiValueSeparator(String multiValueSeparator) {
}

public String getDefaultValue() {
return defaultValue;
if(defaultValue!=null && !defaultValue.isEmpty())
return defaultValue;
return null;
}

public void setDefaultValue(String defaultValue) {
if(this.type != null && this.type.equals(FieldType.DATE_TYPE) && getCompiledDateFormat() != null)
public void setDefaultValue(String defaultValue)
{
if(defaultValue!=null && !defaultValue.isEmpty())
{
if(defaultValue!=null && !defaultValue.isEmpty())
if(this.type != null && this.type.equals(FieldType.DATE_TYPE) && getCompiledDateFormat() != null)
{
try {
this.defaultDate = getCompiledDateFormat().parse(defaultValue);
} catch (ParseException e) {
throw new IllegalArgumentException(e.toString());
}
try {
this.defaultDate = getCompiledDateFormat().parse(defaultValue);
} catch (ParseException e) {
throw new IllegalArgumentException(e.toString());
}
}
this.defaultValue = defaultValue;
}
this.defaultValue = defaultValue;
}

public String getFullyQualifiedName() {
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/com/sforce/dataset/server/CsvUploadWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,12 +216,13 @@ public static void moveInputFile(File inputFile, boolean isSuccess, Session sess
return;

// File parent = inputFile.getAbsoluteFile().getParentFile();

File directory = DatasetUtilConstants.getSuccessDir(session.getOrgId());
if(!isSuccess)
directory = DatasetUtilConstants.getErrorDir(session.getOrgId());

File doneFile = new File(directory, FilenameUtils.getBaseName(inputFile.getName())+"_"+session.getId()+"."+FilenameUtils.getExtension(inputFile.getName()));
// File doneFile = new File(directory, FilenameUtils.getBaseName(inputFile.getName())+"_"+session.getId()+"."+FilenameUtils.getExtension(inputFile.getName()));
File doneFile = new File(directory, inputFile.getName());
try {
FileUtils.moveFile(inputFile, doneFile);
} catch (IOException e) {
Expand All @@ -230,7 +231,8 @@ public static void moveInputFile(File inputFile, boolean isSuccess, Session sess
File sortedtFile = new File(inputFile.getParent(), FilenameUtils.getBaseName(inputFile.getName())+ "_sorted." + FilenameUtils.getExtension(inputFile.getName()));
if(sortedtFile.exists())
{
File sortedDoneFile = new File(directory,FilenameUtils.getBaseName(sortedtFile.getName())+"_"+session.getId()+"."+FilenameUtils.getExtension(sortedtFile.getName()));
// File sortedDoneFile = new File(directory,FilenameUtils.getBaseName(sortedtFile.getName())+"_"+session.getId()+"."+FilenameUtils.getExtension(sortedtFile.getName()));
File sortedDoneFile = new File(directory,sortedtFile.getName());
try {
FileUtils.moveFile(sortedtFile, sortedDoneFile);
} catch (IOException e) {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/sforce/dataset/server/MultipartRequestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ public static List<FileUploadRequest> uploadByApacheFileUpload(List<FileItem> it
fm.setInputCsv(inputCsv);
fm.setInputJson(inputJson);
// File outFile = new File(parent,fm.getInputFileName());
File outFile = new File(parent,session.getId()+".csv");
File outFile = new File(parent,datasetName+"_"+session.getId()+".csv");
if(fm.getInputFileName().equalsIgnoreCase(inputJson))
{
ExternalFileSchema schema = ExternalFileSchema.load(fm.inputFileStream, Charset.forName("UTF-8"), System.out);
Expand Down

0 comments on commit 5b90a8b

Please sign in to comment.