Added check for maximum number of columns in a file

forcedotcom · Mar 25, 2015 · b6b7129 · b6b7129
1 parent e181ddc
commit b6b7129
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 4 deletions.
diff --git a/src/main/java/com/sforce/dataset/loader/EbinFormatWriter.java b/src/main/java/com/sforce/dataset/loader/EbinFormatWriter.java
@@ -100,6 +100,12 @@ public EbinFormatWriter(OutputStream out, FieldType[] dataTypes,PrintStream logg
 	{
 		this(out, logger);
 //		this.numColumns = dataTypes.length;
+
+		if(dataTypes.length>5000)
+		{
+			throw new IllegalArgumentException("Input file cannot contain more than 5000 columns. Found {"+dataTypes.length+"} columns");
+		}
+
 		for (FieldType dataType: dataTypes)
 		{
 			_dataTypes.add(dataType);

diff --git a/src/main/java/com/sforce/dataset/loader/file/schema/ext/DetectFieldTypes.java b/src/main/java/com/sforce/dataset/loader/file/schema/ext/DetectFieldTypes.java
@@ -66,16 +66,25 @@ public LinkedList<FieldType> detect(File inputCsv, ExternalFileSchema userSchema
 		{
 			reader = new CsvListReader(new InputStreamReader(new BOMInputStream(new FileInputStream(inputCsv), false), DatasetUtils.utf8Decoder(null , fileCharset)), CsvPreference.STANDARD_PREFERENCE);
 			header = reader.getHeader(true);
-
+			
 			if(reader!=null)
 			{
 				reader.close();
 				reader = null;
 			}
-			
+
 			List<String> nextLine = null;
 			types = new LinkedList<FieldType>();
 			boolean uniqueColumnFound = false;
+
+			if(header==null)
+				return types;
+
+			if(header.length>5000)
+			{
+				throw new IllegalArgumentException("Input file cannot contain more than 5000 columns. found {"+header.length+"} columns");
+			}
+
 			String devNames[] = ExternalFileSchema.createUniqueDevName(header);
 			boolean first = true;
 			for (int i=0; i< header.length; i++) 

diff --git a/src/main/java/com/sforce/dataset/loader/file/schema/ext/ExternalFileSchema.java b/src/main/java/com/sforce/dataset/loader/file/schema/ext/ExternalFileSchema.java
@@ -379,7 +379,7 @@ private static void validateSchema(ExternalFileSchema schema, PrintStream logger
 						}
 
 
-						if(user_fields!=null && !user_fields.isEmpty())
+						if(user_fields!=null && !user_fields.isEmpty() && user_fields.size()<=5000)
 						{
 							HashSet<String> fieldNames = new HashSet<String>();
 							HashSet<String> uniqueIdfieldNames = new HashSet<String>();
@@ -536,7 +536,13 @@ private static void validateSchema(ExternalFileSchema schema, PrintStream logger
 							}
 						}else
 						{
-							message.append("[objects["+objectCount+"].fields] in schema cannot be null or empty\n");
+							if(user_fields==null || user_fields.isEmpty())
+							{
+								message.append("[objects["+objectCount+"].fields] in schema cannot be null or empty\n");
+							}else if(user_fields.size()>5000)
+							{
+								message.append("[objects["+objectCount+"].fields] in schema cannot contain more than 5000 fields\n");
+							}
 						}
 				}
 			}else