Merge branch 'develop'

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 3.5.0 - 8 August 2022
+
+- Enhancements
+	- \#291 Speed improvement for getAllSheetFormulas()
+	- \#293 Add includeHiddenRows option to read()
+	- \#296 Add readLargeFile()
+
 ## 3.4.4 - 25 March 2022
 
 - \#289 Prevent one-off OSGi bundle errors when the bundle version changes

ModuleConfig.cfc

@@ -4,7 +4,7 @@ component{
     this.author = "Julian Halliwell";
     this.webURL = "https://github.com/cfsimplicity/spreadsheet-cfml";
     this.description = "CFML Spreadsheet Library";
-    this.version = "3.4.4";
+    this.version = "3.5.0";
     this.autoMapModels = false;
 
     function configure(){

README.md

@@ -140,6 +140,7 @@ You may wish to place the spreadsheet library files in a central location with a
 * [isRowHidden](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/isRowHidden)
 * [isStreamingXmlFormat](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/isStreamingXmlFormat)
 * [isXmlFormat](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/isXmlFormat)
+* [readLargeFile](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/readLargeFile)
 * [removePrintGridlines](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/removePrintGridlines)
 * [renameSheet](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/renameSheet)
 * [removeSheetNumber](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/removeSheetNumber)

Spreadsheet.cfc

@@ -1,8 +1,8 @@
 component accessors="true"{
 
 	//"static"
-	property name="version" default="3.4.4" setter="false";
-	property name="osgiLibBundleVersion" default="5.2.2.0" setter="false"; //first 3 octets = POI version; increment 4th with other jar updates
+	property name="version" default="3.5.0" setter="false";
+	property name="osgiLibBundleVersion" default="5.2.2.1" setter="false"; //first 3 octets = POI version; increment 4th with other jar updates
 	property name="osgiLibBundleSymbolicName" default="spreadsheet-cfml" setter="false";
 	property name="exceptionType" default="cfsimplicity.spreadsheet" setter="false";
 	//commonly invoked POI class names
@@ -39,6 +39,7 @@ component accessors="true"{
 	property name="rangeHelper";
 	property name="rowHelper";
 	property name="sheetHelper";
+	property name="streamingReaderHelper";
 	property name="stringHelper";
 	property name="workbookHelper";
 
@@ -80,12 +81,13 @@ component accessors="true"{
 		setRangeHelper( New helpers.range( this ) );
 		setRowHelper( New helpers.row( this ) );
 		setSheetHelper( New helpers.sheet( this ) );
+		setStreamingReaderHelper( New helpers.streamingReader( this ) );
 		setStringHelper( New helpers.string( this ) );
 		setWorkbookHelper( New helpers.workbook( this ) );
 	}
 
 	/* Meta utilities */
-	
+
 	private void function detectEngineProperties(){
 		this.setIsACF( ( server.coldfusion.productname == "ColdFusion Server" ) );
 	}
@@ -196,7 +198,7 @@ component accessors="true"{
 			Throw( type=this.getExceptionType(), message="Invalid argument 'queryColumnTypes'.", detail="When specifying 'queryColumnTypes' as a struct you must also set the 'firstRowIsHeader' argument to true OR provide 'queryColumnNames'" );
 		if( arguments.trim )
 			csvString = csvString.Trim();
-		var format = arguments.KeyExists( "delimiter" )? 
+		var format = arguments.KeyExists( "delimiter" )?
 			getCsvHelper().getCsvFormatForDelimiter( arguments.delimiter )
 			: getClassHelper().loadClass( "org.apache.commons.csv.CSVFormat" )[ JavaCast( "string", "RFC4180" ) ].withIgnoreSurroundingSpaces();
 		var parsed = getClassHelper().loadClass( "org.apache.commons.csv.CSVParser" ).parse( csvString, format );
@@ -385,7 +387,7 @@ component accessors="true"{
 	public Spreadsheet function addAutofilter( required workbook, string cellRange="", numeric row=1 ){
 		arguments.cellRange = arguments.cellRange.Trim();
 		if( arguments.cellRange.IsEmpty() ){
-			//default to all columns in the first (default) or specified row 
+			//default to all columns in the first (default) or specified row
 			var rowIndex = ( Max( 0, arguments.row -1 ) );
 			var indices = {
 				startRow: rowIndex
@@ -692,7 +694,7 @@ component accessors="true"{
 	public Spreadsheet function cleanUpStreamingXml( required workbook ){
 		// SXSSF uses temporary files which MUST be cleaned up, see http://poi.apache.org/components/spreadsheet/how-to.html#sxssf
 		if( isStreamingXmlFormat( arguments.workbook ) )
-			arguments.workbook.dispose(); 
+			arguments.workbook.dispose();
 		return this;
 	}
 
@@ -784,7 +786,7 @@ component accessors="true"{
 			Throw( type=this.getExceptionType(), message="Invalid row value", detail="The value for row must be greater than or equal to 1." );
 		var sheet = getSheetHelper().getActiveSheet( arguments.workbook );
 		var rowIndex = ( arguments.row -1 );
-		if( 
+		if(
 				( rowIndex < getSheetHelper().getFirstRowIndex( sheet ) )
 				||
 				( rowIndex > getSheetHelper().getLastRowIndex( sheet ) )
@@ -1064,9 +1066,8 @@ component accessors="true"{
 	public numeric function getColumnCount( required workbook, sheetNameOrNumber ){
 		if( arguments.KeyExists( "sheetNameOrNumber" ) )
 			getSheetHelper().setActiveSheetNameOrNumber( argumentCollection=arguments );
-		var sheet = getSheetHelper().getActiveSheet( arguments.workbook );
-		var rowIterator = sheet.rowIterator();
 		var result = 0;
+		var rowIterator = getSheetHelper().getActiveSheet( arguments.workbook ).rowIterator();
 		while( rowIterator.hasNext() ){
 			var row = rowIterator.next();
 			result = Max( result, row.getLastCellNum() );
@@ -1159,7 +1160,7 @@ component accessors="true"{
 	}
 
 	public boolean function isRowHidden( required workbook, required numeric row ){
-		return getRowHelper().getRowFromActiveSheet( arguments.workbook, arguments.row ).getZeroHeight();
+		return getRowHelper().isRowHidden( arguments.workbook, arguments.row );
 	}
 
 	public boolean function isSpreadsheetFile( required string path ){
@@ -1256,7 +1257,7 @@ component accessors="true"{
 		return new( sheetName=arguments.sheetName, xmlFormat=true );
 	}
 
-	public string function queryToCsv( required query query, boolean includeHeaderRow=false, string delimiter="," ){		
+	public string function queryToCsv( required query query, boolean includeHeaderRow=false, string delimiter="," ){
 		var data = [];
 		var columns = getQueryHelper()._QueryColumnArray( arguments.query );
 		if( arguments.includeHeaderRow )
@@ -1274,7 +1275,7 @@ component accessors="true"{
 			data.Append( rowValues );
 		}
 		var builder = getStringHelper().newJavaStringBuilder();
-		var csvFormat =  getCsvHelper().delimiterIsTab( arguments.delimiter )?
+		var csvFormat = getCsvHelper().delimiterIsTab( arguments.delimiter )?
 			getClassHelper().loadClass( "org.apache.commons.csv.CSVFormat" )[ JavaCast( "string", "TDF" ) ]
 			: getClassHelper().loadClass( "org.apache.commons.csv.CSVFormat" )[ JavaCast( "string", "EXCEL" ) ]
 				.withDelimiter( JavaCast( "char", arguments.delimiter ) );
@@ -1297,6 +1298,7 @@ component accessors="true"{
 		,boolean includeBlankRows=false
 		,boolean fillMergedCellsWithVisibleValue=false
 		,boolean includeHiddenColumns=true
+		,boolean includeHiddenRows=true
 		,boolean includeRichTextFormatting=false
 		,string password
 		,string csvDelimiter=","
@@ -1305,10 +1307,8 @@ component accessors="true"{
 	){
 		if( arguments.KeyExists( "query" ) )
 			Throw( type=this.getExceptionType(), message="Invalid argument 'query'.", detail="Just use format='query' to return a query object" );
-		if( arguments.KeyExists( "format" ) && !ListFindNoCase( "query,html,csv", arguments.format ) )
-			Throw( type=this.getExceptionType(), message="Invalid format", detail="Supported formats are: 'query', 'html' and 'csv'" );
-		if( arguments.KeyExists( "sheetName" ) && arguments.KeyExists( "sheetNumber" ) )
-			Throw( type=this.getExceptionType(), message="Cannot provide both sheetNumber and sheetName arguments", detail="Only one of either 'sheetNumber' or 'sheetName' arguments may be provided." );
+		getExceptionHelper().throwExceptionIFreadFormatIsInvalid( argumentCollection=arguments );
+		getSheetHelper().throwErrorIFSheetNameAndNumberArgumentsBothPassed( argumentCollection=arguments );
 		getFileHelper().throwErrorIFfileNotExists( arguments.src );
 		var passwordProtected = ( arguments.KeyExists( "password") && !arguments.password.Trim().IsEmpty() );
 		var workbook = passwordProtected? getWorkbookHelper().workbookFromFile( arguments.src, arguments.password ): getWorkbookHelper().workbookFromFile( arguments.src );
@@ -1334,12 +1334,13 @@ component accessors="true"{
 		else if( arguments.KeyExists( "queryColumnNames" ) )
 			args.columnNames = arguments.queryColumnNames;// accept better alias `queryColumnNames` to match csvToQuery
 		if( ( arguments.format == "query" ) && arguments.KeyExists( "queryColumnTypes" ) ){
-			getQueryHelper().throwErrorIFinvalidQueryColumnTypesArgument( argumentCollection=arguments );
 			args.queryColumnTypes = arguments.queryColumnTypes;
+			getQueryHelper().throwErrorIFinvalidQueryColumnTypesArgument( argumentCollection=args );
 		}
 		args.includeBlankRows = arguments.includeBlankRows;
 		args.fillMergedCellsWithVisibleValue = arguments.fillMergedCellsWithVisibleValue;
 		args.includeHiddenColumns = arguments.includeHiddenColumns;
+		args.includeHiddenRows = arguments.includeHiddenRows;
 		args.includeRichTextFormatting = arguments.includeRichTextFormatting;
 		args.makeColumnNamesSafe = arguments.makeColumnNamesSafe;
 		var generatedQuery = getSheetHelper().sheetToQuery( argumentCollection=args );
@@ -1365,6 +1366,68 @@ component accessors="true"{
 		return baos.toByteArray();
 	}
 
+	public any function readLargeFile(
+		required string src
+		,string format="query"
+		,string sheetName
+		,numeric sheetNumber // 1-based
+		,numeric headerRow
+		,boolean includeHeaderRow=false
+		,boolean includeBlankRows=false
+		,boolean includeHiddenColumns=true
+		,boolean includeHiddenRows=true
+		,any queryColumnNames //list or array
+		,any queryColumnTypes //'auto', list of types, or struct of column names/types mapping. Null means no types are specified.
+		,boolean makeColumnNamesSafe=false
+		,string password
+		,string csvDelimiter=","
+		,struct streamingReaderOptions
+	){
+		if( this.getIsACF() ){
+			Throw( type="#this.getExceptionType()#.methodNotSupported", message="'readLargeFile()' is not supported with ColdFusion", detail="'readLargeFile()' currently only works with Lucee." );
+		}
+		getFileHelper().throwErrorIFfileNotExists( arguments.src );
+		getExceptionHelper().throwExceptionIFreadFormatIsInvalid( argumentCollection=arguments );
+		getSheetHelper().throwErrorIFSheetNameAndNumberArgumentsBothPassed( argumentCollection=arguments );
+		var builderOptions = arguments.streamingReaderOptions?:{};
+		if( arguments.KeyExists( "password" ) )
+			builderOptions.password = arguments.password;
+		var sheetToQueryArgs = {
+			includeBlankRows: arguments.includeBlankRows
+			,includeHiddenColumns: arguments.includeHiddenColumns
+			,includeHiddenRows: arguments.includeHiddenRows
+			,makeColumnNamesSafe: arguments.makeColumnNamesSafe
+		};
+		if( arguments.KeyExists( "sheetName" ) )
+			sheetToQueryArgs.sheetName = arguments.sheetName;
+		if( arguments.KeyExists( "sheetNumber" ) )
+			sheetToQueryArgs.sheetNumber = arguments.sheetNumber;
+		if( arguments.KeyExists( "headerRow" ) ){
+			sheetToQueryArgs.headerRow = arguments.headerRow;
+			sheetToQueryArgs.includeHeaderRow = arguments.includeHeaderRow;
+		}
+		if( arguments.KeyExists( "queryColumnNames" ) )
+			sheetToQueryArgs.columnNames = arguments.queryColumnNames;
+		if( ( arguments.format == "query" ) && arguments.KeyExists( "queryColumnTypes" ) ){
+			sheetToQueryArgs.queryColumnTypes = arguments.queryColumnTypes;
+			getQueryHelper().throwErrorIFinvalidQueryColumnTypesArgument( argumentCollection=sheetToQueryArgs );
+		}
+		var generatedQuery = getStreamingReaderHelper().readFileIntoQuery( arguments.src, builderOptions, sheetToQueryArgs );
+		if( arguments.format == "query" )
+			return generatedQuery;
+		var exportArgs = { query: generatedQuery };
+		if( arguments.KeyExists( "headerRow" ) ){
+			exportArgs.headerRow = arguments.headerRow;
+			exportArgs.includeHeaderRow = arguments.includeHeaderRow;
+		}
+		if( arguments.format == "csv" ){
+			exportArgs.delimiter = arguments.csvDelimiter;
+			return queryToCsv( argumentCollection=exportArgs );
+		}
+		// format = html
+		return getQueryHelper().queryToHtml( argumentCollection=exportArgs );
+	}
+
 	public Spreadsheet function removePrintGridlines( required workbook ){
 		getSheetHelper().getActiveSheet( arguments.workbook ).setPrintGridlines( JavaCast( "boolean", false ) );
 		return this;
@@ -1477,7 +1540,7 @@ component accessors="true"{
 		cell.setCellFormula( JavaCast( "string", arguments.formula ) );
 		return this;
 	}
-	
+
 	public Spreadsheet function setCellHyperlink(
 		required workbook
 		,required string link

box.json

@@ -1,10 +1,10 @@
 {
     "name" : "Spreadsheet CFML",
     "slug" : "spreadsheet-cfml",
-    "version" : "3.4.4",
+    "version" : "3.5.0",
     "shortDescription" : "CFML spreadsheet library",
     "author" : "Julian Halliwell",
-    "location" : "https://github.com/cfsimplicity/spreadsheet-cfml/archive/v3.4.4.zip",
+    "location" : "https://github.com/cfsimplicity/spreadsheet-cfml/archive/v3.5.0.zip",
     "homepage" : "https://github.com/cfsimplicity/spreadsheet-cfml",
     "projectURL" : "https://github.com/cfsimplicity/spreadsheet-cfml",
     "documentation" : "https://github.com/cfsimplicity/spreadsheet-cfml/blob/main/README.md",

build/lib-osgi.mf

@@ -2,17 +2,19 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: Spreadsheet CFML
 Bundle-SymbolicName: spreadsheet-cfml
-Bundle-Version: 5.2.2.0
+Bundle-Version: 5.2.2.1
 Bundle-ClassPath: commons-codec-1.15.jar,
  commons-collections4-4.4.jar,
  commons-compress-1.21.jar,
  commons-csv-1.9.0.jar,
  commons-io-2.11.0.jar,
  commons-math3-3.6.1.jar,
+ excel-streaming-reader-4.0.1.jar,
  log4j-api-2.17.2.jar,
  poi-5.2.2.jar,
  poi-ooxml-5.2.2.jar,
  poi-ooxml-full-5.2.2.jar,
+ slf4j-api-1.7.36.jar,
  SparseBitSet-1.2.jar,
  spreadsheet-cfml.jar,
  xmlbeans-5.0.3.jar

helpers/base.cfc

@@ -89,6 +89,10 @@ component accessors="true"{
 		return library().getSheetHelper();
 	}
 
+	any function getStreamingReaderHelper(){
+		return library().getStreamingReaderHelper();
+	}
+
 	any function getStringHelper(){
 		return library().getStringHelper();
 	}

helpers/cell.cfc

@@ -37,6 +37,9 @@ component extends="base" accessors="true"{
 	}
 
 	any function getCellFormulaValue( required workbook, required cell ){
+		// streaming reader cannot calculate formulas: return cached value
+		if( getStreamingReaderHelper().isStreamingReaderFormat( arguments.workbook ) )
+			return arguments.cell.getStringCellValue();
 		var formulaEvaluator = arguments.workbook.getCreationHelper().createFormulaEvaluator();
 		try{
 			return getFormatHelper().getDataFormatter().formatCellValue( arguments.cell, formulaEvaluator );

helpers/exception.cfc

@@ -17,4 +17,13 @@ component extends="base" accessors="true"{
 		Throw( type=library().getExceptionType(), message="Could not determine image type", detail="An image type could not be determined from the image provided" );
 	}
 
+	void function throwExceptionIFreadFormatIsInvalid(){
+		if( arguments.KeyExists( "format" ) && !ListFindNoCase( "query,html,csv", arguments.format ) )
+			Throw( type=library().getExceptionType() & ".invalidReadFormat", message="Invalid format", detail="Supported formats are: 'query', 'html' and 'csv'" );
+	}
+
+	void function throwInvalidFileForReadLargeFileException(){
+		Throw( type=library().getExceptionType() & ".invalidFile", message="Invalid spreadsheet file", detail="readLargeFile() can only be used with XLSX files. The file you are trying to read does not appear to be an XLSX file." );
+	}
+
 }

helpers/row.cfc

@@ -5,21 +5,24 @@ component extends="base" accessors="true"{
 		,required struct sheet
 		,required numeric rowIndex
 		,boolean includeRichTextFormatting=false
+		,any rowObject
 	){
 		if( ( arguments.rowIndex == arguments.sheet.headerRowIndex ) && !arguments.sheet.includeHeaderRow ){
-			var row = arguments.sheet.object.getRow( JavaCast( "int", arguments.rowIndex ) );
+			var row = arguments.rowObject?: arguments.sheet.object.getRow( JavaCast( "int", arguments.rowIndex ) );
 			setSheetColumnCountFromRow( row, arguments.sheet );
 			return this;
 		}
 		var rowData = [];
-		var row = arguments.sheet.object.getRow( JavaCast( "int", arguments.rowIndex ) );
+		var row = arguments.rowObject?: arguments.sheet.object.getRow( JavaCast( "int", arguments.rowIndex ) );
 		if( IsNull( row ) ){
 			if( arguments.sheet.includeBlankRows )
 				arguments.sheet.data.Append( rowData );
 			return this;
 		}
 		if( rowIsEmpty( row ) && !arguments.sheet.includeBlankRows )
 			return this;
+		if( rowIsHidden( row ) && !arguments.sheet.includeHiddenRows )
+			return this;
 		rowData = getRowData( arguments.workbook, row, arguments.sheet.columnRanges, arguments.includeRichTextFormatting );
 		arguments.sheet.data.Append( rowData );
 		setSheetColumnCountFromRow( row, arguments.sheet );
@@ -86,6 +89,18 @@ component extends="base" accessors="true"{
 		return getSheetHelper().getActiveSheet( arguments.workbook ).getRow( JavaCast( "int", rowIndex ) );
 	}
 
+	any function getRowFromSheet( required workbook, required sheet, required numeric rowIndex ){
+		if( !getStreamingReaderHelper().isStreamingReaderFormat( arguments.workbook ) )
+			return arguments.sheet.getRow( JavaCast( "int", arguments.rowIndex ) );
+		//streaming reader sheet, no random access so iterate
+		var rowIterator = arguments.sheet.rowIterator();
+		while( rowIterator.hasNext() ){
+			var rowObject = rowIterator.next();
+			if( rowObject.getRowNum() == arguments.rowIndex )
+				return rowObject;
+		}
+	}
+
 	array function parseListDataToArray( required string line, required string delimiter, boolean handleEmbeddedCommas=true ){
 		var elements = ListToArray( arguments.line, arguments.delimiter );
 		var potentialQuotes = 0;
@@ -210,8 +225,12 @@ component extends="base" accessors="true"{
 		return this;
 	}
 
+	boolean function isRowHidden( required workbook, required numeric row ){
+		return rowIsHidden( getRowFromActiveSheet( arguments.workbook, arguments.row ) );
+	}
+
 	any function toggleRowHidden( required workbook, required numeric rowNumber, required boolean state ){
-		getRowHelper().getRowFromActiveSheet( arguments.workbook, arguments.rowNumber ).setZeroHeight( JavaCast( "boolean", arguments.state ) );
+		getRowFromActiveSheet( arguments.workbook, arguments.rowNumber ).setZeroHeight( JavaCast( "boolean", arguments.state ) );
 		return this;
 	}
 
@@ -226,6 +245,10 @@ component extends="base" accessors="true"{
 	  return true;
 	}
 
+	private boolean function rowIsHidden( required row ){
+		return arguments.row.getZeroHeight() || arguments.row.getHeight() == 0;
+	}
+
 	private void function setSheetColumnCountFromRow( required any row, required struct sheet ){
 		if( arguments.sheet.columnRanges.Len() )//don't change the column count if specific columns have been specified
 			return;