Refactor/various fixes 2

cmd/commands/datasetIngestor.go

@@ -132,7 +132,7 @@ For Windows you need instead to specify -user username:password on the command l
 			return
 		}
 
-		// check for program version
+		// === check for program version ===
 		datasetUtils.CheckForNewVersion(client, CMD, VERSION)
 		datasetUtils.CheckForServiceAvailability(client, testenvFlag, autoarchiveFlag)
 
@@ -214,7 +214,9 @@ For Windows you need instead to specify -user username:password on the command l
 		}
 		color.Set(color.FgYellow)
 		if len(foundList) > 0 {
-			fmt.Println("Warning! The following datasets have been found with the same sourceFolder: ")
+			fmt.Println("Warning! The following datasets have been found with the same sourceFolders: ")
+		} else {
+			log.Println("Finished testing for existing source folders.")
 		}
 		for _, element := range foundList {
 			fmt.Printf("  - PID: \"%s\", sourceFolder: \"%s\"\n", element.Pid, element.SourceFolder)
@@ -277,15 +279,16 @@ For Windows you need instead to specify -user username:password on the command l
 				skipSymlinks = ""
 			}
 
-			// get filelist of dataset
+			// === get filelist of dataset ===
+			log.Printf("Getting filelist for \"%s\"...\n", datasetSourceFolder)
 			fullFileArray, startTime, endTime, owner, numFiles, totalSize, err :=
 				datasetIngestor.GetLocalFileList(datasetSourceFolder, datasetFileListTxt, localSymlinkCallback, localFilepathFilterCallback)
 			if err != nil {
 				log.Fatalf("Can't gather the filelist of \"%s\"", datasetSourceFolder)
 			}
-
+			log.Println("File list collected.")
 			//log.Printf("full fileListing: %v\n Start and end time: %s %s\n ", fullFileArray, startTime, endTime)
-			log.Printf("The dataset contains %v files with a total size of %v bytes.", numFiles, totalSize)
+			log.Printf("The dataset contains %v files with a total size of %v bytes.\n", numFiles, totalSize)
 
 			// filecount checks
 			if totalSize == 0 {
@@ -309,11 +312,13 @@ For Windows you need instead to specify -user username:password on the command l
 				log.Printf("Note: this dataset, if archived, will be copied to two tape copies")
 				color.Unset()
 			}
+			// === update metadata ===
 			datasetIngestor.UpdateMetaData(client, APIServer, user, originalMap, metaDataMap, startTime, endTime, owner, tapecopies)
 			pretty, _ := json.MarshalIndent(metaDataMap, "", "    ")
 
 			log.Printf("Updated metadata object:\n%s\n", pretty)
 
+			// === check central availability of data ===
 			// check if data is accesible at archive server, unless beamline account (assumed to be centrally available always)
 			// and unless (no)copy flag defined via command line
 			if checkCentralAvailability {
@@ -348,6 +353,7 @@ For Windows you need instead to specify -user username:password on the command l
 				}
 			}
 
+			// === ingest dataset ===
 			if ingestFlag {
 				// create ingest . For decentral case delay setting status to archivable until data is copied
 				archivable := false
@@ -380,6 +386,7 @@ For Windows you need instead to specify -user username:password on the command l
 					}
 					log.Printf("Attachment file %v added to dataset %v\n", addAttachment, datasetId)
 				}
+				// === copying files ===
 				if copyFlag {
 					// TODO rewrite SyncDataToFileserver
 					log.Println("Syncing files to cache server...")
@@ -406,7 +413,8 @@ For Windows you need instead to specify -user username:password on the command l
 					archivableDatasetList = append(archivableDatasetList, datasetId)
 				}
 			}
-			datasetIngestor.ResetUpdatedMetaData(originalMap, metaDataMap) // I don't really get this...
+			// reset dataset metadata for next dataset ingestion
+			datasetIngestor.ResetUpdatedMetaData(originalMap, metaDataMap)
 		}
 
 		if !ingestFlag {
@@ -439,7 +447,8 @@ For Windows you need instead to specify -user username:password on the command l
 		if emptyDatasets > 0 || tooLargeDatasets > 0 {
 			os.Exit(1)
 		}
-		// start archive job
+
+		// === create archive job ===
 		if autoarchiveFlag && ingestFlag {
 			log.Printf("Submitting Archive Job for the ingested datasets.\n")
 			// TODO: change param type from pointer to regular as it is unnecessary

datasetIngestor/ingestDataset.go

@@ -176,7 +176,7 @@ func createOrigDatablocks(client *http.Client, APIServer string, fullFileArray [
 
 	if totalFiles > TOTAL_MAXFILES {
 		return fmt.Errorf(
-			"dataset exceeds (%v) the maximum number of files per dataset , which can currently be handled by the archiving system (%v)",
+			"dataset exceeds the maximum number of files that can be handled by the archiving system per dataset (dataset: %v, max: %v)",
 			totalFiles, TOTAL_MAXFILES)
 	}