diff --git a/cmd/commands/datasetIngestor.go b/cmd/commands/datasetIngestor.go index 85f55f0..7fd5b88 100644 --- a/cmd/commands/datasetIngestor.go +++ b/cmd/commands/datasetIngestor.go @@ -132,7 +132,7 @@ For Windows you need instead to specify -user username:password on the command l return } - // check for program version + // === check for program version === datasetUtils.CheckForNewVersion(client, CMD, VERSION) datasetUtils.CheckForServiceAvailability(client, testenvFlag, autoarchiveFlag) @@ -214,7 +214,9 @@ For Windows you need instead to specify -user username:password on the command l } color.Set(color.FgYellow) if len(foundList) > 0 { - fmt.Println("Warning! The following datasets have been found with the same sourceFolder: ") + fmt.Println("Warning! The following datasets have been found with the same sourceFolders: ") + } else { + log.Println("Finished testing for existing source folders.") } for _, element := range foundList { fmt.Printf(" - PID: \"%s\", sourceFolder: \"%s\"\n", element.Pid, element.SourceFolder) @@ -277,15 +279,16 @@ For Windows you need instead to specify -user username:password on the command l skipSymlinks = "" } - // get filelist of dataset + // === get filelist of dataset === + log.Printf("Getting filelist for \"%s\"...\n", datasetSourceFolder) fullFileArray, startTime, endTime, owner, numFiles, totalSize, err := datasetIngestor.GetLocalFileList(datasetSourceFolder, datasetFileListTxt, localSymlinkCallback, localFilepathFilterCallback) if err != nil { log.Fatalf("Can't gather the filelist of \"%s\"", datasetSourceFolder) } - + log.Println("File list collected.") //log.Printf("full fileListing: %v\n Start and end time: %s %s\n ", fullFileArray, startTime, endTime) - log.Printf("The dataset contains %v files with a total size of %v bytes.", numFiles, totalSize) + log.Printf("The dataset contains %v files with a total size of %v bytes.\n", numFiles, totalSize) // filecount checks if totalSize == 0 { @@ -309,11 +312,13 @@ For Windows you need instead to specify -user username:password on the command l log.Printf("Note: this dataset, if archived, will be copied to two tape copies") color.Unset() } + // === update metadata === datasetIngestor.UpdateMetaData(client, APIServer, user, originalMap, metaDataMap, startTime, endTime, owner, tapecopies) pretty, _ := json.MarshalIndent(metaDataMap, "", " ") log.Printf("Updated metadata object:\n%s\n", pretty) + // === check central availability of data === // check if data is accesible at archive server, unless beamline account (assumed to be centrally available always) // and unless (no)copy flag defined via command line if checkCentralAvailability { @@ -348,6 +353,7 @@ For Windows you need instead to specify -user username:password on the command l } } + // === ingest dataset === if ingestFlag { // create ingest . For decentral case delay setting status to archivable until data is copied archivable := false @@ -380,6 +386,7 @@ For Windows you need instead to specify -user username:password on the command l } log.Printf("Attachment file %v added to dataset %v\n", addAttachment, datasetId) } + // === copying files === if copyFlag { // TODO rewrite SyncDataToFileserver log.Println("Syncing files to cache server...") @@ -406,7 +413,8 @@ For Windows you need instead to specify -user username:password on the command l archivableDatasetList = append(archivableDatasetList, datasetId) } } - datasetIngestor.ResetUpdatedMetaData(originalMap, metaDataMap) // I don't really get this... + // reset dataset metadata for next dataset ingestion + datasetIngestor.ResetUpdatedMetaData(originalMap, metaDataMap) } if !ingestFlag { @@ -439,7 +447,8 @@ For Windows you need instead to specify -user username:password on the command l if emptyDatasets > 0 || tooLargeDatasets > 0 { os.Exit(1) } - // start archive job + + // === create archive job === if autoarchiveFlag && ingestFlag { log.Printf("Submitting Archive Job for the ingested datasets.\n") // TODO: change param type from pointer to regular as it is unnecessary diff --git a/datasetIngestor/ingestDataset.go b/datasetIngestor/ingestDataset.go index 8197d45..8ab4977 100644 --- a/datasetIngestor/ingestDataset.go +++ b/datasetIngestor/ingestDataset.go @@ -176,7 +176,7 @@ func createOrigDatablocks(client *http.Client, APIServer string, fullFileArray [ if totalFiles > TOTAL_MAXFILES { return fmt.Errorf( - "dataset exceeds (%v) the maximum number of files per dataset , which can currently be handled by the archiving system (%v)", + "dataset exceeds the maximum number of files that can be handled by the archiving system per dataset (dataset: %v, max: %v)", totalFiles, TOTAL_MAXFILES) }