diff --git a/cmd/commands/datasetIngestor.go b/cmd/commands/datasetIngestor.go index fea8d59..3e7823a 100644 --- a/cmd/commands/datasetIngestor.go +++ b/cmd/commands/datasetIngestor.go @@ -132,7 +132,7 @@ For Windows you need instead to specify -user username:password on the command l return } - // check for program version + // === check for program version === datasetUtils.CheckForNewVersion(client, CMD, VERSION) datasetUtils.CheckForServiceAvailability(client, testenvFlag, autoarchiveFlag) @@ -214,7 +214,9 @@ For Windows you need instead to specify -user username:password on the command l } color.Set(color.FgYellow) if len(foundList) > 0 { - fmt.Println("Warning! The following datasets have been found with the same sourceFolder: ") + fmt.Println("Warning! The following datasets have been found with the same sourceFolders: ") + } else { + log.Println("Finished testing for existing source folders.") } for _, element := range foundList { fmt.Printf(" - PID: \"%s\", sourceFolder: \"%s\"\n", element.Pid, element.SourceFolder) @@ -237,22 +239,21 @@ For Windows you need instead to specify -user username:password on the command l // a destination location is defined by the archive system // for now let the user decide if he needs a copy - // now everything is prepared, prepare to loop over all folders - if nocopyFlag { + if nocopyFlag || beamlineAccount { copyFlag = false } checkCentralAvailability := !(cmd.Flags().Changed("copy") || cmd.Flags().Changed("nocopy") || beamlineAccount || copyFlag) - skip := "" + skipSymlinks := "" // check if skip flag is globally defined via flags: if cmd.Flags().Changed("linkfiles") { switch linkfiles { case "delete": - skip = "sA" + skipSymlinks = "sA" case "keep": - skip = "kA" + skipSymlinks = "kA" default: - skip = "dA" // default behaviour = keep internal for all + skipSymlinks = "dA" // default behaviour = keep internal for all } } @@ -267,10 +268,15 @@ For Windows you need instead to specify -user username:password on the command l metaDataMap["sourceFolder"] = datasetSourceFolder log.Printf("Scanning files in dataset %s", datasetSourceFolder) - // get filelist of dataset + // reset skip var. if not set for all datasets + if !(skipSymlinks == "sA" || skipSymlinks == "kA" || skipSymlinks == "dA") { + skipSymlinks = "" + } + + // === get filelist of dataset === log.Printf("Getting filelist for \"%s\"...\n", datasetSourceFolder) fullFileArray, startTime, endTime, owner, numFiles, totalSize, err := - datasetIngestor.GetLocalFileList(datasetSourceFolder, datasetFileListTxt, &skip) + datasetIngestor.GetLocalFileList(datasetSourceFolder, datasetFileListTxt, &skipSymlinks) if err != nil { log.Fatalf("Can't gather the filelist of \"%s\"", datasetSourceFolder) } @@ -300,11 +306,13 @@ For Windows you need instead to specify -user username:password on the command l log.Printf("Note: this dataset, if archived, will be copied to two tape copies") color.Unset() } + // === update metadata === datasetIngestor.UpdateMetaData(client, APIServer, user, originalMap, metaDataMap, startTime, endTime, owner, tapecopies) pretty, _ := json.MarshalIndent(metaDataMap, "", " ") log.Printf("Updated metadata object:\n%s\n", pretty) + // === check central availability of data === // check if data is accesible at archive server, unless beamline account (assumed to be centrally available always) // and unless (no)copy flag defined via command line if checkCentralAvailability { @@ -336,6 +344,7 @@ For Windows you need instead to specify -user username:password on the command l } } + // === ingest dataset === if ingestFlag { // create ingest . For decentral case delay setting status to archivable until data is copied archivable := false @@ -368,6 +377,7 @@ For Windows you need instead to specify -user username:password on the command l } log.Printf("Attachment file %v added to dataset %v\n", addAttachment, datasetId) } + // === copying files === if copyFlag { // TODO rewrite SyncDataToFileserver log.Println("Syncing files to cache server...") @@ -418,7 +428,8 @@ For Windows you need instead to specify -user username:password on the command l if emptyDatasets > 0 || tooLargeDatasets > 0 { os.Exit(1) } - // start archive job + + // === create archive job === if autoarchiveFlag && ingestFlag { log.Printf("Submitting Archive Job for the ingested datasets.\n") // TODO: change param type from pointer to regular as it is unnecessary