Skip to content

Commit

Permalink
a couple of corrections and comment updates
Browse files Browse the repository at this point in the history
  • Loading branch information
consolethinks committed Aug 23, 2024
1 parent de50300 commit 90d97b0
Showing 1 changed file with 22 additions and 11 deletions.
33 changes: 22 additions & 11 deletions cmd/commands/datasetIngestor.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ For Windows you need instead to specify -user username:password on the command l
return
}

// check for program version
// === check for program version ===
datasetUtils.CheckForNewVersion(client, CMD, VERSION)
datasetUtils.CheckForServiceAvailability(client, testenvFlag, autoarchiveFlag)

Expand Down Expand Up @@ -214,7 +214,9 @@ For Windows you need instead to specify -user username:password on the command l
}
color.Set(color.FgYellow)
if len(foundList) > 0 {
fmt.Println("Warning! The following datasets have been found with the same sourceFolder: ")
fmt.Println("Warning! The following datasets have been found with the same sourceFolders: ")
} else {
log.Println("Finished testing for existing source folders.")
}
for _, element := range foundList {
fmt.Printf(" - PID: \"%s\", sourceFolder: \"%s\"\n", element.Pid, element.SourceFolder)
Expand All @@ -237,22 +239,21 @@ For Windows you need instead to specify -user username:password on the command l
// a destination location is defined by the archive system
// for now let the user decide if he needs a copy

// now everything is prepared, prepare to loop over all folders
if nocopyFlag {
if nocopyFlag || beamlineAccount {
copyFlag = false
}
checkCentralAvailability := !(cmd.Flags().Changed("copy") || cmd.Flags().Changed("nocopy") || beamlineAccount || copyFlag)
skip := ""
skipSymlinks := ""

// check if skip flag is globally defined via flags:
if cmd.Flags().Changed("linkfiles") {
switch linkfiles {
case "delete":
skip = "sA"
skipSymlinks = "sA"
case "keep":
skip = "kA"
skipSymlinks = "kA"
default:
skip = "dA" // default behaviour = keep internal for all
skipSymlinks = "dA" // default behaviour = keep internal for all
}
}

Expand All @@ -267,10 +268,15 @@ For Windows you need instead to specify -user username:password on the command l
metaDataMap["sourceFolder"] = datasetSourceFolder
log.Printf("Scanning files in dataset %s", datasetSourceFolder)

// get filelist of dataset
// reset skip var. if not set for all datasets
if !(skipSymlinks == "sA" || skipSymlinks == "kA" || skipSymlinks == "dA") {
skipSymlinks = ""
}

// === get filelist of dataset ===
log.Printf("Getting filelist for \"%s\"...\n", datasetSourceFolder)
fullFileArray, startTime, endTime, owner, numFiles, totalSize, err :=
datasetIngestor.GetLocalFileList(datasetSourceFolder, datasetFileListTxt, &skip)
datasetIngestor.GetLocalFileList(datasetSourceFolder, datasetFileListTxt, &skipSymlinks)
if err != nil {
log.Fatalf("Can't gather the filelist of \"%s\"", datasetSourceFolder)
}
Expand Down Expand Up @@ -300,11 +306,13 @@ For Windows you need instead to specify -user username:password on the command l
log.Printf("Note: this dataset, if archived, will be copied to two tape copies")
color.Unset()
}
// === update metadata ===
datasetIngestor.UpdateMetaData(client, APIServer, user, originalMap, metaDataMap, startTime, endTime, owner, tapecopies)
pretty, _ := json.MarshalIndent(metaDataMap, "", " ")

log.Printf("Updated metadata object:\n%s\n", pretty)

// === check central availability of data ===
// check if data is accesible at archive server, unless beamline account (assumed to be centrally available always)
// and unless (no)copy flag defined via command line
if checkCentralAvailability {
Expand Down Expand Up @@ -336,6 +344,7 @@ For Windows you need instead to specify -user username:password on the command l
}
}

// === ingest dataset ===
if ingestFlag {
// create ingest . For decentral case delay setting status to archivable until data is copied
archivable := false
Expand Down Expand Up @@ -368,6 +377,7 @@ For Windows you need instead to specify -user username:password on the command l
}
log.Printf("Attachment file %v added to dataset %v\n", addAttachment, datasetId)
}
// === copying files ===
if copyFlag {
// TODO rewrite SyncDataToFileserver
log.Println("Syncing files to cache server...")
Expand Down Expand Up @@ -418,7 +428,8 @@ For Windows you need instead to specify -user username:password on the command l
if emptyDatasets > 0 || tooLargeDatasets > 0 {
os.Exit(1)
}
// start archive job

// === create archive job ===
if autoarchiveFlag && ingestFlag {
log.Printf("Submitting Archive Job for the ingested datasets.\n")
// TODO: change param type from pointer to regular as it is unnecessary
Expand Down

0 comments on commit 90d97b0

Please sign in to comment.