From 3b19c3f56ecdcc2068b22c85c98419788704c0c8 Mon Sep 17 00:00:00 2001 From: Attila Martin Nacsa Date: Tue, 30 Jul 2024 18:26:55 +0200 Subject: [PATCH] updateMetaData: change pointer parameters into pass-by value, avoid logging, better error handling --- datasetIngestor/updateMetaData.go | 103 +++++++++++-------------- datasetIngestor/updateMetaData_test.go | 86 ++++++++++----------- 2 files changed, 89 insertions(+), 100 deletions(-) diff --git a/datasetIngestor/updateMetaData.go b/datasetIngestor/updateMetaData.go index ce883fe..3833c60 100644 --- a/datasetIngestor/updateMetaData.go +++ b/datasetIngestor/updateMetaData.go @@ -3,24 +3,21 @@ package datasetIngestor import ( "encoding/json" "io" - "log" "net/http" "strings" "time" - - "github.com/fatih/color" ) const ( Classification = "classification" - AVLow = "AV=low" - AVMedium = "AV=medium" - INMedium = "IN=medium" - COLow = "CO=low" + AVLow = "AV=low" + AVMedium = "AV=medium" + INMedium = "IN=medium" + COLow = "CO=low" ) /* -getAVFromPolicy retrieves the AV (?) from a policy. +getAVFromPolicy retrieves the AV (?) from a policy. Parameters: - client: An HTTP client used to send requests. @@ -28,35 +25,39 @@ Parameters: - user: A map containing user information. It should contain an "accessToken" key. - owner: The owner of the policy. -The function constructs a URL using the APIServer, owner, and user's access token, and sends a GET request to this URL. -If the response status code is 200, it reads the response body and unmarshals it into a slice of Policy structs. -If there are no policies available for the owner, it logs a warning and sets the level to "low". +The function constructs a URL using the APIServer, owner, and user's access token, and sends a GET request to this URL. +If the response status code is 200, it reads the response body and unmarshals it into a slice of Policy structs. +If there are no policies available for the owner, it logs a warning and sets the level to "low". If there are policies available, it sets the level to the TapeRedundancy of the first policy. Returns: - level: The TapeRedundancy level of the first policy if available, otherwise "low". */ func getAVFromPolicy(client *http.Client, APIServer string, user map[string]string, owner string) (level string) { + level = "low" // default value + var myurl = APIServer + "/Policies?filter=%7B%22where%22%3A%7B%22ownerGroup%22%3A%22" + owner + "%22%7D%7D&access_token=" + user["accessToken"] - resp, _ := client.Get(myurl) + resp, err := client.Get(myurl) + if err != nil { + return level + } defer resp.Body.Close() + if resp.StatusCode != 200 { + return level + } - level = "low" - if resp.StatusCode == 200 { - body, _ := io.ReadAll(resp.Body) - type Policy struct { - TapeRedundancy string - AutoArchive bool - } - var policies []Policy - _ = json.Unmarshal(body, &policies) - if len(policies) == 0 { - color.Set(color.FgYellow) - log.Printf("No Policy available for owner %v\n", owner) - color.Set(color.FgGreen) - } else { - level = policies[0].TapeRedundancy - } + body, err := io.ReadAll(resp.Body) + if err != nil { + return level + } + type Policy struct { + TapeRedundancy string + AutoArchive bool + } + var policies []Policy + _ = json.Unmarshal(body, &policies) + if len(policies) > 0 { + level = policies[0].TapeRedundancy } return level } @@ -88,17 +89,17 @@ The function logs a message each time it updates a field. If the "classification The function does not return a value. */ func UpdateMetaData(client *http.Client, APIServer string, user map[string]string, - originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies *int) { + originalMap map[string]string, metaDataMap map[string]interface{}, startTime time.Time, endTime time.Time, owner string, tapecopies int) { updateFieldIfDummy(metaDataMap, originalMap, "creationTime", DUMMY_TIME, startTime) updateFieldIfDummy(metaDataMap, originalMap, "ownerGroup", DUMMY_OWNER, owner) - + if metaDataMap["type"] == "raw" { updateFieldIfDummy(metaDataMap, originalMap, "endTime", DUMMY_TIME, endTime) } - + addFieldIfNotExists(metaDataMap, "license", "CC BY-SA 4.0") addFieldIfNotExists(metaDataMap, "isPublished", false) - + updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) } @@ -106,37 +107,33 @@ func updateFieldIfDummy(metaDataMap map[string]interface{}, originalMap map[stri if metaDataMap[fieldName] == dummyValue { originalMap[fieldName] = metaDataMap[fieldName].(string) metaDataMap[fieldName] = newValue - log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) + //log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) } } func addFieldIfNotExists(metaDataMap map[string]interface{}, fieldName string, value interface{}) { if _, ok := metaDataMap[fieldName]; !ok { metaDataMap[fieldName] = value - log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) + //log.Printf("%s field added: %v\n", fieldName, metaDataMap[fieldName]) } } -func updateClassificationField(client *http.Client, APIServer string, user map[string]string, metaDataMap map[string]interface{}, tapecopies *int) { +func updateClassificationField(client *http.Client, APIServer string, user map[string]string, metaDataMap map[string]interface{}, tapecopies int) { if _, ok := metaDataMap[Classification]; !ok { addDefaultClassification(client, APIServer, user, metaDataMap) } - - if *tapecopies == 1 || *tapecopies == 2 { + + if tapecopies == 1 || tapecopies == 2 { updateAVField(metaDataMap, tapecopies) } - - if strings.Contains(metaDataMap[Classification].(string), AVMedium) { - logAVMediumMessage() - } } func addDefaultClassification(client *http.Client, APIServer string, user map[string]string, metaDataMap map[string]interface{}) { metaDataMap[Classification] = INMedium + ",AV=" + getAVFromPolicy(client, APIServer, user, metaDataMap["ownerGroup"].(string)) + "," + COLow - log.Printf("classification field added: %v\n", metaDataMap[Classification]) + //log.Printf("classification field added: %v\n", metaDataMap[Classification]) } -func updateAVField(metaDataMap map[string]interface{}, tapecopies *int) { +func updateAVField(metaDataMap map[string]interface{}, tapecopies int) { av := getAVValue(tapecopies) if _, ok := metaDataMap[Classification]; ok { newresult := getUpdatedClassification(metaDataMap, av) @@ -144,11 +141,11 @@ func updateAVField(metaDataMap map[string]interface{}, tapecopies *int) { } else { metaDataMap[Classification] = INMedium + "," + av + "," + COLow } - log.Printf("classification field adjusted: %s\n", metaDataMap[Classification]) + //log.Printf("classification field adjusted: %s\n", metaDataMap[Classification]) } -func getAVValue(tapecopies *int) string { - if *tapecopies == 1 { +func getAVValue(tapecopies int) string { + if tapecopies == 1 { return AVLow } return AVMedium @@ -162,10 +159,10 @@ func getUpdatedClassification(metaDataMap map[string]interface{}, av string) []s for _, element := range result { if element == "" { continue - } - if strings.HasPrefix(element, "AV=") { - newresult = append(newresult, av) - found = true + } + if strings.HasPrefix(element, "AV=") { + newresult = append(newresult, av) + found = true } else { newresult = append(newresult, element) } @@ -175,9 +172,3 @@ func getUpdatedClassification(metaDataMap map[string]interface{}, av string) []s } return newresult } - -func logAVMediumMessage() { - color.Set(color.FgYellow) - log.Printf("Note: this dataset, if archived, will be copied to two tape copies") - color.Unset() -} diff --git a/datasetIngestor/updateMetaData_test.go b/datasetIngestor/updateMetaData_test.go index 945f9fd..501b417 100644 --- a/datasetIngestor/updateMetaData_test.go +++ b/datasetIngestor/updateMetaData_test.go @@ -3,9 +3,9 @@ package datasetIngestor import ( "net/http" "net/http/httptest" + "reflect" "testing" "time" - "reflect" ) func TestGetAVFromPolicy(t *testing.T) { @@ -15,26 +15,26 @@ func TestGetAVFromPolicy(t *testing.T) { w.Write([]byte(`[]`)) // empty policy list })) defer ts1.Close() - + client := ts1.Client() - + level := getAVFromPolicy(client, ts1.URL, map[string]string{"accessToken": "testToken"}, "testOwner") - + if level != "low" { t.Errorf("Expected level to be 'low', got '%s'", level) } - + // Test case 2: Policies available ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(`[{"TapeRedundancy": "medium", "AutoArchive": false}]`)) // policy list with one policy })) defer ts2.Close() - + client = ts2.Client() - + level = getAVFromPolicy(client, ts2.URL, map[string]string{"accessToken": "testToken"}, "testOwner") - + if level != "medium" { t.Errorf("Expected level to be 'medium', got '%s'", level) } @@ -48,13 +48,13 @@ func TestUpdateMetaData(t *testing.T) { w.Write([]byte(`[{"TapeRedundancy": "medium", "AutoArchive": false}]`)) // policy list with one policy })) defer ts.Close() - + // Create a test client client := ts.Client() - + // Define test parameters APIServer := ts.URL // Use the mock server's URL - + user := map[string]string{"accessToken": "testToken"} originalMap := map[string]string{} metaDataMap := map[string]interface{}{ @@ -66,12 +66,11 @@ func TestUpdateMetaData(t *testing.T) { startTime := time.Now() endTime := startTime.Add(time.Hour) owner := "testOwner" - tapecopies := new(int) - *tapecopies = 1 - + tapecopies := 1 + // Call the function UpdateMetaData(client, APIServer, user, originalMap, metaDataMap, startTime, endTime, owner, tapecopies) - + // Check results if metaDataMap["creationTime"] != startTime { t.Errorf("Expected creationTime to be '%v', got '%v'", startTime, metaDataMap["creationTime"]) @@ -103,10 +102,10 @@ func TestUpdateFieldIfDummy(t *testing.T) { fieldName := "testField" dummyValue := "DUMMY" newValue := "newValue" - + // Call the function updateFieldIfDummy(metaDataMap, originalMap, fieldName, dummyValue, newValue) - + // Check results if metaDataMap[fieldName] != newValue { t.Errorf("Expected %s to be '%v', got '%v'", fieldName, newValue, metaDataMap[fieldName]) @@ -122,10 +121,10 @@ func TestAddFieldIfNotExists(t *testing.T) { metaDataMap := map[string]interface{}{} fieldName := "testField" value := "testValue" - + // Call the function addFieldIfNotExists(metaDataMap, fieldName, value) - + // Check results if metaDataMap[fieldName] != value { t.Errorf("Expected %s to be '%v', got '%v'", fieldName, value, metaDataMap[fieldName]) @@ -140,36 +139,35 @@ func TestUpdateClassificationField(t *testing.T) { w.Write([]byte(`[{"TapeRedundancy": "medium", "AutoArchive": false}]`)) // policy list with one policy })) defer ts.Close() - + // Create a test client client := ts.Client() - + // Define test parameters APIServer := ts.URL // Use the mock server's URL user := map[string]string{"accessToken": "testToken"} metaDataMap := map[string]interface{}{ "ownerGroup": "testOwner", } - tapecopies := new(int) - *tapecopies = 1 + tapecopies := 1 expectedValue1 := "IN=medium,AV=low,CO=low" expectedValue2 := "IN=medium,AV=medium,CO=low" - + // Call the function updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) - + // Check results if _, ok := metaDataMap["classification"]; !ok { t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) } else if metaDataMap["classification"] != expectedValue1 { t.Errorf("Expected classification to be '%v', got '%v'", expectedValue1, metaDataMap["classification"]) } - + // Change tapecopies to 2 and call the function again - *tapecopies = 2 + tapecopies = 2 updateClassificationField(client, APIServer, user, metaDataMap, tapecopies) - + // Check results if _, ok := metaDataMap["classification"]; !ok { t.Errorf("Expected classification to be set, got '%v'", metaDataMap["classification"]) @@ -184,10 +182,10 @@ func TestGetUpdatedClassification(t *testing.T) { "classification": "IN=medium,AV=low,CO=low", } av := "AV=medium" - + expected := []string{"IN=medium", "AV=medium", "CO=low"} result := getUpdatedClassification(metaDataMap, av) - + if !reflect.DeepEqual(result, expected) { t.Errorf("getUpdatedClassification() = %v; want %v", result, expected) } @@ -196,17 +194,17 @@ func TestGetUpdatedClassification(t *testing.T) { func TestGetAVValue(t *testing.T) { tapecopies1 := 1 tapecopies2 := 2 - + expected1 := AVLow expected2 := AVMedium - - result1 := getAVValue(&tapecopies1) - result2 := getAVValue(&tapecopies2) - + + result1 := getAVValue(tapecopies1) + result2 := getAVValue(tapecopies2) + if result1 != expected1 { t.Errorf("getAVValue() with tapecopies = 1 = %v; want %v", result1, expected1) } - + if result2 != expected2 { t.Errorf("getAVValue() with tapecopies = 2 = %v; want %v", result2, expected2) } @@ -215,25 +213,25 @@ func TestGetAVValue(t *testing.T) { func TestUpdateAVField(t *testing.T) { tapecopies1 := 1 tapecopies2 := 2 - + metaDataMap1 := map[string]interface{}{ "classification": "IN=medium,AV=medium,CO=low", } - + metaDataMap2 := map[string]interface{}{ "classification": "IN=medium,AV=low,CO=low", } - + expected1 := "IN=medium,AV=low,CO=low" expected2 := "IN=medium,AV=medium,CO=low" - - updateAVField(metaDataMap1, &tapecopies1) - updateAVField(metaDataMap2, &tapecopies2) - + + updateAVField(metaDataMap1, tapecopies1) + updateAVField(metaDataMap2, tapecopies2) + if metaDataMap1["classification"] != expected1 { t.Errorf("updateAVField() with tapecopies = 1 = %v; want %v", metaDataMap1["classification"], expected1) } - + if metaDataMap2["classification"] != expected2 { t.Errorf("updateAVField() with tapecopies = 2 = %v; want %v", metaDataMap2["classification"], expected2) }