diff --git a/db-connector.go b/db-connector.go index 62e45c9..005226e 100755 --- a/db-connector.go +++ b/db-connector.go @@ -96,7 +96,7 @@ func handleDailyCacheUpdate(executionInfo *ExecutionInfo) *ExecutionInfo { } } - log.Printf("[DEBUG] Daily stats not updated for %s in org %s. Only have %d stats so far", timeYesterday, executionInfo.OrgId, len(executionInfo.DailyStatistics)) + log.Printf("[DEBUG] Daily stats not updated for %s in org %s today. Only have %d stats so far - running update.", timeYesterday, executionInfo.OrgId, len(executionInfo.DailyStatistics)) // If we get here, we need to update the daily stats newDay := DailyStatistics{ Date: timeYesterday, @@ -3154,7 +3154,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error) } } - log.Printf("[INFO] Appending workflows (ADMIN) for organization %s. Already have %d workflows for the user. Found %d (%d new) for org. New unique amount: %d (1)", user.ActiveOrg.Id, userWorkflowLen, len(wrapped.Hits.Hits), len(workflows)-userWorkflowLen, len(workflows)) + log.Printf("[INFO] Appending workflows (ADMIN + suborg distribution) for organization %s. Already have %d workflows for the user. Found %d (%d new) for org. New unique amount: %d (1)", user.ActiveOrg.Id, userWorkflowLen, len(wrapped.Hits.Hits), len(workflows)-userWorkflowLen, len(workflows)) } } else { @@ -3217,7 +3217,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error) } } - log.Printf("[INFO] Appending suborg distribution workflows for organization %s (%s)", user.ActiveOrg.Name, user.ActiveOrg.Id) + //log.Printf("[INFO] Appending suborg distribution workflows for organization %s (%s)", user.ActiveOrg.Name, user.ActiveOrg.Id) cursorStr = "" query = datastore.NewQuery(nameKey).Filter("suborg_distribution =", user.ActiveOrg.Id) @@ -3228,10 +3228,12 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error) innerWorkflow := Workflow{} _, err := it.Next(&innerWorkflow) if err != nil { - log.Printf("[ERROR] Error in suborg workflow iterator: %s", err) if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") { log.Printf("[ERROR] Error in workflow loading. Migrating workflow to new workflow handler (1): %s", err) + } else if strings.Contains(fmt.Sprintf("%s", err), "no more items in iterator") { + break } else { + log.Printf("[ERROR] Error in suborg workflow iterator: %s", err) break } } @@ -3251,8 +3253,9 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error) } } + // FIXME: Handle nil? if err != iterator.Done { - log.Printf("[INFO] Failed fetching suborg workflows: %v", err) + //log.Printf("[INFO] Failed fetching suborg workflows: %v", err) break } @@ -6341,8 +6344,6 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { } } - log.Printf("Found Userapps: %d", len(userApps)) - if project.CacheDb { data, err := json.Marshal(userApps) if err == nil { diff --git a/files.go b/files.go index 9ad36ee..bbfa036 100755 --- a/files.go +++ b/files.go @@ -432,8 +432,9 @@ func HandleDeleteFile(resp http.ResponseWriter, request *http.Request) { } func LoadStandardFromGithub(client *github.Client, owner, repo, path, filename string) ([]*github.RepositoryContent, error) { - ctx := context.Background() + var err error + ctx := context.Background() files := []*github.RepositoryContent{} cacheKey := fmt.Sprintf("github_%s_%s_%s", owner, repo, path) @@ -443,36 +444,39 @@ func LoadStandardFromGithub(client *github.Client, owner, repo, path, filename s cacheData := []byte(cache.([]uint8)) err = json.Unmarshal(cacheData, &files) if err == nil { - return files, nil + //return files, nil } } - } + } - _, items, _, err := client.Repositories.GetContents(ctx, owner, repo, path, nil) - if err != nil { - //log.Printf("[WARNING] Failed getting standard list for namespace %s: %s", path, err) - return files, err + if len(files) == 0 { + _, files, _, err = client.Repositories.GetContents(ctx, owner, repo, path, nil) + if err != nil { + log.Printf("[WARNING] Failed getting standard list for namespace %s: %s", path, err) + return []*github.RepositoryContent{}, err + } } - if len(items) == 0 { - log.Printf("[WARNING] No items found in namespace %s", path) - return files, errors.New("No items found for namespace") + if len(files) == 0 { + log.Printf("[ERROR] No files found in namespace '%s' on Github - Used for integration framework", path) + return []*github.RepositoryContent{}, nil } if len(filename) == 0 { - return items, nil + return []*github.RepositoryContent{}, nil } - for _, item := range items { + matchingFiles := []*github.RepositoryContent{} + for _, item := range files { if len(filename) > 0 && strings.HasPrefix(*item.Name, filename) { - files = append(files, item) + matchingFiles = append(matchingFiles, item) } } if project.CacheDb { data, err := json.Marshal(files) if err != nil { - log.Printf("[WARNING] Failed marshalling in getfiles: %s", err) + log.Printf("[WARNING] Failed marshalling in get github files: %s", err) return files, nil } @@ -482,7 +486,7 @@ func LoadStandardFromGithub(client *github.Client, owner, repo, path, filename s } } - return files, nil + return matchingFiles, nil } func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { @@ -527,7 +531,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { user.Username = "Execution File API" } - log.Printf("[AUDIT] User %s (%s) is trying to get files from namespace %#v", user.Username, user.Id, namespace) + log.Printf("[AUDIT] User '%s' (%s) is trying to get files from namespace %#v", user.Username, user.Id, namespace) ctx := GetContext(request) files, err := GetAllFiles(ctx, user.ActiveOrg.Id, namespace) @@ -592,7 +596,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { // also be environment variables / input arguments filename, filenameOk := request.URL.Query()["filename"] if filenameOk && ArrayContains(reservedCategoryNames, namespace) { - log.Printf("[DEBUG] Found name '%s' with reserved category name: %s. Listlength: %d", filename[0], namespace, len(fileResponse.List)) + log.Printf("[DEBUG] Filename '%s' in URL with reserved category name: %s. Listlength: %d", filename[0], namespace, len(fileResponse.List)) // Load from Github repo https://github.com/Shuffle/standards filenameFound := false @@ -601,10 +605,11 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { parsedFilename = strings.Replace(parsedFilename, ".json", "", -1) } + // This is basically a unique handler for _, item := range fileResponse.List { - itemName := strings.TrimSpace(strings.Replace(strings.ToLower(item.Name), " ", "_", -1)) - if itemName == parsedFilename { + + if itemName == parsedFilename || itemName == fmt.Sprintf("%s.json", parsedFilename) { filenameFound = true break } @@ -628,9 +633,9 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed loading file from Github repo %s/%s"}`, owner, repo))) //return } else { - log.Printf("[DEBUG] Found %d file(s) in category %s for filename '%s'", len(foundFiles), namespace, filename[0]) + log.Printf("[DEBUG] Found %d file(s) in category '%s' for filename '%s'", len(foundFiles), namespace, filename[0]) for _, item := range foundFiles { - //log.printf("[DEBUG] %#v: %s", *item.Name, item.Status) + log.Printf("[DEBUG] Found file from Github '%s'", *item.Name) fileContent, _, _, err := client.Repositories.GetContents(ctx, owner, repo, *item.Path, nil) if err != nil { @@ -645,7 +650,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { continue } - //log.Printf("[DEBUG] Decoded file %s with content:\n%s", *item.Path, string(decoded)) + log.Printf("\n\n\n[DEBUG] Decoded file '%s' with content:\n%s\n\n\n", *item.Path, string(decoded)) timeNow := time.Now().Unix() fileId := "file_"+uuid.NewV4().String() @@ -693,7 +698,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { continue } - log.Printf("[DEBUG] Uploaded file %s with ID %s in category %#v", file.Filename, fileId, namespace) + log.Printf("\n\n[DEBUG] Uploaded file %s with ID %s in category %#v\n\n", file.Filename, fileId, namespace) fileResponse.List = append(fileResponse.List, BaseFile{ Name: file.Filename, @@ -867,7 +872,7 @@ func HandleGetFileContent(resp http.ResponseWriter, request *http.Request) { user.Username = "Execution File API" } - log.Printf("[AUDIT] User %s (%s) downloading file %s for org %s", user.Username, user.Id, fileId, user.ActiveOrg.Id) + log.Printf("[AUDIT] User '%s' (%s) downloading file %s in org %s", user.Username, user.Id, fileId, user.ActiveOrg.Id) // 1. Verify if the user has access to the file: org_id and workflow ctx := GetContext(request) @@ -875,7 +880,7 @@ func HandleGetFileContent(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("[ERROR] File %s not found: %s", fileId, err) resp.WriteHeader(400) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(`{"success": false, "reason": "File not found"}`)) return } diff --git a/shared.go b/shared.go index 17ff4ac..d7fa45b 100755 --- a/shared.go +++ b/shared.go @@ -3968,8 +3968,7 @@ func HandleGetTriggers(resp http.ResponseWriter, request *http.Request) { wg.Wait() - log.Println("[INFO] All Go routines Completed") - // this is to check if we got any errors without blocking the entire process + // Checks if we got any errors without blocking the entire process select { case err := <-errChan: if err != nil { @@ -3979,7 +3978,7 @@ func HandleGetTriggers(resp http.ResponseWriter, request *http.Request) { return } default: - log.Println("[INFO] No errors received within Go routines, proceeding with further logic") + //log.Println("[INFO] No errors received within Go routines, proceeding with further logic") } hooks := <-hooksChan @@ -3987,8 +3986,6 @@ func HandleGetTriggers(resp http.ResponseWriter, request *http.Request) { workflows := <-workflowsChan // pipelines := <-pipelinesChan - log.Printf("[INFO] recieved all the data from the channels") - hookMap := map[string]Hook{} scheduleMap := map[string]ScheduleOld{} // pipelineMap := map[string]Pipeline{} @@ -6171,7 +6168,7 @@ func SaveWorkflow(resp http.ResponseWriter, request *http.Request) { } } - if curapp.ID == "" { + if curapp.ID == "" && action.AppID != "integration" { log.Printf("[WARNING] Didn't find the App ID for %s", action.AppID) for _, app := range workflowapps { if app.ID == action.AppID { @@ -20746,6 +20743,8 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { } } + + threadId := value.WorkflowId if len(value.WorkflowId) > 0 { // Should maybe cache this based on the thread? Then reuse and connect? @@ -20837,7 +20836,7 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { foundCategory = org.SecurityFramework.SIEM } else { if len(foundAppType) > 0 { - log.Printf("[WARNING] Unknown app type in category action: %#v", foundAppType) + log.Printf("[ERROR] Unknown app type in category action: %#v", foundAppType) } } @@ -21001,6 +21000,53 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { return } + fieldHash := "" + fieldFileFound := false + fieldFileContentMap := map[string]interface{}{} + if len(value.Fields) > 0 { + sortedKeys := []string{} + for _, field := range value.Fields { + sortedKeys = append(sortedKeys, field.Key) + } + + sort.Strings(sortedKeys) + newFields := []Valuereplace{} + for _, key := range sortedKeys { + for _, field := range value.Fields { + if field.Key == key { + newFields = append(newFields, field) + break + } + } + } + + value.Fields = newFields + + // Md5 based on sortedKeys. Could subhash key search work? + mappedString := fmt.Sprintf("%s-%s", selectedApp.ID, value.Label, strings.Join(sortedKeys, "")) + fieldHash = fmt.Sprintf("%x", md5.Sum([]byte(mappedString))) + file, err := GetFile(ctx, fmt.Sprintf("file_%s", fieldHash)) + if err != nil { + //log.Printf("[DEBUG] Error with getting file in category action: %s", err) + } else { + //log.Printf("[DEBUG] Found file in category action: %#v", file) + if file.Status == "active" { + fieldFileFound = true + fileContent, err := GetFileContent(ctx, file, nil) + if err != nil { + log.Printf("[ERROR] Failed getting file content in category action: %s", err) + fieldFileFound = false + } + + //log.Printf("Output content: %#v", string(fileContent)) + err = json.Unmarshal(fileContent, &fieldFileContentMap) + if err != nil { + log.Printf("[ERROR] Failed unmarshaling file content in category action: %s", err) + } + } + } + } + if strings.Contains(strings.ToLower(strings.Join(selectedApp.ReferenceInfo.Triggers, ",")), "webhook") { availableLabels = append(availableLabels, "Webhook") @@ -21447,13 +21493,15 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { } } - // FIXME: Check if the organisation has a specific set of parameters for this action, mapped to the following fields: + // Check if the organisation has a specific set of parameters for this action, mapped to the following fields: selectedAction.AppID = selectedApp.ID selectedAction.AppName = selectedApp.Name selectedAction = GetOrgspecificParameters(ctx, *org, selectedAction) + log.Printf("[DEBUG] Required bodyfields: %#v", selectedAction.RequiredBodyFields) + + - //log.Printf("[DEBUG] Required bodyfields: %#v", selectedAction.RequiredBodyFields) handledRequiredFields := []string{} missingFields = []string{} for _, param := range selectedAction.Parameters { @@ -21572,13 +21620,88 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { break } + if len(fieldFileContentMap) > 0 { + log.Printf("[DEBUG] Found file content map: %#v", fieldFileContentMap) + + for key, mapValue := range fieldFileContentMap { + if _, ok := mapValue.(string); !ok { + log.Printf("[WARNING] Value for key %s is not a string: %#v", key, mapValue) + continue + } + + mappedFieldSplit := strings.Split(mapValue.(string), ".") + if len(mappedFieldSplit) == 0 { + log.Printf("[WARNING] Failed splitting value for key %s: %#v", key, mapValue) + continue + } + + // Finds the location + for _, field := range value.Fields { + if field.Key != key { + continue + } + + mapValue = field.Value + break + } + + // Check if the key exists in the parameters + for paramIndex, param := range selectedAction.Parameters { + if param.Name != mappedFieldSplit[0] { + continue + } + + foundIndex = paramIndex + if param.Name == "queries" { + if len(param.Value) == 0 { + selectedAction.Parameters[paramIndex].Value = fmt.Sprintf("%s=%s", key, mapValue.(string)) + } else { + selectedAction.Parameters[paramIndex].Value = fmt.Sprintf("%s&%s=%s", param.Value, key, mapValue.(string)) + } + + missingFields = RemoveFromArray(missingFields, key) + } else if param.Name == "body" { + + log.Printf("\n\n\n[DEBUG] Found body field for file content: %s. Location: %#v, Value: %#v\n\n\n", key, strings.Join(mappedFieldSplit, "."), mapValue) + + newBody := param.Value + + mapToSearch := map[string]interface{}{} + err := json.Unmarshal([]byte(newBody), &mapToSearch) + if err != nil { + log.Printf("[WARNING] Failed unmarshalling body for file content: %s. Body: %s", err, string(newBody)) + continue + } + + // Finds where in the body the value should be placed + location := strings.Join(mappedFieldSplit[1:], ".") + outputMap := schemaless.MapValueToLocation(mapToSearch, location, mapValue.(string)) + + // Marshal back to JSON + marshalledMap, err := json.Marshal(outputMap) + if err != nil { + log.Printf("[WARNING] Failed marshalling body for file content: %s", err) + } else { + selectedAction.Parameters[paramIndex].Value = string(marshalledMap) + missingFields = RemoveFromArray(missingFields, key) + } + } else { + log.Printf("\n\n\n[DEBUG] Found map with actionParameter %s with value %s\n\n\n", param.Name, mapValue) + } + + + break + } + } + } + // AI fallback mechanism to handle missing fields // This is in case some fields are not sent in properly orgId := "" authorization := "" optionalExecutionId := "" if len(missingFields) > 0 { - log.Printf("[DEBUG] Missing fields for action: %#v", missingFields) + //log.Printf("\n\n\n[DEBUG] Missing fields for action: %#v\n\n\n", missingFields) formattedQueryFields := []string{} for _, missing := range missingFields { @@ -21594,7 +21717,8 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { } } - formattedQuery := fmt.Sprintf("Use any of the fields '%s' with app %s to '%s'.", strings.Join(formattedQueryFields, "&"), strings.ReplaceAll(selectedApp.Name, "_", " "), strings.ReplaceAll(value.Label, "_", " ")) + //formattedQuery := fmt.Sprintf("Use any of the fields '%s' with app %s to '%s'.", strings.Join(formattedQueryFields, "&"), strings.ReplaceAll(selectedApp.Name, "_", " "), strings.ReplaceAll(value.Label, "_", " ")) + formattedQuery := fmt.Sprintf("Use the fields '%s' with app %s to '%s'.", strings.Join(formattedQueryFields, "&"), strings.ReplaceAll(selectedApp.Name, "_", " "), strings.ReplaceAll(value.Label, "_", " ")) newQueryInput := QueryInput{ Query: formattedQuery, @@ -21859,12 +21983,28 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { } } - if err == nil { if httpOutput.Status < 300 { - log.Printf("\n\n\n[DEBUG] Found VALID status: %d. Should save the current fields as new base\n\n\n", httpOutput.Status) + //log.Printf("\n\n\n[DEBUG] Found VALID status: %d. Should save the current fields as new base\n\n\n", httpOutput.Status) + parsedParameterMap := map[string]interface{}{} for _, param := range secondAction.Parameters { + if strings.Contains(param.Value, "&") && strings.Contains(param.Value, "=") { + // Split by & and then by = + parsedParameterMap[param.Name] = map[string]string{} + paramSplit := strings.Split(param.Value, "&") + for _, paramValue := range paramSplit { + paramValueSplit := strings.Split(paramValue, "=") + if len(paramValueSplit) != 2 { + continue + } + + parsedParameterMap[param.Name].(map[string]string)[paramValueSplit[0]] = paramValueSplit[1] + } + } else { + parsedParameterMap[param.Name] = param.Value + } + // FIXME: Skipping anything but body for now if param.Name != "body" { continue @@ -21875,6 +22015,72 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { log.Printf("[WARNING] Failed uploading parameter base for %s: %s", param.Name, err) } } + + if len(fieldHash) > 0 && fieldFileFound == false { + inputFieldMap := map[string]interface{}{} + for _, field := range value.Fields { + inputFieldMap[field.Key] = field.Value + } + + /* + marshalled1, err := json.Marshal(inputFieldMap) + marshalled2, err := json.Marshal(parsedParameterMap) + log.Printf("[DEBUG] Input field map: %s", string(marshalled1)) + log.Printf("[DEBUG] Parsed parameter map: %s", string(marshalled2)) + */ + + // Finds location of some data in another part of the data. This is to have a predefined location in subsequent requests + reversed, err := schemaless.ReverseTranslate(parsedParameterMap, inputFieldMap) + if err != nil { + log.Printf("[ERROR] Problem with reversing: %s", err) + } else { + finishedFields := 0 + mappedFields := map[string]string{} + err = json.Unmarshal([]byte(reversed), &mappedFields) + if err == nil { + for _, value := range mappedFields { + if len(value) > 0 { + finishedFields++ + } + } + } + + //log.Printf("Reversed (%d): %s", finishedFields, reversed) + if finishedFields > 0 { + timeNow := time.Now().Unix() + + fileId := fmt.Sprintf("file_%s", fieldHash) + encryptionKey := fmt.Sprintf("%s_%s", user.ActiveOrg.Id, fileId) + folderPath := fmt.Sprintf("%s/%s/%s", basepath, user.ActiveOrg.Id, "global") + downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId) + file := &File{ + Id: fileId, + CreatedAt: timeNow, + UpdatedAt: timeNow, + Description: "", + Status: "active", + Filename: fmt.Sprintf("%s.json", fieldHash), + OrgId: user.ActiveOrg.Id, + WorkflowId: "global", + DownloadPath: downloadPath, + Subflows: []string{}, + StorageArea: "local", + Namespace: "translation_output", + Tags: []string{ + "autocomplete", + }, + } + + returnedId, err := uploadFile(ctx, file, encryptionKey, []byte(reversed)) + if err != nil { + log.Printf("[ERROR] Problem uploading file: %s", err) + } else { + log.Printf("[DEBUG] Uploaded file with ID: %s", returnedId) + } + } + } + } + } } else { // Parses out data from the output @@ -21940,7 +22146,7 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { outputmap := make(map[string]interface{}) schemalessOutput, err := schemaless.Translate(ctx, value.Label, marshalledBody, authConfig) if err != nil { - log.Printf("[WARNING] Failed translating schemaless output for label %s: %s", value.Label, err) + log.Printf("[ERROR] Failed translating schemaless output for label '%s': %s", value.Label, err) /* err = json.Unmarshal(marshalledBody, &outputmap)