diff --git a/blobs.go b/blobs.go index db3f6f6..67327d3 100644 --- a/blobs.go +++ b/blobs.go @@ -3,21 +3,31 @@ package shuffle func GetPublicDetections() []DetectionResponse { return []DetectionResponse{ DetectionResponse{ - Title: "Sigma SIEM Detections", + Title: "Sigma SIEM Detections", DetectionName: "Sigma", Category: "SIEM", DetectionInfo: []DetectionFileInfo{}, FolderDisabled: false, IsConnectorActive: false, - DownloadRepo: "https://github.com/satti-hari-krishna-reddy/shuffle_sigma", + DownloadRepo: "https://github.com/shuffle/security-rules", }, DetectionResponse{ - Title: "Sublime Email Detection", + Title: "Sublime Email Detection", DetectionName: "Sublime", Category: "Email", DetectionInfo: []DetectionFileInfo{}, FolderDisabled: false, IsConnectorActive: false, + DownloadRepo: "https://github.com/shuffle/security-rules", + }, + DetectionResponse{ + Title: "File Detection", + DetectionName: "Yara", + Category: "Files", + DetectionInfo: []DetectionFileInfo{}, + FolderDisabled: false, + IsConnectorActive: false, + DownloadRepo: "https://github.com/shuffle/security-rules", }, } } @@ -312,14 +322,14 @@ func GetUsecaseData() string { }, { "name": "ChatOps", - "priority": 70, + "priority": 60, "type": "communication", "last": "cases", "items": {} }, { "name": "Threat Intel received", - "priority": 50, + "priority": 20, "type": "intel", "last": "cases", "items": {} diff --git a/db-connector.go b/db-connector.go index 6155ab0..49f3909 100755 --- a/db-connector.go +++ b/db-connector.go @@ -3060,7 +3060,6 @@ func GetWorkflow(ctx context.Context, id string) (*Workflow, error) { newWorkflow := FixWorkflowPosition(ctx, *workflow) workflow = &newWorkflow - if project.CacheDb && workflow.ID != "" { //log.Printf("[DEBUG] Setting cache for workflow %s", cacheKey) data, err := json.Marshal(workflow) @@ -7904,7 +7903,14 @@ func FixWorkflowPosition(ctx context.Context, workflow Workflow) Workflow { } // Fix branches & triggers + scheduleNotStarted := "" for index, trigger := range workflow.Triggers { + if trigger.TriggerType == "SCHEDULE" { + if trigger.Status != "RUNNING" { + scheduleNotStarted = trigger.ID + } + } + if trigger.ID == "" { workflow.Triggers[index].ID = uuid.NewV4().String() } @@ -7916,6 +7922,35 @@ func FixWorkflowPosition(ctx context.Context, workflow Workflow) Workflow { } } + // Check validation if Schedule is started (?) + if len(scheduleNotStarted) > 0 { + // Add validation problem + found := false + for _, problem := range workflow.Validation.Errors { + if problem.Type == "SCHEDULE" { + found = true + break + } + } + + if !found { + workflow.Validation.Errors = append(workflow.Validation.Errors, ValidationProblem{ + Order: -1, + Type: "SCHEDULE", + ActionId: scheduleNotStarted, + Error: "Schedule not started", + }) + } + } + + if len(workflow.Validation.Errors) == 0 { + workflow.Validation.Errors = []ValidationProblem{} + } + + if len(workflow.Validation.SubflowApps) == 0 { + workflow.Validation.SubflowApps = []ValidationProblem{} + } + return workflow } diff --git a/detection.go b/detection.go index 9db4c67..e646899 100644 --- a/detection.go +++ b/detection.go @@ -10,22 +10,25 @@ import ( "net/http" "os" + "errors" "sort" "strings" "time" - "errors" uuid "github.com/satori/go.uuid" "gopkg.in/yaml.v2" ) -func HandleTenzirHealthUpdate(resp http.ResponseWriter, request *http.Request) { +func HandleDetectionHealthUpdate(resp http.ResponseWriter, request *http.Request) { if request.Method != "POST" { request.Method = "POST" } type HealthUpdate struct { - Status string `json:"status"` + OrgId string `json:"org_id"` + Status string `json:"status"` + Environment string `json:"environment"` + Authorization string `json:"authorization"` } var healthUpdate HealthUpdate @@ -36,25 +39,27 @@ func HandleTenzirHealthUpdate(resp http.ResponseWriter, request *http.Request) { return } + log.Printf("[DEBUG] Tenzir health update: %#v", healthUpdate) + //ctx := context.Background() /* - status := healthUpdate.Status - result, err := GetDisabledRules(ctx, user.ActiveOrg.Id) - if (err != nil && err.Error() == "rules doesn't exist") || err == nil { - result.IsTenzirActive = status - result.LastActive = time.Now().Unix() + status := healthUpdate.Status + result, err := GetDisabledRules(ctx, user.ActiveOrg.Id) + if (err != nil && err.Error() == "rules doesn't exist") || err == nil { + result.IsTenzirActive = status + result.LastActive = time.Now().Unix() - err = StoreDisabledRules(ctx, *result) - if err != nil { - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) + err = StoreDisabledRules(ctx, *result) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) return } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) - return - } */ resp.WriteHeader(500) @@ -634,7 +639,9 @@ func HandleDetectionAutoConnect(resp http.ResponseWriter, request *http.Request) } detectionType := strings.ToLower(location[4]) - log.Printf("[DEBUG] Validating if the org %s (%s) has an %d sandbox handling workflow/system", user.ActiveOrg.Name, user.ActiveOrg.Id, detectionType) + log.Printf("[DEBUG] Validating if the org %s (%s) has a %s sandbox handling workflow/system", user.ActiveOrg.Name, user.ActiveOrg.Id, detectionType) + + workflow := Workflow{} if detectionType == "siem" { log.Printf("[AUDIT] User '%s' (%s) is trying to connect to SIEM", user.Username, user.Id) @@ -642,15 +649,14 @@ func HandleDetectionAutoConnect(resp http.ResponseWriter, request *http.Request) execType := "START_TENZIR" err = SetDetectionOrborusRequest(ctx, user.ActiveOrg.Id, execType, "", "SIGMA", "SHUFFLE_DISCOVER") if err != nil { - if strings.Contains(strings.ToLower(err.Error()), "must be started") { + if strings.Contains(strings.ToLower(err.Error()), "must be started") { resp.WriteHeader(200) resp.Write([]byte(`{"success": true, "reason": "Please start the environment by running the relevant command.", "action": "environment_start"}`)) return } - log.Printf("[ERROR] Failed setting workflow queue for env: %s", err) - if strings.Contains(strings.ToLower(err.Error()), "no valid environments") { + if strings.Contains(strings.ToLower(err.Error()), "no valid environments") { resp.WriteHeader(400) resp.Write([]byte(`{"success": false, "reason": "No valid environments found. Go to /admin?tab=environments to create one.", "action": "environment_create"}`)) return @@ -662,65 +668,36 @@ func HandleDetectionAutoConnect(resp http.ResponseWriter, request *http.Request) } } else if detectionType == "email" { - // FIXME: + // FIXME: // 1. Can we track if it's active based on a workflow + validation? // 2. The workflow should get email // 3. It should track unread AND read emails separately // 4. When a new email is received, we should automatically track the statistics for it ctx := GetContext(request) - workflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "") - if err != nil && len(workflows) == 0 { - log.Printf("[ERROR] Failed to loading workflows to validate email: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed to load workflows"}`)) - return - } - - foundWorkflow := false - workflowValid := false - for _, workflow := range workflows { - if workflow.WorkflowType != "EMAIL-DETECTION" { - continue - } - - if workflow.Validation.Valid { - workflowValid = true - } - - foundWorkflow = true - break - } - - if foundWorkflow { - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "valid": "%v", "reason": "Email workflow found"}`, workflowValid))) - return - } - - err = CreateDetectionWorkflow(ctx, user.ActiveOrg.Id, "EMAIL-DETECTION") + workflow, err = ConfigureDetectionWorkflow(ctx, user.ActiveOrg.Id, "EMAIL-DETECTION") if err != nil { - log.Printf("[ERROR] Failed to create email handling workflow: %s", err) + log.Printf("\n\n\n[ERROR] Failed to create email handling workflow: %s\n\n\n", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed to create email handling workflow. Please try again or contact support@shuffler.io"}`)) return } - log.Printf("[INFO] Email handling workflow created successfully") - } else { resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Detection Type '%s' not implemented"}`, detectionType))) return } - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} + success := true + if len(workflow.ID) == 0 { + success = false + } -func CreateDetectionWorkflow(ctx context.Context, orgId, workflowType string) error { - log.Printf("[DEBUG] Creating detection workflow for org %s (not implemented)", orgId) - return nil + log.Printf("[INFO] '%s' detection workflow in org '%s' ID: %s", detectionType, workflow.OrgId, workflow.ID) + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": %v, "workflow_id": "%s", "workflow_valid": %v}`, success, workflow.ID, workflow.Validation.Valid))) } func SetDetectionOrborusRequest(ctx context.Context, orgId, execType, fileName, executionSource, environmentName string) error { @@ -754,10 +731,10 @@ func SetDetectionOrborusRequest(ctx context.Context, orgId, execType, fileName, log.Printf("[DEBUG] Found %d potentially valid environment(s)", len(selectedEnvironments)) /* - if len(selectedEnvironments) == 0 || environmentName == "SHUFFLE_DISCOVER" { - // FIXME: Get based on the Organisation. This is only tested onprem so far, so there's a lot to do to make this stable ROFL - log.Printf("[DEBUG] Automatically discovering the right environment from '%s'", environmentName) - } + if len(selectedEnvironments) == 0 || environmentName == "SHUFFLE_DISCOVER" { + // FIXME: Get based on the Organisation. This is only tested onprem so far, so there's a lot to do to make this stable ROFL + log.Printf("[DEBUG] Automatically discovering the right environment from '%s'", environmentName) + } */ if len(selectedEnvironments) == 0 { @@ -807,16 +784,16 @@ func HandleListDetectionCategories(resp http.ResponseWriter, request *http.Reque } /* - user, err := HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("[WARNING] Api authentication failed in get detection rules: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } + user, err := HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("[WARNING] Api authentication failed in get detection rules: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } */ - publicDetections := GetPublicDetections() + publicDetections := GetPublicDetections() data, err := json.Marshal(publicDetections) if err != nil { resp.WriteHeader(500) @@ -827,3 +804,196 @@ func HandleListDetectionCategories(resp http.ResponseWriter, request *http.Reque resp.WriteHeader(200) resp.Write(data) } + +func ConfigureDetectionWorkflow(ctx context.Context, orgId, workflowType string) (Workflow, error) { + log.Printf("\n\n[DEBUG] Creating detection workflow for org %s (not implemented)\n\n", orgId) + /* + // FIXME: Use Org to find the correct tools according to the Usecase + // SHOULD map usecase from workflowType -> actual Usecase in blobs + foundOrg, err := GetOrg(ctx, orgId) + if err != nil { + log.Printf("[ERROR] Failed to get org '%s' during detection workflow creation: %s", err) + return err + } + */ + + user := User{ + Role: "admin", + ActiveOrg: OrgMini{ + Id: orgId, + }, + } + + workflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "") + if err != nil && len(workflows) == 0 { + log.Printf("[ERROR] Failed to loading workflows to validate email: %s", err) + return Workflow{}, err + } + + workflow := Workflow{} + workflowValid := false + for _, foundworkflow := range workflows { + if foundworkflow.WorkflowType != workflowType { + continue + } + + if foundworkflow.Validation.Valid { + workflowValid = true + } + + workflow = foundworkflow + break + } + + _ = workflowValid + if len(workflow.ID) > 0 { + return workflow, nil + } + + workflow = Workflow{ + WorkflowType: workflowType, + Actions: []Action{}, + Triggers: []Trigger{}, + } + + // Do this based on public workflows + cloudWorkflowId := "" + usecaseNames := []string{} + if workflowType == "TENZIR-SIGMA" { + } else if workflowType == "EMAIL-DETECTION" { + // How do we check what email tool they use? + //log.Printf("[INFO] Creating email handling workflow for org %s", orgId) + + cloudWorkflowId = "31d1a492-9fe0-4c4a-807d-b44d9cb81fc0" + usecaseNames = []string{"Search emails (Sublime)"} + } + + if len(cloudWorkflowId) == 0 { + return workflow, errors.New("No valid workflow found") + } + + // Load it in from cloud with a normal GET request + url := fmt.Sprintf("https://shuffler.io/api/v1/workflows/%s", cloudWorkflowId) + client := GetExternalClient(url) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + log.Printf("[ERROR] Failed to create request for workflow: %s", err) + return workflow, err + } + + resp, err := client.Do(req) + if err != nil { + log.Printf("[ERROR] Failed to get workflow from cloud: %s", err) + return workflow, err + } + + defer resp.Body.Close() + if resp.StatusCode != 200 { + log.Printf("[ERROR] Failed to get workflow from cloud: %s", resp.Status) + return workflow, errors.New("Failed to get workflow from cloud") + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Printf("[ERROR] Failed to read response body: %s", err) + return workflow, err + } + + err = json.Unmarshal(body, &workflow) + if err != nil { + log.Printf("[ERROR] Failed to unmarshal response body: %s", err) + return workflow, err + } + + // Clear out and reset IDs + workflow.Created = time.Now().Unix() + workflow.ID = uuid.NewV4().String() + workflow.OrgId = orgId + workflow.Org = []OrgMini{ + OrgMini{ + Id: orgId, + }, + } + workflow.ExecutingOrg = OrgMini{ + Id: orgId, + } + workflow.Public = false + workflow.WorkflowType = workflowType + workflow.Validation = TypeValidation{} + + for _, usecaseName := range usecaseNames { + workflow.UsecaseIds = append(workflow.UsecaseIds, usecaseName) + } + + workflow.ParentWorkflowId = "" + for actionIndex, _ := range workflow.Actions { + newId := uuid.NewV4().String() + + if workflow.Start == workflow.Actions[actionIndex].ID { + workflow.Start = newId + } + + for branchIndex, _ := range workflow.Branches { + if workflow.Actions[actionIndex].ID == workflow.Branches[branchIndex].SourceID { + workflow.Branches[branchIndex].SourceID = newId + } + + if workflow.Actions[actionIndex].ID == workflow.Branches[branchIndex].DestinationID { + workflow.Branches[branchIndex].DestinationID = newId + } + } + + workflow.Actions[actionIndex].ID = newId + } + + for triggerIndex, _ := range workflow.Triggers { + newId := uuid.NewV4().String() + + for branchIndex, _ := range workflow.Branches { + if workflow.Triggers[triggerIndex].ID == workflow.Branches[branchIndex].SourceID { + workflow.Branches[branchIndex].SourceID = newId + } + + if workflow.Triggers[triggerIndex].ID == workflow.Branches[branchIndex].DestinationID { + workflow.Branches[branchIndex].DestinationID = newId + } + } + + workflow.Triggers[triggerIndex].ID = newId + + // FIXME: Check if it's a schedule, then set the interval + start it + if workflow.Triggers[triggerIndex].TriggerType == "schedule" { + //workflow.Triggers[triggerIndex].Interval = 60 + for paramIndex, param := range workflow.Triggers[triggerIndex].Parameters { + if param.Name == "interval" { + if project.Environment == "cloud" { + param.Value = "*/5 * * * *" + } else { + param.Value = "300" + } + } + + workflow.Triggers[triggerIndex].Parameters[paramIndex] = param + } + + // FIXME: Start the schedule automatically + } + } + + /* + for branchIndex, _ := range workflow.Branches { + workflow.Branches[branchIndex].ID = uuid.NewV4().String() + } + */ + + // FIXME: Add a changeout for ANY schemaless node to use the correct + // action in it + log.Printf("[DEBUG] Saving workflow for org %s", orgId) + err = SetWorkflow(ctx, workflow, workflow.ID) + if err != nil { + log.Printf("[ERROR] Failed to set workflow during detection save: %s", err) + return Workflow{}, err + } + + return workflow, nil +} diff --git a/shared.go b/shared.go index 4b7230b..c5adf77 100755 --- a/shared.go +++ b/shared.go @@ -162,17 +162,17 @@ func randStr(strSize int, randType string) string { } func isLoop(arg string) bool { - if strings.Contains(arg, "$") && (strings.HasSuffix(arg, ".#") || strings.Contains(arg, ".#.")){ - return true - } + if strings.Contains(arg, "$") && (strings.HasSuffix(arg, ".#") || strings.Contains(arg, ".#.")) { + return true + } - if strings.Contains(arg,"$") && strings.Contains(arg, ".#") { - pattern := `(^|\.)(#(\d+-\d+)?($|\.))` - re := regexp.MustCompile(pattern) - return strings.Contains(arg, "$") && re.MatchString(arg) - } + if strings.Contains(arg, "$") && strings.Contains(arg, ".#") { + pattern := `(^|\.)(#(\d+-\d+)?($|\.))` + re := regexp.MustCompile(pattern) + return strings.Contains(arg, "$") && re.MatchString(arg) + } - return false + return false } func HandleSet2fa(resp http.ResponseWriter, request *http.Request) { @@ -2039,6 +2039,9 @@ func AddAppAuthentication(resp http.ResponseWriter, request *http.Request) { appAuth.App.LargeImage = app.LargeImage } + // If editing, reset verification? + appAuth.Validation = TypeValidation{} + appAuth.OrgId = user.ActiveOrg.Id appAuth.Defined = true err = SetWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) @@ -4190,7 +4193,7 @@ func GetWorkflowExecutionsV2(resp http.ResponseWriter, request *http.Request) { for index, execution := range workflowExecutions { if project.Environment != "cloud" && execution.Status != "FINISHED" && execution.Status != "ABORTED" { - execution, _ = Fixexecution(ctx, execution) + execution, _ = Fixexecution(ctx, execution) } newResults := []ActionResult{} @@ -4241,7 +4244,7 @@ func GetWorkflowExecutionsV2(resp http.ResponseWriter, request *http.Request) { workflowExecutions[index].Workflow.Image = "" workflowExecutions[index].Workflow.Triggers = newTriggers - if workflowExecutions[index].Status != "EXECUTION" && workflowExecutions[index].Workflow.Validation.Valid == false && len(workflowExecutions[index].Workflow.Validation.Problems) == 0 && len(workflowExecutions[index].Workflow.Validation.SubflowApps) == 0 { + if workflowExecutions[index].Status != "EXECUTION" && workflowExecutions[index].Workflow.Validation.Valid == false && len(workflowExecutions[index].Workflow.Validation.Errors) == 0 && len(workflowExecutions[index].Workflow.Validation.SubflowApps) == 0 { validation, err := GetExecutionValidation(ctx, workflowExecutions[index].ExecutionId) if err == nil { workflowExecutions[index].Workflow.Validation = validation @@ -5675,16 +5678,18 @@ func SetNewWorkflow(resp http.ResponseWriter, request *http.Request) { newTriggers = append(newTriggers, item) } - newSchedules := []Schedule{} - for _, item := range workflow.Schedules { - item.Id = uuid.NewV4().String() - newSchedules = append(newSchedules, item) - } + /* + newSchedules := []Schedule{} + for _, item := range workflow.Schedules { + item.Id = uuid.NewV4().String() + newSchedules = append(newSchedules, item) + } + */ timeNow := int64(time.Now().Unix()) workflow.Actions = newActions workflow.Triggers = newTriggers - workflow.Schedules = newSchedules + //workflow.Schedules = newSchedules workflow.IsValid = true workflow.Configuration.ExitOnError = false workflow.Created = timeNow @@ -10824,7 +10829,7 @@ func HandleChangeUserOrg(resp http.ResponseWriter, request *http.Request) { } //if org.SSOConfig.SSORequired == true && !ArrayContains(user.ValidatedSessionOrgs, tmpData.OrgId) && user.SupportAccess == false { - if org.SSOConfig.SSORequired == true && !ArrayContains(user.ValidatedSessionOrgs, tmpData.OrgId) { + if org.SSOConfig.SSORequired == true && !ArrayContains(user.ValidatedSessionOrgs, tmpData.OrgId) && user.SupportAccess == false { baseSSOUrl := org.SSOConfig.SSOEntrypoint redirectKey := "SSO_REDIRECT" @@ -11580,7 +11585,6 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { //if org.LeadInfo != newLeadinfo { log.Printf("[DEBUG] Lead info updated for %s (%s) from %#v to %#v", org.Id, org.Name, org.LeadInfo, newLeadinfo) - // Check for ORG_CHANGE_WEBHOOK orgWebhook := os.Getenv("ORG_CHANGE_WEBHOOK") if orgWebhook != "" && strings.HasPrefix(orgWebhook, "http") { @@ -14020,12 +14024,12 @@ func updateExecutionParent(ctx context.Context, executionParent, returnValue, pa } else { // In here maning no-loop? /* - actionValue := SubflowData{ - Success: true, - ExecutionId: executionParent, - Authorization: parentAuth, - Result: returnValue, - } + actionValue := SubflowData{ + Success: true, + ExecutionId: executionParent, + Authorization: parentAuth, + Result: returnValue, + } */ actionValue := SubflowData{ @@ -14042,7 +14046,7 @@ func updateExecutionParent(ctx context.Context, executionParent, returnValue, pa } else { mappedData, ok := cacheData.([]byte) - if ok { + if ok { // Unmarshal it into actionValue err = json.Unmarshal(mappedData, &actionValue) if err != nil { @@ -15486,8 +15490,6 @@ func ParsedExecutionResult(ctx context.Context, workflowExecution WorkflowExecut newCacheKey := fmt.Sprintf("%s_%s_sinkholed_result", workflowExecution.ExecutionId, actionResult.Action.ID) go SetCache(ctx, newCacheKey, []byte(actionResult.Result), 35) - - if jsonerr == nil && len(subflowData.Result) == 0 && !strings.Contains(actionResult.Result, "\"result\"") { log.Printf("[INFO][%s] NO RESULT FOR SUBFLOW RESULT - SETTING TO EXECUTING. Results: %d. Trying to find subexec in cache onprem", workflowExecution.ExecutionId, len(workflowExecution.Results)) @@ -16883,7 +16885,7 @@ func HandleDeleteCacheKey(resp http.ResponseWriter, request *http.Request) { cacheKey, err = url.QueryUnescape(strings.Trim(cacheKey, " ")) if err != nil { - log.Printf("[WARNING] Failed to unescape cache key %s: %s", cacheKey,err) + log.Printf("[WARNING] Failed to unescape cache key %s: %s", cacheKey, err) cacheKey = strings.Trim(cacheKey, " ") } @@ -17624,7 +17626,7 @@ func CheckHookAuth(request *http.Request, auth string) error { } // Body = The action body received from the user to test. -func PrepareSingleAction(ctx context.Context, user User, fileId string, body []byte) (WorkflowExecution, error) { +func PrepareSingleAction(ctx context.Context, user User, fileId string, body []byte, runValidationAction bool) (WorkflowExecution, error) { workflowExecution := WorkflowExecution{} var action Action @@ -17665,6 +17667,21 @@ func PrepareSingleAction(ctx context.Context, user User, fileId string, body []b } } + if runValidationAction { + log.Printf("[INFO] Running validation action for %s for org %s (%s)", app.Name, user.ActiveOrg.Name, user.ActiveOrg.Id) + + // Find the action tagged to be used for validation: + // 1. Find the action in the app + // 2. Find a GET request that is labeled without required parameters + // 3. Run the default request that is sent in IF possible + + // Validation of the workflow will say whether it was successful or not + + //for _, appAction := range app.Actions { + + //} + } + newParams := []WorkflowAppActionParameter{} /* for _, param := range action.Parameters { @@ -17758,7 +17775,6 @@ func PrepareSingleAction(ctx context.Context, user User, fileId string, body []b */ // Make a fake request object as it's not necessary - badRequest := &http.Request{} if user.ActiveOrg.Id != "" { workflow.Owner = user.Id @@ -17769,6 +17785,7 @@ func PrepareSingleAction(ctx context.Context, user User, fileId string, body []b } // Add fake queries to it + badRequest := &http.Request{} badRequest.URL, _ = url.Parse(fmt.Sprintf("http://localhost:3000/api/v1/workflows/%s/execute", workflow.ID)) badRequest.URL.RawQuery = fmt.Sprintf("") badRequest.Method = "GET" @@ -17802,6 +17819,7 @@ func HandleRetValidation(ctx context.Context, workflowExecution WorkflowExecutio Authorization: workflowExecution.Authorization, Result: "", Errors: []string{}, + Validation: workflowExecution.Workflow.Validation, } // VERY short sleeptime here on purpose @@ -17820,9 +17838,11 @@ func HandleRetValidation(ctx context.Context, workflowExecution WorkflowExecutio break } + returnBody.Validation = newExecution.Workflow.Validation + //log.Printf("[INFO] Checking single execution %s. Status: %s. Len: %d, resultAmount: %d", workflowExecution.ExecutionId, newExecution.Status, len(newExecution.Results), resultAmount-1) - if len(newExecution.Results) > resultAmount-1 { + if len(newExecution.Results) > resultAmount-1 { relevantIndex := len(newExecution.Results) - 1 if len(newExecution.Results[relevantIndex].Result) > 0 || newExecution.Results[relevantIndex].Status == "SUCCESS" { @@ -17838,7 +17858,11 @@ func HandleRetValidation(ctx context.Context, workflowExecution WorkflowExecutio returnBody.Errors = append(returnBody.Errors, param.Value) } } - break + + // Wait for validation to have ran + if newExecution.Workflow.Validation.ValidationRan { + break + } } } @@ -19928,7 +19952,7 @@ func PrepareWorkflowExecution(ctx context.Context, workflow Workflow, request *h if err != nil { log.Printf("[ERROR][%s] Failed marshalling userinput result: %s", oldExecution.ExecutionId, err) - } else { + } else { actionCacheId := fmt.Sprintf("%s_%s_result", result.ExecutionId, result.Action.ID) err = SetCache(ctx, actionCacheId, fullMarshal, 35) if err != nil { @@ -20007,7 +20031,6 @@ func PrepareWorkflowExecution(ctx context.Context, workflow Workflow, request *h backendUrl = fmt.Sprintf("http://localhost:%d", port) } - if project.Environment == "cloud" && len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 { backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) } @@ -23716,7 +23739,6 @@ func GetExternalClient(baseUrl string) *http.Client { } } - if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" { //client = &http.Client{} } else { @@ -28195,12 +28217,21 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo } originalActions := workflow.Actions - workflow.Actions = newActions + + if len(newActions) > 0 { + workflow.Actions = newActions + } + + if len(workflow.Actions) == 0 { + workflow.Actions = exec.Workflow.Actions + } authenticationProblems := []ValidationProblem{} handledAuth := []string{} timenow := time.Now().Unix() * 1000 + + //log.Printf("\n\n[DEBUG][%s] STARTING VALIDATION WITH %d results and %d actions\n\n", exec.ExecutionId, len(exec.Results), len(workflow.Actions)) for _, result := range exec.Results { // FIXME: Skipping anything that outright fails right now if result.Status != "SUCCESS" { @@ -28214,6 +28245,8 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo continue } + found = true + authRequired := false for _, param := range action.Parameters { @@ -28241,7 +28274,6 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo break } - found = true foundAction = action break } @@ -28332,7 +28364,7 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo // FIXME: What do we do here if there is no reason? } - //log.Printf("[DEBUG][%s] Checking result for %s", exec.ExecutionId, result.Action.Label) + //log.Printf("\n\n\n[DEBUG][%s] Checking result for %s\n\n\n", exec.ExecutionId, result.Action.Label) handledAuth = append(handledAuth, foundAction.AuthenticationId) for _, auth := range allAuth { if auth.Id != foundAction.AuthenticationId { @@ -28374,6 +28406,12 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo auth.Validation.ExecutionId = exec.ExecutionId auth.Validation.NodeId = result.Action.ID + auth.Validation.ValidationRan = true + + if len(auth.App.LargeImage) == 0 { + auth.App.LargeImage = result.Action.LargeImage + } + err = SetWorkflowAppAuthDatastore(ctx, auth, auth.Id) if err != nil { log.Printf("[ERROR] Failed updating auth at end of workflow run %s: %s", auth.Id, err) @@ -28423,7 +28461,6 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo } } - if foundWorkflow == "" { continue } @@ -28435,11 +28472,11 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo continue } - marshalledListData := []SubflowData{} + marshalledListData := []SubflowData{} err := json.Unmarshal([]byte(res.Result), &marshalledListData) if err != nil { log.Printf("[ERROR] Failed unmarshalling subflow data for %s: %s", res.Action.Label, err) - marshalledData := SubflowData{} + marshalledData := SubflowData{} err := json.Unmarshal([]byte(res.Result), &marshalledData) if err != nil { log.Printf("[ERROR] Failed unmarshalling subflow data for %s: %s", res.Action.Label, err) @@ -28473,22 +28510,22 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo // FIXME: Check based on the workflow itself instead //log.Printf("[DEBUG] Subflow %s is still executing. Validation: %s", execId, subExec.Workflow.Validation.Valid) - // Loading the Workflows own validation in this case + // Loading the Workflows own validation in this case oldWf, err := GetWorkflow(ctx, subExec.Workflow.ID) if err != nil { log.Printf("[ERROR] Failed getting subflow %s for workflow %s: %s", subExec.Workflow.ID, workflow.ID, err) } else { subExec.Workflow = *oldWf } - } + } // Check validations - //log.Printf("[DEBUG] Subflow %s is finished. Validation: %#v. Validation Problems: %d", execId, subExec.Workflow.Validation.Valid, len(subExec.Workflow.Validation.Problems)) + //log.Printf("[DEBUG] Subflow %s is finished. Validation: %#v. Validation.Errors: %d", execId, subExec.Workflow.Validation.Valid, len(subExec.Workflow.Validation.Errors)) if subExec.Workflow.Validation.Valid { continue } - for _, subProblem := range subExec.Workflow.Validation.Problems { + for _, subProblem := range subExec.Workflow.Validation.Errors { // We keep appending for each level if ArrayContains(appendedActionIds, subProblem.ActionId) { continue @@ -28538,8 +28575,8 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo workflow.Validation.SubflowApps = newApps workflowChanged := false - workflow.Validation.Problems = authenticationProblems - if len(workflow.Validation.Problems) > 0 { + workflow.Validation.Errors = authenticationProblems + if len(workflow.Validation.Errors) > 0 { workflow.Validation.Valid = false } else { workflow.Validation.Valid = true @@ -28552,10 +28589,11 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo workflow.Validation.ExecutionId = exec.ExecutionId } - workflow.Validation.TotalProblems = len(workflow.Validation.Problems) + len(workflow.Validation.SubflowApps) + workflow.Validation.TotalProblems = len(workflow.Validation.Errors) + len(workflow.Validation.SubflowApps) // Updating the workflow to show the right status every time for now workflowChanged = true + workflow.Validation.ValidationRan = true workflow.Validation.ExecutionId = exec.ExecutionId if workflowChanged { workflow.Actions = originalActions @@ -28574,12 +28612,12 @@ func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *Workflo // Force them to work without parent context management backgroundContext := context.Background() - go SetCache(backgroundContext, fmt.Sprintf("validation_workflow_%s", workflow.ID), marshalledValidation, 1440) - go SetCache(backgroundContext, cacheKey, marshalledValidation, 120) + SetCache(backgroundContext, fmt.Sprintf("validation_workflow_%s", workflow.ID), marshalledValidation, 1440) + SetCache(backgroundContext, cacheKey, marshalledValidation, 120) // ALWAYS have correct exec id for current execution, but not always in workflow - //log.Printf("\n\n[DEBUG][%s] Set workflow validation (%d) to '%s'\n\n", exec.ExecutionId, len(workflow.Validation.Problems), marshalledValidation) + //log.Printf("\n\n[DEBUG][%s] Set workflow validation (%d) to '%s'\n\n", exec.ExecutionId, len(workflow.Validation.Errors), marshalledValidation) return exec } @@ -28751,4 +28789,3 @@ func HandleUserPrivateTraining(resp http.ResponseWriter, request *http.Request) resp.WriteHeader(http.StatusOK) resp.Write([]byte(`{"success": true}`)) } - diff --git a/structs.go b/structs.go index cae95b0..b5a60b5 100755 --- a/structs.go +++ b/structs.go @@ -1245,12 +1245,11 @@ type InputQuestion struct { } type Workflow struct { - Actions []Action `json:"actions" datastore:"actions,noindex"` - Branches []Branch `json:"branches" datastore:"branches,noindex"` - VisualBranches []Branch `json:"visual_branches" datastore:"visual_branches,noindex"` - Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` - Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` - Comments []Comment `json:"comments" datastore:"comments,noindex"` + Actions []Action `json:"actions" datastore:"actions,noindex"` + Branches []Branch `json:"branches" datastore:"branches,noindex"` + VisualBranches []Branch `json:"visual_branches" datastore:"visual_branches,noindex"` + Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` + Comments []Comment `json:"comments" datastore:"comments,noindex"` Configuration struct { ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` @@ -1440,10 +1439,10 @@ type File struct { } type DisabledRules struct { - Files []File `json:"files" datastore:"files"` - DisabledFolder bool `json:"disabled_folder" datastore:"disabled_folder"` - IsTenzirActive string `json:"tenzir_active" datastore:"tenzir_active"` - LastActive int64 `json:"last_active" datastore:"last_active"` + Files []File `json:"files" datastore:"files"` + DisabledFolder bool `json:"disabled_folder" datastore:"disabled_folder"` + DetectionActive string `json:"detection_active" datastore:"detection_active"` + LastActive int64 `json:"last_active" datastore:"last_active"` } type SelectedDetectionRules struct { @@ -1487,18 +1486,19 @@ type ValidationProblem struct { } type TypeValidation struct { - Valid bool `json:"valid" datastore:"valid"` - ChangedAt int64 `json:"changed_at" datastore:"changed_at"` - LastValid int64 `json:"last_valid" datastore:"last_valid"` + Valid bool `json:"valid" datastore:"valid"` + ChangedAt int64 `json:"changed_at" datastore:"changed_at"` + LastValid int64 `json:"last_valid" datastore:"last_valid"` + ValidationRan bool `json:"validation_ran" datastore:"validation_ran"` // For the last update, which did it WorkflowId string `json:"workflow_id" datastore:"workflow_id"` ExecutionId string `json:"execution_id" datastore:"execution_id"` NodeId string `json:"node_id" datastore:"node_id"` - TotalProblems int `json:"total_problems" datastore:"total_problems"` - Problems []ValidationProblem `json:"errors" datastore:"errors"` - SubflowApps []ValidationProblem `json:"subflow_apps" datastore:"subflow_apps"` + TotalProblems int `json:"total_problems" datastore:"total_problems"` + Errors []ValidationProblem `json:"errors" datastore:"errors"` + SubflowApps []ValidationProblem `json:"subflow_apps" datastore:"subflow_apps"` } type AppAuthenticationStorage struct { @@ -3742,11 +3742,12 @@ type AppCategory struct { } type SingleResult struct { - Success bool `json:"success"` - Id string `json:"id"` - Authorization string `json:"authorization"` - Result string `json:"result"` - Errors []string `json:"errors"` + Success bool `json:"success"` + Id string `json:"id"` + Authorization string `json:"authorization"` + Result string `json:"result"` + Errors []string `json:"errors"` + Validation TypeValidation `json:"validation"` } type DockerRequestCheck struct {