From 20dffac8b36de41ab5974ff3a1864303c77e4ece Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 22 Sep 2024 13:38:48 +0200 Subject: [PATCH 1/2] Fixed a stat based search system on /api/v1/stats/{statskey} and /api/v1/orgs/{orgid}/stats/{statskey} --- stats.go | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/stats.go b/stats.go index dbdd8db..34d2a5e 100755 --- a/stats.go +++ b/stats.go @@ -303,6 +303,13 @@ func GetSpecificStats(resp http.ResponseWriter, request *http.Request) { } } + if len(statsKey) <= 1 { + log.Printf("[WARNING] Invalid stats key: %s", statsKey) + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "Invalid stats key"}`)) + return + } + user, err := HandleApiAuthentication(resp, request) if err != nil { log.Printf("[WARNING] Api authentication failed in get stats: %s", err) @@ -321,10 +328,36 @@ func GetSpecificStats(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("RESP: %#v", info) + totalEntires := 0 + totalValue := 0 + statEntries := []AdditionalUseConfig{} + + info.DailyStatistics = append(info.DailyStatistics, DailyStatistics{ + Additions: info.Additions, + }) + + for _, daily := range info.DailyStatistics { + for _, addition := range daily.Additions { + if addition.Key == statsKey { + totalEntires++ + totalValue += int(addition.Value) + statEntries = append(statEntries, addition) + + break + } + } + } + + marshalledEntries, err := json.Marshal(statEntries) + if err != nil { + log.Printf("[ERROR] Failed marshal in get org stats: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking data for org stats"}`))) + return + } resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "key": "%s", "value": 2}`, statsKey))) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "key": "%s", "total": %d, "entries": %s}`, statsKey, totalValue, string(marshalledEntries)))) } func HandleGetStatistics(resp http.ResponseWriter, request *http.Request) { From 16ad61626af8c83dd1a979a5aad482952fac8df7 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 23 Sep 2024 01:26:20 +0200 Subject: [PATCH 2/2] Fixed autolabeling and categorization for apps when they are automatically tried --- kms.go | 285 +++++++++++++++++++++++++++++++++++++++++++++-------- shared.go | 79 ++++++++++++--- stats.go | 46 ++++++--- structs.go | 26 ++--- 4 files changed, 355 insertions(+), 81 deletions(-) diff --git a/kms.go b/kms.go index 6e9e8ab..6b36503 100644 --- a/kms.go +++ b/kms.go @@ -24,7 +24,8 @@ import ( ) //var model = "gpt-4-turbo-preview" -var model = "gpt-4o" +//var model = "gpt-4o" +var model = "gpt-4o-mini" func GetKmsCache(ctx context.Context, auth AppAuthenticationStorage, key string) (string, error) { //log.Printf("\n\n[DEBUG] Getting KMS cache for key %s\n\n", key) @@ -533,7 +534,7 @@ func FindNextApiStep(action Action, stepOutput []byte, additionalInfo, inputdata } } - log.Printf("[DEBUG] Status: %d, ok: %t", status, bodyOk) + log.Printf("[DEBUG] Previous Status: %d, ok: %t", status, bodyOk) if bodyOk { if val, ok := body1.(map[string]interface{}); ok { @@ -611,7 +612,8 @@ func RunSelfCorrectingRequest(action Action, status int, additionalInfo, outputB // Add all fields with value from here inputBody := "{\n" for _, param := range action.Parameters { - if param.Name == "headers" || param.Name == "ssl_verify" || param.Name == "to_file" || param.Name == "url" || strings.Contains(param.Name, "username_") || strings.Contains(param.Name, "password_") { + //if param.Name == "headers" || param.Name == "ssl_verify" || param.Name == "to_file" || param.Name == "url" || strings.Contains(param.Name, "username_") || strings.Contains(param.Name, "password_") { + if param.Name == "ssl_verify" || param.Name == "to_file" || param.Name == "url" || strings.Contains(param.Name, "username_") || strings.Contains(param.Name, "password_") { continue } @@ -755,7 +757,6 @@ func RunSelfCorrectingRequest(action Action, status int, additionalInfo, outputB // Make map from val and marshal to byte stringType := reflect.TypeOf(val).String() - log.Printf("STRINGTYPE: %#v", stringType) if stringType == "map[string]interface {}" { valByte, err := json.Marshal(val) if err != nil { @@ -835,26 +836,36 @@ func getBadOutputString(action Action, appname, inputdata, outputBody string, st outputData := fmt.Sprintf("Fields: %s\n\nHTTP Status: %d\nHTTP error: %s", outputParams, status, outputBody) - log.Printf("[DEBUG] Skipping output formatting") + log.Printf("[DEBUG] Skipping output formatting (bad output string)") //errorString := handleOutputFormatting(string(outputData), inputdata, appname) return outputData } func RunAiQuery(systemMessage, userMessage string) (string, error) { - if len(systemMessage) > 10000 || len(userMessage) > 10000 { - return "", errors.New("Message too long for general usage. Max 10000 characters for system & user message") + maxTokens := 5000 + maxCharacters := 100000 + //if len(systemMessage) > maxTokens || len(userMessage) > maxTokens { + // FIXME: Error or just cut it off? + //return "", errors.New("Message too long for general usage. Max 10000 characters for system & user message") + + if len(systemMessage) > maxCharacters { + systemMessage = systemMessage[:maxCharacters] } - //log.Printf("[INFO] System message (find API documentation): %s", systemMessage) + if len(userMessage) > maxCharacters { + log.Printf("[WARNING] User message too long. Cutting off from %d to %d characters", len(userMessage), maxCharacters) + userMessage = userMessage[:maxCharacters] + } + //} + cnt := 0 openaiClient := openai.NewClient(os.Getenv("OPENAI_API_KEY")) - chatCompletion := openai.ChatCompletionRequest{ Model: model, Messages: []openai.ChatCompletionMessage{}, Temperature: 0.8, // A tiny bit of creativity - MaxTokens: 500, + MaxTokens: maxTokens, } if len(systemMessage) > 0 { @@ -1215,16 +1226,41 @@ func uploadParameterBase(ctx context.Context, orgId, appId, actionName, paramNam } func FixContentOutput(contentOutput string) string { - if strings.Contains(contentOutput, "```") { + if strings.Contains(contentOutput, "```json") { // Handle ```json start := strings.Index(contentOutput, "```json") end := strings.Index(contentOutput, "```") if start != -1 { end = strings.Index(contentOutput[start+7:], "```") + + // Shift it so the index is at the correct place + end = end + start+7 } + if start != -1 && end != -1 { - contentOutput = contentOutput[start+7 : end+7] + newend := end+7 + newstart := start+7 + + log.Printf("[INFO] Found ``` in content. Start: %d, end: %d", start, end) + + if newend > len(contentOutput) { + newend = end + } + + if newend > len(contentOutput) { + newend = len(contentOutput) + } + + if newstart > len(contentOutput) { + newstart = start + } + + if newstart > len(contentOutput) { + newstart = len(contentOutput) + } + + contentOutput = contentOutput[start+7 : newend] } } @@ -1233,6 +1269,7 @@ func FixContentOutput(contentOutput string) string { end := strings.Index(contentOutput[start+3:], "```") if start != -1 { end = strings.Index(contentOutput[start+3:], "```") + end = end + start+3 } if start != -1 && end != -1 { @@ -1266,7 +1303,16 @@ func AutofixAppLabels(app WorkflowApp, label string) WorkflowApp { return app } - log.Printf("[DEBUG] Running app fix with ONLY one label for app %s (%s) with %d actions: %s", app.Name, app.ID, len(app.Actions), label) + // Fix the label to be as it is in category (uppercase + spaces) + // fml, there is no consistency to casing + underscores, so we keep the new + //label = strings.ReplaceAll(strings.Title(strings.ToLower(label)), "_", " ") + + log.Printf("[INFO] Running app fix for label '%s' for app %s (%s) with %d actions", label, app.Name, app.ID, len(app.Actions)) + + // Just a reset, as Other doesn't really achieve anything directly + if len(app.Categories) > 0 && app.Categories[0] == "Other" { + app.Categories = []string{} + } // Check if the app has any actions foundCategory := AppCategory{} @@ -1291,9 +1337,8 @@ func AutofixAppLabels(app WorkflowApp, label string) WorkflowApp { } } + updatedIndex := -1 if len(foundCategory.ActionLabels) == 0 { - log.Printf("[DEBUG] No category found for app %s (%s). Checking based on input label, then using that category in app setup", app.Name, app.ID) - for _, category := range availableCategories { for _, actionLabel := range category.ActionLabels { if strings.ToLower(actionLabel) != strings.ToLower(label) { @@ -1305,20 +1350,82 @@ func AutofixAppLabels(app WorkflowApp, label string) WorkflowApp { break } } + + if len(foundCategory.Name) == 0 { + log.Printf("[DEBUG] No category found for app %s (%s). Checking based on input label, then using that category in app setup", app.Name, app.ID) + systemMessage := `Your goal is to find the correct CATEGORY for the app to be in. Synonyms are accepted, and you should be very critical to not make mistakes. If none match, don't add any. A synonym example can be something like: cases = alerts = issues = tasks, or messages = chats = communicate. If it exists, return {"success": true, "category": ""} where is replaced with the category found. If it does not exist, return {"success": false, "category": "Other"}. Output as only JSON."` + + categories := "" + for _, category := range availableCategories { + categories += fmt.Sprintf("%s,", category.Name) + } + + userMessage := fmt.Sprintf("The app name is '%s'. Available categories are: %s. Here are SOME actions it can do:\n", app.Name, strings.Trim(categories, ",")) + for cnt, action := range app.Actions { + userMessage += fmt.Sprintf("%s\n", action.Name) + if cnt > 25 { + break + } + } + + output, err := RunAiQuery(systemMessage, userMessage) + log.Printf("[DEBUG] Autocomplete output for category '%s' in '%s' (%d actions): %s", label, app.Name, len(app.Actions), output) + if err != nil { + log.Printf("[ERROR] Failed to run AI query in AutofixAppLabels for category with app %s (%s): %s", app.Name, app.ID, err) + return app + } + + type ActionStruct struct { + Category string `json:"category"` + } + + output = FixContentOutput(output) + actionStruct := ActionStruct{} + err = json.Unmarshal([]byte(output), &actionStruct) + if err != nil { + log.Printf("[ERROR] FAILED action mapping parsed output: %s", output) + } + + if len(actionStruct.Category) == 0 { + log.Printf("[ERROR] No category found for app %s (%s) based on label %s (1)", app.Name, app.ID, label) + return app + } + + app.Categories = append(app.Categories, actionStruct.Category) + + // Forces app to update + if len(app.Actions) > 0 { + updatedIndex = 0 + } + + for _, category := range availableCategories { + if category.Name != actionStruct.Category { + continue + } + + foundCategory = category + break + } + } } if len(foundCategory.ActionLabels) == 0 { + log.Printf("[ERROR] No category found for app %s (%s) based on label %s", app.Name, app.ID, label) return app } // FIXME: Run AI here to check based on the label which action may be matching - //log.Printf("[DEBUG] Found category %s for app %s (%s) based on app categories", foundCategory.Name, app.Name, app.ID) - systemMessage := fmt.Sprintf(`Find which action is most likely to be used based on the label '%s'. If any match, return their exact name and if none match, write "none" as the name. Return in the JSON format {"action": "action name"}`, label) - userMessage := "The available actions are as follows:\n" + + // Old attempts + //systemMessage := fmt.Sprintf(`Find which action is most likely to be used based on the label '%s'. If any match, return their exact name and if none match, write "none" as the name. Return in the JSON format {"action": "action name"}`, label) + //userMessage := "The available actions are as follows:\n" + + systemMessage := `Your goal is to find the correct action for a specific label if it exists. Synonyms are accepted, and you should be very critical to not make mistakes. A synonym example can be something like: cases = alerts = issues = tasks, or messages = chats = communicate. If it exists, return {"success": true, "action": ""} where is replaced with the action found. If it does not exist, return {"success": false, "action": ""}. Output as only JSON."` + userMessage := fmt.Sprintf("Out of the following actions, which action matches '%s'?\n", label) for _, action := range app.Actions { - userMessage += fmt.Sprintf("Action: %s\n", action.Name) + userMessage += fmt.Sprintf("%s\n", action.Name) } output, err := RunAiQuery(systemMessage, userMessage) @@ -1333,6 +1440,8 @@ func AutofixAppLabels(app WorkflowApp, label string) WorkflowApp { output = FixContentOutput(output) + log.Printf("[DEBUG] Autocomplete output for label '%s' in '%s' (%d actions): %s", label, app.Name, len(app.Actions), output) + actionStruct := ActionStruct{} err = json.Unmarshal([]byte(output), &actionStruct) if err != nil { @@ -1341,43 +1450,133 @@ func AutofixAppLabels(app WorkflowApp, label string) WorkflowApp { if len(actionStruct.Action) == 0 { log.Printf("[ERROR] No action found for app %s (%s) based on label %s (1)", app.Name, app.ID, label) - return app - } + //return app + } else { + newname := strings.Trim(strings.ToLower(strings.Replace(GetCorrectActionName(actionStruct.Action), " ", "_", -1)), " ") - //log.Printf("[DEBUG] Found action %s for app %s (%s) based on label %s", actionStruct.Action, app.Name, app.ID, label) + //log.Printf("[DEBUG] Looking for action: %s\n\n\n\n", newname) - updatedIndex := -1 - newname := strings.Trim(strings.ToLower(strings.Replace(actionStruct.Action, " ", "_", -1)), " ") - for actionIndex, action := range app.Actions { - if strings.Trim(strings.ToLower(strings.Replace(action.Name, " ", "_", -1)), " ") != newname { - continue - } + for actionIndex, action := range app.Actions { + searchName := strings.Trim(strings.ToLower(strings.Replace(GetCorrectActionName(action.Name), " ", "_", -1)), " ") + + // For some reason this doesn't find it properly + if searchName != newname { + continue + } + + log.Printf("[INFO] Found action %s in app %s based on label %s", action.Name, app.Name, label) + + // Avoid duplicates in case validation system fails + foundLabel := false + newLabels := []string{} + for _, categoryLabel := range action.CategoryLabel { + if strings.ToLower(categoryLabel) == "no label" { + continue + } + + newLabels = append(newLabels, categoryLabel) + if strings.ToLower(categoryLabel) == strings.ToLower(label) { + foundLabel = true + } + } - // Avoid duplicates in case validation system fails - foundLabel := false - for _, categoryLabel := range action.CategoryLabel { - if strings.ToLower(categoryLabel) != strings.ToLower(label) { - foundLabel = true + app.Actions[actionIndex].CategoryLabel = newLabels + if foundLabel { + log.Printf("[INFO] %s already has label '%s' in app %s (%s)", action.Name, label, app.Name, app.ID) break } - } - if foundLabel { + updatedIndex = actionIndex + app.Actions[actionIndex].CategoryLabel = append(app.Actions[actionIndex].CategoryLabel, label) + + log.Printf("[DEBUG] Adding label %s to action %s in app %s (%s). New labels: %#v", label, action.Name, app.Name, app.ID, app.Actions[actionIndex].CategoryLabel) break } - - updatedIndex = actionIndex - app.Actions[actionIndex].CategoryLabel = append(app.Actions[actionIndex].CategoryLabel, label) - - log.Printf("[DEBUG] Adding label %s to action %s in app %s (%s). New labels: %#v", label, action.Name, app.Name, app.ID, app.Actions[actionIndex].CategoryLabel) - break } // FIXME: Add the label to the OpenAPI action as well? if updatedIndex >= 0 { - go SetWorkflowAppDatastore(context.Background(), app, app.ID) + err = SetWorkflowAppDatastore(context.Background(), app, app.ID) + if err != nil { + log.Printf("[ERROR] Failed to set app datastore in AutofixAppLabels for app %s (%s): %s", app.Name, app.ID, err) + } + + //log.Printf("\n\n\n[WARNING] Updated app %s (%s) with label %s. SHOULD update OpenAPI action as well\n\n\n", app.Name, app.ID, label) + + // Find the OpenAPI version and update it too + openapiApp, err := GetOpenApiDatastore(context.Background(), app.ID) + if err != nil { + log.Printf("[ERROR] Failed to get openapi datastore in AutofixAppLabels for app %s (%s): %s", app.Name, app.ID, err) + return app + } + + swaggerLoader := openapi3.NewSwaggerLoader() + swaggerLoader.IsExternalRefsAllowed = true + openapi, err := swaggerLoader.LoadSwaggerFromData([]byte(openapiApp.Body)) + if err != nil { + log.Printf("[ERROR] Failed to unmarshal openapi in AutofixAppLabels for app %s (%s): %s", app.Name, app.ID, err) + return app + } + + + // Overwrite categories no matter what? + openapi.Info.Extensions["x-categories"] = app.Categories + + // Find the path + actionName := GetCorrectActionName(app.Actions[updatedIndex].Name) + changed := false + _ = openapi + log.Printf("OPENAPI, ACTIONNAME: %s", actionName) + for pathIndex, path := range openapi.Paths { + _ = pathIndex + + for method, operation := range path.Operations() { + if operation == nil { + continue + } + + correctName := strings.Replace(strings.ToLower(GetCorrectActionName(operation.Summary)), " ", "_", -1) + if correctName != actionName { + //log.Printf("[INFO] Skipping method %s with summary '%s' as it doesn't match action '%s'", method, correctName, actionName) + continue + } + + log.Printf("[INFO] Found method %s for action %s (OPENAPI) during label mapping for '%s' in app '%s'", method, app.Actions[updatedIndex].Name, label, app.Name) + if len(operation.Extensions) == 0 { + operation.Extensions["x-label"] = label + } else { + if _, found := operation.Extensions["x-label"]; !found { + operation.Extensions["x-label"] = label + } else { + // add to it with comma? + operation.Extensions["x-label"] = fmt.Sprintf("%s,%s", operation.Extensions["x-label"], label) + } + } + + changed = true + openapi.Paths[pathIndex].SetOperation(method, operation) + } + + if changed { + break + } + } + + if changed { + parsedOpenapi, err := openapi.MarshalJSON() + if err != nil { + log.Printf("[ERROR] Failed to marshal openapi in AutofixAppLabels for app %s (%s): %s", app.Name, app.ID, err) + } else { + openapiApp.Body = string(parsedOpenapi) + + log.Printf("[INFO] Updated openapi with new label for action %s in app %s", app.Actions[updatedIndex].Name, app.Name) + err = SetOpenApiDatastore(context.Background(), openapiApp.ID, openapiApp) + if err != nil { + log.Printf("[ERROR] Failed to set openapi datastore in AutofixAppLabels for app %s (%s): %s", app.Name, app.ID, err) + } + } + } - log.Printf("\n\n\n[WARNING] Updated app %s (%s) with label %s. SHOULD update OpenAPI action as well\n\n\n", app.Name, app.ID, label) } else { log.Printf("[ERROR] No action found for app %s (%s) based on label %s (2). GPT error most likely. Output: %s", app.Name, app.ID, label, output) } diff --git a/shared.go b/shared.go index 2aa06f6..88567fb 100755 --- a/shared.go +++ b/shared.go @@ -24032,6 +24032,14 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { return } + if len(value.AppName) == 0 && len(value.App) > 0 { + value.AppName = value.App + } + + if len(value.Label) == 0 && len(value.Action) > 0 { + value.Label = value.Action + } + log.Printf("[INFO] Running category-action '%s' in category '%s' with app %s for org %s (%s)", value.Label, value.Category, value.AppName, user.ActiveOrg.Name, user.ActiveOrg.Id) if len(value.Query) > 0 { @@ -24197,6 +24205,8 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { selectedCategory := AppCategory{} selectedAction := WorkflowAppAction{} + //RunAiQuery(systemMessage, userMessage) + availableLabels := []string{} for _, app := range newapps { if app.Name == "" || len(app.Categories) == 0 { @@ -24251,15 +24261,16 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { } else { // If we DONT have a category app already if app.ID == value.AppName || strings.ReplaceAll(strings.ToLower(app.Name), " ", "_") == value.AppName { - log.Printf("[DEBUG] Found app - checking label: %s vs %s (%s)", app.Name, value.AppName, app.ID) - selectedAction, selectedCategory, availableLabels = GetActionFromLabel(selectedApp, value.Label, true) + //log.Printf("[DEBUG] Found app - checking label: %s vs %s (%s)", app.Name, value.AppName, app.ID) + selectedAction, selectedCategory, availableLabels = GetActionFromLabel(ctx, selectedApp, value.Label, true) break + } else if selectedApp.ID == "" && len(value.AppName) > 0 && (strings.Contains(strings.ToLower(app.Name), strings.ToLower(value.AppName)) || strings.Contains(strings.ToLower(value.AppName), strings.ToLower(app.Name))) { selectedApp = app log.Printf("[WARNING] Set selected app to partial match %s (%s) for input %s", selectedApp.Name, selectedApp.ID, value.AppName) - selectedAction, selectedCategory, availableLabels = GetActionFromLabel(selectedApp, value.Label, true) + selectedAction, selectedCategory, availableLabels = GetActionFromLabel(ctx, selectedApp, value.Label, true) } } } @@ -24272,15 +24283,15 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("[ERROR] Failed getting app with ID or name '%s' in category action: %s", value.AppName, err) } else if err == nil && len(foundApp.ObjectID) > 0 { - log.Printf("\n\n[INFO] Found app %s (%s) for name %s in Algolia\n\n", foundApp.Name, foundApp.ObjectID, value.AppName) + log.Printf("[INFO] Found app %s (%s) for name %s in Algolia", foundApp.Name, foundApp.ObjectID, value.AppName) tmpApp, err := GetApp(ctx, foundApp.ObjectID, user, false) if err == nil { selectedApp = *tmpApp failed = false - log.Printf("[DEBUG] Got app %s with %d actions", selectedApp.Name, len(selectedApp.Actions)) - selectedAction, selectedCategory, availableLabels = GetActionFromLabel(selectedApp, value.Label, true) + //log.Printf("[DEBUG] Got app %s with %d actions", selectedApp.Name, len(selectedApp.Actions)) + selectedAction, selectedCategory, availableLabels = GetActionFromLabel(ctx, selectedApp, value.Label, true) } } else { log.Printf("[DEBUG] Found app with ID or name '%s' in Algolia: %#v", value.AppName, foundApp) @@ -24294,6 +24305,7 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { } } + // Section for mapping fields to automatic translation from previous runs fieldHash := "" fieldFileFound := false fieldFileContentMap := map[string]interface{}{} @@ -24353,11 +24365,11 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { } if len(selectedAction.Name) == 0 && value.Label != "discover_app" { - log.Printf("[WARNING] Couldn't find the label '%s' in app '%s'. If authentication/discover_app, ignore this.", value.Label, selectedApp.Name) + log.Printf("[WARNING] Couldn't find the label '%s' in app '%s'.", value.Label, selectedApp.Name) if value.Label != "app_authentication" && value.Label != "authenticate_app" && value.Label != "discover_app" { resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed finding action '%s' labeled in app '%s'. If this is wrong, please contact support@shuffler.io"}`, value.Label, strings.ReplaceAll(selectedApp.Name, "_", " ")))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "app_id": "%s", "reason": "Failed finding action '%s' labeled in app '%s'. If this is wrong, please suggest a label by finding the app in Shuffle, OR contact support@shuffler.io and we can help with labeling."}`, selectedApp.ID, value.Label, strings.ReplaceAll(selectedApp.Name, "_", " ")))) return } else { //log.Printf("[DEBUG] NOT sending back due to label %s", value.Label) @@ -24606,6 +24618,7 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { Action: "app_authentication", Category: discoveredCategory, Reason: fmt.Sprintf("Authenticate %s first.", selectedApp.Name), + Label: value.Label, Apps: []WorkflowApp{ selectedApp, }, @@ -24613,7 +24626,24 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { AvailableLabels: availableLabels, } - jsonFormatted, err := json.Marshal(structuredFeedback) + // Check for user agent including shufflepy + useragent := request.Header.Get("User-Agent") + if strings.Contains(strings.ToLower(useragent), "shufflepy") { + structuredFeedback.Apps = []WorkflowApp{} + + // Find current domain from the url + currentUrl := fmt.Sprintf("%s://%s", request.URL.Scheme, request.URL.Host) + if project.Environment == "cloud" { + currentUrl = "https://shuffler.io" + } + + // FIXME: Implement this. Uses org's auth + orgAuth := org.OrgAuth.Token + + structuredFeedback.Reason = fmt.Sprintf("Authenticate here: %s/appauth?app_id=%s&auth=%s", currentUrl, selectedApp.ID, orgAuth) + } + + jsonFormatted, err := json.MarshalIndent(structuredFeedback, "", " ") if err != nil { log.Printf("[WARNING] Failed marshalling structured feedback: %s", err) resp.WriteHeader(500) @@ -24621,6 +24651,9 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { return } + // Replace \u0026 with & + jsonFormatted = bytes.Replace(jsonFormatted, []byte("\\u0026"), []byte("&"), -1) + resp.WriteHeader(400) resp.Write(jsonFormatted) return @@ -25831,24 +25864,40 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { resp.Write(jsonParsed) } -func GetActionFromLabel(app WorkflowApp, label string, fixLabels bool) (WorkflowAppAction, AppCategory, []string) { - +func GetActionFromLabel(ctx context.Context, app WorkflowApp, label string, fixLabels bool) (WorkflowAppAction, AppCategory, []string) { availableLabels := []string{} selectedCategory := AppCategory{} selectedAction := WorkflowAppAction{} if len(app.ID) == 0 || len(app.Actions) == 0 { + log.Printf("[WARNING] No actions in app %s (%s) for label '%s'", app.Name, app.ID, label) return selectedAction, selectedCategory, availableLabels } + // Reload the app to be the proper one with updated actions instead + // of random cache issues + newApp, err := GetApp(ctx, app.ID, User{}, false) + if err != nil { + log.Printf("[WARNING] Failed getting app in category action: %s", err) + } else { + app = *newApp + } + categories := GetAllAppCategories() lowercaseLabel := strings.ReplaceAll(strings.ToLower(label), " ", "_") for _, action := range app.Actions { if len(action.CategoryLabel) == 0 { + //log.Printf("%s: %#v\n", action.Name, action.CategoryLabel) continue } + //log.Printf("FOUND LABELS: %s -> %#v\n", action.Name, action.CategoryLabel) + for labelIndex, _ := range action.CategoryLabel { + if strings.ReplaceAll(strings.ToLower(action.CategoryLabel[labelIndex]), " ", "_") == "no_label" { + continue + } + availableLabels = append(availableLabels, action.CategoryLabel[labelIndex]) actionCategory := strings.ReplaceAll(strings.ToLower(action.CategoryLabel[labelIndex]), " ", "_") @@ -25865,16 +25914,16 @@ func GetActionFromLabel(app WorkflowApp, label string, fixLabels bool) (Workflow } } + //log.Printf("\n\n[DEBUG] SELECTED: %#v\n\n", selectedAction) + // FIXME: If selectedAction isn't chosen, then we need to try to discover it in the app if len(selectedAction.ID) == 0 { if fixLabels == true { //log.Printf("\n\n[DEBUG] Action not found in app %s (%s) for label '%s'. Autodiscovering and updating the app!!!\n\n", app.Name, app.ID, label) // Make it FORCE look for a specific label if it exists, otherwise - // try any newApp := AutofixAppLabels(app, label) - - return GetActionFromLabel(newApp, label, false) + return GetActionFromLabel(ctx, newApp, label, false) } } @@ -27626,7 +27675,7 @@ func ValidateRequestOverload(resp http.ResponseWriter, request *http.Request) er return errors.New("Too many requests") } - log.Printf("[DEBUG] Adding request to list") + //log.Printf("[DEBUG] Adding request to list") newList = append(newList, userRequest) b, err := json.Marshal(newList) if err != nil { diff --git a/stats.go b/stats.go index 34d2a5e..9320d2b 100755 --- a/stats.go +++ b/stats.go @@ -3,6 +3,7 @@ package shuffle import ( "fmt" "log" + "time" "sort" "strings" @@ -310,6 +311,8 @@ func GetSpecificStats(resp http.ResponseWriter, request *http.Request) { return } + statsKey = strings.ToLower(strings.ReplaceAll(statsKey, " ", "_")) + user, err := HandleApiAuthentication(resp, request) if err != nil { log.Printf("[WARNING] Api authentication failed in get stats: %s", err) @@ -331,23 +334,44 @@ func GetSpecificStats(resp http.ResponseWriter, request *http.Request) { totalEntires := 0 totalValue := 0 statEntries := []AdditionalUseConfig{} - info.DailyStatistics = append(info.DailyStatistics, DailyStatistics{ + Date: time.Now(), Additions: info.Additions, }) + allStats := []string{} for _, daily := range info.DailyStatistics { for _, addition := range daily.Additions { - if addition.Key == statsKey { + if strings.ToLower(strings.ReplaceAll(addition.Key, " ", "_")) == statsKey { totalEntires++ totalValue += int(addition.Value) + + addition.Date = daily.Date statEntries = append(statEntries, addition) break } + + if !ArrayContains(allStats, addition.Key) { + allStats = append(allStats, addition.Key) + } } } + if len(statEntries) == 0 { + marshalledEntries, err := json.Marshal(allStats) + if err != nil { + log.Printf("[ERROR] Failed marshal in get org stats: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking data for org stats"}`))) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "key": "%s", "total": %d, "available_entries": %s, "entries": []}`, statsKey, totalValue, string(marshalledEntries)))) + return + } + marshalledEntries, err := json.Marshal(statEntries) if err != nil { log.Printf("[ERROR] Failed marshal in get org stats: %s", err) @@ -356,6 +380,7 @@ func GetSpecificStats(resp http.ResponseWriter, request *http.Request) { return } + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "key": "%s", "total": %d, "entries": %s}`, statsKey, totalValue, string(marshalledEntries)))) } @@ -370,17 +395,10 @@ func HandleGetStatistics(resp http.ResponseWriter, request *http.Request) { var statsKey string location := strings.Split(request.URL.String(), "/") if location[1] == "api" { + // Just falling back if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - orgId = location[4] - - if len(location) > 6 { - statsKey = location[6] + } else { + orgId = location[4] } } @@ -392,6 +410,10 @@ func HandleGetStatistics(resp http.ResponseWriter, request *http.Request) { return } + if len(orgId) == 0 { + orgId = user.ActiveOrg.Id + } + ctx := GetContext(request) org, err := GetOrg(ctx, orgId) if err != nil { diff --git a/structs.go b/structs.go index b5a60b5..c3f2217 100755 --- a/structs.go +++ b/structs.go @@ -449,6 +449,7 @@ type AdditionalUseConfig struct { Value int64 `json:"value" datastore:"value"` DailyValue int64 `json:"daily_value,omitempty" datastore:"daily_value"` + Date time.Time `json:"date,omitempty" datastore:"date"` } type ParsedOpenApi struct { @@ -3711,6 +3712,8 @@ type CategoryAction struct { SkipWorkflow bool `json:"skip_workflow"` // If true, it will not put it in a workflow, but instead just execute it SkipOutputTranslation bool `json:"skip_output_translation"` // If true, it will not translate the output to the default format for the label Environment string `json:"environment"` // The environment to use for the action (Orborus) + App string `jjson:"app"` // The app to use for the action (Orborus) + Action string `json:"action"` // The action to use for the action (Orborus) } type LabelStruct struct { @@ -3984,23 +3987,24 @@ type WorkflowSearchResult struct { // Used for the integrations API to work with AI well type StructuredCategoryAction struct { - Success bool `json:"success"` - Reason string `json:"reason"` - WorkflowId string `json:"workflow_id"` - ExecutionId string `json:"execution_id"` - Action string `json:"action"` + WorkflowId string `json:"workflow_id,omitempty"` + ExecutionId string `json:"execution_id,omitempty"` Label string `json:"label"` Category string `json:"category"` - Apps []WorkflowApp `json:"apps"` + Apps []WorkflowApp `json:"apps,omitempty"` - Result string `json:"result"` + Result string `json:"result,omitempty"` AvailableLabels []string `json:"available_labels"` - ThreadId string `json:"thread_id"` - RunId string `json:"run_id"` - MissingFields []string `json:"missing_fields"` + ThreadId string `json:"thread_id,omitempty"` + RunId string `json:"run_id,omitempty"` + MissingFields []string `json:"missing_fields,omitempty"` + + Translated bool `json:"translated,omitempty"` - Translated bool `json:"translated"` + Success bool `json:"success"` + Action string `json:"action"` + Reason string `json:"reason"` } type ModelLabelParameter struct {